diff --git a/.github/workflows/rocm_build.yml b/.github/workflows/rocm_build.yml index 37fe17b4ec..ee21e21a90 100644 --- a/.github/workflows/rocm_build.yml +++ b/.github/workflows/rocm_build.yml @@ -24,7 +24,7 @@ jobs: python: ['3.11'] torch_version: ['2.5.1'] toolkit_type: ['rocm'] - toolkit_short_version: ['6.1', '6.2'] + toolkit_short_version: ['6.1', '6.2', '6.3'] uses: ./.github/workflows/wheels_build.yml if: github.repository == 'rocm/xformers' diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index db6ba6572a..b15c04b126 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -32,7 +32,7 @@ jobs: # NOTE: Don't forget to update `upload_pt`'s matrix # when changing the CUDA/ROCM versions below! CU_VERSIONS = ['118', '121', '124'] - ROCM_VERSIONS = ["6.1"] # <- 6.0 broken in `manylinux_2_28` + ROCM_VERSIONS = ['6.1', '6.2', '6.3'] # <- 6.0 broken in `manylinux_2_28` PY_CU = list(itertools.product(PY_VERSIONS, CU_VERSIONS)) PY_ROCM = list(itertools.product(PY_VERSIONS, ROCM_VERSIONS)) print("Full matrix PY_CU", PY_CU) @@ -111,6 +111,8 @@ jobs: - cu121 - cu124 - rocm6.1 + - rocm6.2 + - rocm6.3 uses: ./.github/workflows/wheels_upload_s3.yml with: aws_role: "arn:aws:iam::749337293305:role/pytorch_bot_uploader_role" @@ -118,4 +120,3 @@ jobs: aws_s3_cp_extra_args: --acl public-read filter: "*torch2.5.1+${{ matrix.suffix }}*" execute: ${{ github.repository == 'facebookresearch/xformers' && github.ref_type == 'tag' }} - diff --git a/setup.py b/setup.py index 0a88185867..6f53403b08 100644 --- a/setup.py +++ b/setup.py @@ -522,14 +522,6 @@ def get_extensions(): elif torch.version.hip and ( torch.cuda.is_available() or os.getenv("HIP_ARCHITECTURES", "") != "" ): - disable_hd256_hip_fmha = os.getenv("DISABLE_HD256_HIP_FMHA", "0") - if disable_hd256_hip_fmha == "1": - source_hip_maxk_256 = [] - for ff in source_hip: - if ff.endswith("maxk_256.cpp"): - source_hip_maxk_256 += [ff] - source_hip = list(set(source_hip) - set(source_hip_maxk_256)) - rename_cpp_cu(source_hip) hip_version = get_hip_version(ROCM_HOME) @@ -549,8 +541,6 @@ def get_extensions(): ] generator_flag = [] - if disable_hd256_hip_fmha == "1": - generator_flag += ["-DFMHA_SUPPORT_MAX_HEADDIM_128=1"] cc_flag = ["-DBUILD_PYTHON_PACKAGE"] use_rtn_bf16_convert = os.getenv("ENABLE_HIP_FMHA_RTN_BF16_CONVERT", "0") @@ -575,6 +565,7 @@ def get_extensions(): "-DCK_TILE_FMHA_FWD_FAST_EXP2=1", "-fgpu-flush-denormals-to-zero", "-Werror", + "-Wc++11-narrowing", "-Woverloaded-virtual", "-mllvm", "-enable-post-misched=0", diff --git a/tests/test_mem_eff_attention.py b/tests/test_mem_eff_attention.py index 68b5c13446..f4022a4239 100644 --- a/tests/test_mem_eff_attention.py +++ b/tests/test_mem_eff_attention.py @@ -457,6 +457,16 @@ def test_forward(opFW_device_dtype_biasT_B_Mq_Mkv_H_K_Kv, packed, fmt, **kwargs) if fmt == "BMK" and not fmha.common._is_bias_type_supported_in_BMK(bias_type): pytest.skip("BMK incompatible with this bias") + if op is fmha.ck.FwOp: + if (k > 256 or kv > 256) and issubclass( + bias_type, + ( + fmha.attn_bias.PagedBlockDiagonalPaddedKeysMask, + fmha.attn_bias.PagedBlockDiagonalGappyKeysMask, + ), + ): + pytest.skip("ck.FwOp hdim-512 is not supported when Paged-KVCache is used!") + query, key, value, attn_bias = create_tensors( *opFW_device_dtype_biasT_B_Mq_Mkv_H_K_Kv, fmt="BMHK" if packed else fmt, @@ -545,6 +555,18 @@ def test_logsumexp(opFW_device_dtype_biasT_B_Mq_Mkv_H_K_Kv): kv, ) = opFW_device_dtype_biasT_B_Mq_Mkv_H_K_Kv + if op is fmha.ck.FwOp: + if issubclass( + bias_type, + ( + fmha.attn_bias.PagedBlockDiagonalPaddedKeysMask, + fmha.attn_bias.PagedBlockDiagonalGappyKeysMask, + ), + ): + pytest.skip( + "With ck.FwOp Paged-KVCache has some problem with forward training!" + ) + query, key, value, attn_bias = create_tensors( *opFW_device_dtype_biasT_B_Mq_Mkv_H_K_Kv, fmt="BMHK", diff --git a/third_party/composable_kernel_tiled b/third_party/composable_kernel_tiled index 4e076909b6..4cfb24feb6 160000 --- a/third_party/composable_kernel_tiled +++ b/third_party/composable_kernel_tiled @@ -1 +1 @@ -Subproject commit 4e076909b6c1e1404d9ff5dc0e71e3be1c06569e +Subproject commit 4cfb24feb67602d38b60a1568492c6313bf25a82 diff --git a/xformers/benchmarks/benchmark_attn_decoding.py b/xformers/benchmarks/benchmark_attn_decoding.py index f78fa9806c..f5dfd61e96 100644 --- a/xformers/benchmarks/benchmark_attn_decoding.py +++ b/xformers/benchmarks/benchmark_attn_decoding.py @@ -168,6 +168,73 @@ class AttentionDecodingCUTLASS(AttentionDecodingBase): class AttentionDecodingCK(AttentionDecodingBase): OP = xops.fmha.ck.FwOp + def __init__( + self, + B: int, + Mq: int, + Mkv: int, + Hq: int, + Hkv: int, + K: int, + bw: bool, + attn_bias_type, + ) -> None: + dtype = torch.float16 + torch.manual_seed(10) + self.sub_label = ( + f"B={B} Mq={Mq} Mkv={Mkv} Hq={Hq} Hkv={Hkv} K={K} TotalBytes=" + f"{((B * Mkv * Hkv * K * 2) + (B * Mq * Hq * K) + (B * Mq * Hq * K)) * 2}" + ) + self.label = "attn_decoding" + self.shapes = (B, Mq, Mkv, Hq, Hkv, K) + + assert Hkv <= Hq + assert Hq % Hkv == 0 + self.q = torch.randn( + [B, Mq, Hkv, Hq // Hkv, K], device="cuda", dtype=dtype, requires_grad=bw + ) + self.k = torch.randn( + [B, Mkv, Hkv, 1, K], device="cuda", dtype=dtype, requires_grad=bw + ).expand(-1, -1, -1, Hq // Hkv, -1) + self.v = torch.randn( + [B, Mkv, Hkv, 1, K], device="cuda", dtype=dtype, requires_grad=bw + ).expand(-1, -1, -1, Hq // Hkv, -1) + + if Hq == Hkv: + self.q = self.q[:, :, :, 0] + self.k = self.k[:, :, :, 0] + self.v = self.v[:, :, :, 0] + + self.attn_bias = create_attn_bias( + attn_bias_type, + batch_size=B, + num_heads=Hq, + num_heads_groups=Hq // Hkv, + q_len=Mq, + kv_len=Mkv, + dtype=dtype, + device=device, + requires_grad=False, + fmt="BMHK", + op=self.OP, + ) + + if isinstance( + self.attn_bias, + xops.fmha.attn_bias.BlockDiagonalCausalWithOffsetPaddedKeysMask, + ): + self.q = self.q.view(1, -1, *self.q.shape[2:]) + self.k = self.k.view(1, -1, *self.k.shape[2:]) + self.v = self.v.view(1, -1, *self.v.shape[2:]) + + if hasattr(self.OP, "not_supported_reasons"): + inp = xops.fmha.Inputs( + query=self.q, key=self.k, value=self.v, attn_bias=self.attn_bias + ) + not_supported_reasons = self.OP.not_supported_reasons(inp) + if not_supported_reasons: + raise NotSupportedInputError(not_supported_reasons) + class AttentionDecodingCKDecoder(AttentionDecodingBase): OP = xops.fmha.ck_decoder.FwOp diff --git a/xformers/csrc/attention/hip_decoder/attention_forward_splitk.cpp b/xformers/csrc/attention/hip_decoder/attention_forward_splitk.cpp index 647e540d37..553bd81305 100644 --- a/xformers/csrc/attention/hip_decoder/attention_forward_splitk.cpp +++ b/xformers/csrc/attention/hip_decoder/attention_forward_splitk.cpp @@ -8,7 +8,7 @@ #include #include -#include "ck_attention_forward_decoder_splitk.h" +#include "ck_tile_attention_forward_decoder_splitk.h" namespace { constexpr int32_t kThreadsPerWavefront = 64; @@ -31,12 +31,12 @@ struct c10_to_data_t { template <> struct c10_to_data_t { - using type = ck::half_t; + using type = ck_tile::fp16_t; }; template <> struct c10_to_data_t { - using type = ck::bhalf_t; + using type = ck_tile::bf16_t; }; } // namespace diff --git a/xformers/csrc/attention/hip_decoder/ck_attention_forward_decoder_splitk.h b/xformers/csrc/attention/hip_decoder/ck_tile_attention_forward_decoder_splitk.h similarity index 90% rename from xformers/csrc/attention/hip_decoder/ck_attention_forward_decoder_splitk.h rename to xformers/csrc/attention/hip_decoder/ck_tile_attention_forward_decoder_splitk.h index 5389affacc..52863accd0 100644 --- a/xformers/csrc/attention/hip_decoder/ck_attention_forward_decoder_splitk.h +++ b/xformers/csrc/attention/hip_decoder/ck_tile_attention_forward_decoder_splitk.h @@ -1,17 +1,15 @@ #pragma once -#include -#include +#include -#include "ck_attention_inner_product.h" -#include "ck_attention_math_ext.h" +#include "ck_tile_attention_inner_product.h" namespace { template -__device__ typename ck::vector_type::type scalar_scale_acc( - typename ck::vector_type::type acc, - typename ck::vector_type::type a, +__device__ ck_tile::ext_vector_t scalar_scale_acc( + ck_tile::ext_vector_t acc, + ck_tile::ext_vector_t a, float b) { union { decltype(acc) vec; @@ -24,7 +22,7 @@ __device__ typename ck::vector_type::type scalar_scale_acc( #pragma unroll for (int32_t i = 0; i < vec_size; ++i) { - acc_u.arr[i] += ck::type_convert(a_u.arr[i]) * b; + acc_u.arr[i] += ck_tile::type_convert(a_u.arr[i]) * b; } return acc_u.vec; @@ -99,8 +97,8 @@ struct ForwardDecoderSplitKReduceKernelImpl { const int32_t h = blockIdx.x % arg.Q_size_h; using data_t = scalar_t; - using data_vec_t = typename ck::vector_type::type; - using compute_vec_t = typename ck::vector_type::type; + using data_vec_t = ck_tile::ext_vector_t; + using compute_vec_t = ck_tile::ext_vector_t; union { data_vec_t vec; @@ -129,7 +127,7 @@ struct ForwardDecoderSplitKReduceKernelImpl { } compute_t global_sumexp = 0; - compute_t global_max = ck::NumericLimits::Lowest(); + compute_t global_max = ck_tile::numeric::lowest(); for (int32_t split_idx = 0; split_idx < arg.split_k; ++split_idx) { load_v( @@ -141,7 +139,7 @@ struct ForwardDecoderSplitKReduceKernelImpl { #pragma unroll for (int32_t i = 0; i < vec_size; ++i) { O_split_compute.arr[i] = - ck::type_convert(O_split_data.arr[i]); + ck_tile::type_convert(O_split_data.arr[i]); } compute_t local_max = *(arg.split_max + blockIdx.x * arg.split_k + split_idx); @@ -150,7 +148,7 @@ struct ForwardDecoderSplitKReduceKernelImpl { compute_t log_alpha = -std::abs(local_max - global_max); compute_t alpha = - isnan(log_alpha) ? compute_t{1.} : ck::math::exp(log_alpha); + ck_tile::isnan(log_alpha) ? compute_t{1.} : ck_tile::exp(log_alpha); bool pick_new = local_max < global_max; compute_t pick_current_coef = pick_new ? 1. : alpha; @@ -160,12 +158,13 @@ struct ForwardDecoderSplitKReduceKernelImpl { pick_current_coef * global_sumexp + pick_new_coef * local_sumexp; global_O_compute.vec = pick_current_coef * global_O_compute.vec + pick_new_coef * O_split_compute.vec; - global_max = ck::math::max(local_max, global_max); + global_max = ck_tile::max(local_max, global_max); } global_O_compute.vec /= global_sumexp; #pragma unroll for (int32_t i = 0; i < vec_size; ++i) { - global_O_data.arr[i] = ck::type_convert(global_O_compute.arr[i]); + global_O_data.arr[i] = + ck_tile::type_convert(global_O_compute.arr[i]); } store_v( arg.O + b * arg.XQ_stride_b + m * arg.XQ_stride_m + @@ -223,8 +222,11 @@ struct ForwardDecoderSplitKAttnKernelImpl { const auto* __restrict__ cache_V_base = arg.cache_V + cache_KV_base_offset; using data_t = scalar_t; - using data_vec_t = typename ck::vector_type::type; - using compute_vec_t = typename ck::vector_type::type; + using data_vec_t = std::conditional_t< + vec_size == 1, + data_t, + ck_tile::ext_vector_t>; + using compute_vec_t = ck_tile::ext_vector_t; const bool lane_active_for_io = lane_idx * vec_size < arg.Q_size_k; @@ -237,7 +239,7 @@ struct ForwardDecoderSplitKAttnKernelImpl { load_v(q_, lane_idx, &q_thread); } - compute_t max_qk_acc = ck::NumericLimits::Lowest(); + compute_t max_qk_acc = ck_tile::numeric::lowest(); // Compute S[0:t_max] = // ``` @@ -279,12 +281,12 @@ struct ForwardDecoderSplitKAttnKernelImpl { #pragma unroll n_loop_unroll for (auto ttt = 0; ttt < n_loop_unroll; ++ttt) { compute_t qk_acc = 0; - ck::inner_product( + ck_tile::inner_product( q_thread, k_loads[ttt], qk_acc); qk_acc *= arg.qk_scale; qk_acc = wavefrontReduce(qk_acc, [](auto a, auto b) { return a + b; }); - max_qk_acc = ck::math::max(qk_acc, max_qk_acc); + max_qk_acc = ck_tile::max(qk_acc, max_qk_acc); if (lane_idx == 0) { smem[tt + ttt - n_unrolled_loops * dtt * split_idx] = qk_acc; } @@ -308,13 +310,13 @@ struct ForwardDecoderSplitKAttnKernelImpl { compute_t qk_acc = 0; const int32_t t = tt + ttt; if (t < t_max) { - ck::inner_product( + ck_tile::inner_product( q_thread, k_loads[ttt], qk_acc); qk_acc *= arg.qk_scale; qk_acc = wavefrontReduce(qk_acc, [](auto a, auto b) { return a + b; }); - max_qk_acc = ck::math::max(qk_acc, max_qk_acc); + max_qk_acc = ck_tile::max(qk_acc, max_qk_acc); // write accumulated sums to smem. if (lane_idx == 0) { @@ -331,7 +333,7 @@ struct ForwardDecoderSplitKAttnKernelImpl { } __syncthreads(); if (lane_idx < wavefronts_per_block) { - max_qk_acc = ck::math::max(max_qk_acc, smem[KV_M_MAX + lane_idx]); + max_qk_acc = ck_tile::max(max_qk_acc, smem[KV_M_MAX + lane_idx]); } // shared across all threads in block max_qk_acc = wavefrontReduce( @@ -350,7 +352,7 @@ struct ForwardDecoderSplitKAttnKernelImpl { : t_max; for (int32_t t = t_low + thread_linear_idx; t < t_high; t += threads_per_block) { - const auto s = ck::math::exp(smem[t - t_low] - max_qk_acc); + const auto s = ck_tile::exp(smem[t - t_low] - max_qk_acc); softmax_denominator += s; smem[t - t_low] = s; } @@ -445,7 +447,7 @@ struct ForwardDecoderSplitKAttnKernelImpl { } bf_r; #pragma unroll for (int32_t i = 0; i < vec_size; ++i) { - bf_r.arr[i] = ck::type_convert(r.arr[i]); + bf_r.arr[i] = ck_tile::type_convert(r.arr[i]); } // write output row O[b][m][g][h][:] data_t* __restrict__ o_ = diff --git a/xformers/csrc/attention/hip_decoder/ck_tile_attention_inner_product.h b/xformers/csrc/attention/hip_decoder/ck_tile_attention_inner_product.h new file mode 100644 index 0000000000..39350789bf --- /dev/null +++ b/xformers/csrc/attention/hip_decoder/ck_tile_attention_inner_product.h @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2023, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ +#pragma once + +#include + +namespace ck_tile { + +template +__device__ void inner_product(const TA& a, const TB& b, TC& c); + +template <> +__device__ void inner_product( + const float& a, + const float& b, + float& c) { +#if CK_USE_AMD_V_MAC_INLINE_ASM && defined(CK_USE_AMD_V_MAC_F32) + asm volatile( + "\n \ + v_mac_f32 %0, %1, %2 \n \ + " + : "=v"(c) + : "v"(a), "v"(b), "0"(c)); +#elif CK_USE_AMD_V_MAC_INLINE_ASM && defined(CK_USE_AMD_V_FMAC_F32) + asm volatile( + "\n \ + v_fmac_f32 %0, %1, %2 \n \ + " + : "=v"(c) + : "v"(a), "v"(b), "0"(c)); +#else + c += a * b; +#endif +} + +template <> +__device__ void inner_product( + const fp32x2_t& a, + const fp32x2_t& b, + float& c) { + inner_product(a[0], b[0], c); + inner_product(a[1], b[1], c); +} + +template <> +__device__ void inner_product( + const fp32x4_t& a, + const fp32x4_t& b, + float& c) { + inner_product(a[0], b[0], c); + inner_product(a[1], b[1], c); + inner_product(a[2], b[2], c); + inner_product(a[3], b[3], c); +} + +template <> +__device__ void inner_product( + const bf16_t& a, + const bf16_t& b, + float& c) { + inner_product(type_convert(a), type_convert(b), c); +} + +template <> +__device__ void inner_product( + const fp16_t& a, + const fp16_t& b, + float& c) { + inner_product(type_convert(a), type_convert(b), c); +} + +template <> +__device__ void inner_product( + const fp16x2_t& a, + const fp16x2_t& b, + float& c) { +#if defined(CK_USE_AMD_V_DOT2_F32_F16) +#if CK_USE_AMD_V_DOT_INLINE_ASM + // Use 3 x s_nop to avoid hazard (mi200 cdna2 isa page 47 + // https://www.amd.com/system/files/TechDocs/instinct-mi200-cdna2-instruction-set-architecture.pdf + // ) s_nop with parameter 2 is equal to 3 x s_nop + asm volatile( + "\n \ + v_dot2_f32_f16 %0, %1, %2, %0\n \ + s_nop 2 \n \ + " + : "=v"(c) + : "v"(a), "v"(b), "0"(c)); +#else + c = __builtin_amdgcn_fdot2(a, b, c, false); +#endif +#else + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); +#endif +} + +template <> +__device__ void inner_product( + const fp16x4_t& a, + const fp16x4_t& b, + float& c) { + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); + c += type_convert(a[2]) * type_convert(b[2]); + c += type_convert(a[3]) * type_convert(b[3]); +} + +template <> +__device__ void inner_product( + const fp16x8_t& a, + const fp16x8_t& b, + float& c) { + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); + c += type_convert(a[2]) * type_convert(b[2]); + c += type_convert(a[3]) * type_convert(b[3]); + c += type_convert(a[4]) * type_convert(b[4]); + c += type_convert(a[5]) * type_convert(b[5]); + c += type_convert(a[6]) * type_convert(b[6]); + c += type_convert(a[7]) * type_convert(b[7]); +} + +template <> +__device__ void inner_product( + const bf16x2_t& a, + const bf16x2_t& b, + float& c) { + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); +} + +template <> +__device__ void inner_product( + const bf16x4_t& a, + const bf16x4_t& b, + float& c) { + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); + c += type_convert(a[2]) * type_convert(b[2]); + c += type_convert(a[3]) * type_convert(b[3]); +} + +template <> +__device__ void inner_product( + const int8_t& a, + const int8_t& b, + int32_t& c) { + c += type_convert(a) * type_convert(b); +} + +template <> +__device__ void inner_product( + const int8x2_t& a, + const int8x2_t& b, + int32_t& c) { + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); +} + +template <> +__device__ void inner_product( + const int8x4_t& a, + const int8x4_t& b, + int32_t& c) { +#if defined(CK_USE_AMD_V_DOT4_I32_I8) +#if CK_USE_AMD_V_DOT_INLINE_ASM + // Use 3 x s_nop to avoid hazard (mi200 cdna2 isa page 47 + // https://www.amd.com/system/files/TechDocs/instinct-mi200-cdna2-instruction-set-architecture.pdf + // ) s_nop with parameter 2 is equal to 3 x s_nop + asm volatile( + "\n \ + v_dot4_i32_i8 %0, %1, %2, %0\n \ + s_nop 2 \n \ + " + : "=v"(c) + : "v"(bit_cast(a)), "v"(bit_cast(b)), "0"(c)); +#else + c = __builtin_amdgcn_sdot4( + bit_cast(a), bit_cast(b), c, false); +#endif +#elif defined(CK_USE_AMD_V_DOT4_I32_I8_GFX11) + c = __builtin_amdgcn_sudot4( + true, bit_cast(a), true, bit_cast(b), c, false); +#else + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); + c += type_convert(a[2]) * type_convert(b[2]); + c += type_convert(a[3]) * type_convert(b[3]); +#endif +} + +template <> +__device__ void inner_product( + const int8x8_t& a, + const int8x8_t& b, + int32_t& c) { + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); + c += type_convert(a[2]) * type_convert(b[2]); + c += type_convert(a[3]) * type_convert(b[3]); + c += type_convert(a[4]) * type_convert(b[4]); + c += type_convert(a[5]) * type_convert(b[5]); + c += type_convert(a[6]) * type_convert(b[6]); + c += type_convert(a[7]) * type_convert(b[7]); +} + +template <> +__device__ void inner_product( + const int8x16_t& a, + const int8x16_t& b, + int32_t& c) { + c += type_convert(a[0]) * type_convert(b[0]); + c += type_convert(a[1]) * type_convert(b[1]); + c += type_convert(a[2]) * type_convert(b[2]); + c += type_convert(a[3]) * type_convert(b[3]); + c += type_convert(a[4]) * type_convert(b[4]); + c += type_convert(a[5]) * type_convert(b[5]); + c += type_convert(a[6]) * type_convert(b[6]); + c += type_convert(a[7]) * type_convert(b[7]); + c += type_convert(a[8]) * type_convert(b[8]); + c += type_convert(a[9]) * type_convert(b[9]); + c += type_convert(a[10]) * type_convert(b[10]); + c += type_convert(a[11]) * type_convert(b[11]); + c += type_convert(a[12]) * type_convert(b[12]); + c += type_convert(a[13]) * type_convert(b[13]); + c += type_convert(a[14]) * type_convert(b[14]); + c += type_convert(a[15]) * type_convert(b[15]); +} + +} // namespace ck_tile diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward.h index a79887c55b..434e80a084 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward.h @@ -23,24 +23,31 @@ template < void run_batched_forward_mask_bias_dropout_dispatch( BatchedForwardParams& param, hipStream_t stream) { - // currently split-kv implementation does not support dropout + // currently split-kv implementation does not support: + // (*) dropout + // (*) head dimension > 256 if constexpr (!kHasDropout) { - if (param.use_split_kv) { - if (use_splitkv_smallq(param.M, std::max(param.K, param.Kv))) { - batched_forward_splitkv_smallq_mask_bias_dropout_dispatch< - ScalarType, - kHasMask, - kHasBias, - MaxK>::Run(param, stream); - } else { - FMHA_FWD_SEQLEN_Q_SWITCH(param.M, MaxSeqlenQ, [&] { - batched_forward_splitkv_mask_bias_dropout_dispatch< + if (param.use_split_kv && MaxK <= 256) { + if constexpr (MaxK <= 256) { + if (use_splitkv_smallq(param.M, std::max(param.K, param.Kv))) { + batched_forward_splitkv_smallq_mask_bias_dropout_dispatch< ScalarType, kHasMask, kHasBias, - MaxK, - MaxSeqlenQ>::Run(param, stream); - }); + MaxK>::Run(param, stream); + } else { + FMHA_FWD_SEQLEN_Q_SWITCH(param.M, MaxSeqlenQ, [&] { + batched_forward_splitkv_mask_bias_dropout_dispatch< + ScalarType, + kHasMask, + kHasBias, + MaxK, + MaxSeqlenQ>::Run(param, stream); + }); + } + } else { + // Unreachable. Do not instantiate split-kv pipelines with head + // dimension > 256 } } else { if (get_fmha_fwd_mtile(param.B, param.Hq, param.M) == 128) diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_dispatch.h index 6fdd1c6bb5..34a38bab98 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_dispatch.h @@ -47,7 +47,7 @@ struct batched_forward_mask_bias_dropout_dispatch { using FmhaFwdShape_ = typename FmhaFwdShape::Type; constexpr ck_tile::index_t occupancy = - (MaxK == 64) ? 3 : ((MaxK == 256) ? 1 : 2); + (MaxK == 64) ? 3 : ((MaxK >= 256) ? 1 : 2); constexpr auto kBiasEnum = kHasBias ? ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS @@ -89,8 +89,10 @@ struct batched_forward_mask_bias_dropout_dispatch { using FmhaPipelineProblem = FmhaPipelineProblemTemp; - using FmhaFwdPipeline_ = - ck_tile::BlockFmhaPipelineQRKSVS; + using FmhaFwdPipeline_ = std::conditional_t< + MaxK <= 256, + ck_tile::BlockFmhaPipelineQRKSVS, + ck_tile::BlockFmhaPipelineQSKSVS>; using FmhaFwdEpilogue_ = ck_tile::Default2DEpilogue; if (param.num_kv_splits > 1) { @@ -305,7 +306,7 @@ struct batched_forward_splitkv_mask_bias_dropout_dispatch { }(); dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( - param.B, param.Hq, param.M, param.Kv, param.num_kv_splits); + param.B, param.Hq, param.Hkv, param.M, param.Kv, param.num_kv_splits); constexpr dim3 kBlockSize = FmhaFwdSplitKVKernel::BlockSize(); constexpr ck_tile::index_t kBlockPerCu = FmhaFwdSplitKVKernel::kBlockPerCu; diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_splitkv_smallq_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_splitkv_smallq_dispatch.h index 806a507fd2..c615838cc2 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_splitkv_smallq_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_forward_splitkv_smallq_dispatch.h @@ -98,6 +98,7 @@ struct batched_forward_splitkv_smallq_mask_bias_dropout_dispatch { false, // kDoFp8StaticQuant place-holder false, // kIsPagedKV kHasUnevenSplits, + false, // kMergeNumHeadGroupsSeqLenQ occupancy>; if (param.num_kv_splits > 1) { @@ -304,7 +305,7 @@ struct batched_forward_splitkv_smallq_mask_bias_dropout_dispatch { }(); dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( - param.B, param.Hq, param.M, param.Kv, param.num_kv_splits); + param.B, param.Hq, param.Hkv, param.M, param.Kv, param.num_kv_splits); constexpr dim3 kBlockSize = FmhaFwdSplitKVKernel::BlockSize(); constexpr ck_tile::index_t kBlockPerCu = FmhaFwdSplitKVKernel::kBlockPerCu; diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer.h index 06b3b66232..77ec5f9663 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer.h @@ -23,24 +23,31 @@ template < void run_batched_infer_mask_bias_dropout_dispatch( BatchedForwardParams& param, hipStream_t stream) { - // currently split-kv implementation does not support dropout + // currently split-kv implementation does not support: + // (*) dropout + // (*) head dimension > 256 if constexpr (!kHasDropout) { - if (param.use_split_kv) { - if (use_splitkv_smallq(param.M, std::max(param.K, param.Kv))) { - batched_infer_splitkv_smallq_mask_bias_dropout_dispatch< - ScalarType, - kHasMask, - kHasBias, - MaxK>::Run(param, stream); - } else { - FMHA_FWD_SEQLEN_Q_SWITCH(param.M, MaxSeqlenQ, [&] { - batched_infer_splitkv_mask_bias_dropout_dispatch< + if (param.use_split_kv && MaxK <= 256) { + if constexpr (MaxK <= 256) { + if (use_splitkv_smallq(param.M, std::max(param.K, param.Kv))) { + batched_infer_splitkv_smallq_mask_bias_dropout_dispatch< ScalarType, kHasMask, kHasBias, - MaxK, - MaxSeqlenQ>::Run(param, stream); - }); + MaxK>::Run(param, stream); + } else { + FMHA_FWD_SEQLEN_Q_SWITCH(param.M, MaxSeqlenQ, [&] { + batched_infer_splitkv_mask_bias_dropout_dispatch< + ScalarType, + kHasMask, + kHasBias, + MaxK, + MaxSeqlenQ>::Run(param, stream); + }); + } + } else { + // Unreachable. Do not instantiate split-kv pipelines with head + // dimension > 256 } } else { if (get_fmha_fwd_mtile(param.B, param.Hq, param.M) == 128) diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_dispatch.h index ed49eac35e..02fc125b7e 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_dispatch.h @@ -48,7 +48,7 @@ struct batched_infer_mask_bias_dropout_dispatch { using FmhaShape = typename FmhaFwdShape::Type; constexpr ck_tile::index_t occupancy = - (MaxK == 64) ? 3 : ((MaxK == 256) ? 1 : 2); + (MaxK == 64) ? 3 : ((MaxK >= 256) ? 1 : 2); constexpr auto kBiasEnum = kHasBias ? ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS @@ -92,8 +92,10 @@ struct batched_infer_mask_bias_dropout_dispatch { using FmhaPipelineProblem = FmhaPipelineProblemTemp; - using FmhaPipeline = - ck_tile::BlockFmhaPipelineQRKSVS; + using FmhaPipeline = std::conditional_t< + MaxK <= 256, + ck_tile::BlockFmhaPipelineQRKSVS, + ck_tile::BlockFmhaPipelineQSKSVS>; using FmhaEpilogue = ck_tile::Default2DEpilogue; - using FmhaPipeline = - ck_tile::BlockFmhaPipelineQRKSVSAsync; + using FmhaPipeline = std::conditional_t< + MaxK <= 256, + ck_tile::BlockFmhaPipelineQRKSVSAsync, + ck_tile::BlockFmhaPipelineQSKSVS>; using FmhaEpilogue = ck_tile::Default2DEpilogue; using ODataType = @@ -136,6 +137,7 @@ struct batched_infer_splitkv_mask_bias_dropout_dispatch { false, // kDoFp8StaticQuant place-holder false, // kIsPagedKV kHasUnevenSplits, + false, // kMergeNumHeadGroupsSeqLenQ occupancy>; using ODataType = @@ -318,7 +320,7 @@ struct batched_infer_splitkv_mask_bias_dropout_dispatch { }(); dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( - param.B, param.Hq, param.M, param.Kv, param.num_kv_splits); + param.B, param.Hq, param.Hkv, param.M, param.Kv, param.num_kv_splits); constexpr dim3 kBlockSize = FmhaFwdSplitKVKernel::BlockSize(); constexpr ck_tile::index_t kBlockPerCu = FmhaFwdSplitKVKernel::kBlockPerCu; diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_splitkv_smallq_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_splitkv_smallq_dispatch.h index 9ef7c24424..df9ce0016e 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_splitkv_smallq_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_batched_infer_splitkv_smallq_dispatch.h @@ -77,90 +77,181 @@ struct batched_infer_splitkv_smallq_mask_bias_dropout_dispatch { const bool has_uneven_splits = !(param.N % (param.num_kv_splits * FmhaTileShape::kN0) == 0); - BOOL_SWITCH_3( - pad_seqlen_q, - kPadSeqLenQ, - pad_headdim, - kPadHeadDim, - has_uneven_splits, - kHasUnevenSplits, - [&] { - constexpr bool kPadSeqLenK = kHasUnevenSplits ? true : false; - - if (param.num_kv_splits > 1) { - using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< - kPadSeqLenQ, - kPadSeqLenK, - kPadHeadDim, // kPadHeadDimQ, - kPadHeadDim, // kPadHeadDimV, - kBiasEnum, - false, // kHasBiasGrad place-holder - true, // kStoreLSE - false, // kDoFp8StaticQuant place-holder - false, // kIsPagedKV - kHasUnevenSplits, - occupancy>; - - using ODataType = - typename FmhaFwdTypeConfig::OaccDataType; - using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< - FmhaTraits, - FmhaMask, - ODataType>; - - using FmhaPipeline = - ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< - FmhaPipelineProblem>; - - using FmhaEpilogue = - ck_tile::Default2DEpilogue::OaccDataType, - ODataType, - false, - false>>; - - using FmhaKernel = - ck_tile::FmhaFwdSplitKVKernel; - - RunWithFwdSplitKVKernel(param, stream); - } else { - using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< - kPadSeqLenQ, - kPadSeqLenK, - kPadHeadDim, // kPadHeadDimQ, - kPadHeadDim, // kPadHeadDimV, - kBiasEnum, - false, // kHasBiasGrad place-holder - false, // kStoreLSE - false, // kDoFp8StaticQuant place-holder - false, // kIsPagedKV - kHasUnevenSplits, - occupancy>; - - using ODataType = - typename FmhaFwdTypeConfig::ODataType; - using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< - FmhaTraits, - FmhaMask, - ODataType>; - - using FmhaPipeline = - ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< - FmhaPipelineProblem>; - - using FmhaEpilogue = - ck_tile::Default2DEpilogue::OaccDataType, - ODataType, - false, - false>>; - - using FmhaKernel = - ck_tile::FmhaFwdSplitKVKernel; - - RunWithFwdSplitKVKernel(param, stream); - } - }); + // indicates to the splitkv kernel whether should it merge Hq/Hkv with + // seqlen_q + const bool merge_nhead_groups_seqlen_q = + ((param.M == 1) && (param.Hq > param.Hkv) && !kHasBias); + + if (merge_nhead_groups_seqlen_q) { + using FmhaMaskNone = ck_tile::SimplifiedGenericAttentionMask; + BOOL_SWITCH_2( + pad_headdim, kPadHeadDim, has_uneven_splits, kHasUnevenSplits, [&] { + constexpr bool kPadSeqLenK = kHasUnevenSplits ? true : false; + + if (param.num_kv_splits > 1) { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + true, // kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDim, // kPadHeadDimQ, + kPadHeadDim, // kPadHeadDimV, + ck_tile::BlockAttentionBiasEnum::NO_BIAS, + false, // kHasBiasGrad place-holder + true, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + false, // kIsPagedKV + kHasUnevenSplits, + true, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::OaccDataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMaskNone, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } else { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + true, // kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDim, // kPadHeadDimQ, + kPadHeadDim, // kPadHeadDimV, + ck_tile::BlockAttentionBiasEnum::NO_BIAS, + false, // kHasBiasGrad place-holder + false, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + false, // kIsPagedKV + kHasUnevenSplits, + true, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::ODataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMaskNone, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } + }); + } else { + BOOL_SWITCH_3( + pad_seqlen_q, + kPadSeqLenQ, + pad_headdim, + kPadHeadDim, + has_uneven_splits, + kHasUnevenSplits, + [&] { + constexpr bool kPadSeqLenK = kHasUnevenSplits ? true : false; + + if (param.num_kv_splits > 1) { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDim, // kPadHeadDimQ, + kPadHeadDim, // kPadHeadDimV, + kBiasEnum, + false, // kHasBiasGrad place-holder + true, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + false, // kIsPagedKV + kHasUnevenSplits, + false, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::OaccDataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMask, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } else { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDim, // kPadHeadDimQ, + kPadHeadDim, // kPadHeadDimV, + kBiasEnum, + false, // kHasBiasGrad place-holder + false, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + false, // kIsPagedKV + kHasUnevenSplits, + false, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::ODataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMask, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } + }); + }; }; if (param.num_kv_splits > 1) { @@ -317,7 +408,7 @@ struct batched_infer_splitkv_smallq_mask_bias_dropout_dispatch { }(); dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( - param.B, param.Hq, param.M, param.Kv, param.num_kv_splits); + param.B, param.Hq, param.Hkv, param.M, param.Kv, param.num_kv_splits); constexpr dim3 kBlockSize = FmhaFwdSplitKVKernel::BlockSize(); constexpr ck_tile::index_t kBlockPerCu = FmhaFwdSplitKVKernel::kBlockPerCu; diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_setting.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_setting.h index 922bdd05d6..0045b8b49f 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_setting.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_setting.h @@ -66,6 +66,15 @@ struct FmhaFwdBlockTile<256, MTile> { template struct FmhaFwdBlockTile<256>; +template +struct FmhaFwdBlockTile<512, MTile> { + using type = ck_tile::sequence<64, 128, 32, 512, 32, 512>; + using gemm0_warps = ck_tile::sequence<4, 1, 1>; + using gemm1_warps = ck_tile::sequence<4, 1, 1>; +}; + +template struct FmhaFwdBlockTile<512>; + using FmhaFwdWarpTile1 = ck_tile::sequence<32, 32, 16>; using FmhaFwdWarpTile2 = ck_tile::sequence<16, 16, 16>; @@ -150,6 +159,20 @@ struct FmhaFwdShape<256, MTile> { template struct FmhaFwdShape<256, 64>; template struct FmhaFwdShape<256, 128>; +template +struct FmhaFwdShape<512, MTile> { + using Type = ck_tile::TileFmhaShape< + typename FmhaFwdBlockTile<512>::type, + typename FmhaFwdBlockTile<512>::gemm0_warps, + FmhaFwdWarpTile2, + typename FmhaFwdBlockTile<512>::gemm1_warps, + FmhaFwdWarpTile2, + IsVLayoutRowMajor>; +}; + +template struct FmhaFwdShape<512, 64>; +template struct FmhaFwdShape<512, 128>; + static int get_fmha_fwd_mtile( int num_batches, int num_heads, diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_splitkv_selector.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_splitkv_selector.h index daa281c28d..5ba0e97d67 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_splitkv_selector.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_fwd_splitkv_selector.h @@ -47,6 +47,10 @@ static std::pair get_num_kv_splits_heuristic( mtile_size_for_splitkv_smallq = get_mtile_size_for_splitkv_smallq(max_headdim); + // hdim-512 is not supported by splitkv-kernel at present + if (max_headdim > 256) + return std::make_pair(false, 1); + if (max_seqlen_q >= mtile_size_for_pipeline_default) { int batch_nhead_mblocks = num_batches * num_heads * ceildiv(max_seqlen_q, mtile_size_for_pipeline_default); diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward.h index 5d19d6cc0e..39c3a10fbf 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward.h @@ -23,24 +23,32 @@ template < void run_grouped_forward_mask_bias_dropout_dispatch( GroupedForwardParams& param, hipStream_t stream) { - // currently split-kv implementation does not support dropout + // currently split-kv implementation does not support: + // (*) dropout + // (*) head dimension > 256 if constexpr (!kHasDropout) { - if (param.use_split_kv) { - if (use_splitkv_smallq(param.max_seqlen_q, std::max(param.K, param.Kv))) { - grouped_forward_splitkv_smallq_mask_bias_dropout_dispatch< - ScalarType, - kHasMask, - kHasBias, - MaxK>::Run(param, stream); - } else { - FMHA_FWD_SEQLEN_Q_SWITCH(param.max_seqlen_q, MaxSeqlenQ, [&] { - grouped_forward_splitkv_mask_bias_dropout_dispatch< + if (param.use_split_kv && MaxK <= 256) { + if constexpr (MaxK <= 256) { + if (use_splitkv_smallq( + param.max_seqlen_q, std::max(param.K, param.Kv))) { + grouped_forward_splitkv_smallq_mask_bias_dropout_dispatch< ScalarType, kHasMask, kHasBias, - MaxK, - MaxSeqlenQ>::Run(param, stream); - }); + MaxK>::Run(param, stream); + } else { + FMHA_FWD_SEQLEN_Q_SWITCH(param.max_seqlen_q, MaxSeqlenQ, [&] { + grouped_forward_splitkv_mask_bias_dropout_dispatch< + ScalarType, + kHasMask, + kHasBias, + MaxK, + MaxSeqlenQ>::Run(param, stream); + }); + } + } else { + // Unreachable. Do not instantiate split-kv pipelines with head + // dimension > 256 } } else { if (get_fmha_fwd_mtile(param.num_batches, param.Hq, param.max_seqlen_q) == diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_dispatch.h index 920c093e33..a5bab401b1 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_dispatch.h @@ -48,7 +48,7 @@ struct grouped_forward_mask_bias_dropout_dispatch { using FmhaFwdShape_ = typename FmhaFwdShape::Type; constexpr ck_tile::index_t occupancy = (MaxK == 64) ? 3 - : (MaxK == 256) ? 1 + : (MaxK >= 256) ? 1 : 2; constexpr auto kBiasEnum = kHasBias @@ -78,8 +78,10 @@ struct grouped_forward_mask_bias_dropout_dispatch { using FmhaPipelineProblem = FmhaPipelineProblemTemp; - using FmhaFwdPipeline_ = - ck_tile::BlockFmhaPipelineQRKSVS; + using FmhaFwdPipeline_ = std::conditional_t< + MaxK <= 256, + ck_tile::BlockFmhaPipelineQRKSVS, + ck_tile::BlockFmhaPipelineQSKSVS>; using FmhaFwdEpilogue_ = ck_tile::Default2DEpilogue; if (param.num_kv_splits > 1) { @@ -285,6 +286,7 @@ struct grouped_forward_splitkv_mask_bias_dropout_dispatch { dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( param.num_batches, param.Hq, + param.Hkv, param.max_seqlen_q, param.Kv, param.num_kv_splits); diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_splitkv_smallq_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_splitkv_smallq_dispatch.h index 4f92d2bdf4..f8d4452c54 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_splitkv_smallq_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_forward_splitkv_smallq_dispatch.h @@ -86,6 +86,7 @@ struct grouped_forward_splitkv_smallq_mask_bias_dropout_dispatch { false, // kDoFp8StaticQuant place-holder false, // kIsPagedKV true, // kHasUnevenSplits + false, // kMergeNumHeadGroupsSeqLenQ occupancy>; if (param.num_kv_splits > 1) { @@ -282,6 +283,7 @@ struct grouped_forward_splitkv_smallq_mask_bias_dropout_dispatch { dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( param.num_batches, param.Hq, + param.Hkv, param.max_seqlen_q, param.Kv, param.num_kv_splits); diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer.h index 539e33215e..f990b7218a 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer.h @@ -23,24 +23,32 @@ template < void run_grouped_infer_mask_bias_dropout_dispatch( GroupedForwardParams& param, hipStream_t stream) { - // currently split-kv implementation does not support dropout + // currently split-kv implementation does not support: + // (*) dropout + // (*) head dimension > 256 if constexpr (!kHasDropout) { - if (param.use_split_kv) { - if (use_splitkv_smallq(param.max_seqlen_q, std::max(param.K, param.Kv))) { - grouped_infer_splitkv_smallq_mask_bias_dropout_dispatch< - ScalarType, - kHasMask, - kHasBias, - MaxK>::Run(param, stream); - } else { - FMHA_FWD_SEQLEN_Q_SWITCH(param.max_seqlen_q, MaxSeqlenQ, [&] { - grouped_infer_splitkv_mask_bias_dropout_dispatch< + if (param.use_split_kv && MaxK <= 256) { + if constexpr (MaxK <= 256) { + if (use_splitkv_smallq( + param.max_seqlen_q, std::max(param.K, param.Kv))) { + grouped_infer_splitkv_smallq_mask_bias_dropout_dispatch< ScalarType, kHasMask, kHasBias, - MaxK, - MaxSeqlenQ>::Run(param, stream); - }); + MaxK>::Run(param, stream); + } else { + FMHA_FWD_SEQLEN_Q_SWITCH(param.max_seqlen_q, MaxSeqlenQ, [&] { + grouped_infer_splitkv_mask_bias_dropout_dispatch< + ScalarType, + kHasMask, + kHasBias, + MaxK, + MaxSeqlenQ>::Run(param, stream); + }); + } + } else { + // Unreachable. Do not instantiate split-kv pipelines with head + // dimension > 256 } } else { if (get_fmha_fwd_mtile(param.num_batches, param.Hq, param.max_seqlen_q) == diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_dispatch.h index 6cda6e8233..70973b880d 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_dispatch.h @@ -48,7 +48,7 @@ struct grouped_infer_mask_bias_dropout_dispatch { using FmhaShape = typename FmhaFwdShape::Type; constexpr ck_tile::index_t occupancy = - (MaxK == 64) ? 3 : ((MaxK == 256) ? 1 : 2); + (MaxK == 64) ? 3 : ((MaxK >= 256) ? 1 : 2); constexpr auto kBiasEnum = kHasBias ? ck_tile::BlockAttentionBiasEnum::ELEMENTWISE_BIAS @@ -81,8 +81,10 @@ struct grouped_infer_mask_bias_dropout_dispatch { using FmhaPipelineProblem = FmhaPipelineProblemTemp; - using FmhaPipeline = - ck_tile::BlockFmhaPipelineQRKSVS; + using FmhaPipeline = std::conditional_t< + MaxK <= 256, + ck_tile::BlockFmhaPipelineQRKSVS, + ck_tile::BlockFmhaPipelineQSKSVS>; using FmhaEpilogue = ck_tile::Default2DEpilogue; - using FmhaPipeline = - ck_tile::BlockFmhaPipelineQRKSVSAsync; + using FmhaPipeline = std::conditional_t< + MaxK <= 256, + ck_tile::BlockFmhaPipelineQRKSVSAsync, + ck_tile::BlockFmhaPipelineQSKSVS>; using FmhaEpilogue = ck_tile::Default2DEpilogue; using ODataType = @@ -132,6 +133,7 @@ struct grouped_infer_splitkv_mask_bias_dropout_dispatch { false, // kDoFp8StaticQuant place-holder kIsPagedKV, true, // kHasUnevenSplits + false, // kMergeNumHeadGroupsSeqLenQ occupancy>; using ODataType = @@ -309,6 +311,7 @@ struct grouped_infer_splitkv_mask_bias_dropout_dispatch { dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( param.num_batches, param.Hq, + param.Hkv, param.max_seqlen_q, param.Kv, param.num_kv_splits); diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_splitkv_smallq_dispatch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_splitkv_smallq_dispatch.h index 916c2ab11e..22077833fa 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_splitkv_smallq_dispatch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_fmha_grouped_infer_splitkv_smallq_dispatch.h @@ -75,88 +75,183 @@ struct grouped_infer_splitkv_smallq_mask_bias_dropout_dispatch { bool is_paged_kv = param.use_paged_kvcache; - BOOL_SWITCH_3( - pad_headdim_q, - kPadHeadDimQ, - pad_headdim_v, - kPadHeadDimV, - is_paged_kv, - kIsPagedKV, - [&] { - if (param.num_kv_splits > 1) { - using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< - kPadSeqLenQ, - kPadSeqLenK, - kPadHeadDimQ, - kPadHeadDimV, - kBiasEnum, - false, // kHasBiasGrad place-holder - true, // kStoreLSE - false, // kDoFp8StaticQuant place-holder - kIsPagedKV, - true, // kHasUnevenSplits - occupancy>; - - using ODataType = - typename FmhaFwdTypeConfig::OaccDataType; - using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< - FmhaTraits, - FmhaMask, - ODataType>; - - using FmhaPipeline = - ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< - FmhaPipelineProblem>; - - using FmhaEpilogue = - ck_tile::Default2DEpilogue::OaccDataType, - ODataType, - false, - false>>; - - using FmhaKernel = - ck_tile::FmhaFwdSplitKVKernel; - - RunWithFwdSplitKVKernel(param, stream); - } else { - using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< - kPadSeqLenQ, - kPadSeqLenK, - kPadHeadDimQ, - kPadHeadDimV, - kBiasEnum, - false, // kHasBiasGrad place-holder - false, // kStoreLSE - false, // kDoFp8StaticQuant place-holder - kIsPagedKV, - true, // kHasUnevenSplits - occupancy>; - - using ODataType = - typename FmhaFwdTypeConfig::ODataType; - using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< - FmhaTraits, - FmhaMask, - ODataType>; - - using FmhaPipeline = - ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< - FmhaPipelineProblem>; - - using FmhaEpilogue = - ck_tile::Default2DEpilogue::OaccDataType, - ODataType, - false, - false>>; - - using FmhaKernel = - ck_tile::FmhaFwdSplitKVKernel; - - RunWithFwdSplitKVKernel(param, stream); - } - }); + // indicates to the splitkv kernel whether should it merge Hq/Hkv with + // seqlen_q + const bool merge_nhead_groups_seqlen_q = + ((param.max_seqlen_q == 1) && (param.Hq > param.Hkv) && !kHasBias); + + if (merge_nhead_groups_seqlen_q) { + using FmhaMaskNone = ck_tile::SimplifiedGenericAttentionMask; + BOOL_SWITCH_3( + pad_headdim_q, + kPadHeadDimQ, + pad_headdim_v, + kPadHeadDimV, + is_paged_kv, + kIsPagedKV, + [&] { + if (param.num_kv_splits > 1) { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDimQ, + kPadHeadDimV, + ck_tile::BlockAttentionBiasEnum::NO_BIAS, + false, // kHasBiasGrad place-holder + true, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + kIsPagedKV, + true, // kHasUnevenSplits + true, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::OaccDataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMaskNone, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } else { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDimQ, + kPadHeadDimV, + ck_tile::BlockAttentionBiasEnum::NO_BIAS, + false, // kHasBiasGrad place-holder + false, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + kIsPagedKV, + true, // kHasUnevenSplits + true, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::ODataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMaskNone, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } + }); + } else { + BOOL_SWITCH_3( + pad_headdim_q, + kPadHeadDimQ, + pad_headdim_v, + kPadHeadDimV, + is_paged_kv, + kIsPagedKV, + [&] { + if (param.num_kv_splits > 1) { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDimQ, + kPadHeadDimV, + kBiasEnum, + false, // kHasBiasGrad place-holder + true, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + kIsPagedKV, + true, // kHasUnevenSplits + false, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::OaccDataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMask, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } else { + using FmhaTraits = ck_tile::TileFmhaFwdSplitKVTraits< + kPadSeqLenQ, + kPadSeqLenK, + kPadHeadDimQ, + kPadHeadDimV, + kBiasEnum, + false, // kHasBiasGrad place-holder + false, // kStoreLSE + false, // kDoFp8StaticQuant place-holder + kIsPagedKV, + true, // kHasUnevenSplits + false, // kMergeNumHeadGroupsSeqLenQ + occupancy>; + + using ODataType = + typename FmhaFwdTypeConfig::ODataType; + using FmhaPipelineProblem = FmhaFwdSplitKVPipelineProblemTemp< + FmhaTraits, + FmhaMask, + ODataType>; + + using FmhaPipeline = + ck_tile::BlockFmhaFwdSplitKVPipelineNWarpSShuffleQRKSVS< + FmhaPipelineProblem>; + + using FmhaEpilogue = ck_tile::Default2DEpilogue< + ck_tile::Default2DEpilogueProblem< + typename FmhaFwdTypeConfig::OaccDataType, + ODataType, + false, + false>>; + + using FmhaKernel = + ck_tile::FmhaFwdSplitKVKernel; + + RunWithFwdSplitKVKernel(param, stream); + } + }); + }; }; if (param.num_kv_splits > 1) { @@ -308,6 +403,7 @@ struct grouped_infer_splitkv_smallq_mask_bias_dropout_dispatch { dim3 kGridSize = FmhaFwdSplitKVKernel::GridSize( param.num_batches, param.Hq, + param.Hkv, param.max_seqlen_q, param.Kv, param.num_kv_splits); diff --git a/xformers/csrc/attention/hip_fmha/ck_tiled_headdim_switch.h b/xformers/csrc/attention/hip_fmha/ck_tiled_headdim_switch.h index 498e17f91d..b0f66cc773 100644 --- a/xformers/csrc/attention/hip_fmha/ck_tiled_headdim_switch.h +++ b/xformers/csrc/attention/hip_fmha/ck_tiled_headdim_switch.h @@ -9,11 +9,11 @@ #include #include -#ifndef FMHA_SUPPORT_MAX_HEADDIM_128 -#define FMHA_SUPPORT_MAX_HEADDIM_128 0 +#ifndef FMHA_LIMIT_MAX_HEADDIM_TO_256 +#define FMHA_LIMIT_MAX_HEADDIM_TO_256 0 #endif -#if FMHA_SUPPORT_MAX_HEADDIM_128 +#if FMHA_LIMIT_MAX_HEADDIM_TO_256 #define FMHA_FWD_HEADDIM_SWITCH(HEAD_DIM1, HEAD_DIM2, CONST_NAME, ...) \ [&] { \ @@ -29,24 +29,8 @@ } else if (HEAD_DIM1 <= 128 && HEAD_DIM2 <= 128) { \ constexpr ck_tile::index_t CONST_NAME = 128; \ __VA_ARGS__(); \ - } else { \ - throw std::runtime_error("Head-dim sizes not supported!"); \ - } \ - }() - -#define FMHA_BWD_HEADDIM_SWITCH(HEAD_DIM1, HEAD_DIM2, CONST_NAME, ...) \ - [&] { \ - if (HEAD_DIM1 <= 32 && HEAD_DIM2 <= 32) { \ - constexpr ck_tile::index_t CONST_NAME = 32; \ - __VA_ARGS__(); \ - } else if (HEAD_DIM1 <= 64 && HEAD_DIM2 <= 64) { \ - constexpr ck_tile::index_t CONST_NAME = 64; \ - __VA_ARGS__(); \ - } else if (HEAD_DIM1 <= 96 && HEAD_DIM2 <= 96) { \ - constexpr ck_tile::index_t CONST_NAME = 96; \ - __VA_ARGS__(); \ - } else if (HEAD_DIM1 <= 128 && HEAD_DIM2 <= 128) { \ - constexpr ck_tile::index_t CONST_NAME = 128; \ + } else if (HEAD_DIM1 <= 256 && HEAD_DIM2 <= 256) { \ + constexpr ck_tile::index_t CONST_NAME = 256; \ __VA_ARGS__(); \ } else { \ throw std::runtime_error("Head-dim sizes not supported!"); \ @@ -72,11 +56,16 @@ } else if (HEAD_DIM1 <= 256 && HEAD_DIM2 <= 256) { \ constexpr ck_tile::index_t CONST_NAME = 256; \ __VA_ARGS__(); \ + } else if (HEAD_DIM1 <= 512 && HEAD_DIM2 <= 512) { \ + constexpr ck_tile::index_t CONST_NAME = 512; \ + __VA_ARGS__(); \ } else { \ throw std::runtime_error("Head-dim sizes not supported!"); \ } \ }() +#endif + #define FMHA_BWD_HEADDIM_SWITCH(HEAD_DIM1, HEAD_DIM2, CONST_NAME, ...) \ [&] { \ if (HEAD_DIM1 <= 32 && HEAD_DIM2 <= 32) { \ @@ -98,5 +87,3 @@ throw std::runtime_error("Head-dim sizes not supported!"); \ } \ }() - -#endif diff --git a/xformers/csrc/attention/hip_fmha/generate_instances.py b/xformers/csrc/attention/hip_fmha/generate_instances.py index d769b8b358..9af1d90224 100644 --- a/xformers/csrc/attention/hip_fmha/generate_instances.py +++ b/xformers/csrc/attention/hip_fmha/generate_instances.py @@ -18,8 +18,12 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `{file}` */ -""" +""".format( + file=os.path.relpath(os.path.realpath(__file__), start=Path(__file__).parents[4]) +) FMHA_INFER_INSTANCE_TEMPLATE_INC = """ #include @@ -103,13 +107,7 @@ False: "no_dropout", } -INT_MAP_MAX_K = { - 32: "maxk_32", - 64: "maxk_64", - 96: "maxk_96", - 128: "maxk_128", - 256: "maxk_256", -} +INT_MAP_MAX_K = {hd: f"maxk_{hd}" for hd in [32, 64, 96, 128, 256, 512]} TYPE_CTYPE_MAP = { "fp16": "ck_tile::fp16_t", @@ -356,18 +354,8 @@ def create_backward_instances_ref(instance_dir: Path, headdims: List) -> None: if __name__ == "__main__": - disable_hd256 = False - - for arg in sys.argv: - if arg == "--ignore-hd256": - disable_hd256 = True - - if disable_hd256: - headdims_fwd = [32, 64, 96, 128] - headdims_bwd = [32, 64, 96, 128] - else: - headdims_fwd = [32, 64, 96, 128, 256] - headdims_bwd = [32, 64, 96, 128, 256] + headdims_fwd = [32, 64, 96, 128, 256, 512] + headdims_bwd = [32, 64, 96, 128, 256] this_dir = os.path.dirname(__file__) output_dir = Path(this_dir) / "instances" diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index d6b447d173..deedccf7fc 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index c319629872..ac5fca62ed 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index 6161fc4ae4..c752780d71 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index 08c3ec38a2..160404b382 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index 12c1aa463c..70a3e8c894 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index 8bea77809d..7cc63083d3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 5ed35bbef6..b53beb84b8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index 672d36fe11..99ce1a2c6b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index b70134c681..c72335f2ca 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index e2301db5ec..3a03f2e516 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index c132e77e64..30b2ad6338 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index aac5a1aaf8..d7548a1378 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index a4d5950050..84f3066d31 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index aa88585bc2..8fbe5fe5be 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index 3e99fd87db..d42de53daf 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index 8c95d9392c..fb9c77f840 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index 25e054c6ce..128d68ccc3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index cec2dec8bf..47b6329e24 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index fe59c183f4..968a692d90 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index 9c1dd943e7..f11a9c0044 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index 7603478867..6baa061c50 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index a085a7ab08..94058fe2f3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index 1e0a77cfd4..5f37585118 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index ec28f459b8..f81e6bc878 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index aefdd4d6af..23e1361baa 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index d580e1549e..98077a70ce 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index 6a2ffe01cf..ff974bba43 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index 2fbc707a50..0c90b1ed05 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index 8a8ac48042..2f6730e757 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index ddd9e4ff7e..fb15d26a2c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_instances_ref.h index 607048cbad..c1f1f8c872 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index 6901b50c17..dc03dda716 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index efa38d5329..6c420ff7af 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index 0d21552eee..35a263a012 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index 8366fe3350..b1617dc879 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index f57bb62706..5a00ced069 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index b481351c79..65c2ff3617 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 470a8ee444..404b4a5536 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index 1a58c63720..0bcb409de6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index f5c4d3df3b..9d4f69235b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index 2e8451901e..70f33a6274 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index 8d3e5e0ad2..6549d52b4e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index 69492777b2..4d5237993c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index b25b805768..e37d5c2e5e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index 1f8ac812df..22caa72a11 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index 247dd491cf..9114a2c359 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index d66ebd7d54..36c7ce8f78 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index f71f0a98fd..1e2ef1125b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index 3d001ec57c..3679b8cb84 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index 4ffb7f4193..5330820d47 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index cf9da51fd8..34edd21a43 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index e0e5c1093b..c0028a2307 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index cb039bd893..b843bec00b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index e988f88a63..388e2158cd 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index 6d4f8e8832..d506351108 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index 7bc8fbb70e..f7b20b098a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index b40590e752..a8eba26ed2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index 9e543ce456..99cc8622d8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index d4b4d3d25a..b4c2c9f031 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index 78d157c8b0..5c689eee81 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index c26216d39e..184be2aa70 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index 80f5cbafaa..b9ea123ace 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index e09b3ada17..48a7a26bae 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index c7bb811828..0973b02700 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index 3184149372..d231a533e8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index fe54bed624..f248539133 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index 4285510a6a..625af0ff3c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 86410bafac..8e9f67a09b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index 2c91e6152a..8a4d19bd91 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index 8855ffd887..863fd62ae4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index cc4e57f2d3..09e60aa441 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index 2d98de9388..6338140176 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index 89b21aa7c6..5e08fc0b6e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index 648a99f443..dd91cd7ad2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index fc4e72b84a..8c26c66ea6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index 6c25ae5b80..f6a7bc9ec2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index e77b97fd84..3d2a07bf07 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index 304bdea6ac..f918c4f3e9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index 2aaaa250bb..a8304bd504 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index 82cf516785..ed346c346c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index 744858265b..90b36d8661 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index 71f2f421e0..3381e5e06e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index 8b84758423..c6ddacb818 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index 70ceb95945..0cd13c3b43 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index 54a97cc2c1..4a08919efc 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index 0b5415c041..0a051ea374 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index 217d876bcc..98f7492633 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index 303b93b077..93479691e5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index 74d455fff4..e30446aa70 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index 2783b3be1d..3e4dab172c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index 11f72a7b4c..b1d6b0e716 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_instances_ref.h index 1655e42ce5..b7395d5b2e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index 6748c1b011..7a99081396 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index ecc6392b9c..f059e6e532 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index c9280ecea9..9eec13fe6f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index 4a3fb67186..a32fb482a3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index f54fd36354..8bc6fa542f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index 110394c34d..575e48d0ab 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 161304b8ed..013c62a92d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index 6ec124e26a..e539aa9fd8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index 8d8fa202e0..ae79c2c1da 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index 29c9fb6a4c..3546c27451 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index 671d37710a..27cc051cde 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index 6ba00de55c..773cc3f879 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index 367d9f6e26..010a08c8c0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index 643f6ad5bd..79eeda92ae 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index 4832c97990..5605658204 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index 3712d8cd6a..d5177b1b6b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index ad905cbdf9..4549561f19 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index 777bef0160..8ffd8e0799 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index b748de7b95..fd391f1cd1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index dbb567a280..f2cd0e5bc1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index d76eae7cff..f86da48991 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index 37ded4ac11..776c854d38 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index 0cfc315f8b..76a4d73089 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index 2e95e9082f..79fb4d3703 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index f1d3f39d00..0d2b5cb5ee 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index 4a65054c8f..30fa8733a0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index fb57f88653..b041793ca4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index 3cb6b9d3e2..65ac20faf5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index 53052e40d4..895adc6d34 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index 494f10a720..0f28d75a29 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp index a60963f802..d36c57904b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp index cfe158f63e..d7d4220810 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp index f83330c354..803d5bb55c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..e31e2705fb --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp index d218b55775..bec8dbc157 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp index 1ab50df932..e3b77d2193 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp index 88664056e3..696465c315 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp index 52327df1ce..186055abd9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp index e7576d0c4c..1dd92aa01a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..dd827217a1 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp index eeaf62d6fc..6bc1c864f5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp index ae7317559d..bc5d468bbe 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp index a1544c50a6..bbc7b1bfcd 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp index 565a51e164..e5ba1fe1c4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp index 5a33c64489..200a04b29f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..cb87478f39 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp index 40bfebada2..64f42ef1ee 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp index 96287c4882..630572b7d1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp index 8e071fc747..0177223369 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp index 406c49d6d1..ea75fddc5f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp index 0bf56df8c0..e5d56077ab 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7101d19874 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp index 83ba77748f..5d4d629c08 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp index 43a36ce652..89af8cf6d9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_instances_ref.h index dd1a636a6b..93883a3645 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_batched_forward_mask_bias_dropout_dispatch< false, false, 256>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp index 967c68daa5..e67f241229 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp index 3bbc694732..062d8071db 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp index f4e5f5eb7f..918539dfd2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..4a216ec666 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp index 71569c47c6..7bb04afaed 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp index fa01afbfb2..59a5ac46ba 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp index 0e385e642b..91192edda6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp index 3375f54543..feb34557d0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp index 4cff079b20..269c8fe30b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..dd22cd4a31 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp index 489bad0fad..cbd7cc0f6a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp index 0b955693c0..3aa2725e7a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp index 65d7b902a9..7a06bc2768 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp index 972ad19835..ac4bb9b800 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp index ea7a9926ab..6cc6dc5553 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..d99f24ab2d --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp index 9111ebbbbc..bdb278b191 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp index 5038f0028e..ed5057e5b7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp index 55d50683d0..1c5ffe761d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp index be72e76d24..4b1c355d4f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp index 96d9f212de..04910aec94 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7ee48e7340 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp index 247d27508f..3baf32c407 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp index 8fbe1f0ce6..639a7532d3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp index 8a22e0a124..cdd463cbbd 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp index b523959364..49ada07bb2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp index 3f8d2ea4a0..26c877b926 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..817012a844 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp index c73e76ba54..585e647ee2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp index cb6f657839..6d1ca56400 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp index 3721e1206e..a420333adb 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp index 6449266a26..086ed371f6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp index 98a23c5da4..404c16a9d1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..a483e13d25 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp index c12921f2f4..89921f9714 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp index 3b347a64bd..ab84bb40d0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp index 498c653437..24d6519b5e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp index fd696a20b2..534ea475f2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp index 2660e9f956..a80a833606 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..ce9d9d6b0f --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp index ffd777b0f3..16984afd64 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp index 03e08c45a1..52e3d8d158 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp index fe81619104..1958d334e7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp index 0fc54fd688..597f78ec65 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp index ca9c1aeb5f..e4b09d8ea2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..84adfadc80 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp index bf77caa3a8..622f3944e6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp index 2e56a95123..1a16b67db1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_instances_ref.h index f4fb71af63..0e5c6d7736 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_batched_forward_mask_bias_dropout_dispatch< false, false, 256>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp index fa4ca05fd6..3673aa644c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp index 078fc9a96c..2988bdc9d3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp index 722424784f..7de593d4ec 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..260efb2ca7 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp index c13355df47..a5400cf5d5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp index 63141d2382..4c93dff9e9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp index 640a324464..ec6d0d009a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp index b1d2f9261b..c036a338e0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp index 6be825ead5..c0344fad67 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..eb71d020fa --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp index 82b2d2a37d..966b6acf83 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp index 518d809847..ea19e327a1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp index 5ceff03a83..ef72522307 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp index ec115bde5d..92efd034e9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp index e237d7a1d9..2e8b290184 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..daa4184524 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp index d22f8e5e7c..d56676df6f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp index ada24fa386..8bc1cdce75 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp index bf94d16cac..905eb9ca5e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp index 91f8252bc0..59c7f92776 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp index 2849c4a01e..85d40a2c25 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7ddbcad6d1 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_forward.h" + +template void run_batched_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp index bfb2727b55..189c6e1c55 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp index b2c4b3fc95..85c28e18f4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp index c969aaddd4..869ca5cf2b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp index 4b5c1722f2..6955d189d5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp index 82155df9dc..4cb13e88b5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..3d2a48f3f8 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp index 0f037342f1..1639eff023 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp index 4199f8dfc1..24122b2c10 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp index 4a02de28c9..97f133da18 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp index 33f3521253..8ba2b51f62 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp index 251f3435c7..f7289a2f6a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..61b303a668 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp index db0bcc4905..0333a592b1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp index 84d693dcd4..1346c8f611 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp index 4964bfa57b..88781ead96 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp index d1afa4f97b..9cf29b566a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp index b53ce42583..489f72e614 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..fe0cee2c7a --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp index 10fecb0b1f..5260a0eb46 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp index 9683175ce5..bc2edad6ce 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp index 99ecd3f153..596447317f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp index 9fe1f47000..3c676a5af8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp index 9cb5037ff4..1765ce00a3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..c37303ad16 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp index 688e746c30..dcdc20df76 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp index 9d345eb620..7558821e06 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_instances_ref.h index a0a632332c..bee199f470 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_batched_infer_mask_bias_dropout_dispatch< false, false, 256>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp index 384ed6c7d6..db6d18ab9c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp index 1d14ec3223..2c197f0b0c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp index 38bb1e4898..ed0cae5fb4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..a5837d2a98 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp index 9e01187176..9124fa99d2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp index 94a7b0ecf6..df0b8ea20e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp index f9eee86a38..ee1277e7c3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp index 662850493a..7339b662ec 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp index 809d7fb2fa..7c356ea638 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..c1efa3617f --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp index 2b015348a3..34e7ddce7a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp index 23badfdcb3..07f6aaadbc 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp index 1eb945d8c9..11ca22c8a4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp index bdae23c5f5..6d20fb5d5c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp index abcd6e5054..bbfb06dbb0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7c67a7010d --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp index f91e7d396a..3554cf6543 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp index 6633c2a2d8..6b618c8541 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp index 606f3e51d0..5dd0105bc3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp index f37c3155a5..cbd96d97ee 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp index d05287595e..5125eb7cf8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..b8db9a264f --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp index 931c73fb80..034d75e663 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp index 222818766a..3552f6547b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp index 48d3a2c3f0..c682a64e4e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp index 71e0a40272..940cb764d3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp index 2914d3566b..07d6dae14d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..c1bd53c4d2 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp index 1dc4f4cefe..53fc9026c1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp index 49089a5a2c..2487a2c3fb 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp index 83ee3847ac..7f501bc70f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp index f6d3cd1f9a..5ee8ded1d2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp index 44e794f26d..61342d8ec3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..0f7cb86bb1 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp index 2b8d9371b1..1a8e1eff41 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp index cda89d9882..8041cd4fc3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp index b83806efab..eaa25797bc 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp index c22ec1891b..c31ed24a1a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp index 39d5af11cb..f5356e83c1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..e1ae25703d --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp index 1333e0e3a2..6a6d5640ce 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp index c6dd68fbcf..488888e08a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp index a8c94892ae..0bf9fd1c17 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp index 37abd037a6..543a86611c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp index d45e9747ac..c0021eccc6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..90e02550a4 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp index 4a5b32f1d6..9fbaed81c8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp index 3aded97795..0456cb5fd5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_instances_ref.h index 5b63c0083b..6dee01829b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_batched_infer_mask_bias_dropout_dispatch< false, false, 256>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); + +extern template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp index 215574613c..b611c3f8df 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp index fda3a851a9..7e80d24a97 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp index 3a461d75b8..dbd5885646 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..52fd6090b1 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp index f5de5ab9fc..3a945108ea 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp index 6199c05109..0e96cddec4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp index 8ca40c295b..8bb37690f7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp index 9ea1c82aae..12b6e1b046 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp index 7e6fdd12f1..46fe0af3e8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..24d7a9fe1d --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp index 4eeeafdda9..54ad9ef4e8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp index cba6c7eb6e..9905f219c6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp index a46736ec72..7fa7f35e88 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp index 477836c7c5..c7755e8938 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp index 81dba703d5..e7813a3912 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..78187d962b --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp index 92dd14a639..4d4b716e83 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp index c2780682c6..8eb76218e0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp index 4488da3605..8b33546d49 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp index f38d36564e..eeec380b37 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp index 9025bd9b97..422ed9f8ef 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..cf08ef4f6d --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_batched_infer.h" + +template void run_batched_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(BatchedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp index 8aa5368312..3f8b3e1c40 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp index 3ef3ae0ad2..c36b3ead50 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_batched_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index 52258dd70d..f0858d6834 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index f18614fa08..b76cca7c95 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index ba78d65d3b..ca389eea0b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index 7258831cee..ee9a2be337 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index c37c77d554..81073d727c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index bd10c628ac..60ff8fb6ce 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 99903f6560..e4fb906623 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index fb92ebe6fc..71f2ce2c81 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index 59249a8b03..49d680d3ba 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index db4d2ce297..7dba5c2441 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index bbe5fc4a71..b03f3e338c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index 91f7af8f29..39948d5156 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index 33467b58f8..40a58aee22 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index 628ad56249..b846b688ee 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index 979c39e34a..3cb774e517 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index 67f3bb857b..04661c0b0c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index 5fc15b960f..3937433027 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index be106ab035..4993e17ff6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index 1bc566b34a..df319f8381 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index f17c75ecbc..8415d88583 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index 6ab1929abb..5522f7b861 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index 9153f0a6dd..15471e801c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index f9d2de3cd8..f280b137b0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index 02e6479f99..0b9f321d66 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index 7352541275..4f1c6e934f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index cdf8c64d07..04ab490179 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index ea0cdd8794..8a4f00d52b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index 4b20062e26..ab5fabb07d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index 262fe65ae7..a5b5eff92e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index 342bccf249..1aa78190fb 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_instances_ref.h index 77fd2adfd4..3e336087fc 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index 1ec85b39bd..ba826a4b9a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index 11e98efd9f..bd11debaab 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index 28a019accc..9ddb424d02 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index ea25b5eaff..c1a7db4b9e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index a5e8ac4541..d858b691d4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index fb21b6429d..b2bfdef979 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 90046688f1..b53485b285 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index 8bee1bacd7..6481044da2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index b8a6e10e65..2c8419f57b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index 1f0d4e2d28..c25f0981f3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index fb7617cf96..f61a2b16d5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index 649682a521..2f3a6019bf 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index b7ef701393..196f90be11 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index f043077872..fcc64bd3b5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index 7f5cc32bf8..b7290994c9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index 20f2299474..bc00f54c84 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index 0c5b0899d2..9855baf5c0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index a10ed99695..7422d69252 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index 1778c650af..a6db954636 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index 7f18e6c0d9..d08d076637 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index 90eaf9020c..a4bc3e8c87 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index 6041d88106..d2c2a4d11d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index f4f4a74a29..77c8981856 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index 723dad8b4d..3b4d1346e5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index 725fb3b751..b80c077ec9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index a213e1feea..f3d57bf68a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index 55be37bff0..7ed2852eb3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index 8d4e8157c4..73fbe3b5d3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index 2a11628eaf..f824550b7d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index 37c739e6d6..bb34d3add9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_bf16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index be282c1692..20b7d16d32 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index 16c1a56335..960a73e503 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index 0d126762fb..b1e63b0b6e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index bba62020d6..b6d6f8881f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index b4973f6d4f..0abc11a1df 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index d397432a8b..9522e37a6f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 576f4ec43c..8f7ad5789e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index 9ec9c32a5e..0ee5896212 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index 0e1421f0ba..6b99fa89dd 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index 1cfbb64a6a..967ab8b27f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index 936aceb179..e8859d4b28 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index 2601c44b53..4963659a65 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index db40de8e14..73eb2c1e3f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index 520aef06c0..bf14541ac5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index e11bd53369..945e717314 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index db1a8fe044..184fbcdb3c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index 9a7ae39f16..9b4b4cd36e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index 57b874c858..6a932dff32 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index c542a2c255..1cf08095b4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index 1d22178487..6661f6323a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index a4f08bb7be..1d3a8793ce 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index 9d24093276..f43e9e2f88 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index 3596811967..e5c2576e12 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index a958635127..4926711f59 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index 792825647a..2f23cd8f5e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index 7fb1932394..f7f1e95177 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index a81fe6db2e..425dace404 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index e4940345d3..995ff3c048 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index dad5ec5274..0daacd7e0a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index c0e01a73b9..6c600d0b8b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_has_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_instances_ref.h index 61472494f2..d3ec26812a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp index 70837e9b2b..417f3e06f3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp index 3ad63b3fb7..40f3f97c24 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp index d2ec293abe..64418045ae 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp index 6f988aedf5..0459074656 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp index 170b7dc080..c1b33f7cdf 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp index 060a6b875a..a66e90fc55 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp index 4093a812e7..edc4f875af 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp index ef3521c8bd..0739f859c7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp index 9f76e20d90..3f59a2c3a0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp index 6274a56bb5..245ce567c2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_has_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp index 6b97237665..d7e0383232 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp index fc9b10b1a1..a330c69ff9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp index c166a7bd48..153cac73b8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp index 30cc3c575d..d358169c7e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp index 2f4058c055..3a9982511b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp index dd172a8cd6..75fba8fe0e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp index 4eb6cba1aa..a0435f7d0f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp index 34a1a45a03..2a680f091b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp index 15691115b3..26768fc1ab 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp index 5ea99eb70f..1551b2b71b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_has_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp index 9e72f65f20..82c4d31a43 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp index 143c79b972..b538e40663 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp index e7935d54b7..ad925135c4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp index 0b911129cb..900c8841e4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp index e2ff64c3dc..971c6d6f40 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp index ee07981f0e..cafbf71198 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp index 5e47962a51..277e798a01 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp index 8936424612..549b64c28c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp index b8d022181c..3f900efed8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp index 835604b023..dff6a8b8bb 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_backward_fp16_no_mask_no_bias_no_biasgrad_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp index e221a4df68..fa34129e88 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp index 7708b6be81..0d7ada33ba 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp index f500369249..29c89bbacd 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..bb6e4df67e --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp index 7af9ce737b..77a3d0b1aa 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp index 90ed257288..87f8a6a92a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp index 63d87a7ceb..be2fcf8972 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp index 5ec5b2076d..bc5f99bce7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp index 0202533758..00fcfedc67 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..c54dadf223 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp index d49d2b41de..825d31e3e2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp index 8945954299..a9f4416a55 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp index acc3e80445..54768e673e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp index ef243b0dc5..91eacaad3d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp index 23a3d60725..2a6cd0f5c7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..db6e0cc93c --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp index 2048527030..d882d2c1ef 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp index 9866d6a0b2..7029f0e39f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp index ce742afc08..627a6bba11 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp index 8170a8859c..2338f7c70c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp index 33515ab436..dfdf5ed326 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..5ec50eaf88 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp index c1bfa5227f..d37827666e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp index c0602f9c08..ef9293d352 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_instances_ref.h index ea0947de21..eeced2b284 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_grouped_forward_mask_bias_dropout_dispatch< false, false, 256>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp index 9f5253947f..fc5a832055 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp index 83474e1d76..3ecb974cf1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp index 8e8b152379..3d4c373d0c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..b78db8c558 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp index c542571932..57de1acbae 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp index a5a67b1ad6..fcfca04262 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp index 48a41626a1..7bce3adb1a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp index 41c9d6f57c..be593023d4 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp index 553b1fc8ba..6c56ccb31e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..6622bdaec4 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp index dfe68ffcad..b43a7a0465 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp index 810e671500..db14720016 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp index 2d72bcb6a3..0dd97ddb8d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp index eda1008bf9..1c3532ca6e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp index c101072938..a0287fbb9f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..5b8a117bdb --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp index a67bb0844f..a956f98236 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp index 71182531ad..53ed01fae6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp index 4910d1463e..b445e84fd1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp index ab647a2e7e..4895d01d0f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp index f8c7491ae4..1200f8fdb9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..e4ac09e718 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp index c4cd4e7b88..c52d5aa6ea 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp index 9203a02a35..dec369fdde 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp index 1d130ea119..da8fb43ece 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp index e9525bfd6a..9a1136081a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp index 601415d752..d0496f8e6b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..62e7d3ea2f --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp index 571780c49b..8a8c5f07b3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp index 608cf7b582..2216dbc921 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp index 3841dadae0..829f982744 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp index 3ed3b86656..4de96e1cff 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp index 8f45feab8c..2acae5143f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..d3e92fc835 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp index 8690683e49..73518c2ad0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp index e8ae22495d..d26a148db3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp index 4a985fb011..ce8be64e98 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp index 3420d3aa50..1f27466398 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp index 74849113c9..30cd42e3e9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..ddef229388 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp index 1303aa9b43..4d7dfc6aaa 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp index 213703efeb..35159d0bb0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp index 5ef755ddf6..bec1d5415c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp index 24c5729743..6071d6b43f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp index 6a6952ec63..e9c87a19e7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..23443d0871 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp index 434dcc2693..c0322eaac6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp index 1ecdd0f832..5d102fcda6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_instances_ref.h index e4327e83e5..561b385ca6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_grouped_forward_mask_bias_dropout_dispatch< false, false, 256>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp index 3f5f2707fc..4e429edc3d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp index 3a24dd4611..fa5cdd8fd7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp index b20dcc77ec..ca3ce199a8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..dd4161c783 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp index e93471b9a2..4a460ff479 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp index cbfcdfa07d..1d8707f9b7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp index 4fd11b41bf..a5a825ba0e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp index 5b83a321c2..92c202962b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp index ece97ea1d8..f062b35d88 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..a441b7dda0 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp index a9af6a8ded..07bd49f225 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp index 60f4f7d652..da99265cc2 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp index 94bfe75ea5..1e526137b0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp index 31136ded22..058717a41c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp index 0e79cea140..2ae97dbeaf 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7b185aacf2 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp index c4e8677838..01b4d9d6de 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp index 77d6057173..af851eb90d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp index 25c0c1ac25..911bb57f23 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp index d7d3a36219..9458012051 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp index a49ac26ee6..0a630cfd54 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7cd0e9d6f1 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_forward.h" + +template void run_grouped_forward_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp index fc7ddced9e..b252bddcba 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp index 2942d3e91a..0ca10974fb 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_forward_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp index d50935b1d7..f22a52e5ff 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp index e985ad8805..b5714d2e75 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp index 8f88cf8e63..5892be2944 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..000a5a0981 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp index bcf4508b97..b8a53e83de 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp index e6bbaad9e8..69b7db7106 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp index 82b400f0c6..c00bb8bf13 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp index a3325e6686..52d87e03be 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp index cca4cc5431..ee516d1e7d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7db13a38c8 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp index e033986a24..e479845422 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp index cb80ff6e05..945d259a6a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp index 2f257ffd73..2c9fda9bb0 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp index a772490804..0c6ec53e6a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp index 94b83ea16b..6e8bbf96e9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..b61f5a6f89 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp index 1e0258d11a..a56da016c9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp index b8aecbef49..80d4f80be9 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp index 5c5052773a..ddb0f10b1c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp index f5267d11a7..6377fd5347 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp index 17549b1ff6..7083541875 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..9d5ad0d4f1 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp index 49b14547cd..92976d2e7f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp index 30db8093b3..cebccc9ca5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_instances_ref.h index 6022b79cc3..0a4eec170d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_grouped_infer_mask_bias_dropout_dispatch< false, false, 256>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp index e5fb64fac3..d7e99ee8e1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp index 4eec28e4df..38856c8573 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp index d26e0d4771..0adce36610 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..e653081050 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp index b9498adfc1..67d0e364f3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp index 48530caca9..6c5c178c1a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp index d09cd5a863..e46a9b9c62 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp index acb1b14fef..3f634b4698 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp index 1924525a47..44d02d78b6 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..23499fa191 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp index 818af21711..1ffb92523c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp index a1236ed698..453b550fec 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp index b73fbd3e60..42d7ec40b7 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp index 8e40965635..788f5ca688 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp index 92db0a3bac..6cdcb6660c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..9bc461de90 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp index affb5a980b..9ad85ea237 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp index 75ff69dfec..bdb7be54c5 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp index 7efc0e9203..ca2a94b4ad 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp index c1493d3e44..f632fa38ef 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp index 315429ef08..61ef4515cc 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..d118cc6e9d --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::bf16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp index 8cce00c824..e69e981428 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp index 86f93c2b3a..97e95e2f4f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_bf16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp index cbbd746a8f..84df8eec07 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp index 960634ed47..64a3a61152 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp index d3bbeeaea0..69a97345ee 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..1c00d0c092 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp index 0fda8f6a47..99fa9115b3 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp index 9eac3a46b5..56c6733a30 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp index 91a3b3aec9..62a2d9d46d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp index 8859657b71..c1344d525e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp index ab8ee4823b..b352cfcc1d 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..70684674fe --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp index dea721a634..d1e56628ec 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp index d843caa1ac..4c46a4d274 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp index edecb5ee5b..740f359784 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp index 5aabfa102d..9ed553b85f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp index d4b2a56bd7..5fce53c665 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..61bef88c4a --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp index 5c6b91be17..081cede055 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp index 90175276f0..f1f99083fd 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp index 40d3950944..97e34494ee 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp index 0abf5b79ba..efd0e8a7af 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp index afa07836b9..2527da3e2e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..6db0bc6663 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp index 03fa1e82b9..c01b8f3733 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp index 5efcef2c86..1b581efa76 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_has_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_instances_ref.h b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_instances_ref.h index c38d01ca60..8e19ba6aa1 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_instances_ref.h +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_instances_ref.h @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include @@ -290,3 +292,59 @@ extern template void run_grouped_infer_mask_bias_dropout_dispatch< false, false, 256>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + true, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); + +extern template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp index db687f5110..d64e64affe 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp index d78135bea3..41173ee32f 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp index fd4fea5d62..669822ed25 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..95639331b4 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp index c1c4742435..6ea26c5136 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp index 37d18699ee..03c78c3547 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp index 33dd36ae2a..29a25c2590 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp index 4ed97869a3..b01e1b8b21 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp index 8317354c85..87d2b6480c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..7237f2b476 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + true, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp index f761773b84..5886d5ef64 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp index 3d80d5fd9c..8899702665 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_has_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp index f9ab0be1fa..fcd01f9a47 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp index f4f7fee792..fc1f2c479c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp index a510dfb2b5..608acb2e2b 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp new file mode 100644 index 0000000000..ed59b858e1 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + true, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp index 9d8b8e8987..4521d8efff 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp index 15788edbf7..ca2423cb69 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_has_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp index 3287d5e4ba..825f8b4f7c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_128.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp index b7f99432ce..24dfdb4c1a 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_256.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp index f6d6340842..c6d974745e 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_32.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp new file mode 100644 index 0000000000..87ece0b3d3 --- /dev/null +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_512.cpp @@ -0,0 +1,21 @@ + +/* + Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` + */ + +#include +#include "ck_tiled_fmha_grouped_infer.h" + +template void run_grouped_infer_mask_bias_dropout_dispatch< + ck_tile::fp16_t, + false, + false, + false, + 512>(GroupedForwardParams& param, hipStream_t stream); diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp index 44f3b7d0cb..ec4ce83cd8 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_64.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp index b6e94978f6..f390e8974c 100644 --- a/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp +++ b/xformers/csrc/attention/hip_fmha/instances/fmha_grouped_infer_fp16_no_mask_no_bias_no_dropout_maxk_96.cpp @@ -6,6 +6,8 @@ * LICENSE file in the root directory of this source tree. * * The file is automatically generated, don't modify! + * See the generator script + * `xformers/csrc/attention/hip_fmha/generate_instances.py` */ #include diff --git a/xformers/ops/fmha/ck.py b/xformers/ops/fmha/ck.py index 5d11377117..0d908340c8 100644 --- a/xformers/ops/fmha/ck.py +++ b/xformers/ops/fmha/ck.py @@ -155,7 +155,7 @@ class FwOp(AttentionFwOpBase): OPERATOR = get_operator("xformers", "efficient_attention_forward_ck") SUPPORTED_DEVICES: Set[str] = {"cuda"} SUPPORTED_DTYPES: Set[torch.dtype] = {torch.half, torch.bfloat16} - SUPPORTED_MAX_K = 256 + SUPPORTED_MAX_K = 512 SUPPORTED_ATTN_BIAS_TYPES: Iterable[Any] = ( type(None), @@ -201,6 +201,7 @@ class FwOp(AttentionFwOpBase): 96, 128, # 64x128 kernel 256, # 64x128 with accumulation in gmem + 512, ] @classmethod @@ -216,7 +217,7 @@ def apply( assert inp.query.ndim == 5, f"query has shape {inp.query.shape}" ctx: Optional[Context] = None - # consider for expanded 5-D inputted + # when the input is expanded 5-D, the group dimension has zero stride if inp.key.stride()[3] == 0: assert ( inp.value.stride()[3] == 0 @@ -239,7 +240,13 @@ def apply( [_, _, G, Hq, _] = inp.query.shape attn_bias_replace = inp.attn_bias - if isinstance(inp.attn_bias, torch.Tensor) and inp.attn_bias.ndim != 0: + if isinstance(inp.attn_bias, LowerTriangularMaskWithTensorBias): + bias_tensor = _get_tensor_bias(inp.attn_bias) + if bias_tensor.ndim == 5: + attn_bias_replace = LowerTriangularMaskWithTensorBias( + bias_tensor.flatten(1, 2) + ) + elif isinstance(inp.attn_bias, torch.Tensor) and inp.attn_bias.ndim == 5: attn_bias_replace = inp.attn_bias.flatten(1, 2) inp = replace( inp,