/* SPDX-License-Identifier: AGPL-3.0-or-later */ /* Copyright (C) 2026 David (Corey) Herrera */ /* * fc_cuda_ptx.h — Auto-generated embedded PTX kernels * * DO NOT EDIT — regenerate with: cd cuda_kernels && make * Compiled for sm_37 with CUDA 11.4 */ #ifndef FC_CUDA_PTX_H #define FC_CUDA_PTX_H static const char FC_PTX_ELEMENTWISE[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_add\n" "\n" ".visible .entry fc_add(\n" " .param .u64 fc_add_param_0,\n" " .param .u64 fc_add_param_1,\n" " .param .u64 fc_add_param_2,\n" " .param .u32 fc_add_param_3\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<4>;\n" " .reg .b32 %r<6>;\n" " .reg .b64 %rd<11>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_add_param_0];\n" " ld.param.u64 %rd2, [fc_add_param_1];\n" " ld.param.u64 %rd3, [fc_add_param_2];\n" " ld.param.u32 %r2, [fc_add_param_3];\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r1, %r3, %r4, %r5;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB0_2;\n" "\n" " cvta.to.global.u64 %rd4, %rd1;\n" " mul.wide.s32 %rd5, %r1, 4;\n" " add.s64 %rd6, %rd4, %rd5;\n" " cvta.to.global.u64 %rd7, %rd2;\n" " add.s64 %rd8, %rd7, %rd5;\n" " ld.global.f32 %f1, [%rd8];\n" " ld.global.f32 %f2, [%rd6];\n" " add.f32 %f3, %f2, %f1;\n" " cvta.to.global.u64 %rd9, %rd3;\n" " add.s64 %rd10, %rd9, %rd5;\n" " st.global.f32 [%rd10], %f3;\n" "\n" "$L__BB0_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_mul\n" ".visible .entry fc_mul(\n" " .param .u64 fc_mul_param_0,\n" " .param .u64 fc_mul_param_1,\n" " .param .u64 fc_mul_param_2,\n" " .param .u32 fc_mul_param_3\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<4>;\n" " .reg .b32 %r<6>;\n" " .reg .b64 %rd<11>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_mul_param_0];\n" " ld.param.u64 %rd2, [fc_mul_param_1];\n" " ld.param.u64 %rd3, [fc_mul_param_2];\n" " ld.param.u32 %r2, [fc_mul_param_3];\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r1, %r3, %r4, %r5;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB1_2;\n" "\n" " cvta.to.global.u64 %rd4, %rd1;\n" " mul.wide.s32 %rd5, %r1, 4;\n" " add.s64 %rd6, %rd4, %rd5;\n" " cvta.to.global.u64 %rd7, %rd2;\n" " add.s64 %rd8, %rd7, %rd5;\n" " ld.global.f32 %f1, [%rd8];\n" " ld.global.f32 %f2, [%rd6];\n" " mul.f32 %f3, %f2, %f1;\n" " cvta.to.global.u64 %rd9, %rd3;\n" " add.s64 %rd10, %rd9, %rd5;\n" " st.global.f32 [%rd10], %f3;\n" "\n" "$L__BB1_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_silu\n" ".visible .entry fc_silu(\n" " .param .u64 fc_silu_param_0,\n" " .param .u64 fc_silu_param_1,\n" " .param .u32 fc_silu_param_2\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<20>;\n" " .reg .b32 %r<8>;\n" " .reg .b64 %rd<8>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_silu_param_0];\n" " ld.param.u64 %rd2, [fc_silu_param_1];\n" " ld.param.u32 %r2, [fc_silu_param_2];\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r1, %r3, %r4, %r5;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB2_2;\n" "\n" " cvta.to.global.u64 %rd3, %rd1;\n" " mul.wide.s32 %rd4, %r1, 4;\n" " add.s64 %rd5, %rd3, %rd4;\n" " ld.global.f32 %f1, [%rd5];\n" " neg.f32 %f2, %f1;\n" " mov.f32 %f3, 0f3F000000;\n" " mov.f32 %f4, 0f3BBB989D;\n" " fma.rn.f32 %f5, %f2, %f4, %f3;\n" " mov.f32 %f6, 0f3FB8AA3B;\n" " mov.f32 %f7, 0f437C0000;\n" " cvt.sat.f32.f32 %f8, %f5;\n" " mov.f32 %f9, 0f4B400001;\n" " fma.rm.f32 %f10, %f8, %f7, %f9;\n" " add.f32 %f11, %f10, 0fCB40007F;\n" " neg.f32 %f12, %f11;\n" " fma.rn.f32 %f13, %f2, %f6, %f12;\n" " mov.f32 %f14, 0f32A57060;\n" " fma.rn.f32 %f15, %f2, %f14, %f13;\n" " mov.b32 %r6, %f10;\n" " shl.b32 %r7, %r6, 23;\n" " mov.b32 %f16, %r7;\n" " ex2.approx.ftz.f32 %f17, %f15;\n" " fma.rn.f32 %f18, %f17, %f16, 0f3F800000;\n" " div.rn.f32 %f19, %f1, %f18;\n" " cvta.to.global.u64 %rd6, %rd2;\n" " add.s64 %rd7, %rd6, %rd4;\n" " st.global.f32 [%rd7], %f19;\n" "\n" "$L__BB2_2:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_NORMALIZATION[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_rms_norm\n" ".extern .shared .align 16 .b8 sdata[];\n" "\n" ".visible .entry fc_rms_norm(\n" " .param .u64 fc_rms_norm_param_0,\n" " .param .u64 fc_rms_norm_param_1,\n" " .param .u64 fc_rms_norm_param_2,\n" " .param .u32 fc_rms_norm_param_3,\n" " .param .f32 fc_rms_norm_param_4\n" ")\n" "{\n" " .reg .pred %p<19>;\n" " .reg .f32 %f<65>;\n" " .reg .b32 %r<70>;\n" " .reg .b64 %rd<67>;\n" "\n" "\n" " ld.param.u64 %rd26, [fc_rms_norm_param_0];\n" " ld.param.u64 %rd25, [fc_rms_norm_param_1];\n" " ld.param.u64 %rd27, [fc_rms_norm_param_2];\n" " ld.param.u32 %r35, [fc_rms_norm_param_3];\n" " ld.param.f32 %f9, [fc_rms_norm_param_4];\n" " cvta.to.global.u64 %rd1, %rd27;\n" " cvta.to.global.u64 %rd2, %rd25;\n" " cvta.to.global.u64 %rd3, %rd26;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r68, %tid.x;\n" " setp.ge.s32 %p1, %r68, %r35;\n" " mov.f32 %f64, 0f00000000;\n" " @%p1 bra $L__BB0_7;\n" "\n" " not.b32 %r36, %r68;\n" " add.s32 %r37, %r36, %r35;\n" " div.u32 %r3, %r37, %r1;\n" " add.s32 %r38, %r3, 1;\n" " and.b32 %r58, %r38, 3;\n" " setp.eq.s32 %p2, %r58, 0;\n" " mov.f32 %f64, 0f00000000;\n" " mov.u32 %r59, %r68;\n" " @%p2 bra $L__BB0_4;\n" "\n" " mul.wide.s32 %rd28, %r68, 4;\n" " add.s64 %rd61, %rd3, %rd28;\n" " mul.wide.s32 %rd5, %r1, 4;\n" " mov.f32 %f64, 0f00000000;\n" " mov.u32 %r59, %r68;\n" "\n" "$L__BB0_3:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f14, [%rd61];\n" " fma.rn.f32 %f64, %f14, %f14, %f64;\n" " add.s32 %r59, %r59, %r1;\n" " add.s64 %rd61, %rd61, %rd5;\n" " add.s32 %r58, %r58, -1;\n" " setp.ne.s32 %p3, %r58, 0;\n" " @%p3 bra $L__BB0_3;\n" "\n" "$L__BB0_4:\n" " setp.lt.u32 %p4, %r3, 3;\n" " @%p4 bra $L__BB0_7;\n" "\n" " shl.b32 %r10, %r1, 2;\n" " cvt.s64.s32 %rd31, %r10;\n" "\n" "$L__BB0_6:\n" " mul.wide.s32 %rd29, %r59, 4;\n" " add.s64 %rd30, %rd3, %rd29;\n" " ld.global.f32 %f15, [%rd30];\n" " fma.rn.f32 %f16, %f15, %f15, %f64;\n" " add.s64 %rd32, %rd30, %rd31;\n" " ld.global.f32 %f17, [%rd32];\n" " fma.rn.f32 %f18, %f17, %f17, %f16;\n" " add.s32 %r39, %r59, %r1;\n" " add.s32 %r40, %r39, %r1;\n" " add.s64 %rd33, %rd32, %rd31;\n" " ld.global.f32 %f19, [%rd33];\n" " fma.rn.f32 %f20, %f19, %f19, %f18;\n" " add.s32 %r41, %r40, %r1;\n" " add.s64 %rd34, %rd33, %rd31;\n" " ld.global.f32 %f21, [%rd34];\n" " fma.rn.f32 %f64, %f21, %f21, %f20;\n" " add.s32 %r59, %r41, %r1;\n" " setp.lt.s32 %p5, %r59, %r35;\n" " @%p5 bra $L__BB0_6;\n" "\n" "$L__BB0_7:\n" " shl.b32 %r42, %r68, 2;\n" " mov.u32 %r43, sdata;\n" " add.s32 %r13, %r43, %r42;\n" " st.shared.f32 [%r13], %f64;\n" " bar.sync 0;\n" " setp.lt.s32 %p6, %r1, 2;\n" " @%p6 bra $L__BB0_12;\n" "\n" " shr.u32 %r44, %r1, 31;\n" " add.s32 %r45, %r1, %r44;\n" " shr.s32 %r61, %r45, 1;\n" "\n" "$L__BB0_9:\n" " setp.ge.s32 %p7, %r68, %r61;\n" " @%p7 bra $L__BB0_11;\n" "\n" " shl.b32 %r46, %r61, 2;\n" " add.s32 %r47, %r13, %r46;\n" " ld.shared.f32 %f22, [%r13];\n" " ld.shared.f32 %f23, [%r47];\n" " add.f32 %f24, %f23, %f22;\n" " st.shared.f32 [%r13], %f24;\n" "\n" "$L__BB0_11:\n" " bar.sync 0;\n" " shr.s32 %r16, %r61, 1;\n" " setp.gt.s32 %p8, %r61, 1;\n" " mov.u32 %r61, %r16;\n" " @%p8 bra $L__BB0_9;\n" "\n" "$L__BB0_12:\n" " cvt.rn.f32.s32 %f25, %r35;\n" " ld.shared.f32 %f26, [sdata];\n" " div.rn.f32 %f27, %f26, %f25;\n" " add.f32 %f28, %f27, %f9;\n" " sqrt.rn.f32 %f29, %f28;\n" " rcp.rn.f32 %f8, %f29;\n" " @%p1 bra $L__BB0_26;\n" "\n" " setp.eq.s64 %p10, %rd25, 0;\n" " not.b32 %r48, %r68;\n" " add.s32 %r49, %r48, %r35;\n" " div.u32 %r17, %r49, %r1;\n" " add.s32 %r50, %r17, 1;\n" " and.b32 %r63, %r50, 3;\n" " @%p10 bra $L__BB0_20;\n" "\n" " setp.eq.s32 %p11, %r63, 0;\n" " @%p11 bra $L__BB0_17;\n" "\n" " mul.wide.s32 %rd35, %r68, 4;\n" " add.s64 %rd64, %rd1, %rd35;\n" " mul.wide.s32 %rd9, %r1, 4;\n" " add.s64 %rd63, %rd2, %rd35;\n" " add.s64 %rd62, %rd3, %rd35;\n" "\n" "$L__BB0_16:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f30, [%rd62];\n" " mul.f32 %f31, %f8, %f30;\n" " ld.global.f32 %f32, [%rd63];\n" " mul.f32 %f33, %f31, %f32;\n" " st.global.f32 [%rd64], %f33;\n" " add.s32 %r68, %r68, %r1;\n" " add.s64 %rd64, %rd64, %rd9;\n" " add.s64 %rd63, %rd63, %rd9;\n" " add.s64 %rd62, %rd62, %rd9;\n" " add.s32 %r63, %r63, -1;\n" " setp.ne.s32 %p12, %r63, 0;\n" " @%p12 bra $L__BB0_16;\n" "\n" "$L__BB0_17:\n" " setp.lt.u32 %p13, %r17, 3;\n" " @%p13 bra $L__BB0_26;\n" "\n" " shl.b32 %r24, %r1, 2;\n" " cvt.s64.s32 %rd40, %r24;\n" "\n" "$L__BB0_19:\n" " mul.wide.s32 %rd36, %r68, 4;\n" " add.s64 %rd37, %rd3, %rd36;\n" " ld.global.f32 %f34, [%rd37];\n" " mul.f32 %f35, %f8, %f34;\n" " add.s64 %rd38, %rd2, %rd36;\n" " ld.global.f32 %f36, [%rd38];\n" " mul.f32 %f37, %f35, %f36;\n" " add.s64 %rd39, %rd1, %rd36;\n" " st.global.f32 [%rd39], %f37;\n" " add.s64 %rd41, %rd37, %rd40;\n" " ld.global.f32 %f38, [%rd41];\n" " mul.f32 %f39, %f8, %f38;\n" " add.s64 %rd42, %rd38, %rd40;\n" " ld.global.f32 %f40, [%rd42];\n" " mul.f32 %f41, %f39, %f40;\n" " add.s64 %rd43, %rd39, %rd40;\n" " st.global.f32 [%rd43], %f41;\n" " add.s32 %r51, %r68, %r1;\n" " add.s32 %r52, %r51, %r1;\n" " add.s64 %rd44, %rd41, %rd40;\n" " ld.global.f32 %f42, [%rd44];\n" " mul.f32 %f43, %f8, %f42;\n" " add.s64 %rd45, %rd42, %rd40;\n" " ld.global.f32 %f44, [%rd45];\n" " mul.f32 %f45, %f43, %f44;\n" " add.s64 %rd46, %rd43, %rd40;\n" " st.global.f32 [%rd46], %f45;\n" " add.s32 %r53, %r52, %r1;\n" " add.s64 %rd47, %rd44, %rd40;\n" " ld.global.f32 %f46, [%rd47];\n" " mul.f32 %f47, %f8, %f46;\n" " add.s64 %rd48, %rd45, %rd40;\n" " ld.global.f32 %f48, [%rd48];\n" " mul.f32 %f49, %f47, %f48;\n" " add.s64 %rd49, %rd46, %rd40;\n" " st.global.f32 [%rd49], %f49;\n" " add.s32 %r68, %r53, %r1;\n" " setp.lt.s32 %p14, %r68, %r35;\n" " @%p14 bra $L__BB0_19;\n" " bra.uni $L__BB0_26;\n" "\n" "$L__BB0_20:\n" " setp.eq.s32 %p15, %r63, 0;\n" " @%p15 bra $L__BB0_23;\n" "\n" " mul.wide.s32 %rd50, %r68, 4;\n" " add.s64 %rd66, %rd1, %rd50;\n" " mul.wide.s32 %rd19, %r1, 4;\n" " add.s64 %rd65, %rd3, %rd50;\n" "\n" "$L__BB0_22:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f50, [%rd65];\n" " mul.f32 %f51, %f8, %f50;\n" " st.global.f32 [%rd66], %f51;\n" " add.s32 %r68, %r68, %r1;\n" " add.s64 %rd66, %rd66, %rd19;\n" " add.s64 %rd65, %rd65, %rd19;\n" " add.s32 %r63, %r63, -1;\n" " setp.ne.s32 %p16, %r63, 0;\n" " @%p16 bra $L__BB0_22;\n" "\n" "$L__BB0_23:\n" " setp.lt.u32 %p17, %r17, 3;\n" " @%p17 bra $L__BB0_26;\n" "\n" " shl.b32 %r32, %r1, 2;\n" " cvt.s64.s32 %rd54, %r32;\n" "\n" "$L__BB0_25:\n" " mul.wide.s32 %rd51, %r68, 4;\n" " add.s64 %rd52, %rd3, %rd51;\n" " ld.global.f32 %f52, [%rd52];\n" " mul.f32 %f53, %f8, %f52;\n" " add.s64 %rd53, %rd1, %rd51;\n" " st.global.f32 [%rd53], %f53;\n" " add.s64 %rd55, %rd52, %rd54;\n" " ld.global.f32 %f54, [%rd55];\n" " mul.f32 %f55, %f8, %f54;\n" " add.s64 %rd56, %rd53, %rd54;\n" " st.global.f32 [%rd56], %f55;\n" " add.s32 %r54, %r68, %r1;\n" " add.s32 %r55, %r54, %r1;\n" " add.s64 %rd57, %rd55, %rd54;\n" " ld.global.f32 %f56, [%rd57];\n" " mul.f32 %f57, %f8, %f56;\n" " add.s64 %rd58, %rd56, %rd54;\n" " st.global.f32 [%rd58], %f57;\n" " add.s32 %r56, %r55, %r1;\n" " add.s64 %rd59, %rd57, %rd54;\n" " ld.global.f32 %f58, [%rd59];\n" " mul.f32 %f59, %f8, %f58;\n" " add.s64 %rd60, %rd58, %rd54;\n" " st.global.f32 [%rd60], %f59;\n" " add.s32 %r68, %r56, %r1;\n" " setp.lt.s32 %p18, %r68, %r35;\n" " @%p18 bra $L__BB0_25;\n" "\n" "$L__BB0_26:\n" " ret;\n" "\n" "}\n" " // .globl fc_softmax\n" ".visible .entry fc_softmax(\n" " .param .u64 fc_softmax_param_0,\n" " .param .u64 fc_softmax_param_1,\n" " .param .u32 fc_softmax_param_2,\n" " .param .u32 fc_softmax_param_3\n" ")\n" "{\n" " .reg .pred %p<29>;\n" " .reg .f32 %f<134>;\n" " .reg .b32 %r<104>;\n" " .reg .b64 %rd<49>;\n" "\n" "\n" " ld.param.u64 %rd18, [fc_softmax_param_0];\n" " ld.param.u64 %rd19, [fc_softmax_param_1];\n" " ld.param.u32 %r44, [fc_softmax_param_2];\n" " ld.param.u32 %r43, [fc_softmax_param_3];\n" " cvta.to.global.u64 %rd1, %rd18;\n" " cvta.to.global.u64 %rd2, %rd19;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r44;\n" " @%p1 bra $L__BB1_32;\n" "\n" " mov.u32 %r102, %tid.x;\n" " mul.lo.s32 %r3, %r1, %r43;\n" " mov.u32 %r4, %ntid.x;\n" " setp.ge.s32 %p2, %r102, %r43;\n" " mov.f32 %f128, 0fF149F2CA;\n" " @%p2 bra $L__BB1_8;\n" "\n" " not.b32 %r45, %r102;\n" " add.s32 %r46, %r45, %r43;\n" " div.u32 %r5, %r46, %r4;\n" " add.s32 %r47, %r5, 1;\n" " and.b32 %r91, %r47, 3;\n" " setp.eq.s32 %p3, %r91, 0;\n" " mov.f32 %f128, 0fF149F2CA;\n" " mov.u32 %r92, %r102;\n" " @%p3 bra $L__BB1_5;\n" "\n" " add.s32 %r48, %r102, %r3;\n" " mul.wide.s32 %rd20, %r48, 4;\n" " add.s64 %rd45, %rd1, %rd20;\n" " mul.wide.s32 %rd4, %r4, 4;\n" " mov.f32 %f128, 0fF149F2CA;\n" " mov.u32 %r92, %r102;\n" "\n" "$L__BB1_4:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f22, [%rd45];\n" " setp.gt.f32 %p4, %f22, %f128;\n" " selp.f32 %f128, %f22, %f128, %p4;\n" " add.s32 %r92, %r92, %r4;\n" " add.s64 %rd45, %rd45, %rd4;\n" " add.s32 %r91, %r91, -1;\n" " setp.ne.s32 %p5, %r91, 0;\n" " @%p5 bra $L__BB1_4;\n" "\n" "$L__BB1_5:\n" " setp.lt.u32 %p6, %r5, 3;\n" " @%p6 bra $L__BB1_8;\n" "\n" " shl.b32 %r12, %r4, 2;\n" " cvt.s64.s32 %rd23, %r12;\n" "\n" "$L__BB1_7:\n" " add.s32 %r49, %r92, %r3;\n" " mul.wide.s32 %rd21, %r49, 4;\n" " add.s64 %rd22, %rd1, %rd21;\n" " ld.global.f32 %f23, [%rd22];\n" " setp.gt.f32 %p7, %f23, %f128;\n" " selp.f32 %f24, %f23, %f128, %p7;\n" " add.s64 %rd24, %rd22, %rd23;\n" " ld.global.f32 %f25, [%rd24];\n" " setp.gt.f32 %p8, %f25, %f24;\n" " selp.f32 %f26, %f25, %f24, %p8;\n" " add.s32 %r50, %r92, %r4;\n" " add.s32 %r51, %r50, %r4;\n" " add.s64 %rd25, %rd24, %rd23;\n" " ld.global.f32 %f27, [%rd25];\n" " setp.gt.f32 %p9, %f27, %f26;\n" " selp.f32 %f28, %f27, %f26, %p9;\n" " add.s32 %r52, %r51, %r4;\n" " add.s64 %rd26, %rd25, %rd23;\n" " ld.global.f32 %f29, [%rd26];\n" " setp.gt.f32 %p10, %f29, %f28;\n" " selp.f32 %f128, %f29, %f28, %p10;\n" " add.s32 %r92, %r52, %r4;\n" " setp.lt.s32 %p11, %r92, %r43;\n" " @%p11 bra $L__BB1_7;\n" "\n" "$L__BB1_8:\n" " shl.b32 %r53, %r102, 2;\n" " mov.u32 %r54, sdata;\n" " add.s32 %r15, %r54, %r53;\n" " st.shared.f32 [%r15], %f128;\n" " bar.sync 0;\n" " shr.u32 %r55, %r4, 31;\n" " add.s32 %r56, %r4, %r55;\n" " shr.s32 %r99, %r56, 1;\n" " setp.lt.s32 %p12, %r4, 2;\n" " @%p12 bra $L__BB1_14;\n" "\n" " mov.u32 %r94, %r99;\n" "\n" "$L__BB1_10:\n" " mov.u32 %r17, %r94;\n" " setp.ge.s32 %p13, %r102, %r17;\n" " @%p13 bra $L__BB1_13;\n" "\n" " shl.b32 %r57, %r17, 2;\n" " add.s32 %r58, %r15, %r57;\n" " ld.shared.f32 %f30, [%r15];\n" " ld.shared.f32 %f8, [%r58];\n" " setp.leu.f32 %p14, %f8, %f30;\n" " @%p14 bra $L__BB1_13;\n" "\n" " st.shared.f32 [%r15], %f8;\n" "\n" "$L__BB1_13:\n" " bar.sync 0;\n" " shr.s32 %r94, %r17, 1;\n" " setp.gt.s32 %p15, %r17, 1;\n" " @%p15 bra $L__BB1_10;\n" "\n" "$L__BB1_14:\n" " mov.f32 %f133, 0f00000000;\n" " ld.shared.f32 %f9, [sdata];\n" " @%p2 bra $L__BB1_21;\n" "\n" " not.b32 %r59, %r102;\n" " add.s32 %r60, %r59, %r43;\n" " div.u32 %r19, %r60, %r4;\n" " add.s32 %r61, %r19, 1;\n" " and.b32 %r96, %r61, 3;\n" " setp.eq.s32 %p17, %r96, 0;\n" " mov.f32 %f133, 0f00000000;\n" " mov.u32 %r97, %r102;\n" " @%p17 bra $L__BB1_18;\n" "\n" " add.s32 %r62, %r102, %r3;\n" " mul.wide.s32 %rd27, %r62, 4;\n" " add.s64 %rd47, %rd2, %rd27;\n" " mul.wide.s32 %rd8, %r4, 4;\n" " add.s64 %rd46, %rd1, %rd27;\n" " mov.f32 %f133, 0f00000000;\n" " mov.f32 %f37, 0f3F000000;\n" " mov.f32 %f38, 0f3BBB989D;\n" " mov.u32 %r97, %r102;\n" "\n" "$L__BB1_17:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f35, [%rd46];\n" " sub.f32 %f36, %f35, %f9;\n" " fma.rn.f32 %f39, %f36, %f38, %f37;\n" " mov.f32 %f40, 0f3FB8AA3B;\n" " mov.f32 %f41, 0f437C0000;\n" " cvt.sat.f32.f32 %f42, %f39;\n" " mov.f32 %f43, 0f4B400001;\n" " fma.rm.f32 %f44, %f42, %f41, %f43;\n" " add.f32 %f45, %f44, 0fCB40007F;\n" " neg.f32 %f46, %f45;\n" " fma.rn.f32 %f47, %f36, %f40, %f46;\n" " mov.f32 %f48, 0f32A57060;\n" " fma.rn.f32 %f49, %f36, %f48, %f47;\n" " mov.b32 %r63, %f44;\n" " shl.b32 %r64, %r63, 23;\n" " mov.b32 %f50, %r64;\n" " ex2.approx.ftz.f32 %f51, %f49;\n" " mul.f32 %f52, %f51, %f50;\n" " st.global.f32 [%rd47], %f52;\n" " add.f32 %f133, %f133, %f52;\n" " add.s32 %r97, %r97, %r4;\n" " add.s64 %rd47, %rd47, %rd8;\n" " add.s64 %rd46, %rd46, %rd8;\n" " add.s32 %r96, %r96, -1;\n" " setp.ne.s32 %p18, %r96, 0;\n" " @%p18 bra $L__BB1_17;\n" "\n" "$L__BB1_18:\n" " setp.lt.u32 %p19, %r19, 3;\n" " @%p19 bra $L__BB1_21;\n" "\n" " shl.b32 %r26, %r4, 2;\n" " mov.f32 %f55, 0f3F000000;\n" " mov.f32 %f56, 0f3BBB989D;\n" " cvt.s64.s32 %rd31, %r26;\n" "\n" "$L__BB1_20:\n" " add.s32 %r65, %r97, %r3;\n" " mul.wide.s32 %rd28, %r65, 4;\n" " add.s64 %rd29, %rd1, %rd28;\n" " ld.global.f32 %f53, [%rd29];\n" " sub.f32 %f54, %f53, %f9;\n" " fma.rn.f32 %f57, %f54, %f56, %f55;\n" " mov.f32 %f58, 0f3FB8AA3B;\n" " mov.f32 %f59, 0f437C0000;\n" " cvt.sat.f32.f32 %f60, %f57;\n" " mov.f32 %f61, 0f4B400001;\n" " fma.rm.f32 %f62, %f60, %f59, %f61;\n" " add.f32 %f63, %f62, 0fCB40007F;\n" " neg.f32 %f64, %f63;\n" " fma.rn.f32 %f65, %f54, %f58, %f64;\n" " mov.f32 %f66, 0f32A57060;\n" " fma.rn.f32 %f67, %f54, %f66, %f65;\n" " mov.b32 %r66, %f62;\n" " shl.b32 %r67, %r66, 23;\n" " mov.b32 %f68, %r67;\n" " ex2.approx.ftz.f32 %f69, %f67;\n" " mul.f32 %f70, %f69, %f68;\n" " add.s64 %rd30, %rd2, %rd28;\n" " st.global.f32 [%rd30], %f70;\n" " add.f32 %f71, %f133, %f70;\n" " add.s64 %rd32, %rd29, %rd31;\n" " ld.global.f32 %f72, [%rd32];\n" " sub.f32 %f73, %f72, %f9;\n" " fma.rn.f32 %f74, %f73, %f56, %f55;\n" " cvt.sat.f32.f32 %f75, %f74;\n" " fma.rm.f32 %f76, %f75, %f59, %f61;\n" " add.f32 %f77, %f76, 0fCB40007F;\n" " neg.f32 %f78, %f77;\n" " fma.rn.f32 %f79, %f73, %f58, %f78;\n" " fma.rn.f32 %f80, %f73, %f66, %f79;\n" " mov.b32 %r68, %f76;\n" " shl.b32 %r69, %r68, 23;\n" " mov.b32 %f81, %r69;\n" " ex2.approx.ftz.f32 %f82, %f80;\n" " mul.f32 %f83, %f82, %f81;\n" " add.s64 %rd33, %rd30, %rd31;\n" " st.global.f32 [%rd33], %f83;\n" " add.f32 %f84, %f71, %f83;\n" " add.s32 %r70, %r97, %r4;\n" " add.s32 %r71, %r70, %r4;\n" " add.s64 %rd34, %rd32, %rd31;\n" " ld.global.f32 %f85, [%rd34];\n" " sub.f32 %f86, %f85, %f9;\n" " fma.rn.f32 %f87, %f86, %f56, %f55;\n" " cvt.sat.f32.f32 %f88, %f87;\n" " fma.rm.f32 %f89, %f88, %f59, %f61;\n" " add.f32 %f90, %f89, 0fCB40007F;\n" " neg.f32 %f91, %f90;\n" " fma.rn.f32 %f92, %f86, %f58, %f91;\n" " fma.rn.f32 %f93, %f86, %f66, %f92;\n" " mov.b32 %r72, %f89;\n" " shl.b32 %r73, %r72, 23;\n" " mov.b32 %f94, %r73;\n" " ex2.approx.ftz.f32 %f95, %f93;\n" " mul.f32 %f96, %f95, %f94;\n" " add.s64 %rd35, %rd33, %rd31;\n" " st.global.f32 [%rd35], %f96;\n" " add.f32 %f97, %f84, %f96;\n" " add.s32 %r74, %r71, %r4;\n" " add.s64 %rd36, %rd34, %rd31;\n" " ld.global.f32 %f98, [%rd36];\n" " sub.f32 %f99, %f98, %f9;\n" " fma.rn.f32 %f100, %f99, %f56, %f55;\n" " cvt.sat.f32.f32 %f101, %f100;\n" " fma.rm.f32 %f102, %f101, %f59, %f61;\n" " add.f32 %f103, %f102, 0fCB40007F;\n" " neg.f32 %f104, %f103;\n" " fma.rn.f32 %f105, %f99, %f58, %f104;\n" " fma.rn.f32 %f106, %f99, %f66, %f105;\n" " mov.b32 %r75, %f102;\n" " shl.b32 %r76, %r75, 23;\n" " mov.b32 %f107, %r76;\n" " ex2.approx.ftz.f32 %f108, %f106;\n" " mul.f32 %f109, %f108, %f107;\n" " add.s64 %rd37, %rd35, %rd31;\n" " st.global.f32 [%rd37], %f109;\n" " add.f32 %f133, %f97, %f109;\n" " add.s32 %r97, %r74, %r4;\n" " setp.lt.s32 %p20, %r97, %r43;\n" " @%p20 bra $L__BB1_20;\n" "\n" "$L__BB1_21:\n" " shl.b32 %r77, %r4, 2;\n" " add.s32 %r29, %r54, %r77;\n" " add.s32 %r30, %r29, %r53;\n" " st.shared.f32 [%r30], %f133;\n" " bar.sync 0;\n" " @%p12 bra $L__BB1_25;\n" "\n" "$L__BB1_22:\n" " setp.ge.s32 %p22, %r102, %r99;\n" " @%p22 bra $L__BB1_24;\n" "\n" " shl.b32 %r80, %r99, 2;\n" " add.s32 %r81, %r30, %r80;\n" " ld.shared.f32 %f110, [%r30];\n" " ld.shared.f32 %f111, [%r81];\n" " add.f32 %f112, %f111, %f110;\n" " st.shared.f32 [%r30], %f112;\n" "\n" "$L__BB1_24:\n" " bar.sync 0;\n" " shr.s32 %r32, %r99, 1;\n" " setp.gt.s32 %p23, %r99, 1;\n" " mov.u32 %r99, %r32;\n" " @%p23 bra $L__BB1_22;\n" "\n" "$L__BB1_25:\n" " ld.shared.f32 %f113, [%r29];\n" " rcp.rn.f32 %f17, %f113;\n" " @%p2 bra $L__BB1_32;\n" "\n" " not.b32 %r82, %r102;\n" " add.s32 %r83, %r82, %r43;\n" " div.u32 %r33, %r83, %r4;\n" " add.s32 %r84, %r33, 1;\n" " and.b32 %r101, %r84, 3;\n" " setp.eq.s32 %p25, %r101, 0;\n" " @%p25 bra $L__BB1_29;\n" "\n" " add.s32 %r85, %r102, %r3;\n" " mul.wide.s32 %rd38, %r85, 4;\n" " add.s64 %rd48, %rd2, %rd38;\n" " mul.wide.s32 %rd15, %r4, 4;\n" "\n" "$L__BB1_28:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f114, [%rd48];\n" " mul.f32 %f115, %f17, %f114;\n" " st.global.f32 [%rd48], %f115;\n" " add.s32 %r102, %r102, %r4;\n" " add.s64 %rd48, %rd48, %rd15;\n" " add.s32 %r101, %r101, -1;\n" " setp.ne.s32 %p26, %r101, 0;\n" " @%p26 bra $L__BB1_28;\n" "\n" "$L__BB1_29:\n" " setp.lt.u32 %p27, %r33, 3;\n" " @%p27 bra $L__BB1_32;\n" "\n" " cvt.s64.s32 %rd41, %r77;\n" "\n" "$L__BB1_31:\n" " add.s32 %r86, %r102, %r3;\n" " mul.wide.s32 %rd39, %r86, 4;\n" " add.s64 %rd40, %rd2, %rd39;\n" " ld.global.f32 %f116, [%rd40];\n" " mul.f32 %f117, %f17, %f116;\n" " st.global.f32 [%rd40], %f117;\n" " add.s64 %rd42, %rd40, %rd41;\n" " ld.global.f32 %f118, [%rd42];\n" " mul.f32 %f119, %f17, %f118;\n" " st.global.f32 [%rd42], %f119;\n" " add.s32 %r87, %r102, %r4;\n" " add.s32 %r88, %r87, %r4;\n" " add.s64 %rd43, %rd42, %rd41;\n" " ld.global.f32 %f120, [%rd43];\n" " mul.f32 %f121, %f17, %f120;\n" " st.global.f32 [%rd43], %f121;\n" " add.s32 %r89, %r88, %r4;\n" " add.s64 %rd44, %rd43, %rd41;\n" " ld.global.f32 %f122, [%rd44];\n" " mul.f32 %f123, %f17, %f122;\n" " st.global.f32 [%rd44], %f123;\n" " add.s32 %r102, %r89, %r4;\n" " setp.lt.s32 %p28, %r102, %r43;\n" " @%p28 bra $L__BB1_31;\n" "\n" "$L__BB1_32:\n" " ret;\n" "\n" "}\n" " // .globl fc_qk_rms_norm\n" ".visible .entry fc_qk_rms_norm(\n" " .param .u64 fc_qk_rms_norm_param_0,\n" " .param .u64 fc_qk_rms_norm_param_1,\n" " .param .u32 fc_qk_rms_norm_param_2,\n" " .param .u32 fc_qk_rms_norm_param_3,\n" " .param .f32 fc_qk_rms_norm_param_4\n" ")\n" "{\n" " .reg .pred %p<20>;\n" " .reg .f32 %f<65>;\n" " .reg .b32 %r<69>;\n" " .reg .b64 %rd<64>;\n" "\n" "\n" " ld.param.u64 %rd23, [fc_qk_rms_norm_param_0];\n" " ld.param.u64 %rd22, [fc_qk_rms_norm_param_1];\n" " ld.param.u32 %r34, [fc_qk_rms_norm_param_2];\n" " ld.param.u32 %r33, [fc_qk_rms_norm_param_3];\n" " ld.param.f32 %f9, [fc_qk_rms_norm_param_4];\n" " cvta.to.global.u64 %rd1, %rd22;\n" " cvta.to.global.u64 %rd2, %rd23;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r34;\n" " @%p1 bra $L__BB2_27;\n" "\n" " mov.u32 %r67, %tid.x;\n" " mov.u32 %r3, %ntid.x;\n" " mul.wide.s32 %rd3, %r33, %r1;\n" " setp.ge.s32 %p2, %r67, %r33;\n" " mov.f32 %f64, 0f00000000;\n" " @%p2 bra $L__BB2_8;\n" "\n" " not.b32 %r35, %r67;\n" " add.s32 %r36, %r35, %r33;\n" " div.u32 %r4, %r36, %r3;\n" " add.s32 %r37, %r4, 1;\n" " and.b32 %r57, %r37, 3;\n" " setp.eq.s32 %p3, %r57, 0;\n" " mov.f32 %f64, 0f00000000;\n" " mov.u32 %r58, %r67;\n" " @%p3 bra $L__BB2_5;\n" "\n" " cvt.s64.s32 %rd24, %r67;\n" " add.s64 %rd25, %rd3, %rd24;\n" " shl.b64 %rd26, %rd25, 2;\n" " add.s64 %rd60, %rd2, %rd26;\n" " mul.wide.s32 %rd5, %r3, 4;\n" " mov.f32 %f64, 0f00000000;\n" " mov.u32 %r58, %r67;\n" "\n" "$L__BB2_4:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f14, [%rd60];\n" " fma.rn.f32 %f64, %f14, %f14, %f64;\n" " add.s32 %r58, %r58, %r3;\n" " add.s64 %rd60, %rd60, %rd5;\n" " add.s32 %r57, %r57, -1;\n" " setp.ne.s32 %p4, %r57, 0;\n" " @%p4 bra $L__BB2_4;\n" "\n" "$L__BB2_5:\n" " setp.lt.u32 %p5, %r4, 3;\n" " @%p5 bra $L__BB2_8;\n" "\n" " mul.wide.s32 %rd8, %r3, 4;\n" "\n" "$L__BB2_7:\n" " cvt.s64.s32 %rd27, %r58;\n" " add.s64 %rd28, %rd3, %rd27;\n" " shl.b64 %rd29, %rd28, 2;\n" " add.s64 %rd30, %rd2, %rd29;\n" " ld.global.f32 %f15, [%rd30];\n" " fma.rn.f32 %f16, %f15, %f15, %f64;\n" " add.s64 %rd31, %rd30, %rd8;\n" " ld.global.f32 %f17, [%rd31];\n" " fma.rn.f32 %f18, %f17, %f17, %f16;\n" " add.s32 %r38, %r58, %r3;\n" " add.s32 %r39, %r38, %r3;\n" " add.s64 %rd32, %rd31, %rd8;\n" " ld.global.f32 %f19, [%rd32];\n" " fma.rn.f32 %f20, %f19, %f19, %f18;\n" " add.s32 %r40, %r39, %r3;\n" " add.s64 %rd33, %rd32, %rd8;\n" " ld.global.f32 %f21, [%rd33];\n" " fma.rn.f32 %f64, %f21, %f21, %f20;\n" " add.s32 %r58, %r40, %r3;\n" " setp.lt.s32 %p6, %r58, %r33;\n" " @%p6 bra $L__BB2_7;\n" "\n" "$L__BB2_8:\n" " shl.b32 %r41, %r67, 2;\n" " mov.u32 %r42, sdata;\n" " add.s32 %r13, %r42, %r41;\n" " st.shared.f32 [%r13], %f64;\n" " bar.sync 0;\n" " setp.lt.s32 %p7, %r3, 2;\n" " @%p7 bra $L__BB2_13;\n" "\n" " shr.u32 %r43, %r3, 31;\n" " add.s32 %r44, %r3, %r43;\n" " shr.s32 %r60, %r44, 1;\n" "\n" "$L__BB2_10:\n" " setp.ge.s32 %p8, %r67, %r60;\n" " @%p8 bra $L__BB2_12;\n" "\n" " shl.b32 %r45, %r60, 2;\n" " add.s32 %r46, %r13, %r45;\n" " ld.shared.f32 %f22, [%r13];\n" " ld.shared.f32 %f23, [%r46];\n" " add.f32 %f24, %f23, %f22;\n" " st.shared.f32 [%r13], %f24;\n" "\n" "$L__BB2_12:\n" " bar.sync 0;\n" " shr.s32 %r16, %r60, 1;\n" " setp.gt.s32 %p9, %r60, 1;\n" " mov.u32 %r60, %r16;\n" " @%p9 bra $L__BB2_10;\n" "\n" "$L__BB2_13:\n" " cvt.rn.f32.s32 %f25, %r33;\n" " ld.shared.f32 %f26, [sdata];\n" " div.rn.f32 %f27, %f26, %f25;\n" " add.f32 %f28, %f27, %f9;\n" " sqrt.rn.f32 %f29, %f28;\n" " rcp.rn.f32 %f8, %f29;\n" " @%p2 bra $L__BB2_27;\n" "\n" " setp.eq.s64 %p11, %rd22, 0;\n" " not.b32 %r47, %r67;\n" " add.s32 %r48, %r47, %r33;\n" " div.u32 %r17, %r48, %r3;\n" " add.s32 %r49, %r17, 1;\n" " and.b32 %r62, %r49, 3;\n" " @%p11 bra $L__BB2_21;\n" "\n" " setp.eq.s32 %p12, %r62, 0;\n" " @%p12 bra $L__BB2_18;\n" "\n" " cvt.s64.s32 %rd34, %r67;\n" " mul.wide.s32 %rd35, %r67, 4;\n" " add.s64 %rd62, %rd1, %rd35;\n" " mul.wide.s32 %rd10, %r3, 4;\n" " add.s64 %rd36, %rd3, %rd34;\n" " shl.b64 %rd37, %rd36, 2;\n" " add.s64 %rd61, %rd2, %rd37;\n" "\n" "$L__BB2_17:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f30, [%rd61];\n" " mul.f32 %f31, %f8, %f30;\n" " ld.global.f32 %f32, [%rd62];\n" " mul.f32 %f33, %f31, %f32;\n" " st.global.f32 [%rd61], %f33;\n" " add.s32 %r67, %r67, %r3;\n" " add.s64 %rd62, %rd62, %rd10;\n" " add.s64 %rd61, %rd61, %rd10;\n" " add.s32 %r62, %r62, -1;\n" " setp.ne.s32 %p13, %r62, 0;\n" " @%p13 bra $L__BB2_17;\n" "\n" "$L__BB2_18:\n" " setp.lt.u32 %p14, %r17, 3;\n" " @%p14 bra $L__BB2_27;\n" "\n" " mul.wide.s32 %rd16, %r3, 4;\n" "\n" "$L__BB2_20:\n" " cvt.s64.s32 %rd38, %r67;\n" " add.s64 %rd39, %rd3, %rd38;\n" " shl.b64 %rd40, %rd39, 2;\n" " add.s64 %rd41, %rd2, %rd40;\n" " ld.global.f32 %f34, [%rd41];\n" " mul.f32 %f35, %f8, %f34;\n" " mul.wide.s32 %rd42, %r67, 4;\n" " add.s64 %rd43, %rd1, %rd42;\n" " ld.global.f32 %f36, [%rd43];\n" " mul.f32 %f37, %f35, %f36;\n" " st.global.f32 [%rd41], %f37;\n" " add.s64 %rd44, %rd41, %rd16;\n" " ld.global.f32 %f38, [%rd44];\n" " mul.f32 %f39, %f8, %f38;\n" " add.s64 %rd45, %rd43, %rd16;\n" " ld.global.f32 %f40, [%rd45];\n" " mul.f32 %f41, %f39, %f40;\n" " st.global.f32 [%rd44], %f41;\n" " add.s32 %r50, %r67, %r3;\n" " add.s32 %r51, %r50, %r3;\n" " add.s64 %rd46, %rd44, %rd16;\n" " ld.global.f32 %f42, [%rd46];\n" " mul.f32 %f43, %f8, %f42;\n" " add.s64 %rd47, %rd45, %rd16;\n" " ld.global.f32 %f44, [%rd47];\n" " mul.f32 %f45, %f43, %f44;\n" " st.global.f32 [%rd46], %f45;\n" " add.s32 %r52, %r51, %r3;\n" " add.s64 %rd48, %rd46, %rd16;\n" " ld.global.f32 %f46, [%rd48];\n" " mul.f32 %f47, %f8, %f46;\n" " add.s64 %rd49, %rd47, %rd16;\n" " ld.global.f32 %f48, [%rd49];\n" " mul.f32 %f49, %f47, %f48;\n" " st.global.f32 [%rd48], %f49;\n" " add.s32 %r67, %r52, %r3;\n" " setp.lt.s32 %p15, %r67, %r33;\n" " @%p15 bra $L__BB2_20;\n" " bra.uni $L__BB2_27;\n" "\n" "$L__BB2_21:\n" " setp.eq.s32 %p16, %r62, 0;\n" " @%p16 bra $L__BB2_24;\n" "\n" " cvt.s64.s32 %rd50, %r67;\n" " add.s64 %rd51, %rd3, %rd50;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd63, %rd2, %rd52;\n" " mul.wide.s32 %rd18, %r3, 4;\n" "\n" "$L__BB2_23:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f50, [%rd63];\n" " mul.f32 %f51, %f8, %f50;\n" " st.global.f32 [%rd63], %f51;\n" " add.s32 %r67, %r67, %r3;\n" " add.s64 %rd63, %rd63, %rd18;\n" " add.s32 %r62, %r62, -1;\n" " setp.ne.s32 %p17, %r62, 0;\n" " @%p17 bra $L__BB2_23;\n" "\n" "$L__BB2_24:\n" " setp.lt.u32 %p18, %r17, 3;\n" " @%p18 bra $L__BB2_27;\n" "\n" " mul.wide.s32 %rd21, %r3, 4;\n" "\n" "$L__BB2_26:\n" " cvt.s64.s32 %rd53, %r67;\n" " add.s64 %rd54, %rd3, %rd53;\n" " shl.b64 %rd55, %rd54, 2;\n" " add.s64 %rd56, %rd2, %rd55;\n" " ld.global.f32 %f52, [%rd56];\n" " mul.f32 %f53, %f8, %f52;\n" " st.global.f32 [%rd56], %f53;\n" " add.s64 %rd57, %rd56, %rd21;\n" " ld.global.f32 %f54, [%rd57];\n" " mul.f32 %f55, %f8, %f54;\n" " st.global.f32 [%rd57], %f55;\n" " add.s32 %r53, %r67, %r3;\n" " add.s32 %r54, %r53, %r3;\n" " add.s64 %rd58, %rd57, %rd21;\n" " ld.global.f32 %f56, [%rd58];\n" " mul.f32 %f57, %f8, %f56;\n" " st.global.f32 [%rd58], %f57;\n" " add.s32 %r55, %r54, %r3;\n" " add.s64 %rd59, %rd58, %rd21;\n" " ld.global.f32 %f58, [%rd59];\n" " mul.f32 %f59, %f8, %f58;\n" " st.global.f32 [%rd59], %f59;\n" " add.s32 %r67, %r55, %r3;\n" " setp.lt.s32 %p19, %r67, %r33;\n" " @%p19 bra $L__BB2_26;\n" "\n" "$L__BB2_27:\n" " ret;\n" "\n" "}\n" " // .globl fc_rms_norm_nx\n" ".visible .entry fc_rms_norm_nx(\n" " .param .u64 fc_rms_norm_nx_param_0,\n" " .param .u64 fc_rms_norm_nx_param_1,\n" " .param .u64 fc_rms_norm_nx_param_2,\n" " .param .u32 fc_rms_norm_nx_param_3,\n" " .param .f32 fc_rms_norm_nx_param_4,\n" " .param .u32 fc_rms_norm_nx_param_5\n" ")\n" "{\n" " .reg .pred %p<20>;\n" " .reg .f32 %f<65>;\n" " .reg .b32 %r<69>;\n" " .reg .b64 %rd<82>;\n" "\n" "\n" " ld.param.u64 %rd30, [fc_rms_norm_nx_param_0];\n" " ld.param.u64 %rd29, [fc_rms_norm_nx_param_1];\n" " ld.param.u64 %rd31, [fc_rms_norm_nx_param_2];\n" " ld.param.u32 %r33, [fc_rms_norm_nx_param_3];\n" " ld.param.f32 %f9, [fc_rms_norm_nx_param_4];\n" " ld.param.u32 %r34, [fc_rms_norm_nx_param_5];\n" " cvta.to.global.u64 %rd1, %rd31;\n" " cvta.to.global.u64 %rd2, %rd29;\n" " cvta.to.global.u64 %rd3, %rd30;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r34;\n" " @%p1 bra $L__BB3_27;\n" "\n" " mov.u32 %r67, %tid.x;\n" " mov.u32 %r3, %ntid.x;\n" " mul.wide.s32 %rd4, %r33, %r1;\n" " setp.ge.s32 %p2, %r67, %r33;\n" " mov.f32 %f64, 0f00000000;\n" " @%p2 bra $L__BB3_8;\n" "\n" " not.b32 %r35, %r67;\n" " add.s32 %r36, %r35, %r33;\n" " div.u32 %r4, %r36, %r3;\n" " add.s32 %r37, %r4, 1;\n" " and.b32 %r57, %r37, 3;\n" " setp.eq.s32 %p3, %r57, 0;\n" " mov.f32 %f64, 0f00000000;\n" " mov.u32 %r58, %r67;\n" " @%p3 bra $L__BB3_5;\n" "\n" " cvt.s64.s32 %rd32, %r67;\n" " add.s64 %rd33, %rd4, %rd32;\n" " shl.b64 %rd34, %rd33, 2;\n" " add.s64 %rd76, %rd3, %rd34;\n" " mul.wide.s32 %rd6, %r3, 4;\n" " mov.f32 %f64, 0f00000000;\n" " mov.u32 %r58, %r67;\n" "\n" "$L__BB3_4:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f14, [%rd76];\n" " fma.rn.f32 %f64, %f14, %f14, %f64;\n" " add.s32 %r58, %r58, %r3;\n" " add.s64 %rd76, %rd76, %rd6;\n" " add.s32 %r57, %r57, -1;\n" " setp.ne.s32 %p4, %r57, 0;\n" " @%p4 bra $L__BB3_4;\n" "\n" "$L__BB3_5:\n" " setp.lt.u32 %p5, %r4, 3;\n" " @%p5 bra $L__BB3_8;\n" "\n" " mul.wide.s32 %rd9, %r3, 4;\n" "\n" "$L__BB3_7:\n" " cvt.s64.s32 %rd35, %r58;\n" " add.s64 %rd36, %rd4, %rd35;\n" " shl.b64 %rd37, %rd36, 2;\n" " add.s64 %rd38, %rd3, %rd37;\n" " ld.global.f32 %f15, [%rd38];\n" " fma.rn.f32 %f16, %f15, %f15, %f64;\n" " add.s64 %rd39, %rd38, %rd9;\n" " ld.global.f32 %f17, [%rd39];\n" " fma.rn.f32 %f18, %f17, %f17, %f16;\n" " add.s32 %r38, %r58, %r3;\n" " add.s32 %r39, %r38, %r3;\n" " add.s64 %rd40, %rd39, %rd9;\n" " ld.global.f32 %f19, [%rd40];\n" " fma.rn.f32 %f20, %f19, %f19, %f18;\n" " add.s32 %r40, %r39, %r3;\n" " add.s64 %rd41, %rd40, %rd9;\n" " ld.global.f32 %f21, [%rd41];\n" " fma.rn.f32 %f64, %f21, %f21, %f20;\n" " add.s32 %r58, %r40, %r3;\n" " setp.lt.s32 %p6, %r58, %r33;\n" " @%p6 bra $L__BB3_7;\n" "\n" "$L__BB3_8:\n" " shl.b32 %r41, %r67, 2;\n" " mov.u32 %r42, sdata;\n" " add.s32 %r13, %r42, %r41;\n" " st.shared.f32 [%r13], %f64;\n" " bar.sync 0;\n" " setp.lt.s32 %p7, %r3, 2;\n" " @%p7 bra $L__BB3_13;\n" "\n" " shr.u32 %r43, %r3, 31;\n" " add.s32 %r44, %r3, %r43;\n" " shr.s32 %r60, %r44, 1;\n" "\n" "$L__BB3_10:\n" " setp.ge.s32 %p8, %r67, %r60;\n" " @%p8 bra $L__BB3_12;\n" "\n" " shl.b32 %r45, %r60, 2;\n" " add.s32 %r46, %r13, %r45;\n" " ld.shared.f32 %f22, [%r13];\n" " ld.shared.f32 %f23, [%r46];\n" " add.f32 %f24, %f23, %f22;\n" " st.shared.f32 [%r13], %f24;\n" "\n" "$L__BB3_12:\n" " bar.sync 0;\n" " shr.s32 %r16, %r60, 1;\n" " setp.gt.s32 %p9, %r60, 1;\n" " mov.u32 %r60, %r16;\n" " @%p9 bra $L__BB3_10;\n" "\n" "$L__BB3_13:\n" " cvt.rn.f32.s32 %f25, %r33;\n" " ld.shared.f32 %f26, [sdata];\n" " div.rn.f32 %f27, %f26, %f25;\n" " add.f32 %f28, %f27, %f9;\n" " sqrt.rn.f32 %f29, %f28;\n" " rcp.rn.f32 %f8, %f29;\n" " @%p2 bra $L__BB3_27;\n" "\n" " setp.eq.s64 %p11, %rd29, 0;\n" " not.b32 %r47, %r67;\n" " add.s32 %r48, %r47, %r33;\n" " div.u32 %r17, %r48, %r3;\n" " add.s32 %r49, %r17, 1;\n" " and.b32 %r62, %r49, 3;\n" " @%p11 bra $L__BB3_21;\n" "\n" " setp.eq.s32 %p12, %r62, 0;\n" " @%p12 bra $L__BB3_18;\n" "\n" " cvt.s64.s32 %rd42, %r67;\n" " mul.wide.s32 %rd43, %r67, 4;\n" " add.s64 %rd79, %rd2, %rd43;\n" " mul.wide.s32 %rd11, %r3, 4;\n" " add.s64 %rd44, %rd4, %rd42;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd78, %rd1, %rd45;\n" " add.s64 %rd77, %rd3, %rd45;\n" "\n" "$L__BB3_17:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f30, [%rd77];\n" " mul.f32 %f31, %f8, %f30;\n" " ld.global.f32 %f32, [%rd79];\n" " mul.f32 %f33, %f31, %f32;\n" " st.global.f32 [%rd78], %f33;\n" " add.s32 %r67, %r67, %r3;\n" " add.s64 %rd79, %rd79, %rd11;\n" " add.s64 %rd78, %rd78, %rd11;\n" " add.s64 %rd77, %rd77, %rd11;\n" " add.s32 %r62, %r62, -1;\n" " setp.ne.s32 %p13, %r62, 0;\n" " @%p13 bra $L__BB3_17;\n" "\n" "$L__BB3_18:\n" " setp.lt.u32 %p14, %r17, 3;\n" " @%p14 bra $L__BB3_27;\n" "\n" " mul.wide.s32 %rd20, %r3, 4;\n" "\n" "$L__BB3_20:\n" " cvt.s64.s32 %rd46, %r67;\n" " add.s64 %rd47, %rd4, %rd46;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd49, %rd3, %rd48;\n" " ld.global.f32 %f34, [%rd49];\n" " mul.f32 %f35, %f8, %f34;\n" " mul.wide.s32 %rd50, %r67, 4;\n" " add.s64 %rd51, %rd2, %rd50;\n" " ld.global.f32 %f36, [%rd51];\n" " mul.f32 %f37, %f35, %f36;\n" " add.s64 %rd52, %rd1, %rd48;\n" " st.global.f32 [%rd52], %f37;\n" " add.s64 %rd53, %rd49, %rd20;\n" " ld.global.f32 %f38, [%rd53];\n" " mul.f32 %f39, %f8, %f38;\n" " add.s64 %rd54, %rd51, %rd20;\n" " ld.global.f32 %f40, [%rd54];\n" " mul.f32 %f41, %f39, %f40;\n" " add.s64 %rd55, %rd52, %rd20;\n" " st.global.f32 [%rd55], %f41;\n" " add.s32 %r50, %r67, %r3;\n" " add.s32 %r51, %r50, %r3;\n" " add.s64 %rd56, %rd53, %rd20;\n" " ld.global.f32 %f42, [%rd56];\n" " mul.f32 %f43, %f8, %f42;\n" " add.s64 %rd57, %rd54, %rd20;\n" " ld.global.f32 %f44, [%rd57];\n" " mul.f32 %f45, %f43, %f44;\n" " add.s64 %rd58, %rd55, %rd20;\n" " st.global.f32 [%rd58], %f45;\n" " add.s32 %r52, %r51, %r3;\n" " add.s64 %rd59, %rd56, %rd20;\n" " ld.global.f32 %f46, [%rd59];\n" " mul.f32 %f47, %f8, %f46;\n" " add.s64 %rd60, %rd57, %rd20;\n" " ld.global.f32 %f48, [%rd60];\n" " mul.f32 %f49, %f47, %f48;\n" " add.s64 %rd61, %rd58, %rd20;\n" " st.global.f32 [%rd61], %f49;\n" " add.s32 %r67, %r52, %r3;\n" " setp.lt.s32 %p15, %r67, %r33;\n" " @%p15 bra $L__BB3_20;\n" " bra.uni $L__BB3_27;\n" "\n" "$L__BB3_21:\n" " setp.eq.s32 %p16, %r62, 0;\n" " @%p16 bra $L__BB3_24;\n" "\n" " cvt.s64.s32 %rd62, %r67;\n" " add.s64 %rd63, %rd4, %rd62;\n" " shl.b64 %rd64, %rd63, 2;\n" " add.s64 %rd81, %rd1, %rd64;\n" " mul.wide.s32 %rd22, %r3, 4;\n" " add.s64 %rd80, %rd3, %rd64;\n" "\n" "$L__BB3_23:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f50, [%rd80];\n" " mul.f32 %f51, %f8, %f50;\n" " st.global.f32 [%rd81], %f51;\n" " add.s32 %r67, %r67, %r3;\n" " add.s64 %rd81, %rd81, %rd22;\n" " add.s64 %rd80, %rd80, %rd22;\n" " add.s32 %r62, %r62, -1;\n" " setp.ne.s32 %p17, %r62, 0;\n" " @%p17 bra $L__BB3_23;\n" "\n" "$L__BB3_24:\n" " setp.lt.u32 %p18, %r17, 3;\n" " @%p18 bra $L__BB3_27;\n" "\n" " mul.wide.s32 %rd28, %r3, 4;\n" "\n" "$L__BB3_26:\n" " cvt.s64.s32 %rd65, %r67;\n" " add.s64 %rd66, %rd4, %rd65;\n" " shl.b64 %rd67, %rd66, 2;\n" " add.s64 %rd68, %rd3, %rd67;\n" " ld.global.f32 %f52, [%rd68];\n" " mul.f32 %f53, %f8, %f52;\n" " add.s64 %rd69, %rd1, %rd67;\n" " st.global.f32 [%rd69], %f53;\n" " add.s64 %rd70, %rd68, %rd28;\n" " ld.global.f32 %f54, [%rd70];\n" " mul.f32 %f55, %f8, %f54;\n" " add.s64 %rd71, %rd69, %rd28;\n" " st.global.f32 [%rd71], %f55;\n" " add.s32 %r53, %r67, %r3;\n" " add.s32 %r54, %r53, %r3;\n" " add.s64 %rd72, %rd70, %rd28;\n" " ld.global.f32 %f56, [%rd72];\n" " mul.f32 %f57, %f8, %f56;\n" " add.s64 %rd73, %rd71, %rd28;\n" " st.global.f32 [%rd73], %f57;\n" " add.s32 %r55, %r54, %r3;\n" " add.s64 %rd74, %rd72, %rd28;\n" " ld.global.f32 %f58, [%rd74];\n" " mul.f32 %f59, %f8, %f58;\n" " add.s64 %rd75, %rd73, %rd28;\n" " st.global.f32 [%rd75], %f59;\n" " add.s32 %r67, %r55, %r3;\n" " setp.lt.s32 %p19, %r67, %r33;\n" " @%p19 bra $L__BB3_26;\n" "\n" "$L__BB3_27:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_MATVEC_Q8[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_matvec_q8\n" ".extern .shared .align 16 .b8 sdata[];\n" "\n" ".visible .entry fc_matvec_q8(\n" " .param .u64 fc_matvec_q8_param_0,\n" " .param .u64 fc_matvec_q8_param_1,\n" " .param .u64 fc_matvec_q8_param_2,\n" " .param .u64 fc_matvec_q8_param_3,\n" " .param .f32 fc_matvec_q8_param_4,\n" " .param .u32 fc_matvec_q8_param_5,\n" " .param .u32 fc_matvec_q8_param_6\n" ")\n" "{\n" " .reg .pred %p<11>;\n" " .reg .f32 %f<13>;\n" " .reg .b32 %r<85>;\n" " .reg .b64 %rd<37>;\n" "\n" "\n" " ld.param.u64 %rd12, [fc_matvec_q8_param_0];\n" " ld.param.u64 %rd13, [fc_matvec_q8_param_1];\n" " ld.param.u64 %rd10, [fc_matvec_q8_param_2];\n" " ld.param.u64 %rd11, [fc_matvec_q8_param_3];\n" " ld.param.f32 %f3, [fc_matvec_q8_param_4];\n" " ld.param.u32 %r37, [fc_matvec_q8_param_5];\n" " ld.param.u32 %r36, [fc_matvec_q8_param_6];\n" " cvta.to.global.u64 %rd1, %rd13;\n" " cvta.to.global.u64 %rd2, %rd12;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r37;\n" " @%p1 bra $L__BB0_16;\n" "\n" " mov.u32 %r2, %tid.x;\n" " mov.u32 %r3, %ntid.x;\n" " mul.lo.s32 %r4, %r1, %r36;\n" " setp.ge.s32 %p2, %r2, %r36;\n" " mov.f32 %f12, 0f00000000;\n" " @%p2 bra $L__BB0_9;\n" "\n" " not.b32 %r40, %r2;\n" " add.s32 %r41, %r40, %r36;\n" " div.u32 %r42, %r41, %r3;\n" " add.s32 %r5, %r42, 1;\n" " and.b32 %r82, %r5, 3;\n" " setp.lt.u32 %p3, %r42, 3;\n" " mov.u32 %r83, 0;\n" " mov.u32 %r79, %r2;\n" " @%p3 bra $L__BB0_5;\n" "\n" " sub.s32 %r77, %r5, %r82;\n" " mad.lo.s32 %r74, %r3, 3, %r2;\n" " shl.b32 %r9, %r3, 2;\n" " shl.b32 %r44, %r3, 1;\n" " add.s32 %r73, %r2, %r44;\n" " add.s32 %r72, %r2, %r3;\n" " mov.u32 %r83, 0;\n" " cvt.s64.s32 %rd18, %r3;\n" " mov.u32 %r79, %r2;\n" "\n" "$L__BB0_4:\n" " cvt.s64.s32 %rd14, %r79;\n" " add.s64 %rd15, %rd2, %rd14;\n" " ld.global.s8 %r45, [%rd15];\n" " add.s32 %r46, %r79, %r4;\n" " cvt.s64.s32 %rd16, %r46;\n" " add.s64 %rd17, %rd1, %rd16;\n" " ld.global.s8 %r47, [%rd17];\n" " mad.lo.s32 %r48, %r47, %r45, %r83;\n" " add.s64 %rd19, %rd15, %rd18;\n" " ld.global.s8 %r49, [%rd19];\n" " add.s32 %r50, %r72, %r4;\n" " cvt.s64.s32 %rd20, %r50;\n" " add.s64 %rd21, %rd1, %rd20;\n" " ld.global.s8 %r51, [%rd21];\n" " mad.lo.s32 %r52, %r51, %r49, %r48;\n" " add.s32 %r53, %r79, %r3;\n" " add.s32 %r54, %r53, %r3;\n" " add.s64 %rd22, %rd19, %rd18;\n" " ld.global.s8 %r55, [%rd22];\n" " add.s32 %r56, %r73, %r4;\n" " cvt.s64.s32 %rd23, %r56;\n" " add.s64 %rd24, %rd1, %rd23;\n" " ld.global.s8 %r57, [%rd24];\n" " mad.lo.s32 %r58, %r57, %r55, %r52;\n" " add.s32 %r59, %r54, %r3;\n" " add.s64 %rd25, %rd22, %rd18;\n" " ld.global.s8 %r60, [%rd25];\n" " add.s32 %r61, %r74, %r4;\n" " cvt.s64.s32 %rd26, %r61;\n" " add.s64 %rd27, %rd1, %rd26;\n" " ld.global.s8 %r62, [%rd27];\n" " mad.lo.s32 %r83, %r62, %r60, %r58;\n" " add.s32 %r79, %r59, %r3;\n" " add.s32 %r74, %r74, %r9;\n" " add.s32 %r73, %r73, %r9;\n" " add.s32 %r72, %r72, %r9;\n" " add.s32 %r77, %r77, -4;\n" " setp.ne.s32 %p4, %r77, 0;\n" " @%p4 bra $L__BB0_4;\n" "\n" "$L__BB0_5:\n" " setp.eq.s32 %p5, %r82, 0;\n" " @%p5 bra $L__BB0_8;\n" "\n" " add.s32 %r63, %r79, %r4;\n" " cvt.s64.s32 %rd28, %r63;\n" " add.s64 %rd36, %rd1, %rd28;\n" " cvt.s64.s32 %rd4, %r3;\n" " cvt.s64.s32 %rd29, %r79;\n" " add.s64 %rd35, %rd2, %rd29;\n" "\n" "$L__BB0_7:\n" " .pragma \"nounroll\";\n" " ld.global.s8 %r64, [%rd35];\n" " ld.global.s8 %r65, [%rd36];\n" " mad.lo.s32 %r83, %r65, %r64, %r83;\n" " add.s64 %rd36, %rd36, %rd4;\n" " add.s64 %rd35, %rd35, %rd4;\n" " add.s32 %r82, %r82, -1;\n" " setp.ne.s32 %p6, %r82, 0;\n" " @%p6 bra $L__BB0_7;\n" "\n" "$L__BB0_8:\n" " cvt.rn.f32.s32 %f12, %r83;\n" "\n" "$L__BB0_9:\n" " shl.b32 %r66, %r2, 2;\n" " mov.u32 %r67, sdata;\n" " add.s32 %r32, %r67, %r66;\n" " st.shared.f32 [%r32], %f12;\n" " bar.sync 0;\n" " setp.lt.s32 %p7, %r3, 2;\n" " @%p7 bra $L__BB0_14;\n" "\n" " shr.u32 %r68, %r3, 31;\n" " add.s32 %r69, %r3, %r68;\n" " shr.s32 %r84, %r69, 1;\n" "\n" "$L__BB0_11:\n" " setp.ge.s32 %p8, %r2, %r84;\n" " @%p8 bra $L__BB0_13;\n" "\n" " shl.b32 %r70, %r84, 2;\n" " add.s32 %r71, %r32, %r70;\n" " ld.shared.f32 %f5, [%r32];\n" " ld.shared.f32 %f6, [%r71];\n" " add.f32 %f7, %f6, %f5;\n" " st.shared.f32 [%r32], %f7;\n" "\n" "$L__BB0_13:\n" " bar.sync 0;\n" " shr.s32 %r35, %r84, 1;\n" " setp.gt.s32 %p9, %r84, 1;\n" " mov.u32 %r84, %r35;\n" " @%p9 bra $L__BB0_11;\n" "\n" "$L__BB0_14:\n" " setp.ne.s32 %p10, %r2, 0;\n" " @%p10 bra $L__BB0_16;\n" "\n" " cvta.to.global.u64 %rd30, %rd11;\n" " cvta.to.global.u64 %rd31, %rd10;\n" " mul.wide.s32 %rd32, %r1, 4;\n" " add.s64 %rd33, %rd31, %rd32;\n" " ld.global.f32 %f8, [%rd33];\n" " mul.f32 %f9, %f8, %f3;\n" " ld.shared.f32 %f10, [sdata];\n" " mul.f32 %f11, %f9, %f10;\n" " add.s64 %rd34, %rd30, %rd32;\n" " st.global.f32 [%rd34], %f11;\n" "\n" "$L__BB0_16:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_MATVEC_Q3K[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_matvec_q3k\n" ".extern .shared .align 16 .b8 sh_xps[];\n" ".extern .shared .align 16 .b8 sh_x[];\n" ".extern .shared .align 16 .b8 sh_x$1[];\n" "\n" ".visible .entry fc_matvec_q3k(\n" " .param .u64 fc_matvec_q3k_param_0,\n" " .param .u64 fc_matvec_q3k_param_1,\n" " .param .u64 fc_matvec_q3k_param_2,\n" " .param .u32 fc_matvec_q3k_param_3,\n" " .param .u32 fc_matvec_q3k_param_4,\n" " .param .u32 fc_matvec_q3k_param_5\n" ")\n" "{\n" " .reg .pred %p<19>;\n" " .reg .f32 %f<170>;\n" " .reg .b32 %r<139>;\n" " .reg .b64 %rd<64>;\n" "\n" "\n" " ld.param.u64 %rd26, [fc_matvec_q3k_param_0];\n" " ld.param.u64 %rd24, [fc_matvec_q3k_param_1];\n" " ld.param.u64 %rd25, [fc_matvec_q3k_param_2];\n" " ld.param.u32 %r28, [fc_matvec_q3k_param_3];\n" " ld.param.u32 %r29, [fc_matvec_q3k_param_4];\n" " ld.param.u32 %r30, [fc_matvec_q3k_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r31, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r32, %ctaid.x;\n" " shl.b32 %r33, %r32, 3;\n" " add.s32 %r34, %r33, %r31;\n" " shl.b32 %r35, %r34, 1;\n" " or.b32 %r3, %r35, 1;\n" " setp.ge.s32 %p1, %r3, %r28;\n" " cvt.s64.s32 %rd1, %r35;\n" " cvt.s64.s32 %rd2, %r30;\n" " mul.wide.s32 %rd27, %r30, %r35;\n" " add.s64 %rd62, %rd26, %rd27;\n" " mov.u64 %rd63, %rd62;\n" " @%p1 bra $L__BB0_2;\n" "\n" " add.s64 %rd63, %rd62, %rd2;\n" "\n" "$L__BB0_2:\n" " setp.lt.s32 %p2, %r29, 1;\n" " mov.f32 %f166, 0f00000000;\n" " mov.f32 %f167, %f166;\n" " @%p2 bra $L__BB0_17;\n" "\n" " cvt.u64.u32 %rd6, %r2;\n" " shr.u32 %r38, %r2, 4;\n" " cvt.u64.u32 %rd7, %r38;\n" " mov.u32 %r39, -2;\n" " sub.s32 %r4, %r39, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd8, %r1;\n" " shl.b32 %r40, %r1, 2;\n" " mov.u32 %r41, sh_x$1;\n" " add.s32 %r6, %r41, %r40;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd9, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd10, %r8;\n" " add.s32 %r9, %r1, 768;\n" " mov.f32 %f167, 0f00000000;\n" " mov.u32 %r132, 0;\n" " shl.b64 %rd50, %rd6, 2;\n" " shl.b64 %rd51, %rd7, 3;\n" " mov.u32 %r133, %r132;\n" " mov.f32 %f166, %f167;\n" "\n" "$L__BB0_4:\n" " add.s32 %r42, %r132, %r5;\n" " max.s32 %r43, %r42, -2049;\n" " sub.s32 %r12, %r4, %r43;\n" " sub.s32 %r44, %r29, %r133;\n" " min.s32 %r13, %r44, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r13;\n" " @%p3 bra $L__BB0_12;\n" "\n" " shr.u32 %r45, %r12, 8;\n" " add.s32 %r46, %r45, 1;\n" " cvt.s64.s32 %rd13, %r133;\n" " and.b32 %r14, %r46, 3;\n" " setp.eq.s32 %p4, %r14, 0;\n" " mov.u32 %r134, %r1;\n" " @%p4 bra $L__BB0_9;\n" "\n" " add.s64 %rd29, %rd8, %rd13;\n" " shl.b64 %rd30, %rd29, 2;\n" " add.s64 %rd28, %rd24, %rd30;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd28];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p5, %r14, 1;\n" " mov.u32 %r134, %r7;\n" " @%p5 bra $L__BB0_9;\n" "\n" " add.s64 %rd32, %rd9, %rd13;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd31, %rd24, %rd33;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd31];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p6, %r14, 2;\n" " mov.u32 %r134, %r8;\n" " @%p6 bra $L__BB0_9;\n" "\n" " add.s64 %rd35, %rd10, %rd13;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd34, %rd24, %rd36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd34];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r134, %r9;\n" "\n" "$L__BB0_9:\n" " setp.lt.u32 %p7, %r12, 768;\n" " @%p7 bra $L__BB0_12;\n" "\n" " cvt.s64.s32 %rd37, %r134;\n" " add.s64 %rd38, %rd37, %rd13;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd59, %rd24, %rd39;\n" " shl.b32 %r47, %r134, 2;\n" " mov.u32 %r48, sh_x$1;\n" " add.s32 %r49, %r48, %r47;\n" " add.s32 %r135, %r49, 2048;\n" "\n" "$L__BB0_11:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd59];\n" " // end inline asm\n" " st.shared.f32 [%r135+-2048], %f20;\n" " add.s64 %rd41, %rd59, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r135+-1024], %f21;\n" " add.s64 %rd42, %rd59, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r135], %f22;\n" " add.s64 %rd43, %rd59, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd43];\n" " // end inline asm\n" " st.shared.f32 [%r135+1024], %f23;\n" " add.s64 %rd59, %rd59, 4096;\n" " add.s32 %r135, %r135, 4096;\n" " add.s32 %r134, %r134, 1024;\n" " setp.lt.s32 %p8, %r134, %r13;\n" " @%p8 bra $L__BB0_11;\n" "\n" "$L__BB0_12:\n" " cvt.u32.u64 %r50, %rd1;\n" " setp.ge.s32 %p9, %r50, %r28;\n" " bar.sync 0;\n" " @%p9 bra $L__BB0_16;\n" "\n" " setp.lt.s32 %p10, %r13, 256;\n" " @%p10 bra $L__BB0_16;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r137, 0;\n" " mov.u32 %r138, %r41;\n" "\n" "$L__BB0_15:\n" " add.s64 %rd44, %rd62, %rd50;\n" " // begin inline asm\n" " ld.global.nc.u32 %r53, [%rd44];\n" " // end inline asm\n" " add.s64 %rd45, %rd63, %rd50;\n" " // begin inline asm\n" " ld.global.nc.u32 %r54, [%rd45];\n" " // end inline asm\n" " add.s64 %rd46, %rd62, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd46];\n" " // end inline asm\n" " add.s64 %rd47, %rd63, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd47];\n" " // end inline asm\n" " add.s64 %rd52, %rd62, %rd51;\n" " add.s64 %rd48, %rd52, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r55,%r56}, [%rd48];\n" " // end inline asm\n" " add.s64 %rd53, %rd63, %rd51;\n" " add.s64 %rd49, %rd53, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r57,%r58}, [%rd49];\n" " // end inline asm\n" " shr.u32 %r59, %r53, 16;\n" " shr.u32 %r60, %r54, 16;\n" " and.b32 %r61, %r53, 15;\n" " or.b32 %r62, %r61, 1258291200;\n" " mov.b32 %f26, %r62;\n" " mov.f32 %f27, 0fCB000004;\n" " mov.f32 %f28, 0f3F800000;\n" " fma.rn.f32 %f29, %f26, %f28, %f27;\n" " and.b32 %r63, %r55, 255;\n" " cvt.rn.f32.u32 %f30, %r63;\n" " mov.f32 %f31, 0f42000000;\n" " sub.rn.f32 %f32, %f30, %f31;\n" " and.b32 %r64, %r54, 15;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f33, %r65;\n" " fma.rn.f32 %f34, %f33, %f28, %f27;\n" " and.b32 %r66, %r57, 255;\n" " cvt.rn.f32.u32 %f35, %r66;\n" " sub.rn.f32 %f36, %f35, %f31;\n" " mul.rn.f32 %f37, %f32, %f29;\n" " shl.b32 %r67, %r2, 2;\n" " add.s32 %r68, %r138, %r67;\n" " ld.shared.f32 %f38, [%r68];\n" " mov.f32 %f39, 0f00000000;\n" " fma.rn.f32 %f40, %f37, %f38, %f39;\n" " mul.rn.f32 %f41, %f36, %f34;\n" " fma.rn.f32 %f42, %f41, %f38, %f39;\n" " and.b32 %r69, %r53, 240;\n" " or.b32 %r70, %r69, 1258291200;\n" " mov.b32 %f43, %r70;\n" " mov.f32 %f44, 0fC9000040;\n" " mov.f32 %f45, 0f3D800000;\n" " fma.rn.f32 %f46, %f43, %f45, %f44;\n" " bfe.u32 %r71, %r55, 8, 8;\n" " cvt.rn.f32.u32 %f47, %r71;\n" " sub.rn.f32 %f48, %f47, %f31;\n" " and.b32 %r72, %r54, 240;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f49, %r73;\n" " fma.rn.f32 %f50, %f49, %f45, %f44;\n" " bfe.u32 %r74, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f51, %r74;\n" " sub.rn.f32 %f52, %f51, %f31;\n" " mul.rn.f32 %f53, %f48, %f46;\n" " ld.shared.f32 %f54, [%r68+128];\n" " fma.rn.f32 %f55, %f53, %f54, %f40;\n" " mul.rn.f32 %f56, %f52, %f50;\n" " fma.rn.f32 %f57, %f56, %f54, %f42;\n" " and.b32 %r75, %r53, 3840;\n" " or.b32 %r76, %r75, 1258291200;\n" " mov.b32 %f58, %r76;\n" " mov.f32 %f59, 0fC7000400;\n" " mov.f32 %f60, 0f3B800000;\n" " fma.rn.f32 %f61, %f58, %f60, %f59;\n" " bfe.u32 %r77, %r55, 16, 8;\n" " cvt.rn.f32.u32 %f62, %r77;\n" " sub.rn.f32 %f63, %f62, %f31;\n" " and.b32 %r78, %r54, 3840;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f64, %r79;\n" " fma.rn.f32 %f65, %f64, %f60, %f59;\n" " bfe.u32 %r80, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f66, %r80;\n" " sub.rn.f32 %f67, %f66, %f31;\n" " mul.rn.f32 %f68, %f63, %f61;\n" " ld.shared.f32 %f69, [%r68+256];\n" " fma.rn.f32 %f70, %f68, %f69, %f55;\n" " mul.rn.f32 %f71, %f67, %f65;\n" " fma.rn.f32 %f72, %f71, %f69, %f57;\n" " and.b32 %r81, %r53, 61440;\n" " or.b32 %r82, %r81, 1258291200;\n" " mov.b32 %f73, %r82;\n" " mov.f32 %f74, 0fC5004000;\n" " mov.f32 %f75, 0f39800000;\n" " fma.rn.f32 %f76, %f73, %f75, %f74;\n" " shr.u32 %r83, %r55, 24;\n" " cvt.rn.f32.u32 %f77, %r83;\n" " sub.rn.f32 %f78, %f77, %f31;\n" " and.b32 %r84, %r54, 61440;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f79, %r85;\n" " fma.rn.f32 %f80, %f79, %f75, %f74;\n" " shr.u32 %r86, %r57, 24;\n" " cvt.rn.f32.u32 %f81, %r86;\n" " sub.rn.f32 %f82, %f81, %f31;\n" " mul.rn.f32 %f83, %f78, %f76;\n" " ld.shared.f32 %f84, [%r68+384];\n" " fma.rn.f32 %f85, %f83, %f84, %f70;\n" " mul.rn.f32 %f86, %f82, %f80;\n" " fma.rn.f32 %f87, %f86, %f84, %f72;\n" " and.b32 %r87, %r53, 983040;\n" " or.b32 %r88, %r87, 1258291200;\n" " mov.b32 %f88, %r88;\n" " mov.f32 %f89, 0fC3040000;\n" " mov.f32 %f90, 0f37800000;\n" " fma.rn.f32 %f91, %f88, %f90, %f89;\n" " and.b32 %r89, %r56, 255;\n" " cvt.rn.f32.u32 %f92, %r89;\n" " sub.rn.f32 %f93, %f92, %f31;\n" " and.b32 %r90, %r54, 983040;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f94, %r91;\n" " fma.rn.f32 %f95, %f94, %f90, %f89;\n" " and.b32 %r92, %r58, 255;\n" " cvt.rn.f32.u32 %f96, %r92;\n" " sub.rn.f32 %f97, %f96, %f31;\n" " mul.rn.f32 %f98, %f93, %f91;\n" " ld.shared.f32 %f99, [%r68+512];\n" " fma.rn.f32 %f100, %f98, %f99, %f85;\n" " mul.rn.f32 %f101, %f97, %f95;\n" " fma.rn.f32 %f102, %f101, %f99, %f87;\n" " and.b32 %r93, %r59, 240;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f103, %r94;\n" " fma.rn.f32 %f104, %f103, %f45, %f44;\n" " bfe.u32 %r95, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f105, %r95;\n" " sub.rn.f32 %f106, %f105, %f31;\n" " and.b32 %r96, %r60, 240;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f107, %r97;\n" " fma.rn.f32 %f108, %f107, %f45, %f44;\n" " bfe.u32 %r98, %r58, 8, 8;\n" " cvt.rn.f32.u32 %f109, %r98;\n" " sub.rn.f32 %f110, %f109, %f31;\n" " mul.rn.f32 %f111, %f106, %f104;\n" " ld.shared.f32 %f112, [%r68+640];\n" " fma.rn.f32 %f113, %f111, %f112, %f100;\n" " mul.rn.f32 %f114, %f110, %f108;\n" " fma.rn.f32 %f115, %f114, %f112, %f102;\n" " and.b32 %r99, %r59, 3840;\n" " or.b32 %r100, %r99, 1258291200;\n" " mov.b32 %f116, %r100;\n" " fma.rn.f32 %f117, %f116, %f60, %f59;\n" " bfe.u32 %r101, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f118, %r101;\n" " sub.rn.f32 %f119, %f118, %f31;\n" " and.b32 %r102, %r60, 3840;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f120, %r103;\n" " fma.rn.f32 %f121, %f120, %f60, %f59;\n" " bfe.u32 %r104, %r58, 16, 8;\n" " cvt.rn.f32.u32 %f122, %r104;\n" " sub.rn.f32 %f123, %f122, %f31;\n" " mul.rn.f32 %f124, %f119, %f117;\n" " ld.shared.f32 %f125, [%r68+768];\n" " fma.rn.f32 %f126, %f124, %f125, %f113;\n" " mul.rn.f32 %f127, %f123, %f121;\n" " fma.rn.f32 %f128, %f127, %f125, %f115;\n" " and.b32 %r105, %r59, 61440;\n" " or.b32 %r106, %r105, 1258291200;\n" " mov.b32 %f129, %r106;\n" " fma.rn.f32 %f130, %f129, %f75, %f74;\n" " shr.u32 %r107, %r56, 24;\n" " cvt.rn.f32.u32 %f131, %r107;\n" " sub.rn.f32 %f132, %f131, %f31;\n" " and.b32 %r108, %r60, 61440;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f133, %r109;\n" " fma.rn.f32 %f134, %f133, %f75, %f74;\n" " shr.u32 %r110, %r58, 24;\n" " cvt.rn.f32.u32 %f135, %r110;\n" " sub.rn.f32 %f136, %f135, %f31;\n" " mul.rn.f32 %f137, %f132, %f130;\n" " ld.shared.f32 %f138, [%r68+896];\n" " fma.rn.f32 %f139, %f137, %f138, %f126;\n" " mul.rn.f32 %f140, %f136, %f134;\n" " fma.rn.f32 %f141, %f140, %f138, %f128;\n" " fma.rn.f32 %f166, %f24, %f139, %f166;\n" " fma.rn.f32 %f167, %f25, %f141, %f167;\n" " add.s64 %rd62, %rd62, 152;\n" " add.s64 %rd63, %rd63, 152;\n" " add.s32 %r138, %r138, 1024;\n" " add.s32 %r137, %r137, 1;\n" " setp.lt.s32 %p11, %r137, %r21;\n" " @%p11 bra $L__BB0_15;\n" "\n" "$L__BB0_16:\n" " add.s32 %r133, %r133, 2048;\n" " setp.lt.s32 %p12, %r133, %r29;\n" " add.s32 %r132, %r132, 2048;\n" " @%p12 bra $L__BB0_4;\n" "\n" "$L__BB0_17:\n" " cvt.u32.u64 %r131, %rd1;\n" " setp.ge.s32 %p13, %r131, %r28;\n" " mov.u32 %r121, 16;\n" " mov.u32 %r130, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f142, %f166, %r121, %r130;\n" " // end inline asm\n" " add.f32 %f145, %f166, %f142;\n" " mov.u32 %r123, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f144, %f145, %r123, %r130;\n" " // end inline asm\n" " add.f32 %f147, %f145, %f144;\n" " mov.u32 %r125, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f146, %f147, %r125, %r130;\n" " // end inline asm\n" " add.f32 %f149, %f147, %f146;\n" " mov.u32 %r127, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f148, %f149, %r127, %r130;\n" " // end inline asm\n" " add.f32 %f151, %f149, %f148;\n" " mov.u32 %r129, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f150, %f151, %r129, %r130;\n" " // end inline asm\n" " add.f32 %f11, %f151, %f150;\n" " // begin inline asm\n" " shfl.down.b32 %f152, %f167, %r121, %r130;\n" " // end inline asm\n" " add.f32 %f155, %f167, %f152;\n" " // begin inline asm\n" " shfl.down.b32 %f154, %f155, %r123, %r130;\n" " // end inline asm\n" " add.f32 %f157, %f155, %f154;\n" " // begin inline asm\n" " shfl.down.b32 %f156, %f157, %r125, %r130;\n" " // end inline asm\n" " add.f32 %f159, %f157, %f156;\n" " // begin inline asm\n" " shfl.down.b32 %f158, %f159, %r127, %r130;\n" " // end inline asm\n" " add.f32 %f161, %f159, %f158;\n" " // begin inline asm\n" " shfl.down.b32 %f160, %f161, %r129, %r130;\n" " // end inline asm\n" " add.f32 %f12, %f161, %f160;\n" " setp.ne.s32 %p14, %r2, 0;\n" " cvta.to.global.u64 %rd54, %rd25;\n" " shl.b64 %rd55, %rd1, 2;\n" " add.s64 %rd23, %rd54, %rd55;\n" " or.pred %p15, %p14, %p13;\n" " @%p15 bra $L__BB0_19;\n" "\n" " st.global.f32 [%rd23], %f11;\n" "\n" "$L__BB0_19:\n" " or.pred %p18, %p14, %p1;\n" " @%p18 bra $L__BB0_21;\n" "\n" " st.global.f32 [%rd23+4], %f12;\n" "\n" "$L__BB0_21:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_ps_r2p1\n" ".visible .entry fc_matvec_q3k_ps_r2p1(\n" " .param .u64 fc_matvec_q3k_ps_r2p1_param_0,\n" " .param .u64 fc_matvec_q3k_ps_r2p1_param_1,\n" " .param .u64 fc_matvec_q3k_ps_r2p1_param_2,\n" " .param .u32 fc_matvec_q3k_ps_r2p1_param_3,\n" " .param .u32 fc_matvec_q3k_ps_r2p1_param_4,\n" " .param .u32 fc_matvec_q3k_ps_r2p1_param_5\n" ")\n" "{\n" " .reg .pred %p<30>;\n" " .reg .f32 %f<338>;\n" " .reg .b32 %r<242>;\n" " .reg .b64 %rd<92>;\n" "\n" "\n" " ld.param.u64 %rd29, [fc_matvec_q3k_ps_r2p1_param_0];\n" " ld.param.u64 %rd30, [fc_matvec_q3k_ps_r2p1_param_1];\n" " ld.param.u64 %rd31, [fc_matvec_q3k_ps_r2p1_param_2];\n" " ld.param.u32 %r44, [fc_matvec_q3k_ps_r2p1_param_3];\n" " ld.param.u32 %r45, [fc_matvec_q3k_ps_r2p1_param_4];\n" " ld.param.u32 %r46, [fc_matvec_q3k_ps_r2p1_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r47, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r48, %ctaid.x;\n" " shl.b32 %r49, %r48, 3;\n" " add.s32 %r50, %r49, %r47;\n" " shl.b32 %r3, %r50, 1;\n" " cvt.s64.s32 %rd1, %r3;\n" " or.b32 %r4, %r3, 1;\n" " setp.lt.s32 %p1, %r45, 1;\n" " mov.f32 %f334, 0f00000000;\n" " mov.f32 %f335, %f334;\n" " @%p1 bra $L__BB1_25;\n" "\n" " cvt.s64.s32 %rd32, %r46;\n" " cvt.u64.u32 %rd2, %r2;\n" " shr.u32 %r53, %r2, 4;\n" " cvt.u64.u32 %rd3, %r53;\n" " mov.u32 %r54, -2;\n" " sub.s32 %r5, %r54, %r1;\n" " not.b32 %r6, %r45;\n" " not.b32 %r7, %r1;\n" " cvt.s64.s32 %rd4, %r1;\n" " add.s32 %r8, %r1, 256;\n" " shl.b32 %r55, %r1, 4;\n" " mov.u32 %r56, sh_xps;\n" " add.s32 %r9, %r56, %r55;\n" " mad.lo.s32 %r10, %r1, -12, %r9;\n" " add.s32 %r11, %r1, 512;\n" " add.s32 %r12, %r1, 768;\n" " setp.lt.s32 %p2, %r4, %r44;\n" " selp.u32 %r57, 1, 0, %p2;\n" " add.s32 %r58, %r3, %r57;\n" " mul.wide.s32 %rd33, %r58, %r46;\n" " add.s64 %rd91, %rd29, %rd33;\n" " mul.lo.s64 %rd34, %rd1, %rd32;\n" " add.s64 %rd90, %rd29, %rd34;\n" " mov.u32 %r229, 0;\n" " mov.f32 %f334, 0f00000000;\n" " mov.f32 %f335, %f334;\n" " mov.u32 %r230, %r229;\n" "\n" "$L__BB1_2:\n" " add.s32 %r59, %r229, %r6;\n" " max.s32 %r60, %r59, -2049;\n" " sub.s32 %r15, %r5, %r60;\n" " sub.s32 %r61, %r45, %r230;\n" " min.s32 %r16, %r61, 2048;\n" " bar.sync 0;\n" " cvt.s64.s32 %rd9, %r230;\n" " mul.wide.s32 %rd35, %r230, 4;\n" " add.s64 %rd10, %rd30, %rd35;\n" " and.b32 %r62, %r16, 3;\n" " setp.eq.s32 %p3, %r62, 0;\n" " { .reg .b64 %tmp;\n" " cvt.u64.u32 %tmp, %r56;\n" " cvta.shared.u64 %rd36, %tmp; }\n" " or.b64 %rd37, %rd10, %rd36;\n" " and.b64 %rd38, %rd37, 15;\n" " setp.eq.s64 %p4, %rd38, 0;\n" " and.pred %p5, %p3, %p4;\n" " @%p5 bra $L__BB1_11;\n" " bra.uni $L__BB1_3;\n" "\n" "$L__BB1_11:\n" " shr.s32 %r24, %r16, 2;\n" " setp.ge.s32 %p12, %r1, %r24;\n" " @%p12 bra $L__BB1_18;\n" "\n" " add.s32 %r25, %r24, %r7;\n" " shr.u32 %r69, %r25, 8;\n" " add.s32 %r70, %r69, 1;\n" " and.b32 %r26, %r70, 3;\n" " setp.eq.s32 %p13, %r26, 0;\n" " mov.u32 %r234, %r1;\n" " @%p13 bra $L__BB1_16;\n" "\n" " shl.b64 %rd51, %rd4, 4;\n" " add.s64 %rd50, %rd10, %rd51;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f30,%f31,%f32,%f33}, [%rd50];\n" " // end inline asm\n" " st.shared.v4.f32 [%r9], {%f30, %f31, %f32, %f33};\n" " setp.eq.s32 %p14, %r26, 1;\n" " mov.u32 %r234, %r8;\n" " @%p14 bra $L__BB1_16;\n" "\n" " add.s64 %rd52, %rd50, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f34,%f35,%f36,%f37}, [%rd52];\n" " // end inline asm\n" " st.shared.v4.f32 [%r9+4096], {%f34, %f35, %f36, %f37};\n" " setp.eq.s32 %p15, %r26, 2;\n" " mov.u32 %r234, %r11;\n" " @%p15 bra $L__BB1_16;\n" "\n" " add.s64 %rd53, %rd50, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f38,%f39,%f40,%f41}, [%rd53];\n" " // end inline asm\n" " st.shared.v4.f32 [%r9+8192], {%f38, %f39, %f40, %f41};\n" " mov.u32 %r234, %r12;\n" "\n" "$L__BB1_16:\n" " setp.lt.u32 %p16, %r25, 768;\n" " @%p16 bra $L__BB1_18;\n" "\n" "$L__BB1_17:\n" " mul.wide.s32 %rd58, %r234, 16;\n" " add.s64 %rd54, %rd10, %rd58;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f42,%f43,%f44,%f45}, [%rd54];\n" " // end inline asm\n" " shl.b32 %r71, %r234, 4;\n" " mov.u32 %r72, sh_xps;\n" " add.s32 %r73, %r72, %r71;\n" " st.shared.v4.f32 [%r73], {%f42, %f43, %f44, %f45};\n" " add.s64 %rd55, %rd54, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f46,%f47,%f48,%f49}, [%rd55];\n" " // end inline asm\n" " st.shared.v4.f32 [%r73+4096], {%f46, %f47, %f48, %f49};\n" " add.s64 %rd56, %rd54, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f50,%f51,%f52,%f53}, [%rd56];\n" " // end inline asm\n" " st.shared.v4.f32 [%r73+8192], {%f50, %f51, %f52, %f53};\n" " add.s64 %rd57, %rd54, 12288;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f54,%f55,%f56,%f57}, [%rd57];\n" " // end inline asm\n" " st.shared.v4.f32 [%r73+12288], {%f54, %f55, %f56, %f57};\n" " add.s32 %r234, %r234, 1024;\n" " setp.lt.s32 %p17, %r234, %r24;\n" " @%p17 bra $L__BB1_17;\n" " bra.uni $L__BB1_18;\n" "\n" "$L__BB1_3:\n" " setp.ge.s32 %p6, %r1, %r16;\n" " @%p6 bra $L__BB1_18;\n" "\n" " shr.u32 %r64, %r15, 8;\n" " add.s32 %r65, %r64, 1;\n" " and.b32 %r17, %r65, 3;\n" " setp.eq.s32 %p7, %r17, 0;\n" " mov.u32 %r231, %r1;\n" " @%p7 bra $L__BB1_8;\n" "\n" " shl.b64 %rd40, %rd4, 2;\n" " add.s64 %rd39, %rd10, %rd40;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r10], %f23;\n" " setp.eq.s32 %p8, %r17, 1;\n" " mov.u32 %r231, %r8;\n" " @%p8 bra $L__BB1_8;\n" "\n" " add.s64 %rd41, %rd39, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r10+1024], %f24;\n" " setp.eq.s32 %p9, %r17, 2;\n" " mov.u32 %r231, %r11;\n" " @%p9 bra $L__BB1_8;\n" "\n" " add.s64 %rd42, %rd39, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r10+2048], %f25;\n" " mov.u32 %r231, %r12;\n" "\n" "$L__BB1_8:\n" " setp.lt.u32 %p10, %r15, 768;\n" " @%p10 bra $L__BB1_18;\n" "\n" " cvt.s64.s32 %rd43, %r231;\n" " add.s64 %rd44, %rd43, %rd9;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd83, %rd30, %rd45;\n" " shl.b32 %r66, %r231, 2;\n" " add.s32 %r68, %r56, %r66;\n" " add.s32 %r232, %r68, 2048;\n" "\n" "$L__BB1_10:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd83];\n" " // end inline asm\n" " st.shared.f32 [%r232+-2048], %f26;\n" " add.s64 %rd47, %rd83, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd47];\n" " // end inline asm\n" " st.shared.f32 [%r232+-1024], %f27;\n" " add.s64 %rd48, %rd83, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r232], %f28;\n" " add.s64 %rd49, %rd83, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r232+1024], %f29;\n" " add.s64 %rd83, %rd83, 4096;\n" " add.s32 %r232, %r232, 4096;\n" " add.s32 %r231, %r231, 1024;\n" " setp.lt.s32 %p11, %r231, %r16;\n" " @%p11 bra $L__BB1_10;\n" "\n" "$L__BB1_18:\n" " cvt.u32.u64 %r74, %rd1;\n" " setp.ge.s32 %p18, %r74, %r44;\n" " bar.sync 0;\n" " @%p18 bra $L__BB1_24;\n" "\n" " shr.s32 %r31, %r16, 8;\n" " setp.lt.s32 %p19, %r16, 256;\n" " mov.u32 %r239, 0;\n" " mov.u32 %r238, %r56;\n" " @%p19 bra $L__BB1_22;\n" "\n" " mov.u32 %r238, sh_xps;\n" " mov.u32 %r239, 0;\n" "\n" "$L__BB1_21:\n" " shl.b64 %rd65, %rd2, 2;\n" " add.s64 %rd59, %rd90, %rd65;\n" " // begin inline asm\n" " ld.global.nc.u32 %r85, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd90, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f60, [%rd60];\n" " // end inline asm\n" " shl.b64 %rd66, %rd3, 3;\n" " add.s64 %rd67, %rd90, %rd66;\n" " add.s64 %rd61, %rd67, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r86,%r87}, [%rd61];\n" " // end inline asm\n" " add.s64 %rd62, %rd91, %rd65;\n" " // begin inline asm\n" " ld.global.nc.u32 %r91, [%rd62];\n" " // end inline asm\n" " add.s64 %rd63, %rd91, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f62, [%rd63];\n" " // end inline asm\n" " add.s64 %rd68, %rd91, %rd66;\n" " add.s64 %rd64, %rd68, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r92,%r93}, [%rd64];\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " shr.u32 %r97, %r85, 16;\n" " and.b32 %r98, %r85, 15;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f64, %r99;\n" " mov.f32 %f65, 0fCB000004;\n" " mov.f32 %f66, 0f3F800000;\n" " fma.rn.f32 %f67, %f64, %f66, %f65;\n" " and.b32 %r100, %r86, 255;\n" " cvt.rn.f32.u32 %f68, %r100;\n" " mov.f32 %f69, 0f42000000;\n" " sub.rn.f32 %f70, %f68, %f69;\n" " mul.rn.f32 %f71, %f70, %f67;\n" " shl.b32 %r101, %r2, 2;\n" " add.s32 %r102, %r238, %r101;\n" " ld.shared.f32 %f72, [%r102];\n" " mov.f32 %f73, 0f00000000;\n" " fma.rn.f32 %f74, %f71, %f72, %f73;\n" " and.b32 %r103, %r85, 240;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f75, %r104;\n" " mov.f32 %f76, 0fC9000040;\n" " mov.f32 %f77, 0f3D800000;\n" " fma.rn.f32 %f78, %f75, %f77, %f76;\n" " bfe.u32 %r105, %r86, 8, 8;\n" " cvt.rn.f32.u32 %f79, %r105;\n" " sub.rn.f32 %f80, %f79, %f69;\n" " mul.rn.f32 %f81, %f80, %f78;\n" " ld.shared.f32 %f82, [%r102+128];\n" " fma.rn.f32 %f83, %f81, %f82, %f74;\n" " and.b32 %r106, %r85, 3840;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f84, %r107;\n" " mov.f32 %f85, 0fC7000400;\n" " mov.f32 %f86, 0f3B800000;\n" " fma.rn.f32 %f87, %f84, %f86, %f85;\n" " bfe.u32 %r108, %r86, 16, 8;\n" " cvt.rn.f32.u32 %f88, %r108;\n" " sub.rn.f32 %f89, %f88, %f69;\n" " mul.rn.f32 %f90, %f89, %f87;\n" " ld.shared.f32 %f91, [%r102+256];\n" " fma.rn.f32 %f92, %f90, %f91, %f83;\n" " and.b32 %r109, %r85, 61440;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f93, %r110;\n" " mov.f32 %f94, 0fC5004000;\n" " mov.f32 %f95, 0f39800000;\n" " fma.rn.f32 %f96, %f93, %f95, %f94;\n" " shr.u32 %r111, %r86, 24;\n" " cvt.rn.f32.u32 %f97, %r111;\n" " sub.rn.f32 %f98, %f97, %f69;\n" " mul.rn.f32 %f99, %f98, %f96;\n" " ld.shared.f32 %f100, [%r102+384];\n" " fma.rn.f32 %f101, %f99, %f100, %f92;\n" " and.b32 %r112, %r85, 983040;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f102, %r113;\n" " mov.f32 %f103, 0fC3040000;\n" " mov.f32 %f104, 0f37800000;\n" " fma.rn.f32 %f105, %f102, %f104, %f103;\n" " and.b32 %r114, %r87, 255;\n" " cvt.rn.f32.u32 %f106, %r114;\n" " sub.rn.f32 %f107, %f106, %f69;\n" " mul.rn.f32 %f108, %f107, %f105;\n" " ld.shared.f32 %f109, [%r102+512];\n" " fma.rn.f32 %f110, %f108, %f109, %f101;\n" " and.b32 %r115, %r97, 240;\n" " or.b32 %r116, %r115, 1258291200;\n" " mov.b32 %f111, %r116;\n" " fma.rn.f32 %f112, %f111, %f77, %f76;\n" " bfe.u32 %r117, %r87, 8, 8;\n" " cvt.rn.f32.u32 %f113, %r117;\n" " sub.rn.f32 %f114, %f113, %f69;\n" " mul.rn.f32 %f115, %f114, %f112;\n" " ld.shared.f32 %f116, [%r102+640];\n" " fma.rn.f32 %f117, %f115, %f116, %f110;\n" " and.b32 %r118, %r97, 3840;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f118, %r119;\n" " fma.rn.f32 %f119, %f118, %f86, %f85;\n" " bfe.u32 %r120, %r87, 16, 8;\n" " cvt.rn.f32.u32 %f120, %r120;\n" " sub.rn.f32 %f121, %f120, %f69;\n" " mul.rn.f32 %f122, %f121, %f119;\n" " ld.shared.f32 %f123, [%r102+768];\n" " fma.rn.f32 %f124, %f122, %f123, %f117;\n" " and.b32 %r121, %r97, 61440;\n" " or.b32 %r122, %r121, 1258291200;\n" " mov.b32 %f125, %r122;\n" " fma.rn.f32 %f126, %f125, %f95, %f94;\n" " shr.u32 %r123, %r87, 24;\n" " cvt.rn.f32.u32 %f127, %r123;\n" " sub.rn.f32 %f128, %f127, %f69;\n" " mul.rn.f32 %f129, %f128, %f126;\n" " ld.shared.f32 %f130, [%r102+896];\n" " fma.rn.f32 %f131, %f129, %f130, %f124;\n" " fma.rn.f32 %f334, %f131, %f60, %f334;\n" " shr.u32 %r124, %r91, 16;\n" " and.b32 %r125, %r91, 15;\n" " or.b32 %r126, %r125, 1258291200;\n" " mov.b32 %f132, %r126;\n" " fma.rn.f32 %f133, %f132, %f66, %f65;\n" " and.b32 %r127, %r92, 255;\n" " cvt.rn.f32.u32 %f134, %r127;\n" " sub.rn.f32 %f135, %f134, %f69;\n" " mul.rn.f32 %f136, %f135, %f133;\n" " fma.rn.f32 %f137, %f136, %f72, %f73;\n" " and.b32 %r128, %r91, 240;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f138, %r129;\n" " fma.rn.f32 %f139, %f138, %f77, %f76;\n" " bfe.u32 %r130, %r92, 8, 8;\n" " cvt.rn.f32.u32 %f140, %r130;\n" " sub.rn.f32 %f141, %f140, %f69;\n" " mul.rn.f32 %f142, %f141, %f139;\n" " fma.rn.f32 %f143, %f142, %f82, %f137;\n" " and.b32 %r131, %r91, 3840;\n" " or.b32 %r132, %r131, 1258291200;\n" " mov.b32 %f144, %r132;\n" " fma.rn.f32 %f145, %f144, %f86, %f85;\n" " bfe.u32 %r133, %r92, 16, 8;\n" " cvt.rn.f32.u32 %f146, %r133;\n" " sub.rn.f32 %f147, %f146, %f69;\n" " mul.rn.f32 %f148, %f147, %f145;\n" " fma.rn.f32 %f149, %f148, %f91, %f143;\n" " and.b32 %r134, %r91, 61440;\n" " or.b32 %r135, %r134, 1258291200;\n" " mov.b32 %f150, %r135;\n" " fma.rn.f32 %f151, %f150, %f95, %f94;\n" " shr.u32 %r136, %r92, 24;\n" " cvt.rn.f32.u32 %f152, %r136;\n" " sub.rn.f32 %f153, %f152, %f69;\n" " mul.rn.f32 %f154, %f153, %f151;\n" " fma.rn.f32 %f155, %f154, %f100, %f149;\n" " and.b32 %r137, %r91, 983040;\n" " or.b32 %r138, %r137, 1258291200;\n" " mov.b32 %f156, %r138;\n" " fma.rn.f32 %f157, %f156, %f104, %f103;\n" " and.b32 %r139, %r93, 255;\n" " cvt.rn.f32.u32 %f158, %r139;\n" " sub.rn.f32 %f159, %f158, %f69;\n" " mul.rn.f32 %f160, %f159, %f157;\n" " fma.rn.f32 %f161, %f160, %f109, %f155;\n" " and.b32 %r140, %r124, 240;\n" " or.b32 %r141, %r140, 1258291200;\n" " mov.b32 %f162, %r141;\n" " fma.rn.f32 %f163, %f162, %f77, %f76;\n" " bfe.u32 %r142, %r93, 8, 8;\n" " cvt.rn.f32.u32 %f164, %r142;\n" " sub.rn.f32 %f165, %f164, %f69;\n" " mul.rn.f32 %f166, %f165, %f163;\n" " fma.rn.f32 %f167, %f166, %f116, %f161;\n" " and.b32 %r143, %r124, 3840;\n" " or.b32 %r144, %r143, 1258291200;\n" " mov.b32 %f168, %r144;\n" " fma.rn.f32 %f169, %f168, %f86, %f85;\n" " bfe.u32 %r145, %r93, 16, 8;\n" " cvt.rn.f32.u32 %f170, %r145;\n" " sub.rn.f32 %f171, %f170, %f69;\n" " mul.rn.f32 %f172, %f171, %f169;\n" " fma.rn.f32 %f173, %f172, %f123, %f167;\n" " and.b32 %r146, %r124, 61440;\n" " or.b32 %r147, %r146, 1258291200;\n" " mov.b32 %f174, %r147;\n" " fma.rn.f32 %f175, %f174, %f95, %f94;\n" " shr.u32 %r148, %r93, 24;\n" " cvt.rn.f32.u32 %f176, %r148;\n" " sub.rn.f32 %f177, %f176, %f69;\n" " mul.rn.f32 %f178, %f177, %f175;\n" " fma.rn.f32 %f179, %f178, %f130, %f173;\n" " fma.rn.f32 %f335, %f179, %f62, %f335;\n" " add.s64 %rd90, %rd90, 152;\n" " add.s64 %rd91, %rd91, 152;\n" " add.s32 %r238, %r238, 1024;\n" " add.s32 %r239, %r239, 1;\n" " setp.lt.s32 %p20, %r239, %r31;\n" " @%p20 bra $L__BB1_21;\n" "\n" "$L__BB1_22:\n" " setp.ge.s32 %p21, %r239, %r31;\n" " @%p21 bra $L__BB1_24;\n" "\n" "$L__BB1_23:\n" " shl.b64 %rd75, %rd2, 2;\n" " add.s64 %rd69, %rd90, %rd75;\n" " // begin inline asm\n" " ld.global.nc.u32 %r149, [%rd69];\n" " // end inline asm\n" " add.s64 %rd70, %rd90, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f180, [%rd70];\n" " // end inline asm\n" " shl.b64 %rd76, %rd3, 3;\n" " add.s64 %rd77, %rd90, %rd76;\n" " add.s64 %rd71, %rd77, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r150,%r151}, [%rd71];\n" " // end inline asm\n" " shr.u32 %r155, %r149, 16;\n" " and.b32 %r156, %r149, 15;\n" " or.b32 %r157, %r156, 1258291200;\n" " mov.b32 %f182, %r157;\n" " mov.f32 %f183, 0fCB000004;\n" " mov.f32 %f184, 0f3F800000;\n" " fma.rn.f32 %f185, %f182, %f184, %f183;\n" " and.b32 %r158, %r150, 255;\n" " cvt.rn.f32.u32 %f186, %r158;\n" " mov.f32 %f187, 0f42000000;\n" " sub.rn.f32 %f188, %f186, %f187;\n" " mul.rn.f32 %f189, %f188, %f185;\n" " shl.b32 %r159, %r2, 2;\n" " add.s32 %r160, %r238, %r159;\n" " ld.shared.f32 %f190, [%r160];\n" " mov.f32 %f191, 0f00000000;\n" " fma.rn.f32 %f192, %f189, %f190, %f191;\n" " and.b32 %r161, %r149, 240;\n" " or.b32 %r162, %r161, 1258291200;\n" " mov.b32 %f193, %r162;\n" " mov.f32 %f194, 0fC9000040;\n" " mov.f32 %f195, 0f3D800000;\n" " fma.rn.f32 %f196, %f193, %f195, %f194;\n" " bfe.u32 %r163, %r150, 8, 8;\n" " cvt.rn.f32.u32 %f197, %r163;\n" " sub.rn.f32 %f198, %f197, %f187;\n" " mul.rn.f32 %f199, %f198, %f196;\n" " ld.shared.f32 %f200, [%r160+128];\n" " fma.rn.f32 %f201, %f199, %f200, %f192;\n" " and.b32 %r164, %r149, 3840;\n" " or.b32 %r165, %r164, 1258291200;\n" " mov.b32 %f202, %r165;\n" " mov.f32 %f203, 0fC7000400;\n" " mov.f32 %f204, 0f3B800000;\n" " fma.rn.f32 %f205, %f202, %f204, %f203;\n" " bfe.u32 %r166, %r150, 16, 8;\n" " cvt.rn.f32.u32 %f206, %r166;\n" " sub.rn.f32 %f207, %f206, %f187;\n" " mul.rn.f32 %f208, %f207, %f205;\n" " ld.shared.f32 %f209, [%r160+256];\n" " fma.rn.f32 %f210, %f208, %f209, %f201;\n" " and.b32 %r167, %r149, 61440;\n" " or.b32 %r168, %r167, 1258291200;\n" " mov.b32 %f211, %r168;\n" " mov.f32 %f212, 0fC5004000;\n" " mov.f32 %f213, 0f39800000;\n" " fma.rn.f32 %f214, %f211, %f213, %f212;\n" " shr.u32 %r169, %r150, 24;\n" " cvt.rn.f32.u32 %f215, %r169;\n" " sub.rn.f32 %f216, %f215, %f187;\n" " mul.rn.f32 %f217, %f216, %f214;\n" " ld.shared.f32 %f218, [%r160+384];\n" " fma.rn.f32 %f219, %f217, %f218, %f210;\n" " and.b32 %r170, %r149, 983040;\n" " or.b32 %r171, %r170, 1258291200;\n" " mov.b32 %f220, %r171;\n" " mov.f32 %f221, 0fC3040000;\n" " mov.f32 %f222, 0f37800000;\n" " fma.rn.f32 %f223, %f220, %f222, %f221;\n" " and.b32 %r172, %r151, 255;\n" " cvt.rn.f32.u32 %f224, %r172;\n" " sub.rn.f32 %f225, %f224, %f187;\n" " mul.rn.f32 %f226, %f225, %f223;\n" " ld.shared.f32 %f227, [%r160+512];\n" " fma.rn.f32 %f228, %f226, %f227, %f219;\n" " and.b32 %r173, %r155, 240;\n" " or.b32 %r174, %r173, 1258291200;\n" " mov.b32 %f229, %r174;\n" " fma.rn.f32 %f230, %f229, %f195, %f194;\n" " bfe.u32 %r175, %r151, 8, 8;\n" " cvt.rn.f32.u32 %f231, %r175;\n" " sub.rn.f32 %f232, %f231, %f187;\n" " mul.rn.f32 %f233, %f232, %f230;\n" " ld.shared.f32 %f234, [%r160+640];\n" " fma.rn.f32 %f235, %f233, %f234, %f228;\n" " and.b32 %r176, %r155, 3840;\n" " or.b32 %r177, %r176, 1258291200;\n" " mov.b32 %f236, %r177;\n" " fma.rn.f32 %f237, %f236, %f204, %f203;\n" " bfe.u32 %r178, %r151, 16, 8;\n" " cvt.rn.f32.u32 %f238, %r178;\n" " sub.rn.f32 %f239, %f238, %f187;\n" " mul.rn.f32 %f240, %f239, %f237;\n" " ld.shared.f32 %f241, [%r160+768];\n" " fma.rn.f32 %f242, %f240, %f241, %f235;\n" " and.b32 %r179, %r155, 61440;\n" " or.b32 %r180, %r179, 1258291200;\n" " mov.b32 %f243, %r180;\n" " fma.rn.f32 %f244, %f243, %f213, %f212;\n" " shr.u32 %r181, %r151, 24;\n" " cvt.rn.f32.u32 %f245, %r181;\n" " sub.rn.f32 %f246, %f245, %f187;\n" " mul.rn.f32 %f247, %f246, %f244;\n" " ld.shared.f32 %f248, [%r160+896];\n" " fma.rn.f32 %f249, %f247, %f248, %f242;\n" " fma.rn.f32 %f334, %f180, %f249, %f334;\n" " add.s64 %rd90, %rd90, 152;\n" " add.s64 %rd72, %rd91, %rd75;\n" " // begin inline asm\n" " ld.global.nc.u32 %r152, [%rd72];\n" " // end inline asm\n" " add.s64 %rd73, %rd91, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f181, [%rd73];\n" " // end inline asm\n" " add.s64 %rd78, %rd91, %rd76;\n" " add.s64 %rd74, %rd78, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r153,%r154}, [%rd74];\n" " // end inline asm\n" " shr.u32 %r182, %r152, 16;\n" " and.b32 %r183, %r152, 15;\n" " or.b32 %r184, %r183, 1258291200;\n" " mov.b32 %f250, %r184;\n" " fma.rn.f32 %f251, %f250, %f184, %f183;\n" " and.b32 %r185, %r153, 255;\n" " cvt.rn.f32.u32 %f252, %r185;\n" " sub.rn.f32 %f253, %f252, %f187;\n" " mul.rn.f32 %f254, %f253, %f251;\n" " ld.shared.f32 %f255, [%r160];\n" " fma.rn.f32 %f256, %f254, %f255, %f191;\n" " and.b32 %r186, %r152, 240;\n" " or.b32 %r187, %r186, 1258291200;\n" " mov.b32 %f257, %r187;\n" " fma.rn.f32 %f258, %f257, %f195, %f194;\n" " bfe.u32 %r188, %r153, 8, 8;\n" " cvt.rn.f32.u32 %f259, %r188;\n" " sub.rn.f32 %f260, %f259, %f187;\n" " mul.rn.f32 %f261, %f260, %f258;\n" " ld.shared.f32 %f262, [%r160+128];\n" " fma.rn.f32 %f263, %f261, %f262, %f256;\n" " and.b32 %r189, %r152, 3840;\n" " or.b32 %r190, %r189, 1258291200;\n" " mov.b32 %f264, %r190;\n" " fma.rn.f32 %f265, %f264, %f204, %f203;\n" " bfe.u32 %r191, %r153, 16, 8;\n" " cvt.rn.f32.u32 %f266, %r191;\n" " sub.rn.f32 %f267, %f266, %f187;\n" " mul.rn.f32 %f268, %f267, %f265;\n" " ld.shared.f32 %f269, [%r160+256];\n" " fma.rn.f32 %f270, %f268, %f269, %f263;\n" " and.b32 %r192, %r152, 61440;\n" " or.b32 %r193, %r192, 1258291200;\n" " mov.b32 %f271, %r193;\n" " fma.rn.f32 %f272, %f271, %f213, %f212;\n" " shr.u32 %r194, %r153, 24;\n" " cvt.rn.f32.u32 %f273, %r194;\n" " sub.rn.f32 %f274, %f273, %f187;\n" " mul.rn.f32 %f275, %f274, %f272;\n" " ld.shared.f32 %f276, [%r160+384];\n" " fma.rn.f32 %f277, %f275, %f276, %f270;\n" " and.b32 %r195, %r152, 983040;\n" " or.b32 %r196, %r195, 1258291200;\n" " mov.b32 %f278, %r196;\n" " fma.rn.f32 %f279, %f278, %f222, %f221;\n" " and.b32 %r197, %r154, 255;\n" " cvt.rn.f32.u32 %f280, %r197;\n" " sub.rn.f32 %f281, %f280, %f187;\n" " mul.rn.f32 %f282, %f281, %f279;\n" " ld.shared.f32 %f283, [%r160+512];\n" " fma.rn.f32 %f284, %f282, %f283, %f277;\n" " and.b32 %r198, %r182, 240;\n" " or.b32 %r199, %r198, 1258291200;\n" " mov.b32 %f285, %r199;\n" " fma.rn.f32 %f286, %f285, %f195, %f194;\n" " bfe.u32 %r200, %r154, 8, 8;\n" " cvt.rn.f32.u32 %f287, %r200;\n" " sub.rn.f32 %f288, %f287, %f187;\n" " mul.rn.f32 %f289, %f288, %f286;\n" " ld.shared.f32 %f290, [%r160+640];\n" " fma.rn.f32 %f291, %f289, %f290, %f284;\n" " and.b32 %r201, %r182, 3840;\n" " or.b32 %r202, %r201, 1258291200;\n" " mov.b32 %f292, %r202;\n" " fma.rn.f32 %f293, %f292, %f204, %f203;\n" " bfe.u32 %r203, %r154, 16, 8;\n" " cvt.rn.f32.u32 %f294, %r203;\n" " sub.rn.f32 %f295, %f294, %f187;\n" " mul.rn.f32 %f296, %f295, %f293;\n" " ld.shared.f32 %f297, [%r160+768];\n" " fma.rn.f32 %f298, %f296, %f297, %f291;\n" " and.b32 %r204, %r182, 61440;\n" " or.b32 %r205, %r204, 1258291200;\n" " mov.b32 %f299, %r205;\n" " fma.rn.f32 %f300, %f299, %f213, %f212;\n" " shr.u32 %r206, %r154, 24;\n" " cvt.rn.f32.u32 %f301, %r206;\n" " sub.rn.f32 %f302, %f301, %f187;\n" " mul.rn.f32 %f303, %f302, %f300;\n" " ld.shared.f32 %f304, [%r160+896];\n" " fma.rn.f32 %f305, %f303, %f304, %f298;\n" " fma.rn.f32 %f335, %f181, %f305, %f335;\n" " add.s64 %rd91, %rd91, 152;\n" " add.s32 %r238, %r238, 1024;\n" " add.s32 %r239, %r239, 1;\n" " setp.lt.s32 %p22, %r239, %r31;\n" " @%p22 bra $L__BB1_23;\n" "\n" "$L__BB1_24:\n" " cvt.u32.u64 %r207, %rd9;\n" " add.s32 %r230, %r207, 2048;\n" " setp.lt.s32 %p23, %r230, %r45;\n" " add.s32 %r229, %r229, 2048;\n" " @%p23 bra $L__BB1_2;\n" "\n" "$L__BB1_25:\n" " cvt.u32.u64 %r218, %rd1;\n" " setp.ge.s32 %p24, %r218, %r44;\n" " mov.u32 %r208, 16;\n" " mov.u32 %r217, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f306, %f334, %r208, %r217;\n" " // end inline asm\n" " add.f32 %f309, %f334, %f306;\n" " mov.u32 %r210, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f308, %f309, %r210, %r217;\n" " // end inline asm\n" " add.f32 %f311, %f309, %f308;\n" " mov.u32 %r212, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f310, %f311, %r212, %r217;\n" " // end inline asm\n" " add.f32 %f313, %f311, %f310;\n" " mov.u32 %r214, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f312, %f313, %r214, %r217;\n" " // end inline asm\n" " add.f32 %f315, %f313, %f312;\n" " mov.u32 %r216, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f314, %f315, %r216, %r217;\n" " // end inline asm\n" " add.f32 %f17, %f315, %f314;\n" " setp.ne.s32 %p25, %r2, 0;\n" " cvta.to.global.u64 %rd79, %rd31;\n" " shl.b64 %rd80, %rd1, 2;\n" " add.s64 %rd28, %rd79, %rd80;\n" " or.pred %p26, %p25, %p24;\n" " @%p26 bra $L__BB1_27;\n" "\n" " st.global.f32 [%rd28], %f17;\n" "\n" "$L__BB1_27:\n" " // begin inline asm\n" " shfl.down.b32 %f316, %f335, %r208, %r217;\n" " // end inline asm\n" " add.f32 %f319, %f335, %f316;\n" " // begin inline asm\n" " shfl.down.b32 %f318, %f319, %r210, %r217;\n" " // end inline asm\n" " add.f32 %f321, %f319, %f318;\n" " // begin inline asm\n" " shfl.down.b32 %f320, %f321, %r212, %r217;\n" " // end inline asm\n" " add.f32 %f323, %f321, %f320;\n" " // begin inline asm\n" " shfl.down.b32 %f322, %f323, %r214, %r217;\n" " // end inline asm\n" " add.f32 %f325, %f323, %f322;\n" " // begin inline asm\n" " shfl.down.b32 %f324, %f325, %r216, %r217;\n" " // end inline asm\n" " add.f32 %f18, %f325, %f324;\n" " setp.ge.s32 %p28, %r4, %r44;\n" " or.pred %p29, %p25, %p28;\n" " @%p29 bra $L__BB1_29;\n" "\n" " st.global.f32 [%rd28+4], %f18;\n" "\n" "$L__BB1_29:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_ps_r2p2\n" ".visible .entry fc_matvec_q3k_ps_r2p2(\n" " .param .u64 fc_matvec_q3k_ps_r2p2_param_0,\n" " .param .u64 fc_matvec_q3k_ps_r2p2_param_1,\n" " .param .u64 fc_matvec_q3k_ps_r2p2_param_2,\n" " .param .u32 fc_matvec_q3k_ps_r2p2_param_3,\n" " .param .u32 fc_matvec_q3k_ps_r2p2_param_4,\n" " .param .u32 fc_matvec_q3k_ps_r2p2_param_5\n" ")\n" "{\n" " .reg .pred %p<30>;\n" " .reg .f32 %f<450>;\n" " .reg .b32 %r<310>;\n" " .reg .b64 %rd<98>;\n" "\n" "\n" " ld.param.u64 %rd29, [fc_matvec_q3k_ps_r2p2_param_0];\n" " ld.param.u64 %rd30, [fc_matvec_q3k_ps_r2p2_param_1];\n" " ld.param.u64 %rd31, [fc_matvec_q3k_ps_r2p2_param_2];\n" " ld.param.u32 %r44, [fc_matvec_q3k_ps_r2p2_param_3];\n" " ld.param.u32 %r45, [fc_matvec_q3k_ps_r2p2_param_4];\n" " ld.param.u32 %r46, [fc_matvec_q3k_ps_r2p2_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r47, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r48, %ctaid.x;\n" " shl.b32 %r49, %r48, 3;\n" " add.s32 %r50, %r49, %r47;\n" " shl.b32 %r3, %r50, 1;\n" " cvt.s64.s32 %rd1, %r3;\n" " or.b32 %r4, %r3, 1;\n" " setp.lt.s32 %p1, %r45, 1;\n" " mov.f32 %f446, 0f00000000;\n" " mov.f32 %f447, %f446;\n" " @%p1 bra $L__BB2_25;\n" "\n" " cvt.s64.s32 %rd32, %r46;\n" " cvt.u64.u32 %rd2, %r2;\n" " shr.u32 %r53, %r2, 4;\n" " cvt.u64.u32 %rd3, %r53;\n" " mov.u32 %r54, -2;\n" " sub.s32 %r5, %r54, %r1;\n" " not.b32 %r6, %r45;\n" " not.b32 %r7, %r1;\n" " cvt.s64.s32 %rd4, %r1;\n" " add.s32 %r8, %r1, 256;\n" " shl.b32 %r55, %r1, 4;\n" " mov.u32 %r56, sh_xps;\n" " add.s32 %r9, %r56, %r55;\n" " mad.lo.s32 %r10, %r1, -12, %r9;\n" " add.s32 %r11, %r1, 512;\n" " add.s32 %r12, %r1, 768;\n" " setp.lt.s32 %p2, %r4, %r44;\n" " selp.u32 %r57, 1, 0, %p2;\n" " add.s32 %r58, %r3, %r57;\n" " mul.wide.s32 %rd33, %r58, %r46;\n" " add.s64 %rd97, %rd29, %rd33;\n" " mul.lo.s64 %rd34, %rd1, %rd32;\n" " add.s64 %rd96, %rd29, %rd34;\n" " mov.u32 %r297, 0;\n" " mov.f32 %f446, 0f00000000;\n" " mov.f32 %f447, %f446;\n" " mov.u32 %r298, %r297;\n" "\n" "$L__BB2_2:\n" " add.s32 %r59, %r297, %r6;\n" " max.s32 %r60, %r59, -2049;\n" " sub.s32 %r15, %r5, %r60;\n" " sub.s32 %r61, %r45, %r298;\n" " min.s32 %r16, %r61, 2048;\n" " bar.sync 0;\n" " cvt.s64.s32 %rd9, %r298;\n" " mul.wide.s32 %rd35, %r298, 4;\n" " add.s64 %rd10, %rd30, %rd35;\n" " and.b32 %r62, %r16, 3;\n" " setp.eq.s32 %p3, %r62, 0;\n" " { .reg .b64 %tmp;\n" " cvt.u64.u32 %tmp, %r56;\n" " cvta.shared.u64 %rd36, %tmp; }\n" " or.b64 %rd37, %rd10, %rd36;\n" " and.b64 %rd38, %rd37, 15;\n" " setp.eq.s64 %p4, %rd38, 0;\n" " and.pred %p5, %p3, %p4;\n" " @%p5 bra $L__BB2_11;\n" " bra.uni $L__BB2_3;\n" "\n" "$L__BB2_11:\n" " shr.s32 %r24, %r16, 2;\n" " setp.ge.s32 %p12, %r1, %r24;\n" " @%p12 bra $L__BB2_18;\n" "\n" " add.s32 %r25, %r24, %r7;\n" " shr.u32 %r69, %r25, 8;\n" " add.s32 %r70, %r69, 1;\n" " and.b32 %r26, %r70, 3;\n" " setp.eq.s32 %p13, %r26, 0;\n" " mov.u32 %r302, %r1;\n" " @%p13 bra $L__BB2_16;\n" "\n" " shl.b64 %rd51, %rd4, 4;\n" " add.s64 %rd50, %rd10, %rd51;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f30,%f31,%f32,%f33}, [%rd50];\n" " // end inline asm\n" " st.shared.v4.f32 [%r9], {%f30, %f31, %f32, %f33};\n" " setp.eq.s32 %p14, %r26, 1;\n" " mov.u32 %r302, %r8;\n" " @%p14 bra $L__BB2_16;\n" "\n" " add.s64 %rd52, %rd50, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f34,%f35,%f36,%f37}, [%rd52];\n" " // end inline asm\n" " st.shared.v4.f32 [%r9+4096], {%f34, %f35, %f36, %f37};\n" " setp.eq.s32 %p15, %r26, 2;\n" " mov.u32 %r302, %r11;\n" " @%p15 bra $L__BB2_16;\n" "\n" " add.s64 %rd53, %rd50, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f38,%f39,%f40,%f41}, [%rd53];\n" " // end inline asm\n" " st.shared.v4.f32 [%r9+8192], {%f38, %f39, %f40, %f41};\n" " mov.u32 %r302, %r12;\n" "\n" "$L__BB2_16:\n" " setp.lt.u32 %p16, %r25, 768;\n" " @%p16 bra $L__BB2_18;\n" "\n" "$L__BB2_17:\n" " mul.wide.s32 %rd58, %r302, 16;\n" " add.s64 %rd54, %rd10, %rd58;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f42,%f43,%f44,%f45}, [%rd54];\n" " // end inline asm\n" " shl.b32 %r71, %r302, 4;\n" " mov.u32 %r72, sh_xps;\n" " add.s32 %r73, %r72, %r71;\n" " st.shared.v4.f32 [%r73], {%f42, %f43, %f44, %f45};\n" " add.s64 %rd55, %rd54, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f46,%f47,%f48,%f49}, [%rd55];\n" " // end inline asm\n" " st.shared.v4.f32 [%r73+4096], {%f46, %f47, %f48, %f49};\n" " add.s64 %rd56, %rd54, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f50,%f51,%f52,%f53}, [%rd56];\n" " // end inline asm\n" " st.shared.v4.f32 [%r73+8192], {%f50, %f51, %f52, %f53};\n" " add.s64 %rd57, %rd54, 12288;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f54,%f55,%f56,%f57}, [%rd57];\n" " // end inline asm\n" " st.shared.v4.f32 [%r73+12288], {%f54, %f55, %f56, %f57};\n" " add.s32 %r302, %r302, 1024;\n" " setp.lt.s32 %p17, %r302, %r24;\n" " @%p17 bra $L__BB2_17;\n" " bra.uni $L__BB2_18;\n" "\n" "$L__BB2_3:\n" " setp.ge.s32 %p6, %r1, %r16;\n" " @%p6 bra $L__BB2_18;\n" "\n" " shr.u32 %r64, %r15, 8;\n" " add.s32 %r65, %r64, 1;\n" " and.b32 %r17, %r65, 3;\n" " setp.eq.s32 %p7, %r17, 0;\n" " mov.u32 %r299, %r1;\n" " @%p7 bra $L__BB2_8;\n" "\n" " shl.b64 %rd40, %rd4, 2;\n" " add.s64 %rd39, %rd10, %rd40;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r10], %f23;\n" " setp.eq.s32 %p8, %r17, 1;\n" " mov.u32 %r299, %r8;\n" " @%p8 bra $L__BB2_8;\n" "\n" " add.s64 %rd41, %rd39, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r10+1024], %f24;\n" " setp.eq.s32 %p9, %r17, 2;\n" " mov.u32 %r299, %r11;\n" " @%p9 bra $L__BB2_8;\n" "\n" " add.s64 %rd42, %rd39, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r10+2048], %f25;\n" " mov.u32 %r299, %r12;\n" "\n" "$L__BB2_8:\n" " setp.lt.u32 %p10, %r15, 768;\n" " @%p10 bra $L__BB2_18;\n" "\n" " cvt.s64.s32 %rd43, %r299;\n" " add.s64 %rd44, %rd43, %rd9;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd89, %rd30, %rd45;\n" " shl.b32 %r66, %r299, 2;\n" " add.s32 %r68, %r56, %r66;\n" " add.s32 %r300, %r68, 2048;\n" "\n" "$L__BB2_10:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd89];\n" " // end inline asm\n" " st.shared.f32 [%r300+-2048], %f26;\n" " add.s64 %rd47, %rd89, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd47];\n" " // end inline asm\n" " st.shared.f32 [%r300+-1024], %f27;\n" " add.s64 %rd48, %rd89, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r300], %f28;\n" " add.s64 %rd49, %rd89, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r300+1024], %f29;\n" " add.s64 %rd89, %rd89, 4096;\n" " add.s32 %r300, %r300, 4096;\n" " add.s32 %r299, %r299, 1024;\n" " setp.lt.s32 %p11, %r299, %r16;\n" " @%p11 bra $L__BB2_10;\n" "\n" "$L__BB2_18:\n" " cvt.u32.u64 %r74, %rd1;\n" " setp.ge.s32 %p18, %r74, %r44;\n" " bar.sync 0;\n" " @%p18 bra $L__BB2_24;\n" "\n" " shr.s32 %r31, %r16, 8;\n" " setp.lt.s32 %p19, %r16, 512;\n" " mov.u32 %r308, 0;\n" " mov.u32 %r306, %r56;\n" " @%p19 bra $L__BB2_22;\n" "\n" " mov.u32 %r306, sh_xps;\n" " mov.u32 %r304, 2;\n" "\n" "$L__BB2_21:\n" " mov.u32 %r308, %r304;\n" " shl.b64 %rd71, %rd2, 2;\n" " add.s64 %rd59, %rd96, %rd71;\n" " // begin inline asm\n" " ld.global.nc.u32 %r91, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd96, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f62, [%rd60];\n" " // end inline asm\n" " shl.b64 %rd72, %rd3, 3;\n" " add.s64 %rd73, %rd96, %rd72;\n" " add.s64 %rd61, %rd73, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r92,%r93}, [%rd61];\n" " // end inline asm\n" " add.s64 %rd62, %rd97, %rd71;\n" " // begin inline asm\n" " ld.global.nc.u32 %r97, [%rd62];\n" " // end inline asm\n" " add.s64 %rd63, %rd97, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f64, [%rd63];\n" " // end inline asm\n" " add.s64 %rd74, %rd97, %rd72;\n" " add.s64 %rd64, %rd74, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r98,%r99}, [%rd64];\n" " // end inline asm\n" " add.s64 %rd65, %rd59, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r103, [%rd65];\n" " // end inline asm\n" " add.s64 %rd66, %rd96, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f66, [%rd66];\n" " // end inline asm\n" " add.s64 %rd67, %rd73, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r104,%r105}, [%rd67];\n" " // end inline asm\n" " add.s64 %rd68, %rd62, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r109, [%rd68];\n" " // end inline asm\n" " add.s64 %rd69, %rd97, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f68, [%rd69];\n" " // end inline asm\n" " add.s64 %rd70, %rd74, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r110,%r111}, [%rd70];\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " shr.u32 %r115, %r91, 16;\n" " and.b32 %r116, %r91, 15;\n" " or.b32 %r117, %r116, 1258291200;\n" " mov.b32 %f70, %r117;\n" " mov.f32 %f71, 0fCB000004;\n" " mov.f32 %f72, 0f3F800000;\n" " fma.rn.f32 %f73, %f70, %f72, %f71;\n" " and.b32 %r118, %r92, 255;\n" " cvt.rn.f32.u32 %f74, %r118;\n" " mov.f32 %f75, 0f42000000;\n" " sub.rn.f32 %f76, %f74, %f75;\n" " mul.rn.f32 %f77, %f76, %f73;\n" " shl.b32 %r119, %r2, 2;\n" " add.s32 %r120, %r306, %r119;\n" " ld.shared.f32 %f78, [%r120];\n" " mov.f32 %f79, 0f00000000;\n" " fma.rn.f32 %f80, %f77, %f78, %f79;\n" " and.b32 %r121, %r91, 240;\n" " or.b32 %r122, %r121, 1258291200;\n" " mov.b32 %f81, %r122;\n" " mov.f32 %f82, 0fC9000040;\n" " mov.f32 %f83, 0f3D800000;\n" " fma.rn.f32 %f84, %f81, %f83, %f82;\n" " bfe.u32 %r123, %r92, 8, 8;\n" " cvt.rn.f32.u32 %f85, %r123;\n" " sub.rn.f32 %f86, %f85, %f75;\n" " mul.rn.f32 %f87, %f86, %f84;\n" " ld.shared.f32 %f88, [%r120+128];\n" " fma.rn.f32 %f89, %f87, %f88, %f80;\n" " and.b32 %r124, %r91, 3840;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f90, %r125;\n" " mov.f32 %f91, 0fC7000400;\n" " mov.f32 %f92, 0f3B800000;\n" " fma.rn.f32 %f93, %f90, %f92, %f91;\n" " bfe.u32 %r126, %r92, 16, 8;\n" " cvt.rn.f32.u32 %f94, %r126;\n" " sub.rn.f32 %f95, %f94, %f75;\n" " mul.rn.f32 %f96, %f95, %f93;\n" " ld.shared.f32 %f97, [%r120+256];\n" " fma.rn.f32 %f98, %f96, %f97, %f89;\n" " and.b32 %r127, %r91, 61440;\n" " or.b32 %r128, %r127, 1258291200;\n" " mov.b32 %f99, %r128;\n" " mov.f32 %f100, 0fC5004000;\n" " mov.f32 %f101, 0f39800000;\n" " fma.rn.f32 %f102, %f99, %f101, %f100;\n" " shr.u32 %r129, %r92, 24;\n" " cvt.rn.f32.u32 %f103, %r129;\n" " sub.rn.f32 %f104, %f103, %f75;\n" " mul.rn.f32 %f105, %f104, %f102;\n" " ld.shared.f32 %f106, [%r120+384];\n" " fma.rn.f32 %f107, %f105, %f106, %f98;\n" " and.b32 %r130, %r91, 983040;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f108, %r131;\n" " mov.f32 %f109, 0fC3040000;\n" " mov.f32 %f110, 0f37800000;\n" " fma.rn.f32 %f111, %f108, %f110, %f109;\n" " and.b32 %r132, %r93, 255;\n" " cvt.rn.f32.u32 %f112, %r132;\n" " sub.rn.f32 %f113, %f112, %f75;\n" " mul.rn.f32 %f114, %f113, %f111;\n" " ld.shared.f32 %f115, [%r120+512];\n" " fma.rn.f32 %f116, %f114, %f115, %f107;\n" " and.b32 %r133, %r115, 240;\n" " or.b32 %r134, %r133, 1258291200;\n" " mov.b32 %f117, %r134;\n" " fma.rn.f32 %f118, %f117, %f83, %f82;\n" " bfe.u32 %r135, %r93, 8, 8;\n" " cvt.rn.f32.u32 %f119, %r135;\n" " sub.rn.f32 %f120, %f119, %f75;\n" " mul.rn.f32 %f121, %f120, %f118;\n" " ld.shared.f32 %f122, [%r120+640];\n" " fma.rn.f32 %f123, %f121, %f122, %f116;\n" " and.b32 %r136, %r115, 3840;\n" " or.b32 %r137, %r136, 1258291200;\n" " mov.b32 %f124, %r137;\n" " fma.rn.f32 %f125, %f124, %f92, %f91;\n" " bfe.u32 %r138, %r93, 16, 8;\n" " cvt.rn.f32.u32 %f126, %r138;\n" " sub.rn.f32 %f127, %f126, %f75;\n" " mul.rn.f32 %f128, %f127, %f125;\n" " ld.shared.f32 %f129, [%r120+768];\n" " fma.rn.f32 %f130, %f128, %f129, %f123;\n" " and.b32 %r139, %r115, 61440;\n" " or.b32 %r140, %r139, 1258291200;\n" " mov.b32 %f131, %r140;\n" " fma.rn.f32 %f132, %f131, %f101, %f100;\n" " shr.u32 %r141, %r93, 24;\n" " cvt.rn.f32.u32 %f133, %r141;\n" " sub.rn.f32 %f134, %f133, %f75;\n" " mul.rn.f32 %f135, %f134, %f132;\n" " ld.shared.f32 %f136, [%r120+896];\n" " fma.rn.f32 %f137, %f135, %f136, %f130;\n" " fma.rn.f32 %f138, %f137, %f62, %f446;\n" " shr.u32 %r142, %r97, 16;\n" " and.b32 %r143, %r97, 15;\n" " or.b32 %r144, %r143, 1258291200;\n" " mov.b32 %f139, %r144;\n" " fma.rn.f32 %f140, %f139, %f72, %f71;\n" " and.b32 %r145, %r98, 255;\n" " cvt.rn.f32.u32 %f141, %r145;\n" " sub.rn.f32 %f142, %f141, %f75;\n" " mul.rn.f32 %f143, %f142, %f140;\n" " fma.rn.f32 %f144, %f143, %f78, %f79;\n" " and.b32 %r146, %r97, 240;\n" " or.b32 %r147, %r146, 1258291200;\n" " mov.b32 %f145, %r147;\n" " fma.rn.f32 %f146, %f145, %f83, %f82;\n" " bfe.u32 %r148, %r98, 8, 8;\n" " cvt.rn.f32.u32 %f147, %r148;\n" " sub.rn.f32 %f148, %f147, %f75;\n" " mul.rn.f32 %f149, %f148, %f146;\n" " fma.rn.f32 %f150, %f149, %f88, %f144;\n" " and.b32 %r149, %r97, 3840;\n" " or.b32 %r150, %r149, 1258291200;\n" " mov.b32 %f151, %r150;\n" " fma.rn.f32 %f152, %f151, %f92, %f91;\n" " bfe.u32 %r151, %r98, 16, 8;\n" " cvt.rn.f32.u32 %f153, %r151;\n" " sub.rn.f32 %f154, %f153, %f75;\n" " mul.rn.f32 %f155, %f154, %f152;\n" " fma.rn.f32 %f156, %f155, %f97, %f150;\n" " and.b32 %r152, %r97, 61440;\n" " or.b32 %r153, %r152, 1258291200;\n" " mov.b32 %f157, %r153;\n" " fma.rn.f32 %f158, %f157, %f101, %f100;\n" " shr.u32 %r154, %r98, 24;\n" " cvt.rn.f32.u32 %f159, %r154;\n" " sub.rn.f32 %f160, %f159, %f75;\n" " mul.rn.f32 %f161, %f160, %f158;\n" " fma.rn.f32 %f162, %f161, %f106, %f156;\n" " and.b32 %r155, %r97, 983040;\n" " or.b32 %r156, %r155, 1258291200;\n" " mov.b32 %f163, %r156;\n" " fma.rn.f32 %f164, %f163, %f110, %f109;\n" " and.b32 %r157, %r99, 255;\n" " cvt.rn.f32.u32 %f165, %r157;\n" " sub.rn.f32 %f166, %f165, %f75;\n" " mul.rn.f32 %f167, %f166, %f164;\n" " fma.rn.f32 %f168, %f167, %f115, %f162;\n" " and.b32 %r158, %r142, 240;\n" " or.b32 %r159, %r158, 1258291200;\n" " mov.b32 %f169, %r159;\n" " fma.rn.f32 %f170, %f169, %f83, %f82;\n" " bfe.u32 %r160, %r99, 8, 8;\n" " cvt.rn.f32.u32 %f171, %r160;\n" " sub.rn.f32 %f172, %f171, %f75;\n" " mul.rn.f32 %f173, %f172, %f170;\n" " fma.rn.f32 %f174, %f173, %f122, %f168;\n" " and.b32 %r161, %r142, 3840;\n" " or.b32 %r162, %r161, 1258291200;\n" " mov.b32 %f175, %r162;\n" " fma.rn.f32 %f176, %f175, %f92, %f91;\n" " bfe.u32 %r163, %r99, 16, 8;\n" " cvt.rn.f32.u32 %f177, %r163;\n" " sub.rn.f32 %f178, %f177, %f75;\n" " mul.rn.f32 %f179, %f178, %f176;\n" " fma.rn.f32 %f180, %f179, %f129, %f174;\n" " and.b32 %r164, %r142, 61440;\n" " or.b32 %r165, %r164, 1258291200;\n" " mov.b32 %f181, %r165;\n" " fma.rn.f32 %f182, %f181, %f101, %f100;\n" " shr.u32 %r166, %r99, 24;\n" " cvt.rn.f32.u32 %f183, %r166;\n" " sub.rn.f32 %f184, %f183, %f75;\n" " mul.rn.f32 %f185, %f184, %f182;\n" " fma.rn.f32 %f186, %f185, %f136, %f180;\n" " fma.rn.f32 %f187, %f186, %f64, %f447;\n" " shr.u32 %r167, %r103, 16;\n" " and.b32 %r168, %r103, 15;\n" " or.b32 %r169, %r168, 1258291200;\n" " mov.b32 %f188, %r169;\n" " fma.rn.f32 %f189, %f188, %f72, %f71;\n" " and.b32 %r170, %r104, 255;\n" " cvt.rn.f32.u32 %f190, %r170;\n" " sub.rn.f32 %f191, %f190, %f75;\n" " mul.rn.f32 %f192, %f191, %f189;\n" " ld.shared.f32 %f193, [%r120+1024];\n" " fma.rn.f32 %f194, %f192, %f193, %f79;\n" " and.b32 %r171, %r103, 240;\n" " or.b32 %r172, %r171, 1258291200;\n" " mov.b32 %f195, %r172;\n" " fma.rn.f32 %f196, %f195, %f83, %f82;\n" " bfe.u32 %r173, %r104, 8, 8;\n" " cvt.rn.f32.u32 %f197, %r173;\n" " sub.rn.f32 %f198, %f197, %f75;\n" " mul.rn.f32 %f199, %f198, %f196;\n" " ld.shared.f32 %f200, [%r120+1152];\n" " fma.rn.f32 %f201, %f199, %f200, %f194;\n" " and.b32 %r174, %r103, 3840;\n" " or.b32 %r175, %r174, 1258291200;\n" " mov.b32 %f202, %r175;\n" " fma.rn.f32 %f203, %f202, %f92, %f91;\n" " bfe.u32 %r176, %r104, 16, 8;\n" " cvt.rn.f32.u32 %f204, %r176;\n" " sub.rn.f32 %f205, %f204, %f75;\n" " mul.rn.f32 %f206, %f205, %f203;\n" " ld.shared.f32 %f207, [%r120+1280];\n" " fma.rn.f32 %f208, %f206, %f207, %f201;\n" " and.b32 %r177, %r103, 61440;\n" " or.b32 %r178, %r177, 1258291200;\n" " mov.b32 %f209, %r178;\n" " fma.rn.f32 %f210, %f209, %f101, %f100;\n" " shr.u32 %r179, %r104, 24;\n" " cvt.rn.f32.u32 %f211, %r179;\n" " sub.rn.f32 %f212, %f211, %f75;\n" " mul.rn.f32 %f213, %f212, %f210;\n" " ld.shared.f32 %f214, [%r120+1408];\n" " fma.rn.f32 %f215, %f213, %f214, %f208;\n" " and.b32 %r180, %r103, 983040;\n" " or.b32 %r181, %r180, 1258291200;\n" " mov.b32 %f216, %r181;\n" " fma.rn.f32 %f217, %f216, %f110, %f109;\n" " and.b32 %r182, %r105, 255;\n" " cvt.rn.f32.u32 %f218, %r182;\n" " sub.rn.f32 %f219, %f218, %f75;\n" " mul.rn.f32 %f220, %f219, %f217;\n" " ld.shared.f32 %f221, [%r120+1536];\n" " fma.rn.f32 %f222, %f220, %f221, %f215;\n" " and.b32 %r183, %r167, 240;\n" " or.b32 %r184, %r183, 1258291200;\n" " mov.b32 %f223, %r184;\n" " fma.rn.f32 %f224, %f223, %f83, %f82;\n" " bfe.u32 %r185, %r105, 8, 8;\n" " cvt.rn.f32.u32 %f225, %r185;\n" " sub.rn.f32 %f226, %f225, %f75;\n" " mul.rn.f32 %f227, %f226, %f224;\n" " ld.shared.f32 %f228, [%r120+1664];\n" " fma.rn.f32 %f229, %f227, %f228, %f222;\n" " and.b32 %r186, %r167, 3840;\n" " or.b32 %r187, %r186, 1258291200;\n" " mov.b32 %f230, %r187;\n" " fma.rn.f32 %f231, %f230, %f92, %f91;\n" " bfe.u32 %r188, %r105, 16, 8;\n" " cvt.rn.f32.u32 %f232, %r188;\n" " sub.rn.f32 %f233, %f232, %f75;\n" " mul.rn.f32 %f234, %f233, %f231;\n" " ld.shared.f32 %f235, [%r120+1792];\n" " fma.rn.f32 %f236, %f234, %f235, %f229;\n" " and.b32 %r189, %r167, 61440;\n" " or.b32 %r190, %r189, 1258291200;\n" " mov.b32 %f237, %r190;\n" " fma.rn.f32 %f238, %f237, %f101, %f100;\n" " shr.u32 %r191, %r105, 24;\n" " cvt.rn.f32.u32 %f239, %r191;\n" " sub.rn.f32 %f240, %f239, %f75;\n" " mul.rn.f32 %f241, %f240, %f238;\n" " ld.shared.f32 %f242, [%r120+1920];\n" " fma.rn.f32 %f243, %f241, %f242, %f236;\n" " fma.rn.f32 %f446, %f243, %f66, %f138;\n" " shr.u32 %r192, %r109, 16;\n" " and.b32 %r193, %r109, 15;\n" " or.b32 %r194, %r193, 1258291200;\n" " mov.b32 %f244, %r194;\n" " fma.rn.f32 %f245, %f244, %f72, %f71;\n" " and.b32 %r195, %r110, 255;\n" " cvt.rn.f32.u32 %f246, %r195;\n" " sub.rn.f32 %f247, %f246, %f75;\n" " mul.rn.f32 %f248, %f247, %f245;\n" " fma.rn.f32 %f249, %f248, %f193, %f79;\n" " and.b32 %r196, %r109, 240;\n" " or.b32 %r197, %r196, 1258291200;\n" " mov.b32 %f250, %r197;\n" " fma.rn.f32 %f251, %f250, %f83, %f82;\n" " bfe.u32 %r198, %r110, 8, 8;\n" " cvt.rn.f32.u32 %f252, %r198;\n" " sub.rn.f32 %f253, %f252, %f75;\n" " mul.rn.f32 %f254, %f253, %f251;\n" " fma.rn.f32 %f255, %f254, %f200, %f249;\n" " and.b32 %r199, %r109, 3840;\n" " or.b32 %r200, %r199, 1258291200;\n" " mov.b32 %f256, %r200;\n" " fma.rn.f32 %f257, %f256, %f92, %f91;\n" " bfe.u32 %r201, %r110, 16, 8;\n" " cvt.rn.f32.u32 %f258, %r201;\n" " sub.rn.f32 %f259, %f258, %f75;\n" " mul.rn.f32 %f260, %f259, %f257;\n" " fma.rn.f32 %f261, %f260, %f207, %f255;\n" " and.b32 %r202, %r109, 61440;\n" " or.b32 %r203, %r202, 1258291200;\n" " mov.b32 %f262, %r203;\n" " fma.rn.f32 %f263, %f262, %f101, %f100;\n" " shr.u32 %r204, %r110, 24;\n" " cvt.rn.f32.u32 %f264, %r204;\n" " sub.rn.f32 %f265, %f264, %f75;\n" " mul.rn.f32 %f266, %f265, %f263;\n" " fma.rn.f32 %f267, %f266, %f214, %f261;\n" " and.b32 %r205, %r109, 983040;\n" " or.b32 %r206, %r205, 1258291200;\n" " mov.b32 %f268, %r206;\n" " fma.rn.f32 %f269, %f268, %f110, %f109;\n" " and.b32 %r207, %r111, 255;\n" " cvt.rn.f32.u32 %f270, %r207;\n" " sub.rn.f32 %f271, %f270, %f75;\n" " mul.rn.f32 %f272, %f271, %f269;\n" " fma.rn.f32 %f273, %f272, %f221, %f267;\n" " and.b32 %r208, %r192, 240;\n" " or.b32 %r209, %r208, 1258291200;\n" " mov.b32 %f274, %r209;\n" " fma.rn.f32 %f275, %f274, %f83, %f82;\n" " bfe.u32 %r210, %r111, 8, 8;\n" " cvt.rn.f32.u32 %f276, %r210;\n" " sub.rn.f32 %f277, %f276, %f75;\n" " mul.rn.f32 %f278, %f277, %f275;\n" " fma.rn.f32 %f279, %f278, %f228, %f273;\n" " and.b32 %r211, %r192, 3840;\n" " or.b32 %r212, %r211, 1258291200;\n" " mov.b32 %f280, %r212;\n" " fma.rn.f32 %f281, %f280, %f92, %f91;\n" " bfe.u32 %r213, %r111, 16, 8;\n" " cvt.rn.f32.u32 %f282, %r213;\n" " sub.rn.f32 %f283, %f282, %f75;\n" " mul.rn.f32 %f284, %f283, %f281;\n" " fma.rn.f32 %f285, %f284, %f235, %f279;\n" " and.b32 %r214, %r192, 61440;\n" " or.b32 %r215, %r214, 1258291200;\n" " mov.b32 %f286, %r215;\n" " fma.rn.f32 %f287, %f286, %f101, %f100;\n" " shr.u32 %r216, %r111, 24;\n" " cvt.rn.f32.u32 %f288, %r216;\n" " sub.rn.f32 %f289, %f288, %f75;\n" " mul.rn.f32 %f290, %f289, %f287;\n" " fma.rn.f32 %f291, %f290, %f242, %f285;\n" " fma.rn.f32 %f447, %f291, %f68, %f187;\n" " add.s64 %rd96, %rd96, 304;\n" " add.s64 %rd97, %rd97, 304;\n" " add.s32 %r306, %r306, 2048;\n" " add.s32 %r304, %r308, 2;\n" " setp.le.s32 %p20, %r304, %r31;\n" " @%p20 bra $L__BB2_21;\n" "\n" "$L__BB2_22:\n" " setp.ge.s32 %p21, %r308, %r31;\n" " @%p21 bra $L__BB2_24;\n" "\n" "$L__BB2_23:\n" " shl.b64 %rd81, %rd2, 2;\n" " add.s64 %rd75, %rd96, %rd81;\n" " // begin inline asm\n" " ld.global.nc.u32 %r217, [%rd75];\n" " // end inline asm\n" " add.s64 %rd76, %rd96, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f292, [%rd76];\n" " // end inline asm\n" " shl.b64 %rd82, %rd3, 3;\n" " add.s64 %rd83, %rd96, %rd82;\n" " add.s64 %rd77, %rd83, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r218,%r219}, [%rd77];\n" " // end inline asm\n" " shr.u32 %r223, %r217, 16;\n" " and.b32 %r224, %r217, 15;\n" " or.b32 %r225, %r224, 1258291200;\n" " mov.b32 %f294, %r225;\n" " mov.f32 %f295, 0fCB000004;\n" " mov.f32 %f296, 0f3F800000;\n" " fma.rn.f32 %f297, %f294, %f296, %f295;\n" " and.b32 %r226, %r218, 255;\n" " cvt.rn.f32.u32 %f298, %r226;\n" " mov.f32 %f299, 0f42000000;\n" " sub.rn.f32 %f300, %f298, %f299;\n" " mul.rn.f32 %f301, %f300, %f297;\n" " shl.b32 %r227, %r2, 2;\n" " add.s32 %r228, %r306, %r227;\n" " ld.shared.f32 %f302, [%r228];\n" " mov.f32 %f303, 0f00000000;\n" " fma.rn.f32 %f304, %f301, %f302, %f303;\n" " and.b32 %r229, %r217, 240;\n" " or.b32 %r230, %r229, 1258291200;\n" " mov.b32 %f305, %r230;\n" " mov.f32 %f306, 0fC9000040;\n" " mov.f32 %f307, 0f3D800000;\n" " fma.rn.f32 %f308, %f305, %f307, %f306;\n" " bfe.u32 %r231, %r218, 8, 8;\n" " cvt.rn.f32.u32 %f309, %r231;\n" " sub.rn.f32 %f310, %f309, %f299;\n" " mul.rn.f32 %f311, %f310, %f308;\n" " ld.shared.f32 %f312, [%r228+128];\n" " fma.rn.f32 %f313, %f311, %f312, %f304;\n" " and.b32 %r232, %r217, 3840;\n" " or.b32 %r233, %r232, 1258291200;\n" " mov.b32 %f314, %r233;\n" " mov.f32 %f315, 0fC7000400;\n" " mov.f32 %f316, 0f3B800000;\n" " fma.rn.f32 %f317, %f314, %f316, %f315;\n" " bfe.u32 %r234, %r218, 16, 8;\n" " cvt.rn.f32.u32 %f318, %r234;\n" " sub.rn.f32 %f319, %f318, %f299;\n" " mul.rn.f32 %f320, %f319, %f317;\n" " ld.shared.f32 %f321, [%r228+256];\n" " fma.rn.f32 %f322, %f320, %f321, %f313;\n" " and.b32 %r235, %r217, 61440;\n" " or.b32 %r236, %r235, 1258291200;\n" " mov.b32 %f323, %r236;\n" " mov.f32 %f324, 0fC5004000;\n" " mov.f32 %f325, 0f39800000;\n" " fma.rn.f32 %f326, %f323, %f325, %f324;\n" " shr.u32 %r237, %r218, 24;\n" " cvt.rn.f32.u32 %f327, %r237;\n" " sub.rn.f32 %f328, %f327, %f299;\n" " mul.rn.f32 %f329, %f328, %f326;\n" " ld.shared.f32 %f330, [%r228+384];\n" " fma.rn.f32 %f331, %f329, %f330, %f322;\n" " and.b32 %r238, %r217, 983040;\n" " or.b32 %r239, %r238, 1258291200;\n" " mov.b32 %f332, %r239;\n" " mov.f32 %f333, 0fC3040000;\n" " mov.f32 %f334, 0f37800000;\n" " fma.rn.f32 %f335, %f332, %f334, %f333;\n" " and.b32 %r240, %r219, 255;\n" " cvt.rn.f32.u32 %f336, %r240;\n" " sub.rn.f32 %f337, %f336, %f299;\n" " mul.rn.f32 %f338, %f337, %f335;\n" " ld.shared.f32 %f339, [%r228+512];\n" " fma.rn.f32 %f340, %f338, %f339, %f331;\n" " and.b32 %r241, %r223, 240;\n" " or.b32 %r242, %r241, 1258291200;\n" " mov.b32 %f341, %r242;\n" " fma.rn.f32 %f342, %f341, %f307, %f306;\n" " bfe.u32 %r243, %r219, 8, 8;\n" " cvt.rn.f32.u32 %f343, %r243;\n" " sub.rn.f32 %f344, %f343, %f299;\n" " mul.rn.f32 %f345, %f344, %f342;\n" " ld.shared.f32 %f346, [%r228+640];\n" " fma.rn.f32 %f347, %f345, %f346, %f340;\n" " and.b32 %r244, %r223, 3840;\n" " or.b32 %r245, %r244, 1258291200;\n" " mov.b32 %f348, %r245;\n" " fma.rn.f32 %f349, %f348, %f316, %f315;\n" " bfe.u32 %r246, %r219, 16, 8;\n" " cvt.rn.f32.u32 %f350, %r246;\n" " sub.rn.f32 %f351, %f350, %f299;\n" " mul.rn.f32 %f352, %f351, %f349;\n" " ld.shared.f32 %f353, [%r228+768];\n" " fma.rn.f32 %f354, %f352, %f353, %f347;\n" " and.b32 %r247, %r223, 61440;\n" " or.b32 %r248, %r247, 1258291200;\n" " mov.b32 %f355, %r248;\n" " fma.rn.f32 %f356, %f355, %f325, %f324;\n" " shr.u32 %r249, %r219, 24;\n" " cvt.rn.f32.u32 %f357, %r249;\n" " sub.rn.f32 %f358, %f357, %f299;\n" " mul.rn.f32 %f359, %f358, %f356;\n" " ld.shared.f32 %f360, [%r228+896];\n" " fma.rn.f32 %f361, %f359, %f360, %f354;\n" " fma.rn.f32 %f446, %f292, %f361, %f446;\n" " add.s64 %rd96, %rd96, 152;\n" " add.s64 %rd78, %rd97, %rd81;\n" " // begin inline asm\n" " ld.global.nc.u32 %r220, [%rd78];\n" " // end inline asm\n" " add.s64 %rd79, %rd97, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f293, [%rd79];\n" " // end inline asm\n" " add.s64 %rd84, %rd97, %rd82;\n" " add.s64 %rd80, %rd84, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r221,%r222}, [%rd80];\n" " // end inline asm\n" " shr.u32 %r250, %r220, 16;\n" " and.b32 %r251, %r220, 15;\n" " or.b32 %r252, %r251, 1258291200;\n" " mov.b32 %f362, %r252;\n" " fma.rn.f32 %f363, %f362, %f296, %f295;\n" " and.b32 %r253, %r221, 255;\n" " cvt.rn.f32.u32 %f364, %r253;\n" " sub.rn.f32 %f365, %f364, %f299;\n" " mul.rn.f32 %f366, %f365, %f363;\n" " ld.shared.f32 %f367, [%r228];\n" " fma.rn.f32 %f368, %f366, %f367, %f303;\n" " and.b32 %r254, %r220, 240;\n" " or.b32 %r255, %r254, 1258291200;\n" " mov.b32 %f369, %r255;\n" " fma.rn.f32 %f370, %f369, %f307, %f306;\n" " bfe.u32 %r256, %r221, 8, 8;\n" " cvt.rn.f32.u32 %f371, %r256;\n" " sub.rn.f32 %f372, %f371, %f299;\n" " mul.rn.f32 %f373, %f372, %f370;\n" " ld.shared.f32 %f374, [%r228+128];\n" " fma.rn.f32 %f375, %f373, %f374, %f368;\n" " and.b32 %r257, %r220, 3840;\n" " or.b32 %r258, %r257, 1258291200;\n" " mov.b32 %f376, %r258;\n" " fma.rn.f32 %f377, %f376, %f316, %f315;\n" " bfe.u32 %r259, %r221, 16, 8;\n" " cvt.rn.f32.u32 %f378, %r259;\n" " sub.rn.f32 %f379, %f378, %f299;\n" " mul.rn.f32 %f380, %f379, %f377;\n" " ld.shared.f32 %f381, [%r228+256];\n" " fma.rn.f32 %f382, %f380, %f381, %f375;\n" " and.b32 %r260, %r220, 61440;\n" " or.b32 %r261, %r260, 1258291200;\n" " mov.b32 %f383, %r261;\n" " fma.rn.f32 %f384, %f383, %f325, %f324;\n" " shr.u32 %r262, %r221, 24;\n" " cvt.rn.f32.u32 %f385, %r262;\n" " sub.rn.f32 %f386, %f385, %f299;\n" " mul.rn.f32 %f387, %f386, %f384;\n" " ld.shared.f32 %f388, [%r228+384];\n" " fma.rn.f32 %f389, %f387, %f388, %f382;\n" " and.b32 %r263, %r220, 983040;\n" " or.b32 %r264, %r263, 1258291200;\n" " mov.b32 %f390, %r264;\n" " fma.rn.f32 %f391, %f390, %f334, %f333;\n" " and.b32 %r265, %r222, 255;\n" " cvt.rn.f32.u32 %f392, %r265;\n" " sub.rn.f32 %f393, %f392, %f299;\n" " mul.rn.f32 %f394, %f393, %f391;\n" " ld.shared.f32 %f395, [%r228+512];\n" " fma.rn.f32 %f396, %f394, %f395, %f389;\n" " and.b32 %r266, %r250, 240;\n" " or.b32 %r267, %r266, 1258291200;\n" " mov.b32 %f397, %r267;\n" " fma.rn.f32 %f398, %f397, %f307, %f306;\n" " bfe.u32 %r268, %r222, 8, 8;\n" " cvt.rn.f32.u32 %f399, %r268;\n" " sub.rn.f32 %f400, %f399, %f299;\n" " mul.rn.f32 %f401, %f400, %f398;\n" " ld.shared.f32 %f402, [%r228+640];\n" " fma.rn.f32 %f403, %f401, %f402, %f396;\n" " and.b32 %r269, %r250, 3840;\n" " or.b32 %r270, %r269, 1258291200;\n" " mov.b32 %f404, %r270;\n" " fma.rn.f32 %f405, %f404, %f316, %f315;\n" " bfe.u32 %r271, %r222, 16, 8;\n" " cvt.rn.f32.u32 %f406, %r271;\n" " sub.rn.f32 %f407, %f406, %f299;\n" " mul.rn.f32 %f408, %f407, %f405;\n" " ld.shared.f32 %f409, [%r228+768];\n" " fma.rn.f32 %f410, %f408, %f409, %f403;\n" " and.b32 %r272, %r250, 61440;\n" " or.b32 %r273, %r272, 1258291200;\n" " mov.b32 %f411, %r273;\n" " fma.rn.f32 %f412, %f411, %f325, %f324;\n" " shr.u32 %r274, %r222, 24;\n" " cvt.rn.f32.u32 %f413, %r274;\n" " sub.rn.f32 %f414, %f413, %f299;\n" " mul.rn.f32 %f415, %f414, %f412;\n" " ld.shared.f32 %f416, [%r228+896];\n" " fma.rn.f32 %f417, %f415, %f416, %f410;\n" " fma.rn.f32 %f447, %f293, %f417, %f447;\n" " add.s64 %rd97, %rd97, 152;\n" " add.s32 %r306, %r306, 1024;\n" " add.s32 %r308, %r308, 1;\n" " setp.lt.s32 %p22, %r308, %r31;\n" " @%p22 bra $L__BB2_23;\n" "\n" "$L__BB2_24:\n" " cvt.u32.u64 %r275, %rd9;\n" " add.s32 %r298, %r275, 2048;\n" " setp.lt.s32 %p23, %r298, %r45;\n" " add.s32 %r297, %r297, 2048;\n" " @%p23 bra $L__BB2_2;\n" "\n" "$L__BB2_25:\n" " cvt.u32.u64 %r286, %rd1;\n" " setp.ge.s32 %p24, %r286, %r44;\n" " mov.u32 %r276, 16;\n" " mov.u32 %r285, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f418, %f446, %r276, %r285;\n" " // end inline asm\n" " add.f32 %f421, %f446, %f418;\n" " mov.u32 %r278, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f420, %f421, %r278, %r285;\n" " // end inline asm\n" " add.f32 %f423, %f421, %f420;\n" " mov.u32 %r280, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f422, %f423, %r280, %r285;\n" " // end inline asm\n" " add.f32 %f425, %f423, %f422;\n" " mov.u32 %r282, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f424, %f425, %r282, %r285;\n" " // end inline asm\n" " add.f32 %f427, %f425, %f424;\n" " mov.u32 %r284, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f426, %f427, %r284, %r285;\n" " // end inline asm\n" " add.f32 %f17, %f427, %f426;\n" " setp.ne.s32 %p25, %r2, 0;\n" " cvta.to.global.u64 %rd85, %rd31;\n" " shl.b64 %rd86, %rd1, 2;\n" " add.s64 %rd28, %rd85, %rd86;\n" " or.pred %p26, %p25, %p24;\n" " @%p26 bra $L__BB2_27;\n" "\n" " st.global.f32 [%rd28], %f17;\n" "\n" "$L__BB2_27:\n" " // begin inline asm\n" " shfl.down.b32 %f428, %f447, %r276, %r285;\n" " // end inline asm\n" " add.f32 %f431, %f447, %f428;\n" " // begin inline asm\n" " shfl.down.b32 %f430, %f431, %r278, %r285;\n" " // end inline asm\n" " add.f32 %f433, %f431, %f430;\n" " // begin inline asm\n" " shfl.down.b32 %f432, %f433, %r280, %r285;\n" " // end inline asm\n" " add.f32 %f435, %f433, %f432;\n" " // begin inline asm\n" " shfl.down.b32 %f434, %f435, %r282, %r285;\n" " // end inline asm\n" " add.f32 %f437, %f435, %f434;\n" " // begin inline asm\n" " shfl.down.b32 %f436, %f437, %r284, %r285;\n" " // end inline asm\n" " add.f32 %f18, %f437, %f436;\n" " setp.ge.s32 %p28, %r4, %r44;\n" " or.pred %p29, %p25, %p28;\n" " @%p29 bra $L__BB2_29;\n" "\n" " st.global.f32 [%rd28+4], %f18;\n" "\n" "$L__BB2_29:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_ps_r2p4\n" ".visible .entry fc_matvec_q3k_ps_r2p4(\n" " .param .u64 fc_matvec_q3k_ps_r2p4_param_0,\n" " .param .u64 fc_matvec_q3k_ps_r2p4_param_1,\n" " .param .u64 fc_matvec_q3k_ps_r2p4_param_2,\n" " .param .u32 fc_matvec_q3k_ps_r2p4_param_3,\n" " .param .u32 fc_matvec_q3k_ps_r2p4_param_4,\n" " .param .u32 fc_matvec_q3k_ps_r2p4_param_5\n" ")\n" "{\n" " .reg .pred %p<30>;\n" " .reg .f32 %f<674>;\n" " .reg .b32 %r<447>;\n" " .reg .b64 %rd<110>;\n" "\n" "\n" " ld.param.u64 %rd29, [fc_matvec_q3k_ps_r2p4_param_0];\n" " ld.param.u64 %rd30, [fc_matvec_q3k_ps_r2p4_param_1];\n" " ld.param.u64 %rd31, [fc_matvec_q3k_ps_r2p4_param_2];\n" " ld.param.u32 %r44, [fc_matvec_q3k_ps_r2p4_param_3];\n" " ld.param.u32 %r45, [fc_matvec_q3k_ps_r2p4_param_4];\n" " ld.param.u32 %r46, [fc_matvec_q3k_ps_r2p4_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r47, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r48, %ctaid.x;\n" " shl.b32 %r49, %r48, 3;\n" " add.s32 %r50, %r49, %r47;\n" " shl.b32 %r3, %r50, 1;\n" " cvt.s64.s32 %rd1, %r3;\n" " or.b32 %r4, %r3, 1;\n" " setp.lt.s32 %p1, %r45, 1;\n" " mov.f32 %f670, 0f00000000;\n" " mov.f32 %f671, %f670;\n" " @%p1 bra $L__BB3_25;\n" "\n" " cvt.s64.s32 %rd32, %r46;\n" " cvt.u64.u32 %rd2, %r2;\n" " shr.u32 %r53, %r2, 4;\n" " cvt.u64.u32 %rd3, %r53;\n" " mov.u32 %r54, -2;\n" " sub.s32 %r5, %r54, %r1;\n" " not.b32 %r6, %r45;\n" " not.b32 %r7, %r1;\n" " cvt.s64.s32 %rd4, %r1;\n" " shl.b32 %r55, %r1, 4;\n" " mov.u32 %r56, sh_xps;\n" " add.s32 %r8, %r56, %r55;\n" " add.s32 %r9, %r1, 256;\n" " shl.b32 %r57, %r1, 2;\n" " add.s32 %r10, %r56, %r57;\n" " add.s32 %r11, %r1, 512;\n" " add.s32 %r12, %r1, 768;\n" " setp.lt.s32 %p2, %r4, %r44;\n" " selp.u32 %r58, 1, 0, %p2;\n" " add.s32 %r59, %r3, %r58;\n" " mul.wide.s32 %rd33, %r59, %r46;\n" " add.s64 %rd109, %rd29, %rd33;\n" " mul.lo.s64 %rd34, %rd1, %rd32;\n" " add.s64 %rd108, %rd29, %rd34;\n" " mov.u32 %r434, 0;\n" " mov.f32 %f670, 0f00000000;\n" " mov.f32 %f671, %f670;\n" " mov.u32 %r435, %r434;\n" "\n" "$L__BB3_2:\n" " add.s32 %r60, %r434, %r6;\n" " max.s32 %r61, %r60, -2049;\n" " sub.s32 %r15, %r5, %r61;\n" " sub.s32 %r62, %r45, %r435;\n" " min.s32 %r16, %r62, 2048;\n" " bar.sync 0;\n" " cvt.s64.s32 %rd9, %r435;\n" " mul.wide.s32 %rd35, %r435, 4;\n" " add.s64 %rd10, %rd30, %rd35;\n" " and.b32 %r63, %r16, 3;\n" " setp.eq.s32 %p3, %r63, 0;\n" " { .reg .b64 %tmp;\n" " cvt.u64.u32 %tmp, %r56;\n" " cvta.shared.u64 %rd36, %tmp; }\n" " or.b64 %rd37, %rd10, %rd36;\n" " and.b64 %rd38, %rd37, 15;\n" " setp.eq.s64 %p4, %rd38, 0;\n" " and.pred %p5, %p3, %p4;\n" " @%p5 bra $L__BB3_11;\n" " bra.uni $L__BB3_3;\n" "\n" "$L__BB3_11:\n" " shr.s32 %r24, %r16, 2;\n" " setp.ge.s32 %p12, %r1, %r24;\n" " @%p12 bra $L__BB3_18;\n" "\n" " add.s32 %r25, %r24, %r7;\n" " shr.u32 %r70, %r25, 8;\n" " add.s32 %r71, %r70, 1;\n" " and.b32 %r26, %r71, 3;\n" " setp.eq.s32 %p13, %r26, 0;\n" " mov.u32 %r439, %r1;\n" " @%p13 bra $L__BB3_16;\n" "\n" " shl.b64 %rd51, %rd4, 4;\n" " add.s64 %rd50, %rd10, %rd51;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f30,%f31,%f32,%f33}, [%rd50];\n" " // end inline asm\n" " st.shared.v4.f32 [%r8], {%f30, %f31, %f32, %f33};\n" " setp.eq.s32 %p14, %r26, 1;\n" " mov.u32 %r439, %r9;\n" " @%p14 bra $L__BB3_16;\n" "\n" " add.s64 %rd52, %rd50, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f34,%f35,%f36,%f37}, [%rd52];\n" " // end inline asm\n" " st.shared.v4.f32 [%r8+4096], {%f34, %f35, %f36, %f37};\n" " setp.eq.s32 %p15, %r26, 2;\n" " mov.u32 %r439, %r11;\n" " @%p15 bra $L__BB3_16;\n" "\n" " add.s64 %rd53, %rd50, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f38,%f39,%f40,%f41}, [%rd53];\n" " // end inline asm\n" " st.shared.v4.f32 [%r8+8192], {%f38, %f39, %f40, %f41};\n" " mov.u32 %r439, %r12;\n" "\n" "$L__BB3_16:\n" " setp.lt.u32 %p16, %r25, 768;\n" " @%p16 bra $L__BB3_18;\n" "\n" "$L__BB3_17:\n" " mul.wide.s32 %rd58, %r439, 16;\n" " add.s64 %rd54, %rd10, %rd58;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f42,%f43,%f44,%f45}, [%rd54];\n" " // end inline asm\n" " shl.b32 %r72, %r439, 4;\n" " mov.u32 %r73, sh_xps;\n" " add.s32 %r74, %r73, %r72;\n" " st.shared.v4.f32 [%r74], {%f42, %f43, %f44, %f45};\n" " add.s64 %rd55, %rd54, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f46,%f47,%f48,%f49}, [%rd55];\n" " // end inline asm\n" " st.shared.v4.f32 [%r74+4096], {%f46, %f47, %f48, %f49};\n" " add.s64 %rd56, %rd54, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f50,%f51,%f52,%f53}, [%rd56];\n" " // end inline asm\n" " st.shared.v4.f32 [%r74+8192], {%f50, %f51, %f52, %f53};\n" " add.s64 %rd57, %rd54, 12288;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f54,%f55,%f56,%f57}, [%rd57];\n" " // end inline asm\n" " st.shared.v4.f32 [%r74+12288], {%f54, %f55, %f56, %f57};\n" " add.s32 %r439, %r439, 1024;\n" " setp.lt.s32 %p17, %r439, %r24;\n" " @%p17 bra $L__BB3_17;\n" " bra.uni $L__BB3_18;\n" "\n" "$L__BB3_3:\n" " setp.ge.s32 %p6, %r1, %r16;\n" " @%p6 bra $L__BB3_18;\n" "\n" " shr.u32 %r65, %r15, 8;\n" " add.s32 %r66, %r65, 1;\n" " and.b32 %r17, %r66, 3;\n" " setp.eq.s32 %p7, %r17, 0;\n" " mov.u32 %r436, %r1;\n" " @%p7 bra $L__BB3_8;\n" "\n" " shl.b64 %rd40, %rd4, 2;\n" " add.s64 %rd39, %rd10, %rd40;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r10], %f23;\n" " setp.eq.s32 %p8, %r17, 1;\n" " mov.u32 %r436, %r9;\n" " @%p8 bra $L__BB3_8;\n" "\n" " add.s64 %rd41, %rd39, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r10+1024], %f24;\n" " setp.eq.s32 %p9, %r17, 2;\n" " mov.u32 %r436, %r11;\n" " @%p9 bra $L__BB3_8;\n" "\n" " add.s64 %rd42, %rd39, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r10+2048], %f25;\n" " mov.u32 %r436, %r12;\n" "\n" "$L__BB3_8:\n" " setp.lt.u32 %p10, %r15, 768;\n" " @%p10 bra $L__BB3_18;\n" "\n" " cvt.s64.s32 %rd43, %r436;\n" " add.s64 %rd44, %rd43, %rd9;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd101, %rd30, %rd45;\n" " shl.b32 %r67, %r436, 2;\n" " add.s32 %r69, %r56, %r67;\n" " add.s32 %r437, %r69, 2048;\n" "\n" "$L__BB3_10:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd101];\n" " // end inline asm\n" " st.shared.f32 [%r437+-2048], %f26;\n" " add.s64 %rd47, %rd101, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd47];\n" " // end inline asm\n" " st.shared.f32 [%r437+-1024], %f27;\n" " add.s64 %rd48, %rd101, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r437], %f28;\n" " add.s64 %rd49, %rd101, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r437+1024], %f29;\n" " add.s64 %rd101, %rd101, 4096;\n" " add.s32 %r437, %r437, 4096;\n" " add.s32 %r436, %r436, 1024;\n" " setp.lt.s32 %p11, %r436, %r16;\n" " @%p11 bra $L__BB3_10;\n" "\n" "$L__BB3_18:\n" " cvt.u32.u64 %r75, %rd1;\n" " setp.ge.s32 %p18, %r75, %r44;\n" " bar.sync 0;\n" " @%p18 bra $L__BB3_24;\n" "\n" " shr.s32 %r31, %r16, 8;\n" " setp.lt.s32 %p19, %r16, 1024;\n" " mov.u32 %r445, 0;\n" " mov.u32 %r443, %r56;\n" " @%p19 bra $L__BB3_22;\n" "\n" " mov.u32 %r443, sh_xps;\n" " mov.u32 %r441, 4;\n" "\n" "$L__BB3_21:\n" " mov.u32 %r445, %r441;\n" " shl.b64 %rd83, %rd2, 2;\n" " add.s64 %rd59, %rd108, %rd83;\n" " // begin inline asm\n" " ld.global.nc.u32 %r104, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd108, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f66, [%rd60];\n" " // end inline asm\n" " shl.b64 %rd84, %rd3, 3;\n" " add.s64 %rd85, %rd108, %rd84;\n" " add.s64 %rd61, %rd85, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r105,%r106}, [%rd61];\n" " // end inline asm\n" " add.s64 %rd62, %rd109, %rd83;\n" " // begin inline asm\n" " ld.global.nc.u32 %r110, [%rd62];\n" " // end inline asm\n" " add.s64 %rd63, %rd109, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f68, [%rd63];\n" " // end inline asm\n" " add.s64 %rd86, %rd109, %rd84;\n" " add.s64 %rd64, %rd86, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r111,%r112}, [%rd64];\n" " // end inline asm\n" " add.s64 %rd65, %rd59, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r116, [%rd65];\n" " // end inline asm\n" " add.s64 %rd66, %rd108, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f70, [%rd66];\n" " // end inline asm\n" " add.s64 %rd67, %rd85, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r117,%r118}, [%rd67];\n" " // end inline asm\n" " add.s64 %rd68, %rd62, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r122, [%rd68];\n" " // end inline asm\n" " add.s64 %rd69, %rd109, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f72, [%rd69];\n" " // end inline asm\n" " add.s64 %rd70, %rd86, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r123,%r124}, [%rd70];\n" " // end inline asm\n" " add.s64 %rd71, %rd59, 304;\n" " // begin inline asm\n" " ld.global.nc.u32 %r128, [%rd71];\n" " // end inline asm\n" " add.s64 %rd72, %rd108, 448;\n" " // begin inline asm\n" " ld.global.nc.f32 %f74, [%rd72];\n" " // end inline asm\n" " add.s64 %rd73, %rd85, 432;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r129,%r130}, [%rd73];\n" " // end inline asm\n" " add.s64 %rd74, %rd62, 304;\n" " // begin inline asm\n" " ld.global.nc.u32 %r134, [%rd74];\n" " // end inline asm\n" " add.s64 %rd75, %rd109, 448;\n" " // begin inline asm\n" " ld.global.nc.f32 %f76, [%rd75];\n" " // end inline asm\n" " add.s64 %rd76, %rd86, 432;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r135,%r136}, [%rd76];\n" " // end inline asm\n" " add.s64 %rd77, %rd59, 456;\n" " // begin inline asm\n" " ld.global.nc.u32 %r140, [%rd77];\n" " // end inline asm\n" " add.s64 %rd78, %rd108, 600;\n" " // begin inline asm\n" " ld.global.nc.f32 %f78, [%rd78];\n" " // end inline asm\n" " add.s64 %rd79, %rd85, 584;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r141,%r142}, [%rd79];\n" " // end inline asm\n" " add.s64 %rd80, %rd62, 456;\n" " // begin inline asm\n" " ld.global.nc.u32 %r146, [%rd80];\n" " // end inline asm\n" " add.s64 %rd81, %rd109, 600;\n" " // begin inline asm\n" " ld.global.nc.f32 %f80, [%rd81];\n" " // end inline asm\n" " add.s64 %rd82, %rd86, 584;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r147,%r148}, [%rd82];\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " shr.u32 %r152, %r104, 16;\n" " and.b32 %r153, %r104, 15;\n" " or.b32 %r154, %r153, 1258291200;\n" " mov.b32 %f82, %r154;\n" " mov.f32 %f83, 0fCB000004;\n" " mov.f32 %f84, 0f3F800000;\n" " fma.rn.f32 %f85, %f82, %f84, %f83;\n" " and.b32 %r155, %r105, 255;\n" " cvt.rn.f32.u32 %f86, %r155;\n" " mov.f32 %f87, 0f42000000;\n" " sub.rn.f32 %f88, %f86, %f87;\n" " mul.rn.f32 %f89, %f88, %f85;\n" " shl.b32 %r156, %r2, 2;\n" " add.s32 %r157, %r443, %r156;\n" " ld.shared.f32 %f90, [%r157];\n" " mov.f32 %f91, 0f00000000;\n" " fma.rn.f32 %f92, %f89, %f90, %f91;\n" " and.b32 %r158, %r104, 240;\n" " or.b32 %r159, %r158, 1258291200;\n" " mov.b32 %f93, %r159;\n" " mov.f32 %f94, 0fC9000040;\n" " mov.f32 %f95, 0f3D800000;\n" " fma.rn.f32 %f96, %f93, %f95, %f94;\n" " bfe.u32 %r160, %r105, 8, 8;\n" " cvt.rn.f32.u32 %f97, %r160;\n" " sub.rn.f32 %f98, %f97, %f87;\n" " mul.rn.f32 %f99, %f98, %f96;\n" " ld.shared.f32 %f100, [%r157+128];\n" " fma.rn.f32 %f101, %f99, %f100, %f92;\n" " and.b32 %r161, %r104, 3840;\n" " or.b32 %r162, %r161, 1258291200;\n" " mov.b32 %f102, %r162;\n" " mov.f32 %f103, 0fC7000400;\n" " mov.f32 %f104, 0f3B800000;\n" " fma.rn.f32 %f105, %f102, %f104, %f103;\n" " bfe.u32 %r163, %r105, 16, 8;\n" " cvt.rn.f32.u32 %f106, %r163;\n" " sub.rn.f32 %f107, %f106, %f87;\n" " mul.rn.f32 %f108, %f107, %f105;\n" " ld.shared.f32 %f109, [%r157+256];\n" " fma.rn.f32 %f110, %f108, %f109, %f101;\n" " and.b32 %r164, %r104, 61440;\n" " or.b32 %r165, %r164, 1258291200;\n" " mov.b32 %f111, %r165;\n" " mov.f32 %f112, 0fC5004000;\n" " mov.f32 %f113, 0f39800000;\n" " fma.rn.f32 %f114, %f111, %f113, %f112;\n" " shr.u32 %r166, %r105, 24;\n" " cvt.rn.f32.u32 %f115, %r166;\n" " sub.rn.f32 %f116, %f115, %f87;\n" " mul.rn.f32 %f117, %f116, %f114;\n" " ld.shared.f32 %f118, [%r157+384];\n" " fma.rn.f32 %f119, %f117, %f118, %f110;\n" " and.b32 %r167, %r104, 983040;\n" " or.b32 %r168, %r167, 1258291200;\n" " mov.b32 %f120, %r168;\n" " mov.f32 %f121, 0fC3040000;\n" " mov.f32 %f122, 0f37800000;\n" " fma.rn.f32 %f123, %f120, %f122, %f121;\n" " and.b32 %r169, %r106, 255;\n" " cvt.rn.f32.u32 %f124, %r169;\n" " sub.rn.f32 %f125, %f124, %f87;\n" " mul.rn.f32 %f126, %f125, %f123;\n" " ld.shared.f32 %f127, [%r157+512];\n" " fma.rn.f32 %f128, %f126, %f127, %f119;\n" " and.b32 %r170, %r152, 240;\n" " or.b32 %r171, %r170, 1258291200;\n" " mov.b32 %f129, %r171;\n" " fma.rn.f32 %f130, %f129, %f95, %f94;\n" " bfe.u32 %r172, %r106, 8, 8;\n" " cvt.rn.f32.u32 %f131, %r172;\n" " sub.rn.f32 %f132, %f131, %f87;\n" " mul.rn.f32 %f133, %f132, %f130;\n" " ld.shared.f32 %f134, [%r157+640];\n" " fma.rn.f32 %f135, %f133, %f134, %f128;\n" " and.b32 %r173, %r152, 3840;\n" " or.b32 %r174, %r173, 1258291200;\n" " mov.b32 %f136, %r174;\n" " fma.rn.f32 %f137, %f136, %f104, %f103;\n" " bfe.u32 %r175, %r106, 16, 8;\n" " cvt.rn.f32.u32 %f138, %r175;\n" " sub.rn.f32 %f139, %f138, %f87;\n" " mul.rn.f32 %f140, %f139, %f137;\n" " ld.shared.f32 %f141, [%r157+768];\n" " fma.rn.f32 %f142, %f140, %f141, %f135;\n" " and.b32 %r176, %r152, 61440;\n" " or.b32 %r177, %r176, 1258291200;\n" " mov.b32 %f143, %r177;\n" " fma.rn.f32 %f144, %f143, %f113, %f112;\n" " shr.u32 %r178, %r106, 24;\n" " cvt.rn.f32.u32 %f145, %r178;\n" " sub.rn.f32 %f146, %f145, %f87;\n" " mul.rn.f32 %f147, %f146, %f144;\n" " ld.shared.f32 %f148, [%r157+896];\n" " fma.rn.f32 %f149, %f147, %f148, %f142;\n" " fma.rn.f32 %f150, %f149, %f66, %f670;\n" " shr.u32 %r179, %r110, 16;\n" " and.b32 %r180, %r110, 15;\n" " or.b32 %r181, %r180, 1258291200;\n" " mov.b32 %f151, %r181;\n" " fma.rn.f32 %f152, %f151, %f84, %f83;\n" " and.b32 %r182, %r111, 255;\n" " cvt.rn.f32.u32 %f153, %r182;\n" " sub.rn.f32 %f154, %f153, %f87;\n" " mul.rn.f32 %f155, %f154, %f152;\n" " fma.rn.f32 %f156, %f155, %f90, %f91;\n" " and.b32 %r183, %r110, 240;\n" " or.b32 %r184, %r183, 1258291200;\n" " mov.b32 %f157, %r184;\n" " fma.rn.f32 %f158, %f157, %f95, %f94;\n" " bfe.u32 %r185, %r111, 8, 8;\n" " cvt.rn.f32.u32 %f159, %r185;\n" " sub.rn.f32 %f160, %f159, %f87;\n" " mul.rn.f32 %f161, %f160, %f158;\n" " fma.rn.f32 %f162, %f161, %f100, %f156;\n" " and.b32 %r186, %r110, 3840;\n" " or.b32 %r187, %r186, 1258291200;\n" " mov.b32 %f163, %r187;\n" " fma.rn.f32 %f164, %f163, %f104, %f103;\n" " bfe.u32 %r188, %r111, 16, 8;\n" " cvt.rn.f32.u32 %f165, %r188;\n" " sub.rn.f32 %f166, %f165, %f87;\n" " mul.rn.f32 %f167, %f166, %f164;\n" " fma.rn.f32 %f168, %f167, %f109, %f162;\n" " and.b32 %r189, %r110, 61440;\n" " or.b32 %r190, %r189, 1258291200;\n" " mov.b32 %f169, %r190;\n" " fma.rn.f32 %f170, %f169, %f113, %f112;\n" " shr.u32 %r191, %r111, 24;\n" " cvt.rn.f32.u32 %f171, %r191;\n" " sub.rn.f32 %f172, %f171, %f87;\n" " mul.rn.f32 %f173, %f172, %f170;\n" " fma.rn.f32 %f174, %f173, %f118, %f168;\n" " and.b32 %r192, %r110, 983040;\n" " or.b32 %r193, %r192, 1258291200;\n" " mov.b32 %f175, %r193;\n" " fma.rn.f32 %f176, %f175, %f122, %f121;\n" " and.b32 %r194, %r112, 255;\n" " cvt.rn.f32.u32 %f177, %r194;\n" " sub.rn.f32 %f178, %f177, %f87;\n" " mul.rn.f32 %f179, %f178, %f176;\n" " fma.rn.f32 %f180, %f179, %f127, %f174;\n" " and.b32 %r195, %r179, 240;\n" " or.b32 %r196, %r195, 1258291200;\n" " mov.b32 %f181, %r196;\n" " fma.rn.f32 %f182, %f181, %f95, %f94;\n" " bfe.u32 %r197, %r112, 8, 8;\n" " cvt.rn.f32.u32 %f183, %r197;\n" " sub.rn.f32 %f184, %f183, %f87;\n" " mul.rn.f32 %f185, %f184, %f182;\n" " fma.rn.f32 %f186, %f185, %f134, %f180;\n" " and.b32 %r198, %r179, 3840;\n" " or.b32 %r199, %r198, 1258291200;\n" " mov.b32 %f187, %r199;\n" " fma.rn.f32 %f188, %f187, %f104, %f103;\n" " bfe.u32 %r200, %r112, 16, 8;\n" " cvt.rn.f32.u32 %f189, %r200;\n" " sub.rn.f32 %f190, %f189, %f87;\n" " mul.rn.f32 %f191, %f190, %f188;\n" " fma.rn.f32 %f192, %f191, %f141, %f186;\n" " and.b32 %r201, %r179, 61440;\n" " or.b32 %r202, %r201, 1258291200;\n" " mov.b32 %f193, %r202;\n" " fma.rn.f32 %f194, %f193, %f113, %f112;\n" " shr.u32 %r203, %r112, 24;\n" " cvt.rn.f32.u32 %f195, %r203;\n" " sub.rn.f32 %f196, %f195, %f87;\n" " mul.rn.f32 %f197, %f196, %f194;\n" " fma.rn.f32 %f198, %f197, %f148, %f192;\n" " fma.rn.f32 %f199, %f198, %f68, %f671;\n" " shr.u32 %r204, %r116, 16;\n" " and.b32 %r205, %r116, 15;\n" " or.b32 %r206, %r205, 1258291200;\n" " mov.b32 %f200, %r206;\n" " fma.rn.f32 %f201, %f200, %f84, %f83;\n" " and.b32 %r207, %r117, 255;\n" " cvt.rn.f32.u32 %f202, %r207;\n" " sub.rn.f32 %f203, %f202, %f87;\n" " mul.rn.f32 %f204, %f203, %f201;\n" " ld.shared.f32 %f205, [%r157+1024];\n" " fma.rn.f32 %f206, %f204, %f205, %f91;\n" " and.b32 %r208, %r116, 240;\n" " or.b32 %r209, %r208, 1258291200;\n" " mov.b32 %f207, %r209;\n" " fma.rn.f32 %f208, %f207, %f95, %f94;\n" " bfe.u32 %r210, %r117, 8, 8;\n" " cvt.rn.f32.u32 %f209, %r210;\n" " sub.rn.f32 %f210, %f209, %f87;\n" " mul.rn.f32 %f211, %f210, %f208;\n" " ld.shared.f32 %f212, [%r157+1152];\n" " fma.rn.f32 %f213, %f211, %f212, %f206;\n" " and.b32 %r211, %r116, 3840;\n" " or.b32 %r212, %r211, 1258291200;\n" " mov.b32 %f214, %r212;\n" " fma.rn.f32 %f215, %f214, %f104, %f103;\n" " bfe.u32 %r213, %r117, 16, 8;\n" " cvt.rn.f32.u32 %f216, %r213;\n" " sub.rn.f32 %f217, %f216, %f87;\n" " mul.rn.f32 %f218, %f217, %f215;\n" " ld.shared.f32 %f219, [%r157+1280];\n" " fma.rn.f32 %f220, %f218, %f219, %f213;\n" " and.b32 %r214, %r116, 61440;\n" " or.b32 %r215, %r214, 1258291200;\n" " mov.b32 %f221, %r215;\n" " fma.rn.f32 %f222, %f221, %f113, %f112;\n" " shr.u32 %r216, %r117, 24;\n" " cvt.rn.f32.u32 %f223, %r216;\n" " sub.rn.f32 %f224, %f223, %f87;\n" " mul.rn.f32 %f225, %f224, %f222;\n" " ld.shared.f32 %f226, [%r157+1408];\n" " fma.rn.f32 %f227, %f225, %f226, %f220;\n" " and.b32 %r217, %r116, 983040;\n" " or.b32 %r218, %r217, 1258291200;\n" " mov.b32 %f228, %r218;\n" " fma.rn.f32 %f229, %f228, %f122, %f121;\n" " and.b32 %r219, %r118, 255;\n" " cvt.rn.f32.u32 %f230, %r219;\n" " sub.rn.f32 %f231, %f230, %f87;\n" " mul.rn.f32 %f232, %f231, %f229;\n" " ld.shared.f32 %f233, [%r157+1536];\n" " fma.rn.f32 %f234, %f232, %f233, %f227;\n" " and.b32 %r220, %r204, 240;\n" " or.b32 %r221, %r220, 1258291200;\n" " mov.b32 %f235, %r221;\n" " fma.rn.f32 %f236, %f235, %f95, %f94;\n" " bfe.u32 %r222, %r118, 8, 8;\n" " cvt.rn.f32.u32 %f237, %r222;\n" " sub.rn.f32 %f238, %f237, %f87;\n" " mul.rn.f32 %f239, %f238, %f236;\n" " ld.shared.f32 %f240, [%r157+1664];\n" " fma.rn.f32 %f241, %f239, %f240, %f234;\n" " and.b32 %r223, %r204, 3840;\n" " or.b32 %r224, %r223, 1258291200;\n" " mov.b32 %f242, %r224;\n" " fma.rn.f32 %f243, %f242, %f104, %f103;\n" " bfe.u32 %r225, %r118, 16, 8;\n" " cvt.rn.f32.u32 %f244, %r225;\n" " sub.rn.f32 %f245, %f244, %f87;\n" " mul.rn.f32 %f246, %f245, %f243;\n" " ld.shared.f32 %f247, [%r157+1792];\n" " fma.rn.f32 %f248, %f246, %f247, %f241;\n" " and.b32 %r226, %r204, 61440;\n" " or.b32 %r227, %r226, 1258291200;\n" " mov.b32 %f249, %r227;\n" " fma.rn.f32 %f250, %f249, %f113, %f112;\n" " shr.u32 %r228, %r118, 24;\n" " cvt.rn.f32.u32 %f251, %r228;\n" " sub.rn.f32 %f252, %f251, %f87;\n" " mul.rn.f32 %f253, %f252, %f250;\n" " ld.shared.f32 %f254, [%r157+1920];\n" " fma.rn.f32 %f255, %f253, %f254, %f248;\n" " fma.rn.f32 %f256, %f255, %f70, %f150;\n" " shr.u32 %r229, %r122, 16;\n" " and.b32 %r230, %r122, 15;\n" " or.b32 %r231, %r230, 1258291200;\n" " mov.b32 %f257, %r231;\n" " fma.rn.f32 %f258, %f257, %f84, %f83;\n" " and.b32 %r232, %r123, 255;\n" " cvt.rn.f32.u32 %f259, %r232;\n" " sub.rn.f32 %f260, %f259, %f87;\n" " mul.rn.f32 %f261, %f260, %f258;\n" " fma.rn.f32 %f262, %f261, %f205, %f91;\n" " and.b32 %r233, %r122, 240;\n" " or.b32 %r234, %r233, 1258291200;\n" " mov.b32 %f263, %r234;\n" " fma.rn.f32 %f264, %f263, %f95, %f94;\n" " bfe.u32 %r235, %r123, 8, 8;\n" " cvt.rn.f32.u32 %f265, %r235;\n" " sub.rn.f32 %f266, %f265, %f87;\n" " mul.rn.f32 %f267, %f266, %f264;\n" " fma.rn.f32 %f268, %f267, %f212, %f262;\n" " and.b32 %r236, %r122, 3840;\n" " or.b32 %r237, %r236, 1258291200;\n" " mov.b32 %f269, %r237;\n" " fma.rn.f32 %f270, %f269, %f104, %f103;\n" " bfe.u32 %r238, %r123, 16, 8;\n" " cvt.rn.f32.u32 %f271, %r238;\n" " sub.rn.f32 %f272, %f271, %f87;\n" " mul.rn.f32 %f273, %f272, %f270;\n" " fma.rn.f32 %f274, %f273, %f219, %f268;\n" " and.b32 %r239, %r122, 61440;\n" " or.b32 %r240, %r239, 1258291200;\n" " mov.b32 %f275, %r240;\n" " fma.rn.f32 %f276, %f275, %f113, %f112;\n" " shr.u32 %r241, %r123, 24;\n" " cvt.rn.f32.u32 %f277, %r241;\n" " sub.rn.f32 %f278, %f277, %f87;\n" " mul.rn.f32 %f279, %f278, %f276;\n" " fma.rn.f32 %f280, %f279, %f226, %f274;\n" " and.b32 %r242, %r122, 983040;\n" " or.b32 %r243, %r242, 1258291200;\n" " mov.b32 %f281, %r243;\n" " fma.rn.f32 %f282, %f281, %f122, %f121;\n" " and.b32 %r244, %r124, 255;\n" " cvt.rn.f32.u32 %f283, %r244;\n" " sub.rn.f32 %f284, %f283, %f87;\n" " mul.rn.f32 %f285, %f284, %f282;\n" " fma.rn.f32 %f286, %f285, %f233, %f280;\n" " and.b32 %r245, %r229, 240;\n" " or.b32 %r246, %r245, 1258291200;\n" " mov.b32 %f287, %r246;\n" " fma.rn.f32 %f288, %f287, %f95, %f94;\n" " bfe.u32 %r247, %r124, 8, 8;\n" " cvt.rn.f32.u32 %f289, %r247;\n" " sub.rn.f32 %f290, %f289, %f87;\n" " mul.rn.f32 %f291, %f290, %f288;\n" " fma.rn.f32 %f292, %f291, %f240, %f286;\n" " and.b32 %r248, %r229, 3840;\n" " or.b32 %r249, %r248, 1258291200;\n" " mov.b32 %f293, %r249;\n" " fma.rn.f32 %f294, %f293, %f104, %f103;\n" " bfe.u32 %r250, %r124, 16, 8;\n" " cvt.rn.f32.u32 %f295, %r250;\n" " sub.rn.f32 %f296, %f295, %f87;\n" " mul.rn.f32 %f297, %f296, %f294;\n" " fma.rn.f32 %f298, %f297, %f247, %f292;\n" " and.b32 %r251, %r229, 61440;\n" " or.b32 %r252, %r251, 1258291200;\n" " mov.b32 %f299, %r252;\n" " fma.rn.f32 %f300, %f299, %f113, %f112;\n" " shr.u32 %r253, %r124, 24;\n" " cvt.rn.f32.u32 %f301, %r253;\n" " sub.rn.f32 %f302, %f301, %f87;\n" " mul.rn.f32 %f303, %f302, %f300;\n" " fma.rn.f32 %f304, %f303, %f254, %f298;\n" " fma.rn.f32 %f305, %f304, %f72, %f199;\n" " shr.u32 %r254, %r128, 16;\n" " and.b32 %r255, %r128, 15;\n" " or.b32 %r256, %r255, 1258291200;\n" " mov.b32 %f306, %r256;\n" " fma.rn.f32 %f307, %f306, %f84, %f83;\n" " and.b32 %r257, %r129, 255;\n" " cvt.rn.f32.u32 %f308, %r257;\n" " sub.rn.f32 %f309, %f308, %f87;\n" " mul.rn.f32 %f310, %f309, %f307;\n" " ld.shared.f32 %f311, [%r157+2048];\n" " fma.rn.f32 %f312, %f310, %f311, %f91;\n" " and.b32 %r258, %r128, 240;\n" " or.b32 %r259, %r258, 1258291200;\n" " mov.b32 %f313, %r259;\n" " fma.rn.f32 %f314, %f313, %f95, %f94;\n" " bfe.u32 %r260, %r129, 8, 8;\n" " cvt.rn.f32.u32 %f315, %r260;\n" " sub.rn.f32 %f316, %f315, %f87;\n" " mul.rn.f32 %f317, %f316, %f314;\n" " ld.shared.f32 %f318, [%r157+2176];\n" " fma.rn.f32 %f319, %f317, %f318, %f312;\n" " and.b32 %r261, %r128, 3840;\n" " or.b32 %r262, %r261, 1258291200;\n" " mov.b32 %f320, %r262;\n" " fma.rn.f32 %f321, %f320, %f104, %f103;\n" " bfe.u32 %r263, %r129, 16, 8;\n" " cvt.rn.f32.u32 %f322, %r263;\n" " sub.rn.f32 %f323, %f322, %f87;\n" " mul.rn.f32 %f324, %f323, %f321;\n" " ld.shared.f32 %f325, [%r157+2304];\n" " fma.rn.f32 %f326, %f324, %f325, %f319;\n" " and.b32 %r264, %r128, 61440;\n" " or.b32 %r265, %r264, 1258291200;\n" " mov.b32 %f327, %r265;\n" " fma.rn.f32 %f328, %f327, %f113, %f112;\n" " shr.u32 %r266, %r129, 24;\n" " cvt.rn.f32.u32 %f329, %r266;\n" " sub.rn.f32 %f330, %f329, %f87;\n" " mul.rn.f32 %f331, %f330, %f328;\n" " ld.shared.f32 %f332, [%r157+2432];\n" " fma.rn.f32 %f333, %f331, %f332, %f326;\n" " and.b32 %r267, %r128, 983040;\n" " or.b32 %r268, %r267, 1258291200;\n" " mov.b32 %f334, %r268;\n" " fma.rn.f32 %f335, %f334, %f122, %f121;\n" " and.b32 %r269, %r130, 255;\n" " cvt.rn.f32.u32 %f336, %r269;\n" " sub.rn.f32 %f337, %f336, %f87;\n" " mul.rn.f32 %f338, %f337, %f335;\n" " ld.shared.f32 %f339, [%r157+2560];\n" " fma.rn.f32 %f340, %f338, %f339, %f333;\n" " and.b32 %r270, %r254, 240;\n" " or.b32 %r271, %r270, 1258291200;\n" " mov.b32 %f341, %r271;\n" " fma.rn.f32 %f342, %f341, %f95, %f94;\n" " bfe.u32 %r272, %r130, 8, 8;\n" " cvt.rn.f32.u32 %f343, %r272;\n" " sub.rn.f32 %f344, %f343, %f87;\n" " mul.rn.f32 %f345, %f344, %f342;\n" " ld.shared.f32 %f346, [%r157+2688];\n" " fma.rn.f32 %f347, %f345, %f346, %f340;\n" " and.b32 %r273, %r254, 3840;\n" " or.b32 %r274, %r273, 1258291200;\n" " mov.b32 %f348, %r274;\n" " fma.rn.f32 %f349, %f348, %f104, %f103;\n" " bfe.u32 %r275, %r130, 16, 8;\n" " cvt.rn.f32.u32 %f350, %r275;\n" " sub.rn.f32 %f351, %f350, %f87;\n" " mul.rn.f32 %f352, %f351, %f349;\n" " ld.shared.f32 %f353, [%r157+2816];\n" " fma.rn.f32 %f354, %f352, %f353, %f347;\n" " and.b32 %r276, %r254, 61440;\n" " or.b32 %r277, %r276, 1258291200;\n" " mov.b32 %f355, %r277;\n" " fma.rn.f32 %f356, %f355, %f113, %f112;\n" " shr.u32 %r278, %r130, 24;\n" " cvt.rn.f32.u32 %f357, %r278;\n" " sub.rn.f32 %f358, %f357, %f87;\n" " mul.rn.f32 %f359, %f358, %f356;\n" " ld.shared.f32 %f360, [%r157+2944];\n" " fma.rn.f32 %f361, %f359, %f360, %f354;\n" " fma.rn.f32 %f362, %f361, %f74, %f256;\n" " shr.u32 %r279, %r134, 16;\n" " and.b32 %r280, %r134, 15;\n" " or.b32 %r281, %r280, 1258291200;\n" " mov.b32 %f363, %r281;\n" " fma.rn.f32 %f364, %f363, %f84, %f83;\n" " and.b32 %r282, %r135, 255;\n" " cvt.rn.f32.u32 %f365, %r282;\n" " sub.rn.f32 %f366, %f365, %f87;\n" " mul.rn.f32 %f367, %f366, %f364;\n" " fma.rn.f32 %f368, %f367, %f311, %f91;\n" " and.b32 %r283, %r134, 240;\n" " or.b32 %r284, %r283, 1258291200;\n" " mov.b32 %f369, %r284;\n" " fma.rn.f32 %f370, %f369, %f95, %f94;\n" " bfe.u32 %r285, %r135, 8, 8;\n" " cvt.rn.f32.u32 %f371, %r285;\n" " sub.rn.f32 %f372, %f371, %f87;\n" " mul.rn.f32 %f373, %f372, %f370;\n" " fma.rn.f32 %f374, %f373, %f318, %f368;\n" " and.b32 %r286, %r134, 3840;\n" " or.b32 %r287, %r286, 1258291200;\n" " mov.b32 %f375, %r287;\n" " fma.rn.f32 %f376, %f375, %f104, %f103;\n" " bfe.u32 %r288, %r135, 16, 8;\n" " cvt.rn.f32.u32 %f377, %r288;\n" " sub.rn.f32 %f378, %f377, %f87;\n" " mul.rn.f32 %f379, %f378, %f376;\n" " fma.rn.f32 %f380, %f379, %f325, %f374;\n" " and.b32 %r289, %r134, 61440;\n" " or.b32 %r290, %r289, 1258291200;\n" " mov.b32 %f381, %r290;\n" " fma.rn.f32 %f382, %f381, %f113, %f112;\n" " shr.u32 %r291, %r135, 24;\n" " cvt.rn.f32.u32 %f383, %r291;\n" " sub.rn.f32 %f384, %f383, %f87;\n" " mul.rn.f32 %f385, %f384, %f382;\n" " fma.rn.f32 %f386, %f385, %f332, %f380;\n" " and.b32 %r292, %r134, 983040;\n" " or.b32 %r293, %r292, 1258291200;\n" " mov.b32 %f387, %r293;\n" " fma.rn.f32 %f388, %f387, %f122, %f121;\n" " and.b32 %r294, %r136, 255;\n" " cvt.rn.f32.u32 %f389, %r294;\n" " sub.rn.f32 %f390, %f389, %f87;\n" " mul.rn.f32 %f391, %f390, %f388;\n" " fma.rn.f32 %f392, %f391, %f339, %f386;\n" " and.b32 %r295, %r279, 240;\n" " or.b32 %r296, %r295, 1258291200;\n" " mov.b32 %f393, %r296;\n" " fma.rn.f32 %f394, %f393, %f95, %f94;\n" " bfe.u32 %r297, %r136, 8, 8;\n" " cvt.rn.f32.u32 %f395, %r297;\n" " sub.rn.f32 %f396, %f395, %f87;\n" " mul.rn.f32 %f397, %f396, %f394;\n" " fma.rn.f32 %f398, %f397, %f346, %f392;\n" " and.b32 %r298, %r279, 3840;\n" " or.b32 %r299, %r298, 1258291200;\n" " mov.b32 %f399, %r299;\n" " fma.rn.f32 %f400, %f399, %f104, %f103;\n" " bfe.u32 %r300, %r136, 16, 8;\n" " cvt.rn.f32.u32 %f401, %r300;\n" " sub.rn.f32 %f402, %f401, %f87;\n" " mul.rn.f32 %f403, %f402, %f400;\n" " fma.rn.f32 %f404, %f403, %f353, %f398;\n" " and.b32 %r301, %r279, 61440;\n" " or.b32 %r302, %r301, 1258291200;\n" " mov.b32 %f405, %r302;\n" " fma.rn.f32 %f406, %f405, %f113, %f112;\n" " shr.u32 %r303, %r136, 24;\n" " cvt.rn.f32.u32 %f407, %r303;\n" " sub.rn.f32 %f408, %f407, %f87;\n" " mul.rn.f32 %f409, %f408, %f406;\n" " fma.rn.f32 %f410, %f409, %f360, %f404;\n" " fma.rn.f32 %f411, %f410, %f76, %f305;\n" " shr.u32 %r304, %r140, 16;\n" " and.b32 %r305, %r140, 15;\n" " or.b32 %r306, %r305, 1258291200;\n" " mov.b32 %f412, %r306;\n" " fma.rn.f32 %f413, %f412, %f84, %f83;\n" " and.b32 %r307, %r141, 255;\n" " cvt.rn.f32.u32 %f414, %r307;\n" " sub.rn.f32 %f415, %f414, %f87;\n" " mul.rn.f32 %f416, %f415, %f413;\n" " ld.shared.f32 %f417, [%r157+3072];\n" " fma.rn.f32 %f418, %f416, %f417, %f91;\n" " and.b32 %r308, %r140, 240;\n" " or.b32 %r309, %r308, 1258291200;\n" " mov.b32 %f419, %r309;\n" " fma.rn.f32 %f420, %f419, %f95, %f94;\n" " bfe.u32 %r310, %r141, 8, 8;\n" " cvt.rn.f32.u32 %f421, %r310;\n" " sub.rn.f32 %f422, %f421, %f87;\n" " mul.rn.f32 %f423, %f422, %f420;\n" " ld.shared.f32 %f424, [%r157+3200];\n" " fma.rn.f32 %f425, %f423, %f424, %f418;\n" " and.b32 %r311, %r140, 3840;\n" " or.b32 %r312, %r311, 1258291200;\n" " mov.b32 %f426, %r312;\n" " fma.rn.f32 %f427, %f426, %f104, %f103;\n" " bfe.u32 %r313, %r141, 16, 8;\n" " cvt.rn.f32.u32 %f428, %r313;\n" " sub.rn.f32 %f429, %f428, %f87;\n" " mul.rn.f32 %f430, %f429, %f427;\n" " ld.shared.f32 %f431, [%r157+3328];\n" " fma.rn.f32 %f432, %f430, %f431, %f425;\n" " and.b32 %r314, %r140, 61440;\n" " or.b32 %r315, %r314, 1258291200;\n" " mov.b32 %f433, %r315;\n" " fma.rn.f32 %f434, %f433, %f113, %f112;\n" " shr.u32 %r316, %r141, 24;\n" " cvt.rn.f32.u32 %f435, %r316;\n" " sub.rn.f32 %f436, %f435, %f87;\n" " mul.rn.f32 %f437, %f436, %f434;\n" " ld.shared.f32 %f438, [%r157+3456];\n" " fma.rn.f32 %f439, %f437, %f438, %f432;\n" " and.b32 %r317, %r140, 983040;\n" " or.b32 %r318, %r317, 1258291200;\n" " mov.b32 %f440, %r318;\n" " fma.rn.f32 %f441, %f440, %f122, %f121;\n" " and.b32 %r319, %r142, 255;\n" " cvt.rn.f32.u32 %f442, %r319;\n" " sub.rn.f32 %f443, %f442, %f87;\n" " mul.rn.f32 %f444, %f443, %f441;\n" " ld.shared.f32 %f445, [%r157+3584];\n" " fma.rn.f32 %f446, %f444, %f445, %f439;\n" " and.b32 %r320, %r304, 240;\n" " or.b32 %r321, %r320, 1258291200;\n" " mov.b32 %f447, %r321;\n" " fma.rn.f32 %f448, %f447, %f95, %f94;\n" " bfe.u32 %r322, %r142, 8, 8;\n" " cvt.rn.f32.u32 %f449, %r322;\n" " sub.rn.f32 %f450, %f449, %f87;\n" " mul.rn.f32 %f451, %f450, %f448;\n" " ld.shared.f32 %f452, [%r157+3712];\n" " fma.rn.f32 %f453, %f451, %f452, %f446;\n" " and.b32 %r323, %r304, 3840;\n" " or.b32 %r324, %r323, 1258291200;\n" " mov.b32 %f454, %r324;\n" " fma.rn.f32 %f455, %f454, %f104, %f103;\n" " bfe.u32 %r325, %r142, 16, 8;\n" " cvt.rn.f32.u32 %f456, %r325;\n" " sub.rn.f32 %f457, %f456, %f87;\n" " mul.rn.f32 %f458, %f457, %f455;\n" " ld.shared.f32 %f459, [%r157+3840];\n" " fma.rn.f32 %f460, %f458, %f459, %f453;\n" " and.b32 %r326, %r304, 61440;\n" " or.b32 %r327, %r326, 1258291200;\n" " mov.b32 %f461, %r327;\n" " fma.rn.f32 %f462, %f461, %f113, %f112;\n" " shr.u32 %r328, %r142, 24;\n" " cvt.rn.f32.u32 %f463, %r328;\n" " sub.rn.f32 %f464, %f463, %f87;\n" " mul.rn.f32 %f465, %f464, %f462;\n" " ld.shared.f32 %f466, [%r157+3968];\n" " fma.rn.f32 %f467, %f465, %f466, %f460;\n" " fma.rn.f32 %f670, %f467, %f78, %f362;\n" " shr.u32 %r329, %r146, 16;\n" " and.b32 %r330, %r146, 15;\n" " or.b32 %r331, %r330, 1258291200;\n" " mov.b32 %f468, %r331;\n" " fma.rn.f32 %f469, %f468, %f84, %f83;\n" " and.b32 %r332, %r147, 255;\n" " cvt.rn.f32.u32 %f470, %r332;\n" " sub.rn.f32 %f471, %f470, %f87;\n" " mul.rn.f32 %f472, %f471, %f469;\n" " fma.rn.f32 %f473, %f472, %f417, %f91;\n" " and.b32 %r333, %r146, 240;\n" " or.b32 %r334, %r333, 1258291200;\n" " mov.b32 %f474, %r334;\n" " fma.rn.f32 %f475, %f474, %f95, %f94;\n" " bfe.u32 %r335, %r147, 8, 8;\n" " cvt.rn.f32.u32 %f476, %r335;\n" " sub.rn.f32 %f477, %f476, %f87;\n" " mul.rn.f32 %f478, %f477, %f475;\n" " fma.rn.f32 %f479, %f478, %f424, %f473;\n" " and.b32 %r336, %r146, 3840;\n" " or.b32 %r337, %r336, 1258291200;\n" " mov.b32 %f480, %r337;\n" " fma.rn.f32 %f481, %f480, %f104, %f103;\n" " bfe.u32 %r338, %r147, 16, 8;\n" " cvt.rn.f32.u32 %f482, %r338;\n" " sub.rn.f32 %f483, %f482, %f87;\n" " mul.rn.f32 %f484, %f483, %f481;\n" " fma.rn.f32 %f485, %f484, %f431, %f479;\n" " and.b32 %r339, %r146, 61440;\n" " or.b32 %r340, %r339, 1258291200;\n" " mov.b32 %f486, %r340;\n" " fma.rn.f32 %f487, %f486, %f113, %f112;\n" " shr.u32 %r341, %r147, 24;\n" " cvt.rn.f32.u32 %f488, %r341;\n" " sub.rn.f32 %f489, %f488, %f87;\n" " mul.rn.f32 %f490, %f489, %f487;\n" " fma.rn.f32 %f491, %f490, %f438, %f485;\n" " and.b32 %r342, %r146, 983040;\n" " or.b32 %r343, %r342, 1258291200;\n" " mov.b32 %f492, %r343;\n" " fma.rn.f32 %f493, %f492, %f122, %f121;\n" " and.b32 %r344, %r148, 255;\n" " cvt.rn.f32.u32 %f494, %r344;\n" " sub.rn.f32 %f495, %f494, %f87;\n" " mul.rn.f32 %f496, %f495, %f493;\n" " fma.rn.f32 %f497, %f496, %f445, %f491;\n" " and.b32 %r345, %r329, 240;\n" " or.b32 %r346, %r345, 1258291200;\n" " mov.b32 %f498, %r346;\n" " fma.rn.f32 %f499, %f498, %f95, %f94;\n" " bfe.u32 %r347, %r148, 8, 8;\n" " cvt.rn.f32.u32 %f500, %r347;\n" " sub.rn.f32 %f501, %f500, %f87;\n" " mul.rn.f32 %f502, %f501, %f499;\n" " fma.rn.f32 %f503, %f502, %f452, %f497;\n" " and.b32 %r348, %r329, 3840;\n" " or.b32 %r349, %r348, 1258291200;\n" " mov.b32 %f504, %r349;\n" " fma.rn.f32 %f505, %f504, %f104, %f103;\n" " bfe.u32 %r350, %r148, 16, 8;\n" " cvt.rn.f32.u32 %f506, %r350;\n" " sub.rn.f32 %f507, %f506, %f87;\n" " mul.rn.f32 %f508, %f507, %f505;\n" " fma.rn.f32 %f509, %f508, %f459, %f503;\n" " and.b32 %r351, %r329, 61440;\n" " or.b32 %r352, %r351, 1258291200;\n" " mov.b32 %f510, %r352;\n" " fma.rn.f32 %f511, %f510, %f113, %f112;\n" " shr.u32 %r353, %r148, 24;\n" " cvt.rn.f32.u32 %f512, %r353;\n" " sub.rn.f32 %f513, %f512, %f87;\n" " mul.rn.f32 %f514, %f513, %f511;\n" " fma.rn.f32 %f515, %f514, %f466, %f509;\n" " fma.rn.f32 %f671, %f515, %f80, %f411;\n" " add.s64 %rd108, %rd108, 608;\n" " add.s64 %rd109, %rd109, 608;\n" " add.s32 %r443, %r443, 4096;\n" " add.s32 %r441, %r445, 4;\n" " setp.le.s32 %p20, %r441, %r31;\n" " @%p20 bra $L__BB3_21;\n" "\n" "$L__BB3_22:\n" " setp.ge.s32 %p21, %r445, %r31;\n" " @%p21 bra $L__BB3_24;\n" "\n" "$L__BB3_23:\n" " shl.b64 %rd93, %rd2, 2;\n" " add.s64 %rd87, %rd108, %rd93;\n" " // begin inline asm\n" " ld.global.nc.u32 %r354, [%rd87];\n" " // end inline asm\n" " add.s64 %rd88, %rd108, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f516, [%rd88];\n" " // end inline asm\n" " shl.b64 %rd94, %rd3, 3;\n" " add.s64 %rd95, %rd108, %rd94;\n" " add.s64 %rd89, %rd95, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r355,%r356}, [%rd89];\n" " // end inline asm\n" " shr.u32 %r360, %r354, 16;\n" " and.b32 %r361, %r354, 15;\n" " or.b32 %r362, %r361, 1258291200;\n" " mov.b32 %f518, %r362;\n" " mov.f32 %f519, 0fCB000004;\n" " mov.f32 %f520, 0f3F800000;\n" " fma.rn.f32 %f521, %f518, %f520, %f519;\n" " and.b32 %r363, %r355, 255;\n" " cvt.rn.f32.u32 %f522, %r363;\n" " mov.f32 %f523, 0f42000000;\n" " sub.rn.f32 %f524, %f522, %f523;\n" " mul.rn.f32 %f525, %f524, %f521;\n" " shl.b32 %r364, %r2, 2;\n" " add.s32 %r365, %r443, %r364;\n" " ld.shared.f32 %f526, [%r365];\n" " mov.f32 %f527, 0f00000000;\n" " fma.rn.f32 %f528, %f525, %f526, %f527;\n" " and.b32 %r366, %r354, 240;\n" " or.b32 %r367, %r366, 1258291200;\n" " mov.b32 %f529, %r367;\n" " mov.f32 %f530, 0fC9000040;\n" " mov.f32 %f531, 0f3D800000;\n" " fma.rn.f32 %f532, %f529, %f531, %f530;\n" " bfe.u32 %r368, %r355, 8, 8;\n" " cvt.rn.f32.u32 %f533, %r368;\n" " sub.rn.f32 %f534, %f533, %f523;\n" " mul.rn.f32 %f535, %f534, %f532;\n" " ld.shared.f32 %f536, [%r365+128];\n" " fma.rn.f32 %f537, %f535, %f536, %f528;\n" " and.b32 %r369, %r354, 3840;\n" " or.b32 %r370, %r369, 1258291200;\n" " mov.b32 %f538, %r370;\n" " mov.f32 %f539, 0fC7000400;\n" " mov.f32 %f540, 0f3B800000;\n" " fma.rn.f32 %f541, %f538, %f540, %f539;\n" " bfe.u32 %r371, %r355, 16, 8;\n" " cvt.rn.f32.u32 %f542, %r371;\n" " sub.rn.f32 %f543, %f542, %f523;\n" " mul.rn.f32 %f544, %f543, %f541;\n" " ld.shared.f32 %f545, [%r365+256];\n" " fma.rn.f32 %f546, %f544, %f545, %f537;\n" " and.b32 %r372, %r354, 61440;\n" " or.b32 %r373, %r372, 1258291200;\n" " mov.b32 %f547, %r373;\n" " mov.f32 %f548, 0fC5004000;\n" " mov.f32 %f549, 0f39800000;\n" " fma.rn.f32 %f550, %f547, %f549, %f548;\n" " shr.u32 %r374, %r355, 24;\n" " cvt.rn.f32.u32 %f551, %r374;\n" " sub.rn.f32 %f552, %f551, %f523;\n" " mul.rn.f32 %f553, %f552, %f550;\n" " ld.shared.f32 %f554, [%r365+384];\n" " fma.rn.f32 %f555, %f553, %f554, %f546;\n" " and.b32 %r375, %r354, 983040;\n" " or.b32 %r376, %r375, 1258291200;\n" " mov.b32 %f556, %r376;\n" " mov.f32 %f557, 0fC3040000;\n" " mov.f32 %f558, 0f37800000;\n" " fma.rn.f32 %f559, %f556, %f558, %f557;\n" " and.b32 %r377, %r356, 255;\n" " cvt.rn.f32.u32 %f560, %r377;\n" " sub.rn.f32 %f561, %f560, %f523;\n" " mul.rn.f32 %f562, %f561, %f559;\n" " ld.shared.f32 %f563, [%r365+512];\n" " fma.rn.f32 %f564, %f562, %f563, %f555;\n" " and.b32 %r378, %r360, 240;\n" " or.b32 %r379, %r378, 1258291200;\n" " mov.b32 %f565, %r379;\n" " fma.rn.f32 %f566, %f565, %f531, %f530;\n" " bfe.u32 %r380, %r356, 8, 8;\n" " cvt.rn.f32.u32 %f567, %r380;\n" " sub.rn.f32 %f568, %f567, %f523;\n" " mul.rn.f32 %f569, %f568, %f566;\n" " ld.shared.f32 %f570, [%r365+640];\n" " fma.rn.f32 %f571, %f569, %f570, %f564;\n" " and.b32 %r381, %r360, 3840;\n" " or.b32 %r382, %r381, 1258291200;\n" " mov.b32 %f572, %r382;\n" " fma.rn.f32 %f573, %f572, %f540, %f539;\n" " bfe.u32 %r383, %r356, 16, 8;\n" " cvt.rn.f32.u32 %f574, %r383;\n" " sub.rn.f32 %f575, %f574, %f523;\n" " mul.rn.f32 %f576, %f575, %f573;\n" " ld.shared.f32 %f577, [%r365+768];\n" " fma.rn.f32 %f578, %f576, %f577, %f571;\n" " and.b32 %r384, %r360, 61440;\n" " or.b32 %r385, %r384, 1258291200;\n" " mov.b32 %f579, %r385;\n" " fma.rn.f32 %f580, %f579, %f549, %f548;\n" " shr.u32 %r386, %r356, 24;\n" " cvt.rn.f32.u32 %f581, %r386;\n" " sub.rn.f32 %f582, %f581, %f523;\n" " mul.rn.f32 %f583, %f582, %f580;\n" " ld.shared.f32 %f584, [%r365+896];\n" " fma.rn.f32 %f585, %f583, %f584, %f578;\n" " fma.rn.f32 %f670, %f516, %f585, %f670;\n" " add.s64 %rd108, %rd108, 152;\n" " add.s64 %rd90, %rd109, %rd93;\n" " // begin inline asm\n" " ld.global.nc.u32 %r357, [%rd90];\n" " // end inline asm\n" " add.s64 %rd91, %rd109, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f517, [%rd91];\n" " // end inline asm\n" " add.s64 %rd96, %rd109, %rd94;\n" " add.s64 %rd92, %rd96, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r358,%r359}, [%rd92];\n" " // end inline asm\n" " shr.u32 %r387, %r357, 16;\n" " and.b32 %r388, %r357, 15;\n" " or.b32 %r389, %r388, 1258291200;\n" " mov.b32 %f586, %r389;\n" " fma.rn.f32 %f587, %f586, %f520, %f519;\n" " and.b32 %r390, %r358, 255;\n" " cvt.rn.f32.u32 %f588, %r390;\n" " sub.rn.f32 %f589, %f588, %f523;\n" " mul.rn.f32 %f590, %f589, %f587;\n" " ld.shared.f32 %f591, [%r365];\n" " fma.rn.f32 %f592, %f590, %f591, %f527;\n" " and.b32 %r391, %r357, 240;\n" " or.b32 %r392, %r391, 1258291200;\n" " mov.b32 %f593, %r392;\n" " fma.rn.f32 %f594, %f593, %f531, %f530;\n" " bfe.u32 %r393, %r358, 8, 8;\n" " cvt.rn.f32.u32 %f595, %r393;\n" " sub.rn.f32 %f596, %f595, %f523;\n" " mul.rn.f32 %f597, %f596, %f594;\n" " ld.shared.f32 %f598, [%r365+128];\n" " fma.rn.f32 %f599, %f597, %f598, %f592;\n" " and.b32 %r394, %r357, 3840;\n" " or.b32 %r395, %r394, 1258291200;\n" " mov.b32 %f600, %r395;\n" " fma.rn.f32 %f601, %f600, %f540, %f539;\n" " bfe.u32 %r396, %r358, 16, 8;\n" " cvt.rn.f32.u32 %f602, %r396;\n" " sub.rn.f32 %f603, %f602, %f523;\n" " mul.rn.f32 %f604, %f603, %f601;\n" " ld.shared.f32 %f605, [%r365+256];\n" " fma.rn.f32 %f606, %f604, %f605, %f599;\n" " and.b32 %r397, %r357, 61440;\n" " or.b32 %r398, %r397, 1258291200;\n" " mov.b32 %f607, %r398;\n" " fma.rn.f32 %f608, %f607, %f549, %f548;\n" " shr.u32 %r399, %r358, 24;\n" " cvt.rn.f32.u32 %f609, %r399;\n" " sub.rn.f32 %f610, %f609, %f523;\n" " mul.rn.f32 %f611, %f610, %f608;\n" " ld.shared.f32 %f612, [%r365+384];\n" " fma.rn.f32 %f613, %f611, %f612, %f606;\n" " and.b32 %r400, %r357, 983040;\n" " or.b32 %r401, %r400, 1258291200;\n" " mov.b32 %f614, %r401;\n" " fma.rn.f32 %f615, %f614, %f558, %f557;\n" " and.b32 %r402, %r359, 255;\n" " cvt.rn.f32.u32 %f616, %r402;\n" " sub.rn.f32 %f617, %f616, %f523;\n" " mul.rn.f32 %f618, %f617, %f615;\n" " ld.shared.f32 %f619, [%r365+512];\n" " fma.rn.f32 %f620, %f618, %f619, %f613;\n" " and.b32 %r403, %r387, 240;\n" " or.b32 %r404, %r403, 1258291200;\n" " mov.b32 %f621, %r404;\n" " fma.rn.f32 %f622, %f621, %f531, %f530;\n" " bfe.u32 %r405, %r359, 8, 8;\n" " cvt.rn.f32.u32 %f623, %r405;\n" " sub.rn.f32 %f624, %f623, %f523;\n" " mul.rn.f32 %f625, %f624, %f622;\n" " ld.shared.f32 %f626, [%r365+640];\n" " fma.rn.f32 %f627, %f625, %f626, %f620;\n" " and.b32 %r406, %r387, 3840;\n" " or.b32 %r407, %r406, 1258291200;\n" " mov.b32 %f628, %r407;\n" " fma.rn.f32 %f629, %f628, %f540, %f539;\n" " bfe.u32 %r408, %r359, 16, 8;\n" " cvt.rn.f32.u32 %f630, %r408;\n" " sub.rn.f32 %f631, %f630, %f523;\n" " mul.rn.f32 %f632, %f631, %f629;\n" " ld.shared.f32 %f633, [%r365+768];\n" " fma.rn.f32 %f634, %f632, %f633, %f627;\n" " and.b32 %r409, %r387, 61440;\n" " or.b32 %r410, %r409, 1258291200;\n" " mov.b32 %f635, %r410;\n" " fma.rn.f32 %f636, %f635, %f549, %f548;\n" " shr.u32 %r411, %r359, 24;\n" " cvt.rn.f32.u32 %f637, %r411;\n" " sub.rn.f32 %f638, %f637, %f523;\n" " mul.rn.f32 %f639, %f638, %f636;\n" " ld.shared.f32 %f640, [%r365+896];\n" " fma.rn.f32 %f641, %f639, %f640, %f634;\n" " fma.rn.f32 %f671, %f517, %f641, %f671;\n" " add.s64 %rd109, %rd109, 152;\n" " add.s32 %r443, %r443, 1024;\n" " add.s32 %r445, %r445, 1;\n" " setp.lt.s32 %p22, %r445, %r31;\n" " @%p22 bra $L__BB3_23;\n" "\n" "$L__BB3_24:\n" " cvt.u32.u64 %r412, %rd9;\n" " add.s32 %r435, %r412, 2048;\n" " setp.lt.s32 %p23, %r435, %r45;\n" " add.s32 %r434, %r434, 2048;\n" " @%p23 bra $L__BB3_2;\n" "\n" "$L__BB3_25:\n" " cvt.u32.u64 %r423, %rd1;\n" " setp.ge.s32 %p24, %r423, %r44;\n" " mov.u32 %r413, 16;\n" " mov.u32 %r422, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f642, %f670, %r413, %r422;\n" " // end inline asm\n" " add.f32 %f645, %f670, %f642;\n" " mov.u32 %r415, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f644, %f645, %r415, %r422;\n" " // end inline asm\n" " add.f32 %f647, %f645, %f644;\n" " mov.u32 %r417, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f646, %f647, %r417, %r422;\n" " // end inline asm\n" " add.f32 %f649, %f647, %f646;\n" " mov.u32 %r419, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f648, %f649, %r419, %r422;\n" " // end inline asm\n" " add.f32 %f651, %f649, %f648;\n" " mov.u32 %r421, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f650, %f651, %r421, %r422;\n" " // end inline asm\n" " add.f32 %f17, %f651, %f650;\n" " setp.ne.s32 %p25, %r2, 0;\n" " cvta.to.global.u64 %rd97, %rd31;\n" " shl.b64 %rd98, %rd1, 2;\n" " add.s64 %rd28, %rd97, %rd98;\n" " or.pred %p26, %p25, %p24;\n" " @%p26 bra $L__BB3_27;\n" "\n" " st.global.f32 [%rd28], %f17;\n" "\n" "$L__BB3_27:\n" " // begin inline asm\n" " shfl.down.b32 %f652, %f671, %r413, %r422;\n" " // end inline asm\n" " add.f32 %f655, %f671, %f652;\n" " // begin inline asm\n" " shfl.down.b32 %f654, %f655, %r415, %r422;\n" " // end inline asm\n" " add.f32 %f657, %f655, %f654;\n" " // begin inline asm\n" " shfl.down.b32 %f656, %f657, %r417, %r422;\n" " // end inline asm\n" " add.f32 %f659, %f657, %f656;\n" " // begin inline asm\n" " shfl.down.b32 %f658, %f659, %r419, %r422;\n" " // end inline asm\n" " add.f32 %f661, %f659, %f658;\n" " // begin inline asm\n" " shfl.down.b32 %f660, %f661, %r421, %r422;\n" " // end inline asm\n" " add.f32 %f18, %f661, %f660;\n" " setp.ge.s32 %p28, %r4, %r44;\n" " or.pred %p29, %p25, %p28;\n" " @%p29 bra $L__BB3_29;\n" "\n" " st.global.f32 [%rd28+4], %f18;\n" "\n" "$L__BB3_29:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_ps_r2p8\n" ".visible .entry fc_matvec_q3k_ps_r2p8(\n" " .param .u64 fc_matvec_q3k_ps_r2p8_param_0,\n" " .param .u64 fc_matvec_q3k_ps_r2p8_param_1,\n" " .param .u64 fc_matvec_q3k_ps_r2p8_param_2,\n" " .param .u32 fc_matvec_q3k_ps_r2p8_param_3,\n" " .param .u32 fc_matvec_q3k_ps_r2p8_param_4,\n" " .param .u32 fc_matvec_q3k_ps_r2p8_param_5\n" ")\n" "{\n" " .reg .pred %p<30>;\n" " .reg .f32 %f<1122>;\n" " .reg .b32 %r<719>;\n" " .reg .b64 %rd<134>;\n" "\n" "\n" " ld.param.u64 %rd29, [fc_matvec_q3k_ps_r2p8_param_0];\n" " ld.param.u64 %rd30, [fc_matvec_q3k_ps_r2p8_param_1];\n" " ld.param.u64 %rd31, [fc_matvec_q3k_ps_r2p8_param_2];\n" " ld.param.u32 %r45, [fc_matvec_q3k_ps_r2p8_param_3];\n" " ld.param.u32 %r46, [fc_matvec_q3k_ps_r2p8_param_4];\n" " ld.param.u32 %r47, [fc_matvec_q3k_ps_r2p8_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r48, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r49, %ctaid.x;\n" " shl.b32 %r50, %r49, 3;\n" " add.s32 %r51, %r50, %r48;\n" " shl.b32 %r3, %r51, 1;\n" " cvt.s64.s32 %rd1, %r3;\n" " or.b32 %r4, %r3, 1;\n" " setp.lt.s32 %p1, %r46, 1;\n" " mov.f32 %f1118, 0f00000000;\n" " mov.f32 %f1119, %f1118;\n" " @%p1 bra $L__BB4_25;\n" "\n" " cvt.s64.s32 %rd32, %r47;\n" " cvt.u64.u32 %rd2, %r2;\n" " shr.u32 %r54, %r2, 4;\n" " cvt.u64.u32 %rd3, %r54;\n" " mov.u32 %r55, -2;\n" " sub.s32 %r5, %r55, %r1;\n" " not.b32 %r6, %r46;\n" " not.b32 %r7, %r1;\n" " cvt.s64.s32 %rd4, %r1;\n" " shl.b32 %r56, %r1, 4;\n" " mov.u32 %r57, sh_xps;\n" " add.s32 %r8, %r57, %r56;\n" " add.s32 %r9, %r1, 256;\n" " shl.b32 %r58, %r1, 2;\n" " add.s32 %r10, %r57, %r58;\n" " add.s32 %r11, %r1, 512;\n" " add.s32 %r12, %r1, 768;\n" " setp.lt.s32 %p2, %r4, %r45;\n" " selp.u32 %r59, 1, 0, %p2;\n" " add.s32 %r60, %r3, %r59;\n" " mul.wide.s32 %rd33, %r60, %r47;\n" " add.s64 %rd133, %rd29, %rd33;\n" " mul.lo.s64 %rd34, %rd1, %rd32;\n" " add.s64 %rd132, %rd29, %rd34;\n" " mov.u32 %r706, 0;\n" " mov.f32 %f1118, 0f00000000;\n" " mov.f32 %f1119, %f1118;\n" " mov.u32 %r707, %r706;\n" "\n" "$L__BB4_2:\n" " add.s32 %r61, %r706, %r6;\n" " max.s32 %r62, %r61, -2049;\n" " sub.s32 %r15, %r5, %r62;\n" " sub.s32 %r16, %r46, %r707;\n" " min.s32 %r17, %r16, 2048;\n" " bar.sync 0;\n" " cvt.s64.s32 %rd9, %r707;\n" " mul.wide.s32 %rd35, %r707, 4;\n" " add.s64 %rd10, %rd30, %rd35;\n" " and.b32 %r63, %r17, 3;\n" " setp.eq.s32 %p3, %r63, 0;\n" " { .reg .b64 %tmp;\n" " cvt.u64.u32 %tmp, %r57;\n" " cvta.shared.u64 %rd36, %tmp; }\n" " or.b64 %rd37, %rd10, %rd36;\n" " and.b64 %rd38, %rd37, 15;\n" " setp.eq.s64 %p4, %rd38, 0;\n" " and.pred %p5, %p3, %p4;\n" " @%p5 bra $L__BB4_11;\n" " bra.uni $L__BB4_3;\n" "\n" "$L__BB4_11:\n" " shr.s32 %r25, %r17, 2;\n" " setp.ge.s32 %p12, %r1, %r25;\n" " @%p12 bra $L__BB4_18;\n" "\n" " add.s32 %r26, %r25, %r7;\n" " shr.u32 %r70, %r26, 8;\n" " add.s32 %r71, %r70, 1;\n" " and.b32 %r27, %r71, 3;\n" " setp.eq.s32 %p13, %r27, 0;\n" " mov.u32 %r711, %r1;\n" " @%p13 bra $L__BB4_16;\n" "\n" " shl.b64 %rd51, %rd4, 4;\n" " add.s64 %rd50, %rd10, %rd51;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f30,%f31,%f32,%f33}, [%rd50];\n" " // end inline asm\n" " st.shared.v4.f32 [%r8], {%f30, %f31, %f32, %f33};\n" " setp.eq.s32 %p14, %r27, 1;\n" " mov.u32 %r711, %r9;\n" " @%p14 bra $L__BB4_16;\n" "\n" " add.s64 %rd52, %rd50, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f34,%f35,%f36,%f37}, [%rd52];\n" " // end inline asm\n" " st.shared.v4.f32 [%r8+4096], {%f34, %f35, %f36, %f37};\n" " setp.eq.s32 %p15, %r27, 2;\n" " mov.u32 %r711, %r11;\n" " @%p15 bra $L__BB4_16;\n" "\n" " add.s64 %rd53, %rd50, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f38,%f39,%f40,%f41}, [%rd53];\n" " // end inline asm\n" " st.shared.v4.f32 [%r8+8192], {%f38, %f39, %f40, %f41};\n" " mov.u32 %r711, %r12;\n" "\n" "$L__BB4_16:\n" " setp.lt.u32 %p16, %r26, 768;\n" " @%p16 bra $L__BB4_18;\n" "\n" "$L__BB4_17:\n" " mul.wide.s32 %rd58, %r711, 16;\n" " add.s64 %rd54, %rd10, %rd58;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f42,%f43,%f44,%f45}, [%rd54];\n" " // end inline asm\n" " shl.b32 %r72, %r711, 4;\n" " mov.u32 %r73, sh_xps;\n" " add.s32 %r74, %r73, %r72;\n" " st.shared.v4.f32 [%r74], {%f42, %f43, %f44, %f45};\n" " add.s64 %rd55, %rd54, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f46,%f47,%f48,%f49}, [%rd55];\n" " // end inline asm\n" " st.shared.v4.f32 [%r74+4096], {%f46, %f47, %f48, %f49};\n" " add.s64 %rd56, %rd54, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f50,%f51,%f52,%f53}, [%rd56];\n" " // end inline asm\n" " st.shared.v4.f32 [%r74+8192], {%f50, %f51, %f52, %f53};\n" " add.s64 %rd57, %rd54, 12288;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f54,%f55,%f56,%f57}, [%rd57];\n" " // end inline asm\n" " st.shared.v4.f32 [%r74+12288], {%f54, %f55, %f56, %f57};\n" " add.s32 %r711, %r711, 1024;\n" " setp.lt.s32 %p17, %r711, %r25;\n" " @%p17 bra $L__BB4_17;\n" " bra.uni $L__BB4_18;\n" "\n" "$L__BB4_3:\n" " setp.ge.s32 %p6, %r1, %r17;\n" " @%p6 bra $L__BB4_18;\n" "\n" " shr.u32 %r65, %r15, 8;\n" " add.s32 %r66, %r65, 1;\n" " and.b32 %r18, %r66, 3;\n" " setp.eq.s32 %p7, %r18, 0;\n" " mov.u32 %r708, %r1;\n" " @%p7 bra $L__BB4_8;\n" "\n" " shl.b64 %rd40, %rd4, 2;\n" " add.s64 %rd39, %rd10, %rd40;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r10], %f23;\n" " setp.eq.s32 %p8, %r18, 1;\n" " mov.u32 %r708, %r9;\n" " @%p8 bra $L__BB4_8;\n" "\n" " add.s64 %rd41, %rd39, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r10+1024], %f24;\n" " setp.eq.s32 %p9, %r18, 2;\n" " mov.u32 %r708, %r11;\n" " @%p9 bra $L__BB4_8;\n" "\n" " add.s64 %rd42, %rd39, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r10+2048], %f25;\n" " mov.u32 %r708, %r12;\n" "\n" "$L__BB4_8:\n" " setp.lt.u32 %p10, %r15, 768;\n" " @%p10 bra $L__BB4_18;\n" "\n" " cvt.s64.s32 %rd43, %r708;\n" " add.s64 %rd44, %rd43, %rd9;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd125, %rd30, %rd45;\n" " shl.b32 %r67, %r708, 2;\n" " add.s32 %r69, %r57, %r67;\n" " add.s32 %r709, %r69, 2048;\n" "\n" "$L__BB4_10:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd125];\n" " // end inline asm\n" " st.shared.f32 [%r709+-2048], %f26;\n" " add.s64 %rd47, %rd125, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd47];\n" " // end inline asm\n" " st.shared.f32 [%r709+-1024], %f27;\n" " add.s64 %rd48, %rd125, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r709], %f28;\n" " add.s64 %rd49, %rd125, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r709+1024], %f29;\n" " add.s64 %rd125, %rd125, 4096;\n" " add.s32 %r709, %r709, 4096;\n" " add.s32 %r708, %r708, 1024;\n" " setp.lt.s32 %p11, %r708, %r17;\n" " @%p11 bra $L__BB4_10;\n" "\n" "$L__BB4_18:\n" " cvt.u32.u64 %r75, %rd1;\n" " setp.ge.s32 %p18, %r75, %r45;\n" " bar.sync 0;\n" " @%p18 bra $L__BB4_24;\n" "\n" " setp.lt.s32 %p19, %r16, 2048;\n" " shr.s32 %r32, %r17, 8;\n" " mov.u32 %r717, 0;\n" " mov.u32 %r715, %r57;\n" " @%p19 bra $L__BB4_22;\n" "\n" " mov.u32 %r715, sh_xps;\n" " mov.u32 %r713, 8;\n" "\n" "$L__BB4_21:\n" " mov.u32 %r717, %r713;\n" " shl.b64 %rd107, %rd2, 2;\n" " add.s64 %rd59, %rd132, %rd107;\n" " // begin inline asm\n" " ld.global.nc.u32 %r128, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd132, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f74, [%rd60];\n" " // end inline asm\n" " shl.b64 %rd108, %rd3, 3;\n" " add.s64 %rd109, %rd132, %rd108;\n" " add.s64 %rd61, %rd109, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r129,%r130}, [%rd61];\n" " // end inline asm\n" " add.s64 %rd62, %rd133, %rd107;\n" " // begin inline asm\n" " ld.global.nc.u32 %r134, [%rd62];\n" " // end inline asm\n" " add.s64 %rd63, %rd133, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f76, [%rd63];\n" " // end inline asm\n" " add.s64 %rd110, %rd133, %rd108;\n" " add.s64 %rd64, %rd110, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r135,%r136}, [%rd64];\n" " // end inline asm\n" " add.s64 %rd65, %rd59, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r140, [%rd65];\n" " // end inline asm\n" " add.s64 %rd66, %rd132, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f78, [%rd66];\n" " // end inline asm\n" " add.s64 %rd67, %rd109, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r141,%r142}, [%rd67];\n" " // end inline asm\n" " add.s64 %rd68, %rd62, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r146, [%rd68];\n" " // end inline asm\n" " add.s64 %rd69, %rd133, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f80, [%rd69];\n" " // end inline asm\n" " add.s64 %rd70, %rd110, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r147,%r148}, [%rd70];\n" " // end inline asm\n" " add.s64 %rd71, %rd59, 304;\n" " // begin inline asm\n" " ld.global.nc.u32 %r152, [%rd71];\n" " // end inline asm\n" " add.s64 %rd72, %rd132, 448;\n" " // begin inline asm\n" " ld.global.nc.f32 %f82, [%rd72];\n" " // end inline asm\n" " add.s64 %rd73, %rd109, 432;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r153,%r154}, [%rd73];\n" " // end inline asm\n" " add.s64 %rd74, %rd62, 304;\n" " // begin inline asm\n" " ld.global.nc.u32 %r158, [%rd74];\n" " // end inline asm\n" " add.s64 %rd75, %rd133, 448;\n" " // begin inline asm\n" " ld.global.nc.f32 %f84, [%rd75];\n" " // end inline asm\n" " add.s64 %rd76, %rd110, 432;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r159,%r160}, [%rd76];\n" " // end inline asm\n" " add.s64 %rd77, %rd59, 456;\n" " // begin inline asm\n" " ld.global.nc.u32 %r164, [%rd77];\n" " // end inline asm\n" " add.s64 %rd78, %rd132, 600;\n" " // begin inline asm\n" " ld.global.nc.f32 %f86, [%rd78];\n" " // end inline asm\n" " add.s64 %rd79, %rd109, 584;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r165,%r166}, [%rd79];\n" " // end inline asm\n" " add.s64 %rd80, %rd62, 456;\n" " // begin inline asm\n" " ld.global.nc.u32 %r170, [%rd80];\n" " // end inline asm\n" " add.s64 %rd81, %rd133, 600;\n" " // begin inline asm\n" " ld.global.nc.f32 %f88, [%rd81];\n" " // end inline asm\n" " add.s64 %rd82, %rd110, 584;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r171,%r172}, [%rd82];\n" " // end inline asm\n" " add.s64 %rd83, %rd59, 608;\n" " // begin inline asm\n" " ld.global.nc.u32 %r176, [%rd83];\n" " // end inline asm\n" " add.s64 %rd84, %rd132, 752;\n" " // begin inline asm\n" " ld.global.nc.f32 %f90, [%rd84];\n" " // end inline asm\n" " add.s64 %rd85, %rd109, 736;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r177,%r178}, [%rd85];\n" " // end inline asm\n" " add.s64 %rd86, %rd62, 608;\n" " // begin inline asm\n" " ld.global.nc.u32 %r182, [%rd86];\n" " // end inline asm\n" " add.s64 %rd87, %rd133, 752;\n" " // begin inline asm\n" " ld.global.nc.f32 %f92, [%rd87];\n" " // end inline asm\n" " add.s64 %rd88, %rd110, 736;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r183,%r184}, [%rd88];\n" " // end inline asm\n" " add.s64 %rd89, %rd59, 760;\n" " // begin inline asm\n" " ld.global.nc.u32 %r188, [%rd89];\n" " // end inline asm\n" " add.s64 %rd90, %rd132, 904;\n" " // begin inline asm\n" " ld.global.nc.f32 %f94, [%rd90];\n" " // end inline asm\n" " add.s64 %rd91, %rd109, 888;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r189,%r190}, [%rd91];\n" " // end inline asm\n" " add.s64 %rd92, %rd62, 760;\n" " // begin inline asm\n" " ld.global.nc.u32 %r194, [%rd92];\n" " // end inline asm\n" " add.s64 %rd93, %rd133, 904;\n" " // begin inline asm\n" " ld.global.nc.f32 %f96, [%rd93];\n" " // end inline asm\n" " add.s64 %rd94, %rd110, 888;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r195,%r196}, [%rd94];\n" " // end inline asm\n" " add.s64 %rd95, %rd59, 912;\n" " // begin inline asm\n" " ld.global.nc.u32 %r200, [%rd95];\n" " // end inline asm\n" " add.s64 %rd96, %rd132, 1056;\n" " // begin inline asm\n" " ld.global.nc.f32 %f98, [%rd96];\n" " // end inline asm\n" " add.s64 %rd97, %rd109, 1040;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r201,%r202}, [%rd97];\n" " // end inline asm\n" " add.s64 %rd98, %rd62, 912;\n" " // begin inline asm\n" " ld.global.nc.u32 %r206, [%rd98];\n" " // end inline asm\n" " add.s64 %rd99, %rd133, 1056;\n" " // begin inline asm\n" " ld.global.nc.f32 %f100, [%rd99];\n" " // end inline asm\n" " add.s64 %rd100, %rd110, 1040;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r207,%r208}, [%rd100];\n" " // end inline asm\n" " add.s64 %rd101, %rd59, 1064;\n" " // begin inline asm\n" " ld.global.nc.u32 %r212, [%rd101];\n" " // end inline asm\n" " add.s64 %rd102, %rd132, 1208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f102, [%rd102];\n" " // end inline asm\n" " add.s64 %rd103, %rd109, 1192;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r213,%r214}, [%rd103];\n" " // end inline asm\n" " add.s64 %rd104, %rd62, 1064;\n" " // begin inline asm\n" " ld.global.nc.u32 %r218, [%rd104];\n" " // end inline asm\n" " add.s64 %rd105, %rd133, 1208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f104, [%rd105];\n" " // end inline asm\n" " add.s64 %rd106, %rd110, 1192;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r219,%r220}, [%rd106];\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " shr.u32 %r224, %r128, 16;\n" " and.b32 %r225, %r128, 15;\n" " or.b32 %r226, %r225, 1258291200;\n" " mov.b32 %f106, %r226;\n" " mov.f32 %f107, 0fCB000004;\n" " mov.f32 %f108, 0f3F800000;\n" " fma.rn.f32 %f109, %f106, %f108, %f107;\n" " and.b32 %r227, %r129, 255;\n" " cvt.rn.f32.u32 %f110, %r227;\n" " mov.f32 %f111, 0f42000000;\n" " sub.rn.f32 %f112, %f110, %f111;\n" " mul.rn.f32 %f113, %f112, %f109;\n" " shl.b32 %r228, %r2, 2;\n" " add.s32 %r229, %r715, %r228;\n" " ld.shared.f32 %f114, [%r229];\n" " mov.f32 %f115, 0f00000000;\n" " fma.rn.f32 %f116, %f113, %f114, %f115;\n" " and.b32 %r230, %r128, 240;\n" " or.b32 %r231, %r230, 1258291200;\n" " mov.b32 %f117, %r231;\n" " mov.f32 %f118, 0fC9000040;\n" " mov.f32 %f119, 0f3D800000;\n" " fma.rn.f32 %f120, %f117, %f119, %f118;\n" " bfe.u32 %r232, %r129, 8, 8;\n" " cvt.rn.f32.u32 %f121, %r232;\n" " sub.rn.f32 %f122, %f121, %f111;\n" " mul.rn.f32 %f123, %f122, %f120;\n" " ld.shared.f32 %f124, [%r229+128];\n" " fma.rn.f32 %f125, %f123, %f124, %f116;\n" " and.b32 %r233, %r128, 3840;\n" " or.b32 %r234, %r233, 1258291200;\n" " mov.b32 %f126, %r234;\n" " mov.f32 %f127, 0fC7000400;\n" " mov.f32 %f128, 0f3B800000;\n" " fma.rn.f32 %f129, %f126, %f128, %f127;\n" " bfe.u32 %r235, %r129, 16, 8;\n" " cvt.rn.f32.u32 %f130, %r235;\n" " sub.rn.f32 %f131, %f130, %f111;\n" " mul.rn.f32 %f132, %f131, %f129;\n" " ld.shared.f32 %f133, [%r229+256];\n" " fma.rn.f32 %f134, %f132, %f133, %f125;\n" " and.b32 %r236, %r128, 61440;\n" " or.b32 %r237, %r236, 1258291200;\n" " mov.b32 %f135, %r237;\n" " mov.f32 %f136, 0fC5004000;\n" " mov.f32 %f137, 0f39800000;\n" " fma.rn.f32 %f138, %f135, %f137, %f136;\n" " shr.u32 %r238, %r129, 24;\n" " cvt.rn.f32.u32 %f139, %r238;\n" " sub.rn.f32 %f140, %f139, %f111;\n" " mul.rn.f32 %f141, %f140, %f138;\n" " ld.shared.f32 %f142, [%r229+384];\n" " fma.rn.f32 %f143, %f141, %f142, %f134;\n" " and.b32 %r239, %r128, 983040;\n" " or.b32 %r240, %r239, 1258291200;\n" " mov.b32 %f144, %r240;\n" " mov.f32 %f145, 0fC3040000;\n" " mov.f32 %f146, 0f37800000;\n" " fma.rn.f32 %f147, %f144, %f146, %f145;\n" " and.b32 %r241, %r130, 255;\n" " cvt.rn.f32.u32 %f148, %r241;\n" " sub.rn.f32 %f149, %f148, %f111;\n" " mul.rn.f32 %f150, %f149, %f147;\n" " ld.shared.f32 %f151, [%r229+512];\n" " fma.rn.f32 %f152, %f150, %f151, %f143;\n" " and.b32 %r242, %r224, 240;\n" " or.b32 %r243, %r242, 1258291200;\n" " mov.b32 %f153, %r243;\n" " fma.rn.f32 %f154, %f153, %f119, %f118;\n" " bfe.u32 %r244, %r130, 8, 8;\n" " cvt.rn.f32.u32 %f155, %r244;\n" " sub.rn.f32 %f156, %f155, %f111;\n" " mul.rn.f32 %f157, %f156, %f154;\n" " ld.shared.f32 %f158, [%r229+640];\n" " fma.rn.f32 %f159, %f157, %f158, %f152;\n" " and.b32 %r245, %r224, 3840;\n" " or.b32 %r246, %r245, 1258291200;\n" " mov.b32 %f160, %r246;\n" " fma.rn.f32 %f161, %f160, %f128, %f127;\n" " bfe.u32 %r247, %r130, 16, 8;\n" " cvt.rn.f32.u32 %f162, %r247;\n" " sub.rn.f32 %f163, %f162, %f111;\n" " mul.rn.f32 %f164, %f163, %f161;\n" " ld.shared.f32 %f165, [%r229+768];\n" " fma.rn.f32 %f166, %f164, %f165, %f159;\n" " and.b32 %r248, %r224, 61440;\n" " or.b32 %r249, %r248, 1258291200;\n" " mov.b32 %f167, %r249;\n" " fma.rn.f32 %f168, %f167, %f137, %f136;\n" " shr.u32 %r250, %r130, 24;\n" " cvt.rn.f32.u32 %f169, %r250;\n" " sub.rn.f32 %f170, %f169, %f111;\n" " mul.rn.f32 %f171, %f170, %f168;\n" " ld.shared.f32 %f172, [%r229+896];\n" " fma.rn.f32 %f173, %f171, %f172, %f166;\n" " fma.rn.f32 %f174, %f173, %f74, %f1118;\n" " shr.u32 %r251, %r134, 16;\n" " and.b32 %r252, %r134, 15;\n" " or.b32 %r253, %r252, 1258291200;\n" " mov.b32 %f175, %r253;\n" " fma.rn.f32 %f176, %f175, %f108, %f107;\n" " and.b32 %r254, %r135, 255;\n" " cvt.rn.f32.u32 %f177, %r254;\n" " sub.rn.f32 %f178, %f177, %f111;\n" " mul.rn.f32 %f179, %f178, %f176;\n" " fma.rn.f32 %f180, %f179, %f114, %f115;\n" " and.b32 %r255, %r134, 240;\n" " or.b32 %r256, %r255, 1258291200;\n" " mov.b32 %f181, %r256;\n" " fma.rn.f32 %f182, %f181, %f119, %f118;\n" " bfe.u32 %r257, %r135, 8, 8;\n" " cvt.rn.f32.u32 %f183, %r257;\n" " sub.rn.f32 %f184, %f183, %f111;\n" " mul.rn.f32 %f185, %f184, %f182;\n" " fma.rn.f32 %f186, %f185, %f124, %f180;\n" " and.b32 %r258, %r134, 3840;\n" " or.b32 %r259, %r258, 1258291200;\n" " mov.b32 %f187, %r259;\n" " fma.rn.f32 %f188, %f187, %f128, %f127;\n" " bfe.u32 %r260, %r135, 16, 8;\n" " cvt.rn.f32.u32 %f189, %r260;\n" " sub.rn.f32 %f190, %f189, %f111;\n" " mul.rn.f32 %f191, %f190, %f188;\n" " fma.rn.f32 %f192, %f191, %f133, %f186;\n" " and.b32 %r261, %r134, 61440;\n" " or.b32 %r262, %r261, 1258291200;\n" " mov.b32 %f193, %r262;\n" " fma.rn.f32 %f194, %f193, %f137, %f136;\n" " shr.u32 %r263, %r135, 24;\n" " cvt.rn.f32.u32 %f195, %r263;\n" " sub.rn.f32 %f196, %f195, %f111;\n" " mul.rn.f32 %f197, %f196, %f194;\n" " fma.rn.f32 %f198, %f197, %f142, %f192;\n" " and.b32 %r264, %r134, 983040;\n" " or.b32 %r265, %r264, 1258291200;\n" " mov.b32 %f199, %r265;\n" " fma.rn.f32 %f200, %f199, %f146, %f145;\n" " and.b32 %r266, %r136, 255;\n" " cvt.rn.f32.u32 %f201, %r266;\n" " sub.rn.f32 %f202, %f201, %f111;\n" " mul.rn.f32 %f203, %f202, %f200;\n" " fma.rn.f32 %f204, %f203, %f151, %f198;\n" " and.b32 %r267, %r251, 240;\n" " or.b32 %r268, %r267, 1258291200;\n" " mov.b32 %f205, %r268;\n" " fma.rn.f32 %f206, %f205, %f119, %f118;\n" " bfe.u32 %r269, %r136, 8, 8;\n" " cvt.rn.f32.u32 %f207, %r269;\n" " sub.rn.f32 %f208, %f207, %f111;\n" " mul.rn.f32 %f209, %f208, %f206;\n" " fma.rn.f32 %f210, %f209, %f158, %f204;\n" " and.b32 %r270, %r251, 3840;\n" " or.b32 %r271, %r270, 1258291200;\n" " mov.b32 %f211, %r271;\n" " fma.rn.f32 %f212, %f211, %f128, %f127;\n" " bfe.u32 %r272, %r136, 16, 8;\n" " cvt.rn.f32.u32 %f213, %r272;\n" " sub.rn.f32 %f214, %f213, %f111;\n" " mul.rn.f32 %f215, %f214, %f212;\n" " fma.rn.f32 %f216, %f215, %f165, %f210;\n" " and.b32 %r273, %r251, 61440;\n" " or.b32 %r274, %r273, 1258291200;\n" " mov.b32 %f217, %r274;\n" " fma.rn.f32 %f218, %f217, %f137, %f136;\n" " shr.u32 %r275, %r136, 24;\n" " cvt.rn.f32.u32 %f219, %r275;\n" " sub.rn.f32 %f220, %f219, %f111;\n" " mul.rn.f32 %f221, %f220, %f218;\n" " fma.rn.f32 %f222, %f221, %f172, %f216;\n" " fma.rn.f32 %f223, %f222, %f76, %f1119;\n" " shr.u32 %r276, %r140, 16;\n" " and.b32 %r277, %r140, 15;\n" " or.b32 %r278, %r277, 1258291200;\n" " mov.b32 %f224, %r278;\n" " fma.rn.f32 %f225, %f224, %f108, %f107;\n" " and.b32 %r279, %r141, 255;\n" " cvt.rn.f32.u32 %f226, %r279;\n" " sub.rn.f32 %f227, %f226, %f111;\n" " mul.rn.f32 %f228, %f227, %f225;\n" " ld.shared.f32 %f229, [%r229+1024];\n" " fma.rn.f32 %f230, %f228, %f229, %f115;\n" " and.b32 %r280, %r140, 240;\n" " or.b32 %r281, %r280, 1258291200;\n" " mov.b32 %f231, %r281;\n" " fma.rn.f32 %f232, %f231, %f119, %f118;\n" " bfe.u32 %r282, %r141, 8, 8;\n" " cvt.rn.f32.u32 %f233, %r282;\n" " sub.rn.f32 %f234, %f233, %f111;\n" " mul.rn.f32 %f235, %f234, %f232;\n" " ld.shared.f32 %f236, [%r229+1152];\n" " fma.rn.f32 %f237, %f235, %f236, %f230;\n" " and.b32 %r283, %r140, 3840;\n" " or.b32 %r284, %r283, 1258291200;\n" " mov.b32 %f238, %r284;\n" " fma.rn.f32 %f239, %f238, %f128, %f127;\n" " bfe.u32 %r285, %r141, 16, 8;\n" " cvt.rn.f32.u32 %f240, %r285;\n" " sub.rn.f32 %f241, %f240, %f111;\n" " mul.rn.f32 %f242, %f241, %f239;\n" " ld.shared.f32 %f243, [%r229+1280];\n" " fma.rn.f32 %f244, %f242, %f243, %f237;\n" " and.b32 %r286, %r140, 61440;\n" " or.b32 %r287, %r286, 1258291200;\n" " mov.b32 %f245, %r287;\n" " fma.rn.f32 %f246, %f245, %f137, %f136;\n" " shr.u32 %r288, %r141, 24;\n" " cvt.rn.f32.u32 %f247, %r288;\n" " sub.rn.f32 %f248, %f247, %f111;\n" " mul.rn.f32 %f249, %f248, %f246;\n" " ld.shared.f32 %f250, [%r229+1408];\n" " fma.rn.f32 %f251, %f249, %f250, %f244;\n" " and.b32 %r289, %r140, 983040;\n" " or.b32 %r290, %r289, 1258291200;\n" " mov.b32 %f252, %r290;\n" " fma.rn.f32 %f253, %f252, %f146, %f145;\n" " and.b32 %r291, %r142, 255;\n" " cvt.rn.f32.u32 %f254, %r291;\n" " sub.rn.f32 %f255, %f254, %f111;\n" " mul.rn.f32 %f256, %f255, %f253;\n" " ld.shared.f32 %f257, [%r229+1536];\n" " fma.rn.f32 %f258, %f256, %f257, %f251;\n" " and.b32 %r292, %r276, 240;\n" " or.b32 %r293, %r292, 1258291200;\n" " mov.b32 %f259, %r293;\n" " fma.rn.f32 %f260, %f259, %f119, %f118;\n" " bfe.u32 %r294, %r142, 8, 8;\n" " cvt.rn.f32.u32 %f261, %r294;\n" " sub.rn.f32 %f262, %f261, %f111;\n" " mul.rn.f32 %f263, %f262, %f260;\n" " ld.shared.f32 %f264, [%r229+1664];\n" " fma.rn.f32 %f265, %f263, %f264, %f258;\n" " and.b32 %r295, %r276, 3840;\n" " or.b32 %r296, %r295, 1258291200;\n" " mov.b32 %f266, %r296;\n" " fma.rn.f32 %f267, %f266, %f128, %f127;\n" " bfe.u32 %r297, %r142, 16, 8;\n" " cvt.rn.f32.u32 %f268, %r297;\n" " sub.rn.f32 %f269, %f268, %f111;\n" " mul.rn.f32 %f270, %f269, %f267;\n" " ld.shared.f32 %f271, [%r229+1792];\n" " fma.rn.f32 %f272, %f270, %f271, %f265;\n" " and.b32 %r298, %r276, 61440;\n" " or.b32 %r299, %r298, 1258291200;\n" " mov.b32 %f273, %r299;\n" " fma.rn.f32 %f274, %f273, %f137, %f136;\n" " shr.u32 %r300, %r142, 24;\n" " cvt.rn.f32.u32 %f275, %r300;\n" " sub.rn.f32 %f276, %f275, %f111;\n" " mul.rn.f32 %f277, %f276, %f274;\n" " ld.shared.f32 %f278, [%r229+1920];\n" " fma.rn.f32 %f279, %f277, %f278, %f272;\n" " fma.rn.f32 %f280, %f279, %f78, %f174;\n" " shr.u32 %r301, %r146, 16;\n" " and.b32 %r302, %r146, 15;\n" " or.b32 %r303, %r302, 1258291200;\n" " mov.b32 %f281, %r303;\n" " fma.rn.f32 %f282, %f281, %f108, %f107;\n" " and.b32 %r304, %r147, 255;\n" " cvt.rn.f32.u32 %f283, %r304;\n" " sub.rn.f32 %f284, %f283, %f111;\n" " mul.rn.f32 %f285, %f284, %f282;\n" " fma.rn.f32 %f286, %f285, %f229, %f115;\n" " and.b32 %r305, %r146, 240;\n" " or.b32 %r306, %r305, 1258291200;\n" " mov.b32 %f287, %r306;\n" " fma.rn.f32 %f288, %f287, %f119, %f118;\n" " bfe.u32 %r307, %r147, 8, 8;\n" " cvt.rn.f32.u32 %f289, %r307;\n" " sub.rn.f32 %f290, %f289, %f111;\n" " mul.rn.f32 %f291, %f290, %f288;\n" " fma.rn.f32 %f292, %f291, %f236, %f286;\n" " and.b32 %r308, %r146, 3840;\n" " or.b32 %r309, %r308, 1258291200;\n" " mov.b32 %f293, %r309;\n" " fma.rn.f32 %f294, %f293, %f128, %f127;\n" " bfe.u32 %r310, %r147, 16, 8;\n" " cvt.rn.f32.u32 %f295, %r310;\n" " sub.rn.f32 %f296, %f295, %f111;\n" " mul.rn.f32 %f297, %f296, %f294;\n" " fma.rn.f32 %f298, %f297, %f243, %f292;\n" " and.b32 %r311, %r146, 61440;\n" " or.b32 %r312, %r311, 1258291200;\n" " mov.b32 %f299, %r312;\n" " fma.rn.f32 %f300, %f299, %f137, %f136;\n" " shr.u32 %r313, %r147, 24;\n" " cvt.rn.f32.u32 %f301, %r313;\n" " sub.rn.f32 %f302, %f301, %f111;\n" " mul.rn.f32 %f303, %f302, %f300;\n" " fma.rn.f32 %f304, %f303, %f250, %f298;\n" " and.b32 %r314, %r146, 983040;\n" " or.b32 %r315, %r314, 1258291200;\n" " mov.b32 %f305, %r315;\n" " fma.rn.f32 %f306, %f305, %f146, %f145;\n" " and.b32 %r316, %r148, 255;\n" " cvt.rn.f32.u32 %f307, %r316;\n" " sub.rn.f32 %f308, %f307, %f111;\n" " mul.rn.f32 %f309, %f308, %f306;\n" " fma.rn.f32 %f310, %f309, %f257, %f304;\n" " and.b32 %r317, %r301, 240;\n" " or.b32 %r318, %r317, 1258291200;\n" " mov.b32 %f311, %r318;\n" " fma.rn.f32 %f312, %f311, %f119, %f118;\n" " bfe.u32 %r319, %r148, 8, 8;\n" " cvt.rn.f32.u32 %f313, %r319;\n" " sub.rn.f32 %f314, %f313, %f111;\n" " mul.rn.f32 %f315, %f314, %f312;\n" " fma.rn.f32 %f316, %f315, %f264, %f310;\n" " and.b32 %r320, %r301, 3840;\n" " or.b32 %r321, %r320, 1258291200;\n" " mov.b32 %f317, %r321;\n" " fma.rn.f32 %f318, %f317, %f128, %f127;\n" " bfe.u32 %r322, %r148, 16, 8;\n" " cvt.rn.f32.u32 %f319, %r322;\n" " sub.rn.f32 %f320, %f319, %f111;\n" " mul.rn.f32 %f321, %f320, %f318;\n" " fma.rn.f32 %f322, %f321, %f271, %f316;\n" " and.b32 %r323, %r301, 61440;\n" " or.b32 %r324, %r323, 1258291200;\n" " mov.b32 %f323, %r324;\n" " fma.rn.f32 %f324, %f323, %f137, %f136;\n" " shr.u32 %r325, %r148, 24;\n" " cvt.rn.f32.u32 %f325, %r325;\n" " sub.rn.f32 %f326, %f325, %f111;\n" " mul.rn.f32 %f327, %f326, %f324;\n" " fma.rn.f32 %f328, %f327, %f278, %f322;\n" " fma.rn.f32 %f329, %f328, %f80, %f223;\n" " shr.u32 %r326, %r152, 16;\n" " and.b32 %r327, %r152, 15;\n" " or.b32 %r328, %r327, 1258291200;\n" " mov.b32 %f330, %r328;\n" " fma.rn.f32 %f331, %f330, %f108, %f107;\n" " and.b32 %r329, %r153, 255;\n" " cvt.rn.f32.u32 %f332, %r329;\n" " sub.rn.f32 %f333, %f332, %f111;\n" " mul.rn.f32 %f334, %f333, %f331;\n" " ld.shared.f32 %f335, [%r229+2048];\n" " fma.rn.f32 %f336, %f334, %f335, %f115;\n" " and.b32 %r330, %r152, 240;\n" " or.b32 %r331, %r330, 1258291200;\n" " mov.b32 %f337, %r331;\n" " fma.rn.f32 %f338, %f337, %f119, %f118;\n" " bfe.u32 %r332, %r153, 8, 8;\n" " cvt.rn.f32.u32 %f339, %r332;\n" " sub.rn.f32 %f340, %f339, %f111;\n" " mul.rn.f32 %f341, %f340, %f338;\n" " ld.shared.f32 %f342, [%r229+2176];\n" " fma.rn.f32 %f343, %f341, %f342, %f336;\n" " and.b32 %r333, %r152, 3840;\n" " or.b32 %r334, %r333, 1258291200;\n" " mov.b32 %f344, %r334;\n" " fma.rn.f32 %f345, %f344, %f128, %f127;\n" " bfe.u32 %r335, %r153, 16, 8;\n" " cvt.rn.f32.u32 %f346, %r335;\n" " sub.rn.f32 %f347, %f346, %f111;\n" " mul.rn.f32 %f348, %f347, %f345;\n" " ld.shared.f32 %f349, [%r229+2304];\n" " fma.rn.f32 %f350, %f348, %f349, %f343;\n" " and.b32 %r336, %r152, 61440;\n" " or.b32 %r337, %r336, 1258291200;\n" " mov.b32 %f351, %r337;\n" " fma.rn.f32 %f352, %f351, %f137, %f136;\n" " shr.u32 %r338, %r153, 24;\n" " cvt.rn.f32.u32 %f353, %r338;\n" " sub.rn.f32 %f354, %f353, %f111;\n" " mul.rn.f32 %f355, %f354, %f352;\n" " ld.shared.f32 %f356, [%r229+2432];\n" " fma.rn.f32 %f357, %f355, %f356, %f350;\n" " and.b32 %r339, %r152, 983040;\n" " or.b32 %r340, %r339, 1258291200;\n" " mov.b32 %f358, %r340;\n" " fma.rn.f32 %f359, %f358, %f146, %f145;\n" " and.b32 %r341, %r154, 255;\n" " cvt.rn.f32.u32 %f360, %r341;\n" " sub.rn.f32 %f361, %f360, %f111;\n" " mul.rn.f32 %f362, %f361, %f359;\n" " ld.shared.f32 %f363, [%r229+2560];\n" " fma.rn.f32 %f364, %f362, %f363, %f357;\n" " and.b32 %r342, %r326, 240;\n" " or.b32 %r343, %r342, 1258291200;\n" " mov.b32 %f365, %r343;\n" " fma.rn.f32 %f366, %f365, %f119, %f118;\n" " bfe.u32 %r344, %r154, 8, 8;\n" " cvt.rn.f32.u32 %f367, %r344;\n" " sub.rn.f32 %f368, %f367, %f111;\n" " mul.rn.f32 %f369, %f368, %f366;\n" " ld.shared.f32 %f370, [%r229+2688];\n" " fma.rn.f32 %f371, %f369, %f370, %f364;\n" " and.b32 %r345, %r326, 3840;\n" " or.b32 %r346, %r345, 1258291200;\n" " mov.b32 %f372, %r346;\n" " fma.rn.f32 %f373, %f372, %f128, %f127;\n" " bfe.u32 %r347, %r154, 16, 8;\n" " cvt.rn.f32.u32 %f374, %r347;\n" " sub.rn.f32 %f375, %f374, %f111;\n" " mul.rn.f32 %f376, %f375, %f373;\n" " ld.shared.f32 %f377, [%r229+2816];\n" " fma.rn.f32 %f378, %f376, %f377, %f371;\n" " and.b32 %r348, %r326, 61440;\n" " or.b32 %r349, %r348, 1258291200;\n" " mov.b32 %f379, %r349;\n" " fma.rn.f32 %f380, %f379, %f137, %f136;\n" " shr.u32 %r350, %r154, 24;\n" " cvt.rn.f32.u32 %f381, %r350;\n" " sub.rn.f32 %f382, %f381, %f111;\n" " mul.rn.f32 %f383, %f382, %f380;\n" " ld.shared.f32 %f384, [%r229+2944];\n" " fma.rn.f32 %f385, %f383, %f384, %f378;\n" " fma.rn.f32 %f386, %f385, %f82, %f280;\n" " shr.u32 %r351, %r158, 16;\n" " and.b32 %r352, %r158, 15;\n" " or.b32 %r353, %r352, 1258291200;\n" " mov.b32 %f387, %r353;\n" " fma.rn.f32 %f388, %f387, %f108, %f107;\n" " and.b32 %r354, %r159, 255;\n" " cvt.rn.f32.u32 %f389, %r354;\n" " sub.rn.f32 %f390, %f389, %f111;\n" " mul.rn.f32 %f391, %f390, %f388;\n" " fma.rn.f32 %f392, %f391, %f335, %f115;\n" " and.b32 %r355, %r158, 240;\n" " or.b32 %r356, %r355, 1258291200;\n" " mov.b32 %f393, %r356;\n" " fma.rn.f32 %f394, %f393, %f119, %f118;\n" " bfe.u32 %r357, %r159, 8, 8;\n" " cvt.rn.f32.u32 %f395, %r357;\n" " sub.rn.f32 %f396, %f395, %f111;\n" " mul.rn.f32 %f397, %f396, %f394;\n" " fma.rn.f32 %f398, %f397, %f342, %f392;\n" " and.b32 %r358, %r158, 3840;\n" " or.b32 %r359, %r358, 1258291200;\n" " mov.b32 %f399, %r359;\n" " fma.rn.f32 %f400, %f399, %f128, %f127;\n" " bfe.u32 %r360, %r159, 16, 8;\n" " cvt.rn.f32.u32 %f401, %r360;\n" " sub.rn.f32 %f402, %f401, %f111;\n" " mul.rn.f32 %f403, %f402, %f400;\n" " fma.rn.f32 %f404, %f403, %f349, %f398;\n" " and.b32 %r361, %r158, 61440;\n" " or.b32 %r362, %r361, 1258291200;\n" " mov.b32 %f405, %r362;\n" " fma.rn.f32 %f406, %f405, %f137, %f136;\n" " shr.u32 %r363, %r159, 24;\n" " cvt.rn.f32.u32 %f407, %r363;\n" " sub.rn.f32 %f408, %f407, %f111;\n" " mul.rn.f32 %f409, %f408, %f406;\n" " fma.rn.f32 %f410, %f409, %f356, %f404;\n" " and.b32 %r364, %r158, 983040;\n" " or.b32 %r365, %r364, 1258291200;\n" " mov.b32 %f411, %r365;\n" " fma.rn.f32 %f412, %f411, %f146, %f145;\n" " and.b32 %r366, %r160, 255;\n" " cvt.rn.f32.u32 %f413, %r366;\n" " sub.rn.f32 %f414, %f413, %f111;\n" " mul.rn.f32 %f415, %f414, %f412;\n" " fma.rn.f32 %f416, %f415, %f363, %f410;\n" " and.b32 %r367, %r351, 240;\n" " or.b32 %r368, %r367, 1258291200;\n" " mov.b32 %f417, %r368;\n" " fma.rn.f32 %f418, %f417, %f119, %f118;\n" " bfe.u32 %r369, %r160, 8, 8;\n" " cvt.rn.f32.u32 %f419, %r369;\n" " sub.rn.f32 %f420, %f419, %f111;\n" " mul.rn.f32 %f421, %f420, %f418;\n" " fma.rn.f32 %f422, %f421, %f370, %f416;\n" " and.b32 %r370, %r351, 3840;\n" " or.b32 %r371, %r370, 1258291200;\n" " mov.b32 %f423, %r371;\n" " fma.rn.f32 %f424, %f423, %f128, %f127;\n" " bfe.u32 %r372, %r160, 16, 8;\n" " cvt.rn.f32.u32 %f425, %r372;\n" " sub.rn.f32 %f426, %f425, %f111;\n" " mul.rn.f32 %f427, %f426, %f424;\n" " fma.rn.f32 %f428, %f427, %f377, %f422;\n" " and.b32 %r373, %r351, 61440;\n" " or.b32 %r374, %r373, 1258291200;\n" " mov.b32 %f429, %r374;\n" " fma.rn.f32 %f430, %f429, %f137, %f136;\n" " shr.u32 %r375, %r160, 24;\n" " cvt.rn.f32.u32 %f431, %r375;\n" " sub.rn.f32 %f432, %f431, %f111;\n" " mul.rn.f32 %f433, %f432, %f430;\n" " fma.rn.f32 %f434, %f433, %f384, %f428;\n" " fma.rn.f32 %f435, %f434, %f84, %f329;\n" " shr.u32 %r376, %r164, 16;\n" " and.b32 %r377, %r164, 15;\n" " or.b32 %r378, %r377, 1258291200;\n" " mov.b32 %f436, %r378;\n" " fma.rn.f32 %f437, %f436, %f108, %f107;\n" " and.b32 %r379, %r165, 255;\n" " cvt.rn.f32.u32 %f438, %r379;\n" " sub.rn.f32 %f439, %f438, %f111;\n" " mul.rn.f32 %f440, %f439, %f437;\n" " ld.shared.f32 %f441, [%r229+3072];\n" " fma.rn.f32 %f442, %f440, %f441, %f115;\n" " and.b32 %r380, %r164, 240;\n" " or.b32 %r381, %r380, 1258291200;\n" " mov.b32 %f443, %r381;\n" " fma.rn.f32 %f444, %f443, %f119, %f118;\n" " bfe.u32 %r382, %r165, 8, 8;\n" " cvt.rn.f32.u32 %f445, %r382;\n" " sub.rn.f32 %f446, %f445, %f111;\n" " mul.rn.f32 %f447, %f446, %f444;\n" " ld.shared.f32 %f448, [%r229+3200];\n" " fma.rn.f32 %f449, %f447, %f448, %f442;\n" " and.b32 %r383, %r164, 3840;\n" " or.b32 %r384, %r383, 1258291200;\n" " mov.b32 %f450, %r384;\n" " fma.rn.f32 %f451, %f450, %f128, %f127;\n" " bfe.u32 %r385, %r165, 16, 8;\n" " cvt.rn.f32.u32 %f452, %r385;\n" " sub.rn.f32 %f453, %f452, %f111;\n" " mul.rn.f32 %f454, %f453, %f451;\n" " ld.shared.f32 %f455, [%r229+3328];\n" " fma.rn.f32 %f456, %f454, %f455, %f449;\n" " and.b32 %r386, %r164, 61440;\n" " or.b32 %r387, %r386, 1258291200;\n" " mov.b32 %f457, %r387;\n" " fma.rn.f32 %f458, %f457, %f137, %f136;\n" " shr.u32 %r388, %r165, 24;\n" " cvt.rn.f32.u32 %f459, %r388;\n" " sub.rn.f32 %f460, %f459, %f111;\n" " mul.rn.f32 %f461, %f460, %f458;\n" " ld.shared.f32 %f462, [%r229+3456];\n" " fma.rn.f32 %f463, %f461, %f462, %f456;\n" " and.b32 %r389, %r164, 983040;\n" " or.b32 %r390, %r389, 1258291200;\n" " mov.b32 %f464, %r390;\n" " fma.rn.f32 %f465, %f464, %f146, %f145;\n" " and.b32 %r391, %r166, 255;\n" " cvt.rn.f32.u32 %f466, %r391;\n" " sub.rn.f32 %f467, %f466, %f111;\n" " mul.rn.f32 %f468, %f467, %f465;\n" " ld.shared.f32 %f469, [%r229+3584];\n" " fma.rn.f32 %f470, %f468, %f469, %f463;\n" " and.b32 %r392, %r376, 240;\n" " or.b32 %r393, %r392, 1258291200;\n" " mov.b32 %f471, %r393;\n" " fma.rn.f32 %f472, %f471, %f119, %f118;\n" " bfe.u32 %r394, %r166, 8, 8;\n" " cvt.rn.f32.u32 %f473, %r394;\n" " sub.rn.f32 %f474, %f473, %f111;\n" " mul.rn.f32 %f475, %f474, %f472;\n" " ld.shared.f32 %f476, [%r229+3712];\n" " fma.rn.f32 %f477, %f475, %f476, %f470;\n" " and.b32 %r395, %r376, 3840;\n" " or.b32 %r396, %r395, 1258291200;\n" " mov.b32 %f478, %r396;\n" " fma.rn.f32 %f479, %f478, %f128, %f127;\n" " bfe.u32 %r397, %r166, 16, 8;\n" " cvt.rn.f32.u32 %f480, %r397;\n" " sub.rn.f32 %f481, %f480, %f111;\n" " mul.rn.f32 %f482, %f481, %f479;\n" " ld.shared.f32 %f483, [%r229+3840];\n" " fma.rn.f32 %f484, %f482, %f483, %f477;\n" " and.b32 %r398, %r376, 61440;\n" " or.b32 %r399, %r398, 1258291200;\n" " mov.b32 %f485, %r399;\n" " fma.rn.f32 %f486, %f485, %f137, %f136;\n" " shr.u32 %r400, %r166, 24;\n" " cvt.rn.f32.u32 %f487, %r400;\n" " sub.rn.f32 %f488, %f487, %f111;\n" " mul.rn.f32 %f489, %f488, %f486;\n" " ld.shared.f32 %f490, [%r229+3968];\n" " fma.rn.f32 %f491, %f489, %f490, %f484;\n" " fma.rn.f32 %f492, %f491, %f86, %f386;\n" " shr.u32 %r401, %r170, 16;\n" " and.b32 %r402, %r170, 15;\n" " or.b32 %r403, %r402, 1258291200;\n" " mov.b32 %f493, %r403;\n" " fma.rn.f32 %f494, %f493, %f108, %f107;\n" " and.b32 %r404, %r171, 255;\n" " cvt.rn.f32.u32 %f495, %r404;\n" " sub.rn.f32 %f496, %f495, %f111;\n" " mul.rn.f32 %f497, %f496, %f494;\n" " fma.rn.f32 %f498, %f497, %f441, %f115;\n" " and.b32 %r405, %r170, 240;\n" " or.b32 %r406, %r405, 1258291200;\n" " mov.b32 %f499, %r406;\n" " fma.rn.f32 %f500, %f499, %f119, %f118;\n" " bfe.u32 %r407, %r171, 8, 8;\n" " cvt.rn.f32.u32 %f501, %r407;\n" " sub.rn.f32 %f502, %f501, %f111;\n" " mul.rn.f32 %f503, %f502, %f500;\n" " fma.rn.f32 %f504, %f503, %f448, %f498;\n" " and.b32 %r408, %r170, 3840;\n" " or.b32 %r409, %r408, 1258291200;\n" " mov.b32 %f505, %r409;\n" " fma.rn.f32 %f506, %f505, %f128, %f127;\n" " bfe.u32 %r410, %r171, 16, 8;\n" " cvt.rn.f32.u32 %f507, %r410;\n" " sub.rn.f32 %f508, %f507, %f111;\n" " mul.rn.f32 %f509, %f508, %f506;\n" " fma.rn.f32 %f510, %f509, %f455, %f504;\n" " and.b32 %r411, %r170, 61440;\n" " or.b32 %r412, %r411, 1258291200;\n" " mov.b32 %f511, %r412;\n" " fma.rn.f32 %f512, %f511, %f137, %f136;\n" " shr.u32 %r413, %r171, 24;\n" " cvt.rn.f32.u32 %f513, %r413;\n" " sub.rn.f32 %f514, %f513, %f111;\n" " mul.rn.f32 %f515, %f514, %f512;\n" " fma.rn.f32 %f516, %f515, %f462, %f510;\n" " and.b32 %r414, %r170, 983040;\n" " or.b32 %r415, %r414, 1258291200;\n" " mov.b32 %f517, %r415;\n" " fma.rn.f32 %f518, %f517, %f146, %f145;\n" " and.b32 %r416, %r172, 255;\n" " cvt.rn.f32.u32 %f519, %r416;\n" " sub.rn.f32 %f520, %f519, %f111;\n" " mul.rn.f32 %f521, %f520, %f518;\n" " fma.rn.f32 %f522, %f521, %f469, %f516;\n" " and.b32 %r417, %r401, 240;\n" " or.b32 %r418, %r417, 1258291200;\n" " mov.b32 %f523, %r418;\n" " fma.rn.f32 %f524, %f523, %f119, %f118;\n" " bfe.u32 %r419, %r172, 8, 8;\n" " cvt.rn.f32.u32 %f525, %r419;\n" " sub.rn.f32 %f526, %f525, %f111;\n" " mul.rn.f32 %f527, %f526, %f524;\n" " fma.rn.f32 %f528, %f527, %f476, %f522;\n" " and.b32 %r420, %r401, 3840;\n" " or.b32 %r421, %r420, 1258291200;\n" " mov.b32 %f529, %r421;\n" " fma.rn.f32 %f530, %f529, %f128, %f127;\n" " bfe.u32 %r422, %r172, 16, 8;\n" " cvt.rn.f32.u32 %f531, %r422;\n" " sub.rn.f32 %f532, %f531, %f111;\n" " mul.rn.f32 %f533, %f532, %f530;\n" " fma.rn.f32 %f534, %f533, %f483, %f528;\n" " and.b32 %r423, %r401, 61440;\n" " or.b32 %r424, %r423, 1258291200;\n" " mov.b32 %f535, %r424;\n" " fma.rn.f32 %f536, %f535, %f137, %f136;\n" " shr.u32 %r425, %r172, 24;\n" " cvt.rn.f32.u32 %f537, %r425;\n" " sub.rn.f32 %f538, %f537, %f111;\n" " mul.rn.f32 %f539, %f538, %f536;\n" " fma.rn.f32 %f540, %f539, %f490, %f534;\n" " fma.rn.f32 %f541, %f540, %f88, %f435;\n" " shr.u32 %r426, %r176, 16;\n" " and.b32 %r427, %r176, 15;\n" " or.b32 %r428, %r427, 1258291200;\n" " mov.b32 %f542, %r428;\n" " fma.rn.f32 %f543, %f542, %f108, %f107;\n" " and.b32 %r429, %r177, 255;\n" " cvt.rn.f32.u32 %f544, %r429;\n" " sub.rn.f32 %f545, %f544, %f111;\n" " mul.rn.f32 %f546, %f545, %f543;\n" " ld.shared.f32 %f547, [%r229+4096];\n" " fma.rn.f32 %f548, %f546, %f547, %f115;\n" " and.b32 %r430, %r176, 240;\n" " or.b32 %r431, %r430, 1258291200;\n" " mov.b32 %f549, %r431;\n" " fma.rn.f32 %f550, %f549, %f119, %f118;\n" " bfe.u32 %r432, %r177, 8, 8;\n" " cvt.rn.f32.u32 %f551, %r432;\n" " sub.rn.f32 %f552, %f551, %f111;\n" " mul.rn.f32 %f553, %f552, %f550;\n" " ld.shared.f32 %f554, [%r229+4224];\n" " fma.rn.f32 %f555, %f553, %f554, %f548;\n" " and.b32 %r433, %r176, 3840;\n" " or.b32 %r434, %r433, 1258291200;\n" " mov.b32 %f556, %r434;\n" " fma.rn.f32 %f557, %f556, %f128, %f127;\n" " bfe.u32 %r435, %r177, 16, 8;\n" " cvt.rn.f32.u32 %f558, %r435;\n" " sub.rn.f32 %f559, %f558, %f111;\n" " mul.rn.f32 %f560, %f559, %f557;\n" " ld.shared.f32 %f561, [%r229+4352];\n" " fma.rn.f32 %f562, %f560, %f561, %f555;\n" " and.b32 %r436, %r176, 61440;\n" " or.b32 %r437, %r436, 1258291200;\n" " mov.b32 %f563, %r437;\n" " fma.rn.f32 %f564, %f563, %f137, %f136;\n" " shr.u32 %r438, %r177, 24;\n" " cvt.rn.f32.u32 %f565, %r438;\n" " sub.rn.f32 %f566, %f565, %f111;\n" " mul.rn.f32 %f567, %f566, %f564;\n" " ld.shared.f32 %f568, [%r229+4480];\n" " fma.rn.f32 %f569, %f567, %f568, %f562;\n" " and.b32 %r439, %r176, 983040;\n" " or.b32 %r440, %r439, 1258291200;\n" " mov.b32 %f570, %r440;\n" " fma.rn.f32 %f571, %f570, %f146, %f145;\n" " and.b32 %r441, %r178, 255;\n" " cvt.rn.f32.u32 %f572, %r441;\n" " sub.rn.f32 %f573, %f572, %f111;\n" " mul.rn.f32 %f574, %f573, %f571;\n" " ld.shared.f32 %f575, [%r229+4608];\n" " fma.rn.f32 %f576, %f574, %f575, %f569;\n" " and.b32 %r442, %r426, 240;\n" " or.b32 %r443, %r442, 1258291200;\n" " mov.b32 %f577, %r443;\n" " fma.rn.f32 %f578, %f577, %f119, %f118;\n" " bfe.u32 %r444, %r178, 8, 8;\n" " cvt.rn.f32.u32 %f579, %r444;\n" " sub.rn.f32 %f580, %f579, %f111;\n" " mul.rn.f32 %f581, %f580, %f578;\n" " ld.shared.f32 %f582, [%r229+4736];\n" " fma.rn.f32 %f583, %f581, %f582, %f576;\n" " and.b32 %r445, %r426, 3840;\n" " or.b32 %r446, %r445, 1258291200;\n" " mov.b32 %f584, %r446;\n" " fma.rn.f32 %f585, %f584, %f128, %f127;\n" " bfe.u32 %r447, %r178, 16, 8;\n" " cvt.rn.f32.u32 %f586, %r447;\n" " sub.rn.f32 %f587, %f586, %f111;\n" " mul.rn.f32 %f588, %f587, %f585;\n" " ld.shared.f32 %f589, [%r229+4864];\n" " fma.rn.f32 %f590, %f588, %f589, %f583;\n" " and.b32 %r448, %r426, 61440;\n" " or.b32 %r449, %r448, 1258291200;\n" " mov.b32 %f591, %r449;\n" " fma.rn.f32 %f592, %f591, %f137, %f136;\n" " shr.u32 %r450, %r178, 24;\n" " cvt.rn.f32.u32 %f593, %r450;\n" " sub.rn.f32 %f594, %f593, %f111;\n" " mul.rn.f32 %f595, %f594, %f592;\n" " ld.shared.f32 %f596, [%r229+4992];\n" " fma.rn.f32 %f597, %f595, %f596, %f590;\n" " fma.rn.f32 %f598, %f597, %f90, %f492;\n" " shr.u32 %r451, %r182, 16;\n" " and.b32 %r452, %r182, 15;\n" " or.b32 %r453, %r452, 1258291200;\n" " mov.b32 %f599, %r453;\n" " fma.rn.f32 %f600, %f599, %f108, %f107;\n" " and.b32 %r454, %r183, 255;\n" " cvt.rn.f32.u32 %f601, %r454;\n" " sub.rn.f32 %f602, %f601, %f111;\n" " mul.rn.f32 %f603, %f602, %f600;\n" " fma.rn.f32 %f604, %f603, %f547, %f115;\n" " and.b32 %r455, %r182, 240;\n" " or.b32 %r456, %r455, 1258291200;\n" " mov.b32 %f605, %r456;\n" " fma.rn.f32 %f606, %f605, %f119, %f118;\n" " bfe.u32 %r457, %r183, 8, 8;\n" " cvt.rn.f32.u32 %f607, %r457;\n" " sub.rn.f32 %f608, %f607, %f111;\n" " mul.rn.f32 %f609, %f608, %f606;\n" " fma.rn.f32 %f610, %f609, %f554, %f604;\n" " and.b32 %r458, %r182, 3840;\n" " or.b32 %r459, %r458, 1258291200;\n" " mov.b32 %f611, %r459;\n" " fma.rn.f32 %f612, %f611, %f128, %f127;\n" " bfe.u32 %r460, %r183, 16, 8;\n" " cvt.rn.f32.u32 %f613, %r460;\n" " sub.rn.f32 %f614, %f613, %f111;\n" " mul.rn.f32 %f615, %f614, %f612;\n" " fma.rn.f32 %f616, %f615, %f561, %f610;\n" " and.b32 %r461, %r182, 61440;\n" " or.b32 %r462, %r461, 1258291200;\n" " mov.b32 %f617, %r462;\n" " fma.rn.f32 %f618, %f617, %f137, %f136;\n" " shr.u32 %r463, %r183, 24;\n" " cvt.rn.f32.u32 %f619, %r463;\n" " sub.rn.f32 %f620, %f619, %f111;\n" " mul.rn.f32 %f621, %f620, %f618;\n" " fma.rn.f32 %f622, %f621, %f568, %f616;\n" " and.b32 %r464, %r182, 983040;\n" " or.b32 %r465, %r464, 1258291200;\n" " mov.b32 %f623, %r465;\n" " fma.rn.f32 %f624, %f623, %f146, %f145;\n" " and.b32 %r466, %r184, 255;\n" " cvt.rn.f32.u32 %f625, %r466;\n" " sub.rn.f32 %f626, %f625, %f111;\n" " mul.rn.f32 %f627, %f626, %f624;\n" " fma.rn.f32 %f628, %f627, %f575, %f622;\n" " and.b32 %r467, %r451, 240;\n" " or.b32 %r468, %r467, 1258291200;\n" " mov.b32 %f629, %r468;\n" " fma.rn.f32 %f630, %f629, %f119, %f118;\n" " bfe.u32 %r469, %r184, 8, 8;\n" " cvt.rn.f32.u32 %f631, %r469;\n" " sub.rn.f32 %f632, %f631, %f111;\n" " mul.rn.f32 %f633, %f632, %f630;\n" " fma.rn.f32 %f634, %f633, %f582, %f628;\n" " and.b32 %r470, %r451, 3840;\n" " or.b32 %r471, %r470, 1258291200;\n" " mov.b32 %f635, %r471;\n" " fma.rn.f32 %f636, %f635, %f128, %f127;\n" " bfe.u32 %r472, %r184, 16, 8;\n" " cvt.rn.f32.u32 %f637, %r472;\n" " sub.rn.f32 %f638, %f637, %f111;\n" " mul.rn.f32 %f639, %f638, %f636;\n" " fma.rn.f32 %f640, %f639, %f589, %f634;\n" " and.b32 %r473, %r451, 61440;\n" " or.b32 %r474, %r473, 1258291200;\n" " mov.b32 %f641, %r474;\n" " fma.rn.f32 %f642, %f641, %f137, %f136;\n" " shr.u32 %r475, %r184, 24;\n" " cvt.rn.f32.u32 %f643, %r475;\n" " sub.rn.f32 %f644, %f643, %f111;\n" " mul.rn.f32 %f645, %f644, %f642;\n" " fma.rn.f32 %f646, %f645, %f596, %f640;\n" " fma.rn.f32 %f647, %f646, %f92, %f541;\n" " shr.u32 %r476, %r188, 16;\n" " and.b32 %r477, %r188, 15;\n" " or.b32 %r478, %r477, 1258291200;\n" " mov.b32 %f648, %r478;\n" " fma.rn.f32 %f649, %f648, %f108, %f107;\n" " and.b32 %r479, %r189, 255;\n" " cvt.rn.f32.u32 %f650, %r479;\n" " sub.rn.f32 %f651, %f650, %f111;\n" " mul.rn.f32 %f652, %f651, %f649;\n" " ld.shared.f32 %f653, [%r229+5120];\n" " fma.rn.f32 %f654, %f652, %f653, %f115;\n" " and.b32 %r480, %r188, 240;\n" " or.b32 %r481, %r480, 1258291200;\n" " mov.b32 %f655, %r481;\n" " fma.rn.f32 %f656, %f655, %f119, %f118;\n" " bfe.u32 %r482, %r189, 8, 8;\n" " cvt.rn.f32.u32 %f657, %r482;\n" " sub.rn.f32 %f658, %f657, %f111;\n" " mul.rn.f32 %f659, %f658, %f656;\n" " ld.shared.f32 %f660, [%r229+5248];\n" " fma.rn.f32 %f661, %f659, %f660, %f654;\n" " and.b32 %r483, %r188, 3840;\n" " or.b32 %r484, %r483, 1258291200;\n" " mov.b32 %f662, %r484;\n" " fma.rn.f32 %f663, %f662, %f128, %f127;\n" " bfe.u32 %r485, %r189, 16, 8;\n" " cvt.rn.f32.u32 %f664, %r485;\n" " sub.rn.f32 %f665, %f664, %f111;\n" " mul.rn.f32 %f666, %f665, %f663;\n" " ld.shared.f32 %f667, [%r229+5376];\n" " fma.rn.f32 %f668, %f666, %f667, %f661;\n" " and.b32 %r486, %r188, 61440;\n" " or.b32 %r487, %r486, 1258291200;\n" " mov.b32 %f669, %r487;\n" " fma.rn.f32 %f670, %f669, %f137, %f136;\n" " shr.u32 %r488, %r189, 24;\n" " cvt.rn.f32.u32 %f671, %r488;\n" " sub.rn.f32 %f672, %f671, %f111;\n" " mul.rn.f32 %f673, %f672, %f670;\n" " ld.shared.f32 %f674, [%r229+5504];\n" " fma.rn.f32 %f675, %f673, %f674, %f668;\n" " and.b32 %r489, %r188, 983040;\n" " or.b32 %r490, %r489, 1258291200;\n" " mov.b32 %f676, %r490;\n" " fma.rn.f32 %f677, %f676, %f146, %f145;\n" " and.b32 %r491, %r190, 255;\n" " cvt.rn.f32.u32 %f678, %r491;\n" " sub.rn.f32 %f679, %f678, %f111;\n" " mul.rn.f32 %f680, %f679, %f677;\n" " ld.shared.f32 %f681, [%r229+5632];\n" " fma.rn.f32 %f682, %f680, %f681, %f675;\n" " and.b32 %r492, %r476, 240;\n" " or.b32 %r493, %r492, 1258291200;\n" " mov.b32 %f683, %r493;\n" " fma.rn.f32 %f684, %f683, %f119, %f118;\n" " bfe.u32 %r494, %r190, 8, 8;\n" " cvt.rn.f32.u32 %f685, %r494;\n" " sub.rn.f32 %f686, %f685, %f111;\n" " mul.rn.f32 %f687, %f686, %f684;\n" " ld.shared.f32 %f688, [%r229+5760];\n" " fma.rn.f32 %f689, %f687, %f688, %f682;\n" " and.b32 %r495, %r476, 3840;\n" " or.b32 %r496, %r495, 1258291200;\n" " mov.b32 %f690, %r496;\n" " fma.rn.f32 %f691, %f690, %f128, %f127;\n" " bfe.u32 %r497, %r190, 16, 8;\n" " cvt.rn.f32.u32 %f692, %r497;\n" " sub.rn.f32 %f693, %f692, %f111;\n" " mul.rn.f32 %f694, %f693, %f691;\n" " ld.shared.f32 %f695, [%r229+5888];\n" " fma.rn.f32 %f696, %f694, %f695, %f689;\n" " and.b32 %r498, %r476, 61440;\n" " or.b32 %r499, %r498, 1258291200;\n" " mov.b32 %f697, %r499;\n" " fma.rn.f32 %f698, %f697, %f137, %f136;\n" " shr.u32 %r500, %r190, 24;\n" " cvt.rn.f32.u32 %f699, %r500;\n" " sub.rn.f32 %f700, %f699, %f111;\n" " mul.rn.f32 %f701, %f700, %f698;\n" " ld.shared.f32 %f702, [%r229+6016];\n" " fma.rn.f32 %f703, %f701, %f702, %f696;\n" " fma.rn.f32 %f704, %f703, %f94, %f598;\n" " shr.u32 %r501, %r194, 16;\n" " and.b32 %r502, %r194, 15;\n" " or.b32 %r503, %r502, 1258291200;\n" " mov.b32 %f705, %r503;\n" " fma.rn.f32 %f706, %f705, %f108, %f107;\n" " and.b32 %r504, %r195, 255;\n" " cvt.rn.f32.u32 %f707, %r504;\n" " sub.rn.f32 %f708, %f707, %f111;\n" " mul.rn.f32 %f709, %f708, %f706;\n" " fma.rn.f32 %f710, %f709, %f653, %f115;\n" " and.b32 %r505, %r194, 240;\n" " or.b32 %r506, %r505, 1258291200;\n" " mov.b32 %f711, %r506;\n" " fma.rn.f32 %f712, %f711, %f119, %f118;\n" " bfe.u32 %r507, %r195, 8, 8;\n" " cvt.rn.f32.u32 %f713, %r507;\n" " sub.rn.f32 %f714, %f713, %f111;\n" " mul.rn.f32 %f715, %f714, %f712;\n" " fma.rn.f32 %f716, %f715, %f660, %f710;\n" " and.b32 %r508, %r194, 3840;\n" " or.b32 %r509, %r508, 1258291200;\n" " mov.b32 %f717, %r509;\n" " fma.rn.f32 %f718, %f717, %f128, %f127;\n" " bfe.u32 %r510, %r195, 16, 8;\n" " cvt.rn.f32.u32 %f719, %r510;\n" " sub.rn.f32 %f720, %f719, %f111;\n" " mul.rn.f32 %f721, %f720, %f718;\n" " fma.rn.f32 %f722, %f721, %f667, %f716;\n" " and.b32 %r511, %r194, 61440;\n" " or.b32 %r512, %r511, 1258291200;\n" " mov.b32 %f723, %r512;\n" " fma.rn.f32 %f724, %f723, %f137, %f136;\n" " shr.u32 %r513, %r195, 24;\n" " cvt.rn.f32.u32 %f725, %r513;\n" " sub.rn.f32 %f726, %f725, %f111;\n" " mul.rn.f32 %f727, %f726, %f724;\n" " fma.rn.f32 %f728, %f727, %f674, %f722;\n" " and.b32 %r514, %r194, 983040;\n" " or.b32 %r515, %r514, 1258291200;\n" " mov.b32 %f729, %r515;\n" " fma.rn.f32 %f730, %f729, %f146, %f145;\n" " and.b32 %r516, %r196, 255;\n" " cvt.rn.f32.u32 %f731, %r516;\n" " sub.rn.f32 %f732, %f731, %f111;\n" " mul.rn.f32 %f733, %f732, %f730;\n" " fma.rn.f32 %f734, %f733, %f681, %f728;\n" " and.b32 %r517, %r501, 240;\n" " or.b32 %r518, %r517, 1258291200;\n" " mov.b32 %f735, %r518;\n" " fma.rn.f32 %f736, %f735, %f119, %f118;\n" " bfe.u32 %r519, %r196, 8, 8;\n" " cvt.rn.f32.u32 %f737, %r519;\n" " sub.rn.f32 %f738, %f737, %f111;\n" " mul.rn.f32 %f739, %f738, %f736;\n" " fma.rn.f32 %f740, %f739, %f688, %f734;\n" " and.b32 %r520, %r501, 3840;\n" " or.b32 %r521, %r520, 1258291200;\n" " mov.b32 %f741, %r521;\n" " fma.rn.f32 %f742, %f741, %f128, %f127;\n" " bfe.u32 %r522, %r196, 16, 8;\n" " cvt.rn.f32.u32 %f743, %r522;\n" " sub.rn.f32 %f744, %f743, %f111;\n" " mul.rn.f32 %f745, %f744, %f742;\n" " fma.rn.f32 %f746, %f745, %f695, %f740;\n" " and.b32 %r523, %r501, 61440;\n" " or.b32 %r524, %r523, 1258291200;\n" " mov.b32 %f747, %r524;\n" " fma.rn.f32 %f748, %f747, %f137, %f136;\n" " shr.u32 %r525, %r196, 24;\n" " cvt.rn.f32.u32 %f749, %r525;\n" " sub.rn.f32 %f750, %f749, %f111;\n" " mul.rn.f32 %f751, %f750, %f748;\n" " fma.rn.f32 %f752, %f751, %f702, %f746;\n" " fma.rn.f32 %f753, %f752, %f96, %f647;\n" " shr.u32 %r526, %r200, 16;\n" " and.b32 %r527, %r200, 15;\n" " or.b32 %r528, %r527, 1258291200;\n" " mov.b32 %f754, %r528;\n" " fma.rn.f32 %f755, %f754, %f108, %f107;\n" " and.b32 %r529, %r201, 255;\n" " cvt.rn.f32.u32 %f756, %r529;\n" " sub.rn.f32 %f757, %f756, %f111;\n" " mul.rn.f32 %f758, %f757, %f755;\n" " ld.shared.f32 %f759, [%r229+6144];\n" " fma.rn.f32 %f760, %f758, %f759, %f115;\n" " and.b32 %r530, %r200, 240;\n" " or.b32 %r531, %r530, 1258291200;\n" " mov.b32 %f761, %r531;\n" " fma.rn.f32 %f762, %f761, %f119, %f118;\n" " bfe.u32 %r532, %r201, 8, 8;\n" " cvt.rn.f32.u32 %f763, %r532;\n" " sub.rn.f32 %f764, %f763, %f111;\n" " mul.rn.f32 %f765, %f764, %f762;\n" " ld.shared.f32 %f766, [%r229+6272];\n" " fma.rn.f32 %f767, %f765, %f766, %f760;\n" " and.b32 %r533, %r200, 3840;\n" " or.b32 %r534, %r533, 1258291200;\n" " mov.b32 %f768, %r534;\n" " fma.rn.f32 %f769, %f768, %f128, %f127;\n" " bfe.u32 %r535, %r201, 16, 8;\n" " cvt.rn.f32.u32 %f770, %r535;\n" " sub.rn.f32 %f771, %f770, %f111;\n" " mul.rn.f32 %f772, %f771, %f769;\n" " ld.shared.f32 %f773, [%r229+6400];\n" " fma.rn.f32 %f774, %f772, %f773, %f767;\n" " and.b32 %r536, %r200, 61440;\n" " or.b32 %r537, %r536, 1258291200;\n" " mov.b32 %f775, %r537;\n" " fma.rn.f32 %f776, %f775, %f137, %f136;\n" " shr.u32 %r538, %r201, 24;\n" " cvt.rn.f32.u32 %f777, %r538;\n" " sub.rn.f32 %f778, %f777, %f111;\n" " mul.rn.f32 %f779, %f778, %f776;\n" " ld.shared.f32 %f780, [%r229+6528];\n" " fma.rn.f32 %f781, %f779, %f780, %f774;\n" " and.b32 %r539, %r200, 983040;\n" " or.b32 %r540, %r539, 1258291200;\n" " mov.b32 %f782, %r540;\n" " fma.rn.f32 %f783, %f782, %f146, %f145;\n" " and.b32 %r541, %r202, 255;\n" " cvt.rn.f32.u32 %f784, %r541;\n" " sub.rn.f32 %f785, %f784, %f111;\n" " mul.rn.f32 %f786, %f785, %f783;\n" " ld.shared.f32 %f787, [%r229+6656];\n" " fma.rn.f32 %f788, %f786, %f787, %f781;\n" " and.b32 %r542, %r526, 240;\n" " or.b32 %r543, %r542, 1258291200;\n" " mov.b32 %f789, %r543;\n" " fma.rn.f32 %f790, %f789, %f119, %f118;\n" " bfe.u32 %r544, %r202, 8, 8;\n" " cvt.rn.f32.u32 %f791, %r544;\n" " sub.rn.f32 %f792, %f791, %f111;\n" " mul.rn.f32 %f793, %f792, %f790;\n" " ld.shared.f32 %f794, [%r229+6784];\n" " fma.rn.f32 %f795, %f793, %f794, %f788;\n" " and.b32 %r545, %r526, 3840;\n" " or.b32 %r546, %r545, 1258291200;\n" " mov.b32 %f796, %r546;\n" " fma.rn.f32 %f797, %f796, %f128, %f127;\n" " bfe.u32 %r547, %r202, 16, 8;\n" " cvt.rn.f32.u32 %f798, %r547;\n" " sub.rn.f32 %f799, %f798, %f111;\n" " mul.rn.f32 %f800, %f799, %f797;\n" " ld.shared.f32 %f801, [%r229+6912];\n" " fma.rn.f32 %f802, %f800, %f801, %f795;\n" " and.b32 %r548, %r526, 61440;\n" " or.b32 %r549, %r548, 1258291200;\n" " mov.b32 %f803, %r549;\n" " fma.rn.f32 %f804, %f803, %f137, %f136;\n" " shr.u32 %r550, %r202, 24;\n" " cvt.rn.f32.u32 %f805, %r550;\n" " sub.rn.f32 %f806, %f805, %f111;\n" " mul.rn.f32 %f807, %f806, %f804;\n" " ld.shared.f32 %f808, [%r229+7040];\n" " fma.rn.f32 %f809, %f807, %f808, %f802;\n" " fma.rn.f32 %f810, %f809, %f98, %f704;\n" " shr.u32 %r551, %r206, 16;\n" " and.b32 %r552, %r206, 15;\n" " or.b32 %r553, %r552, 1258291200;\n" " mov.b32 %f811, %r553;\n" " fma.rn.f32 %f812, %f811, %f108, %f107;\n" " and.b32 %r554, %r207, 255;\n" " cvt.rn.f32.u32 %f813, %r554;\n" " sub.rn.f32 %f814, %f813, %f111;\n" " mul.rn.f32 %f815, %f814, %f812;\n" " fma.rn.f32 %f816, %f815, %f759, %f115;\n" " and.b32 %r555, %r206, 240;\n" " or.b32 %r556, %r555, 1258291200;\n" " mov.b32 %f817, %r556;\n" " fma.rn.f32 %f818, %f817, %f119, %f118;\n" " bfe.u32 %r557, %r207, 8, 8;\n" " cvt.rn.f32.u32 %f819, %r557;\n" " sub.rn.f32 %f820, %f819, %f111;\n" " mul.rn.f32 %f821, %f820, %f818;\n" " fma.rn.f32 %f822, %f821, %f766, %f816;\n" " and.b32 %r558, %r206, 3840;\n" " or.b32 %r559, %r558, 1258291200;\n" " mov.b32 %f823, %r559;\n" " fma.rn.f32 %f824, %f823, %f128, %f127;\n" " bfe.u32 %r560, %r207, 16, 8;\n" " cvt.rn.f32.u32 %f825, %r560;\n" " sub.rn.f32 %f826, %f825, %f111;\n" " mul.rn.f32 %f827, %f826, %f824;\n" " fma.rn.f32 %f828, %f827, %f773, %f822;\n" " and.b32 %r561, %r206, 61440;\n" " or.b32 %r562, %r561, 1258291200;\n" " mov.b32 %f829, %r562;\n" " fma.rn.f32 %f830, %f829, %f137, %f136;\n" " shr.u32 %r563, %r207, 24;\n" " cvt.rn.f32.u32 %f831, %r563;\n" " sub.rn.f32 %f832, %f831, %f111;\n" " mul.rn.f32 %f833, %f832, %f830;\n" " fma.rn.f32 %f834, %f833, %f780, %f828;\n" " and.b32 %r564, %r206, 983040;\n" " or.b32 %r565, %r564, 1258291200;\n" " mov.b32 %f835, %r565;\n" " fma.rn.f32 %f836, %f835, %f146, %f145;\n" " and.b32 %r566, %r208, 255;\n" " cvt.rn.f32.u32 %f837, %r566;\n" " sub.rn.f32 %f838, %f837, %f111;\n" " mul.rn.f32 %f839, %f838, %f836;\n" " fma.rn.f32 %f840, %f839, %f787, %f834;\n" " and.b32 %r567, %r551, 240;\n" " or.b32 %r568, %r567, 1258291200;\n" " mov.b32 %f841, %r568;\n" " fma.rn.f32 %f842, %f841, %f119, %f118;\n" " bfe.u32 %r569, %r208, 8, 8;\n" " cvt.rn.f32.u32 %f843, %r569;\n" " sub.rn.f32 %f844, %f843, %f111;\n" " mul.rn.f32 %f845, %f844, %f842;\n" " fma.rn.f32 %f846, %f845, %f794, %f840;\n" " and.b32 %r570, %r551, 3840;\n" " or.b32 %r571, %r570, 1258291200;\n" " mov.b32 %f847, %r571;\n" " fma.rn.f32 %f848, %f847, %f128, %f127;\n" " bfe.u32 %r572, %r208, 16, 8;\n" " cvt.rn.f32.u32 %f849, %r572;\n" " sub.rn.f32 %f850, %f849, %f111;\n" " mul.rn.f32 %f851, %f850, %f848;\n" " fma.rn.f32 %f852, %f851, %f801, %f846;\n" " and.b32 %r573, %r551, 61440;\n" " or.b32 %r574, %r573, 1258291200;\n" " mov.b32 %f853, %r574;\n" " fma.rn.f32 %f854, %f853, %f137, %f136;\n" " shr.u32 %r575, %r208, 24;\n" " cvt.rn.f32.u32 %f855, %r575;\n" " sub.rn.f32 %f856, %f855, %f111;\n" " mul.rn.f32 %f857, %f856, %f854;\n" " fma.rn.f32 %f858, %f857, %f808, %f852;\n" " fma.rn.f32 %f859, %f858, %f100, %f753;\n" " shr.u32 %r576, %r212, 16;\n" " and.b32 %r577, %r212, 15;\n" " or.b32 %r578, %r577, 1258291200;\n" " mov.b32 %f860, %r578;\n" " fma.rn.f32 %f861, %f860, %f108, %f107;\n" " and.b32 %r579, %r213, 255;\n" " cvt.rn.f32.u32 %f862, %r579;\n" " sub.rn.f32 %f863, %f862, %f111;\n" " mul.rn.f32 %f864, %f863, %f861;\n" " ld.shared.f32 %f865, [%r229+7168];\n" " fma.rn.f32 %f866, %f864, %f865, %f115;\n" " and.b32 %r580, %r212, 240;\n" " or.b32 %r581, %r580, 1258291200;\n" " mov.b32 %f867, %r581;\n" " fma.rn.f32 %f868, %f867, %f119, %f118;\n" " bfe.u32 %r582, %r213, 8, 8;\n" " cvt.rn.f32.u32 %f869, %r582;\n" " sub.rn.f32 %f870, %f869, %f111;\n" " mul.rn.f32 %f871, %f870, %f868;\n" " ld.shared.f32 %f872, [%r229+7296];\n" " fma.rn.f32 %f873, %f871, %f872, %f866;\n" " and.b32 %r583, %r212, 3840;\n" " or.b32 %r584, %r583, 1258291200;\n" " mov.b32 %f874, %r584;\n" " fma.rn.f32 %f875, %f874, %f128, %f127;\n" " bfe.u32 %r585, %r213, 16, 8;\n" " cvt.rn.f32.u32 %f876, %r585;\n" " sub.rn.f32 %f877, %f876, %f111;\n" " mul.rn.f32 %f878, %f877, %f875;\n" " ld.shared.f32 %f879, [%r229+7424];\n" " fma.rn.f32 %f880, %f878, %f879, %f873;\n" " and.b32 %r586, %r212, 61440;\n" " or.b32 %r587, %r586, 1258291200;\n" " mov.b32 %f881, %r587;\n" " fma.rn.f32 %f882, %f881, %f137, %f136;\n" " shr.u32 %r588, %r213, 24;\n" " cvt.rn.f32.u32 %f883, %r588;\n" " sub.rn.f32 %f884, %f883, %f111;\n" " mul.rn.f32 %f885, %f884, %f882;\n" " ld.shared.f32 %f886, [%r229+7552];\n" " fma.rn.f32 %f887, %f885, %f886, %f880;\n" " and.b32 %r589, %r212, 983040;\n" " or.b32 %r590, %r589, 1258291200;\n" " mov.b32 %f888, %r590;\n" " fma.rn.f32 %f889, %f888, %f146, %f145;\n" " and.b32 %r591, %r214, 255;\n" " cvt.rn.f32.u32 %f890, %r591;\n" " sub.rn.f32 %f891, %f890, %f111;\n" " mul.rn.f32 %f892, %f891, %f889;\n" " ld.shared.f32 %f893, [%r229+7680];\n" " fma.rn.f32 %f894, %f892, %f893, %f887;\n" " and.b32 %r592, %r576, 240;\n" " or.b32 %r593, %r592, 1258291200;\n" " mov.b32 %f895, %r593;\n" " fma.rn.f32 %f896, %f895, %f119, %f118;\n" " bfe.u32 %r594, %r214, 8, 8;\n" " cvt.rn.f32.u32 %f897, %r594;\n" " sub.rn.f32 %f898, %f897, %f111;\n" " mul.rn.f32 %f899, %f898, %f896;\n" " ld.shared.f32 %f900, [%r229+7808];\n" " fma.rn.f32 %f901, %f899, %f900, %f894;\n" " and.b32 %r595, %r576, 3840;\n" " or.b32 %r596, %r595, 1258291200;\n" " mov.b32 %f902, %r596;\n" " fma.rn.f32 %f903, %f902, %f128, %f127;\n" " bfe.u32 %r597, %r214, 16, 8;\n" " cvt.rn.f32.u32 %f904, %r597;\n" " sub.rn.f32 %f905, %f904, %f111;\n" " mul.rn.f32 %f906, %f905, %f903;\n" " ld.shared.f32 %f907, [%r229+7936];\n" " fma.rn.f32 %f908, %f906, %f907, %f901;\n" " and.b32 %r598, %r576, 61440;\n" " or.b32 %r599, %r598, 1258291200;\n" " mov.b32 %f909, %r599;\n" " fma.rn.f32 %f910, %f909, %f137, %f136;\n" " shr.u32 %r600, %r214, 24;\n" " cvt.rn.f32.u32 %f911, %r600;\n" " sub.rn.f32 %f912, %f911, %f111;\n" " mul.rn.f32 %f913, %f912, %f910;\n" " ld.shared.f32 %f914, [%r229+8064];\n" " fma.rn.f32 %f915, %f913, %f914, %f908;\n" " fma.rn.f32 %f1118, %f915, %f102, %f810;\n" " shr.u32 %r601, %r218, 16;\n" " and.b32 %r602, %r218, 15;\n" " or.b32 %r603, %r602, 1258291200;\n" " mov.b32 %f916, %r603;\n" " fma.rn.f32 %f917, %f916, %f108, %f107;\n" " and.b32 %r604, %r219, 255;\n" " cvt.rn.f32.u32 %f918, %r604;\n" " sub.rn.f32 %f919, %f918, %f111;\n" " mul.rn.f32 %f920, %f919, %f917;\n" " fma.rn.f32 %f921, %f920, %f865, %f115;\n" " and.b32 %r605, %r218, 240;\n" " or.b32 %r606, %r605, 1258291200;\n" " mov.b32 %f922, %r606;\n" " fma.rn.f32 %f923, %f922, %f119, %f118;\n" " bfe.u32 %r607, %r219, 8, 8;\n" " cvt.rn.f32.u32 %f924, %r607;\n" " sub.rn.f32 %f925, %f924, %f111;\n" " mul.rn.f32 %f926, %f925, %f923;\n" " fma.rn.f32 %f927, %f926, %f872, %f921;\n" " and.b32 %r608, %r218, 3840;\n" " or.b32 %r609, %r608, 1258291200;\n" " mov.b32 %f928, %r609;\n" " fma.rn.f32 %f929, %f928, %f128, %f127;\n" " bfe.u32 %r610, %r219, 16, 8;\n" " cvt.rn.f32.u32 %f930, %r610;\n" " sub.rn.f32 %f931, %f930, %f111;\n" " mul.rn.f32 %f932, %f931, %f929;\n" " fma.rn.f32 %f933, %f932, %f879, %f927;\n" " and.b32 %r611, %r218, 61440;\n" " or.b32 %r612, %r611, 1258291200;\n" " mov.b32 %f934, %r612;\n" " fma.rn.f32 %f935, %f934, %f137, %f136;\n" " shr.u32 %r613, %r219, 24;\n" " cvt.rn.f32.u32 %f936, %r613;\n" " sub.rn.f32 %f937, %f936, %f111;\n" " mul.rn.f32 %f938, %f937, %f935;\n" " fma.rn.f32 %f939, %f938, %f886, %f933;\n" " and.b32 %r614, %r218, 983040;\n" " or.b32 %r615, %r614, 1258291200;\n" " mov.b32 %f940, %r615;\n" " fma.rn.f32 %f941, %f940, %f146, %f145;\n" " and.b32 %r616, %r220, 255;\n" " cvt.rn.f32.u32 %f942, %r616;\n" " sub.rn.f32 %f943, %f942, %f111;\n" " mul.rn.f32 %f944, %f943, %f941;\n" " fma.rn.f32 %f945, %f944, %f893, %f939;\n" " and.b32 %r617, %r601, 240;\n" " or.b32 %r618, %r617, 1258291200;\n" " mov.b32 %f946, %r618;\n" " fma.rn.f32 %f947, %f946, %f119, %f118;\n" " bfe.u32 %r619, %r220, 8, 8;\n" " cvt.rn.f32.u32 %f948, %r619;\n" " sub.rn.f32 %f949, %f948, %f111;\n" " mul.rn.f32 %f950, %f949, %f947;\n" " fma.rn.f32 %f951, %f950, %f900, %f945;\n" " and.b32 %r620, %r601, 3840;\n" " or.b32 %r621, %r620, 1258291200;\n" " mov.b32 %f952, %r621;\n" " fma.rn.f32 %f953, %f952, %f128, %f127;\n" " bfe.u32 %r622, %r220, 16, 8;\n" " cvt.rn.f32.u32 %f954, %r622;\n" " sub.rn.f32 %f955, %f954, %f111;\n" " mul.rn.f32 %f956, %f955, %f953;\n" " fma.rn.f32 %f957, %f956, %f907, %f951;\n" " and.b32 %r623, %r601, 61440;\n" " or.b32 %r624, %r623, 1258291200;\n" " mov.b32 %f958, %r624;\n" " fma.rn.f32 %f959, %f958, %f137, %f136;\n" " shr.u32 %r625, %r220, 24;\n" " cvt.rn.f32.u32 %f960, %r625;\n" " sub.rn.f32 %f961, %f960, %f111;\n" " mul.rn.f32 %f962, %f961, %f959;\n" " fma.rn.f32 %f963, %f962, %f914, %f957;\n" " fma.rn.f32 %f1119, %f963, %f104, %f859;\n" " add.s64 %rd132, %rd132, 1216;\n" " add.s64 %rd133, %rd133, 1216;\n" " add.s32 %r715, %r715, 8192;\n" " add.s32 %r713, %r717, 8;\n" " setp.le.s32 %p20, %r713, %r32;\n" " @%p20 bra $L__BB4_21;\n" "\n" "$L__BB4_22:\n" " setp.ge.s32 %p21, %r717, %r32;\n" " @%p21 bra $L__BB4_24;\n" "\n" "$L__BB4_23:\n" " shl.b64 %rd117, %rd2, 2;\n" " add.s64 %rd111, %rd132, %rd117;\n" " // begin inline asm\n" " ld.global.nc.u32 %r626, [%rd111];\n" " // end inline asm\n" " add.s64 %rd112, %rd132, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f964, [%rd112];\n" " // end inline asm\n" " shl.b64 %rd118, %rd3, 3;\n" " add.s64 %rd119, %rd132, %rd118;\n" " add.s64 %rd113, %rd119, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r627,%r628}, [%rd113];\n" " // end inline asm\n" " shr.u32 %r632, %r626, 16;\n" " and.b32 %r633, %r626, 15;\n" " or.b32 %r634, %r633, 1258291200;\n" " mov.b32 %f966, %r634;\n" " mov.f32 %f967, 0fCB000004;\n" " mov.f32 %f968, 0f3F800000;\n" " fma.rn.f32 %f969, %f966, %f968, %f967;\n" " and.b32 %r635, %r627, 255;\n" " cvt.rn.f32.u32 %f970, %r635;\n" " mov.f32 %f971, 0f42000000;\n" " sub.rn.f32 %f972, %f970, %f971;\n" " mul.rn.f32 %f973, %f972, %f969;\n" " shl.b32 %r636, %r2, 2;\n" " add.s32 %r637, %r715, %r636;\n" " ld.shared.f32 %f974, [%r637];\n" " mov.f32 %f975, 0f00000000;\n" " fma.rn.f32 %f976, %f973, %f974, %f975;\n" " and.b32 %r638, %r626, 240;\n" " or.b32 %r639, %r638, 1258291200;\n" " mov.b32 %f977, %r639;\n" " mov.f32 %f978, 0fC9000040;\n" " mov.f32 %f979, 0f3D800000;\n" " fma.rn.f32 %f980, %f977, %f979, %f978;\n" " bfe.u32 %r640, %r627, 8, 8;\n" " cvt.rn.f32.u32 %f981, %r640;\n" " sub.rn.f32 %f982, %f981, %f971;\n" " mul.rn.f32 %f983, %f982, %f980;\n" " ld.shared.f32 %f984, [%r637+128];\n" " fma.rn.f32 %f985, %f983, %f984, %f976;\n" " and.b32 %r641, %r626, 3840;\n" " or.b32 %r642, %r641, 1258291200;\n" " mov.b32 %f986, %r642;\n" " mov.f32 %f987, 0fC7000400;\n" " mov.f32 %f988, 0f3B800000;\n" " fma.rn.f32 %f989, %f986, %f988, %f987;\n" " bfe.u32 %r643, %r627, 16, 8;\n" " cvt.rn.f32.u32 %f990, %r643;\n" " sub.rn.f32 %f991, %f990, %f971;\n" " mul.rn.f32 %f992, %f991, %f989;\n" " ld.shared.f32 %f993, [%r637+256];\n" " fma.rn.f32 %f994, %f992, %f993, %f985;\n" " and.b32 %r644, %r626, 61440;\n" " or.b32 %r645, %r644, 1258291200;\n" " mov.b32 %f995, %r645;\n" " mov.f32 %f996, 0fC5004000;\n" " mov.f32 %f997, 0f39800000;\n" " fma.rn.f32 %f998, %f995, %f997, %f996;\n" " shr.u32 %r646, %r627, 24;\n" " cvt.rn.f32.u32 %f999, %r646;\n" " sub.rn.f32 %f1000, %f999, %f971;\n" " mul.rn.f32 %f1001, %f1000, %f998;\n" " ld.shared.f32 %f1002, [%r637+384];\n" " fma.rn.f32 %f1003, %f1001, %f1002, %f994;\n" " and.b32 %r647, %r626, 983040;\n" " or.b32 %r648, %r647, 1258291200;\n" " mov.b32 %f1004, %r648;\n" " mov.f32 %f1005, 0fC3040000;\n" " mov.f32 %f1006, 0f37800000;\n" " fma.rn.f32 %f1007, %f1004, %f1006, %f1005;\n" " and.b32 %r649, %r628, 255;\n" " cvt.rn.f32.u32 %f1008, %r649;\n" " sub.rn.f32 %f1009, %f1008, %f971;\n" " mul.rn.f32 %f1010, %f1009, %f1007;\n" " ld.shared.f32 %f1011, [%r637+512];\n" " fma.rn.f32 %f1012, %f1010, %f1011, %f1003;\n" " and.b32 %r650, %r632, 240;\n" " or.b32 %r651, %r650, 1258291200;\n" " mov.b32 %f1013, %r651;\n" " fma.rn.f32 %f1014, %f1013, %f979, %f978;\n" " bfe.u32 %r652, %r628, 8, 8;\n" " cvt.rn.f32.u32 %f1015, %r652;\n" " sub.rn.f32 %f1016, %f1015, %f971;\n" " mul.rn.f32 %f1017, %f1016, %f1014;\n" " ld.shared.f32 %f1018, [%r637+640];\n" " fma.rn.f32 %f1019, %f1017, %f1018, %f1012;\n" " and.b32 %r653, %r632, 3840;\n" " or.b32 %r654, %r653, 1258291200;\n" " mov.b32 %f1020, %r654;\n" " fma.rn.f32 %f1021, %f1020, %f988, %f987;\n" " bfe.u32 %r655, %r628, 16, 8;\n" " cvt.rn.f32.u32 %f1022, %r655;\n" " sub.rn.f32 %f1023, %f1022, %f971;\n" " mul.rn.f32 %f1024, %f1023, %f1021;\n" " ld.shared.f32 %f1025, [%r637+768];\n" " fma.rn.f32 %f1026, %f1024, %f1025, %f1019;\n" " and.b32 %r656, %r632, 61440;\n" " or.b32 %r657, %r656, 1258291200;\n" " mov.b32 %f1027, %r657;\n" " fma.rn.f32 %f1028, %f1027, %f997, %f996;\n" " shr.u32 %r658, %r628, 24;\n" " cvt.rn.f32.u32 %f1029, %r658;\n" " sub.rn.f32 %f1030, %f1029, %f971;\n" " mul.rn.f32 %f1031, %f1030, %f1028;\n" " ld.shared.f32 %f1032, [%r637+896];\n" " fma.rn.f32 %f1033, %f1031, %f1032, %f1026;\n" " fma.rn.f32 %f1118, %f964, %f1033, %f1118;\n" " add.s64 %rd132, %rd132, 152;\n" " add.s64 %rd114, %rd133, %rd117;\n" " // begin inline asm\n" " ld.global.nc.u32 %r629, [%rd114];\n" " // end inline asm\n" " add.s64 %rd115, %rd133, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f965, [%rd115];\n" " // end inline asm\n" " add.s64 %rd120, %rd133, %rd118;\n" " add.s64 %rd116, %rd120, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r630,%r631}, [%rd116];\n" " // end inline asm\n" " shr.u32 %r659, %r629, 16;\n" " and.b32 %r660, %r629, 15;\n" " or.b32 %r661, %r660, 1258291200;\n" " mov.b32 %f1034, %r661;\n" " fma.rn.f32 %f1035, %f1034, %f968, %f967;\n" " and.b32 %r662, %r630, 255;\n" " cvt.rn.f32.u32 %f1036, %r662;\n" " sub.rn.f32 %f1037, %f1036, %f971;\n" " mul.rn.f32 %f1038, %f1037, %f1035;\n" " ld.shared.f32 %f1039, [%r637];\n" " fma.rn.f32 %f1040, %f1038, %f1039, %f975;\n" " and.b32 %r663, %r629, 240;\n" " or.b32 %r664, %r663, 1258291200;\n" " mov.b32 %f1041, %r664;\n" " fma.rn.f32 %f1042, %f1041, %f979, %f978;\n" " bfe.u32 %r665, %r630, 8, 8;\n" " cvt.rn.f32.u32 %f1043, %r665;\n" " sub.rn.f32 %f1044, %f1043, %f971;\n" " mul.rn.f32 %f1045, %f1044, %f1042;\n" " ld.shared.f32 %f1046, [%r637+128];\n" " fma.rn.f32 %f1047, %f1045, %f1046, %f1040;\n" " and.b32 %r666, %r629, 3840;\n" " or.b32 %r667, %r666, 1258291200;\n" " mov.b32 %f1048, %r667;\n" " fma.rn.f32 %f1049, %f1048, %f988, %f987;\n" " bfe.u32 %r668, %r630, 16, 8;\n" " cvt.rn.f32.u32 %f1050, %r668;\n" " sub.rn.f32 %f1051, %f1050, %f971;\n" " mul.rn.f32 %f1052, %f1051, %f1049;\n" " ld.shared.f32 %f1053, [%r637+256];\n" " fma.rn.f32 %f1054, %f1052, %f1053, %f1047;\n" " and.b32 %r669, %r629, 61440;\n" " or.b32 %r670, %r669, 1258291200;\n" " mov.b32 %f1055, %r670;\n" " fma.rn.f32 %f1056, %f1055, %f997, %f996;\n" " shr.u32 %r671, %r630, 24;\n" " cvt.rn.f32.u32 %f1057, %r671;\n" " sub.rn.f32 %f1058, %f1057, %f971;\n" " mul.rn.f32 %f1059, %f1058, %f1056;\n" " ld.shared.f32 %f1060, [%r637+384];\n" " fma.rn.f32 %f1061, %f1059, %f1060, %f1054;\n" " and.b32 %r672, %r629, 983040;\n" " or.b32 %r673, %r672, 1258291200;\n" " mov.b32 %f1062, %r673;\n" " fma.rn.f32 %f1063, %f1062, %f1006, %f1005;\n" " and.b32 %r674, %r631, 255;\n" " cvt.rn.f32.u32 %f1064, %r674;\n" " sub.rn.f32 %f1065, %f1064, %f971;\n" " mul.rn.f32 %f1066, %f1065, %f1063;\n" " ld.shared.f32 %f1067, [%r637+512];\n" " fma.rn.f32 %f1068, %f1066, %f1067, %f1061;\n" " and.b32 %r675, %r659, 240;\n" " or.b32 %r676, %r675, 1258291200;\n" " mov.b32 %f1069, %r676;\n" " fma.rn.f32 %f1070, %f1069, %f979, %f978;\n" " bfe.u32 %r677, %r631, 8, 8;\n" " cvt.rn.f32.u32 %f1071, %r677;\n" " sub.rn.f32 %f1072, %f1071, %f971;\n" " mul.rn.f32 %f1073, %f1072, %f1070;\n" " ld.shared.f32 %f1074, [%r637+640];\n" " fma.rn.f32 %f1075, %f1073, %f1074, %f1068;\n" " and.b32 %r678, %r659, 3840;\n" " or.b32 %r679, %r678, 1258291200;\n" " mov.b32 %f1076, %r679;\n" " fma.rn.f32 %f1077, %f1076, %f988, %f987;\n" " bfe.u32 %r680, %r631, 16, 8;\n" " cvt.rn.f32.u32 %f1078, %r680;\n" " sub.rn.f32 %f1079, %f1078, %f971;\n" " mul.rn.f32 %f1080, %f1079, %f1077;\n" " ld.shared.f32 %f1081, [%r637+768];\n" " fma.rn.f32 %f1082, %f1080, %f1081, %f1075;\n" " and.b32 %r681, %r659, 61440;\n" " or.b32 %r682, %r681, 1258291200;\n" " mov.b32 %f1083, %r682;\n" " fma.rn.f32 %f1084, %f1083, %f997, %f996;\n" " shr.u32 %r683, %r631, 24;\n" " cvt.rn.f32.u32 %f1085, %r683;\n" " sub.rn.f32 %f1086, %f1085, %f971;\n" " mul.rn.f32 %f1087, %f1086, %f1084;\n" " ld.shared.f32 %f1088, [%r637+896];\n" " fma.rn.f32 %f1089, %f1087, %f1088, %f1082;\n" " fma.rn.f32 %f1119, %f965, %f1089, %f1119;\n" " add.s64 %rd133, %rd133, 152;\n" " add.s32 %r715, %r715, 1024;\n" " add.s32 %r717, %r717, 1;\n" " setp.lt.s32 %p22, %r717, %r32;\n" " @%p22 bra $L__BB4_23;\n" "\n" "$L__BB4_24:\n" " cvt.u32.u64 %r684, %rd9;\n" " add.s32 %r707, %r684, 2048;\n" " setp.lt.s32 %p23, %r707, %r46;\n" " add.s32 %r706, %r706, 2048;\n" " @%p23 bra $L__BB4_2;\n" "\n" "$L__BB4_25:\n" " cvt.u32.u64 %r695, %rd1;\n" " setp.ge.s32 %p24, %r695, %r45;\n" " mov.u32 %r685, 16;\n" " mov.u32 %r694, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f1090, %f1118, %r685, %r694;\n" " // end inline asm\n" " add.f32 %f1093, %f1118, %f1090;\n" " mov.u32 %r687, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f1092, %f1093, %r687, %r694;\n" " // end inline asm\n" " add.f32 %f1095, %f1093, %f1092;\n" " mov.u32 %r689, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f1094, %f1095, %r689, %r694;\n" " // end inline asm\n" " add.f32 %f1097, %f1095, %f1094;\n" " mov.u32 %r691, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f1096, %f1097, %r691, %r694;\n" " // end inline asm\n" " add.f32 %f1099, %f1097, %f1096;\n" " mov.u32 %r693, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f1098, %f1099, %r693, %r694;\n" " // end inline asm\n" " add.f32 %f17, %f1099, %f1098;\n" " setp.ne.s32 %p25, %r2, 0;\n" " cvta.to.global.u64 %rd121, %rd31;\n" " shl.b64 %rd122, %rd1, 2;\n" " add.s64 %rd28, %rd121, %rd122;\n" " or.pred %p26, %p25, %p24;\n" " @%p26 bra $L__BB4_27;\n" "\n" " st.global.f32 [%rd28], %f17;\n" "\n" "$L__BB4_27:\n" " // begin inline asm\n" " shfl.down.b32 %f1100, %f1119, %r685, %r694;\n" " // end inline asm\n" " add.f32 %f1103, %f1119, %f1100;\n" " // begin inline asm\n" " shfl.down.b32 %f1102, %f1103, %r687, %r694;\n" " // end inline asm\n" " add.f32 %f1105, %f1103, %f1102;\n" " // begin inline asm\n" " shfl.down.b32 %f1104, %f1105, %r689, %r694;\n" " // end inline asm\n" " add.f32 %f1107, %f1105, %f1104;\n" " // begin inline asm\n" " shfl.down.b32 %f1106, %f1107, %r691, %r694;\n" " // end inline asm\n" " add.f32 %f1109, %f1107, %f1106;\n" " // begin inline asm\n" " shfl.down.b32 %f1108, %f1109, %r693, %r694;\n" " // end inline asm\n" " add.f32 %f18, %f1109, %f1108;\n" " setp.ge.s32 %p28, %r4, %r45;\n" " or.pred %p29, %p25, %p28;\n" " @%p29 bra $L__BB4_29;\n" "\n" " st.global.f32 [%rd28+4], %f18;\n" "\n" "$L__BB4_29:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_ps_r1p1\n" ".visible .entry fc_matvec_q3k_ps_r1p1(\n" " .param .u64 fc_matvec_q3k_ps_r1p1_param_0,\n" " .param .u64 fc_matvec_q3k_ps_r1p1_param_1,\n" " .param .u64 fc_matvec_q3k_ps_r1p1_param_2,\n" " .param .u32 fc_matvec_q3k_ps_r1p1_param_3,\n" " .param .u32 fc_matvec_q3k_ps_r1p1_param_4,\n" " .param .u32 fc_matvec_q3k_ps_r1p1_param_5\n" ")\n" "{\n" " .reg .pred %p<26>;\n" " .reg .f32 %f<203>;\n" " .reg .b32 %r<166>;\n" " .reg .b64 %rd<70>;\n" "\n" "\n" " ld.param.u64 %rd20, [fc_matvec_q3k_ps_r1p1_param_0];\n" " ld.param.u64 %rd21, [fc_matvec_q3k_ps_r1p1_param_1];\n" " ld.param.u64 %rd22, [fc_matvec_q3k_ps_r1p1_param_2];\n" " ld.param.u32 %r42, [fc_matvec_q3k_ps_r1p1_param_3];\n" " ld.param.u32 %r43, [fc_matvec_q3k_ps_r1p1_param_4];\n" " ld.param.u32 %r44, [fc_matvec_q3k_ps_r1p1_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r45, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r46, %ctaid.x;\n" " shl.b32 %r47, %r46, 3;\n" " add.s32 %r48, %r47, %r45;\n" " cvt.s64.s32 %rd1, %r48;\n" " setp.lt.s32 %p1, %r43, 1;\n" " mov.f32 %f201, 0f00000000;\n" " @%p1 bra $L__BB5_25;\n" "\n" " cvt.u64.u32 %rd2, %r2;\n" " shr.u32 %r51, %r2, 4;\n" " cvt.u64.u32 %rd3, %r51;\n" " mov.u32 %r52, -2;\n" " sub.s32 %r3, %r52, %r1;\n" " not.b32 %r4, %r43;\n" " not.b32 %r5, %r1;\n" " cvt.s64.s32 %rd4, %r1;\n" " add.s32 %r6, %r1, 256;\n" " shl.b32 %r53, %r1, 4;\n" " mov.u32 %r54, sh_xps;\n" " add.s32 %r7, %r54, %r53;\n" " mad.lo.s32 %r8, %r1, -12, %r7;\n" " add.s32 %r9, %r1, 512;\n" " add.s32 %r10, %r1, 768;\n" " cvt.s64.s32 %rd23, %r44;\n" " mul.lo.s64 %rd24, %rd23, %rd1;\n" " add.s64 %rd69, %rd20, %rd24;\n" " mov.f32 %f201, 0f00000000;\n" " mov.u32 %r153, 0;\n" " mov.f32 %f51, 0fCB000004;\n" " mov.u32 %r154, %r153;\n" "\n" "$L__BB5_2:\n" " add.s32 %r55, %r153, %r4;\n" " max.s32 %r56, %r55, -2049;\n" " sub.s32 %r13, %r3, %r56;\n" " sub.s32 %r57, %r43, %r154;\n" " min.s32 %r14, %r57, 2048;\n" " bar.sync 0;\n" " cvt.s64.s32 %rd7, %r154;\n" " mul.wide.s32 %rd25, %r154, 4;\n" " add.s64 %rd8, %rd21, %rd25;\n" " and.b32 %r58, %r14, 3;\n" " setp.eq.s32 %p2, %r58, 0;\n" " { .reg .b64 %tmp;\n" " cvt.u64.u32 %tmp, %r54;\n" " cvta.shared.u64 %rd26, %tmp; }\n" " or.b64 %rd27, %rd8, %rd26;\n" " and.b64 %rd28, %rd27, 15;\n" " setp.eq.s64 %p3, %rd28, 0;\n" " and.pred %p4, %p2, %p3;\n" " @%p4 bra $L__BB5_11;\n" " bra.uni $L__BB5_3;\n" "\n" "$L__BB5_11:\n" " shr.s32 %r22, %r14, 2;\n" " setp.ge.s32 %p11, %r1, %r22;\n" " @%p11 bra $L__BB5_18;\n" "\n" " add.s32 %r23, %r22, %r5;\n" " shr.u32 %r65, %r23, 8;\n" " add.s32 %r66, %r65, 1;\n" " and.b32 %r24, %r66, 3;\n" " setp.eq.s32 %p12, %r24, 0;\n" " mov.u32 %r158, %r1;\n" " @%p12 bra $L__BB5_16;\n" "\n" " shl.b64 %rd41, %rd4, 4;\n" " add.s64 %rd40, %rd8, %rd41;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f19,%f20,%f21,%f22}, [%rd40];\n" " // end inline asm\n" " st.shared.v4.f32 [%r7], {%f19, %f20, %f21, %f22};\n" " setp.eq.s32 %p13, %r24, 1;\n" " mov.u32 %r158, %r6;\n" " @%p13 bra $L__BB5_16;\n" "\n" " add.s64 %rd42, %rd40, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f23,%f24,%f25,%f26}, [%rd42];\n" " // end inline asm\n" " st.shared.v4.f32 [%r7+4096], {%f23, %f24, %f25, %f26};\n" " setp.eq.s32 %p14, %r24, 2;\n" " mov.u32 %r158, %r9;\n" " @%p14 bra $L__BB5_16;\n" "\n" " add.s64 %rd43, %rd40, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f27,%f28,%f29,%f30}, [%rd43];\n" " // end inline asm\n" " st.shared.v4.f32 [%r7+8192], {%f27, %f28, %f29, %f30};\n" " mov.u32 %r158, %r10;\n" "\n" "$L__BB5_16:\n" " setp.lt.u32 %p15, %r23, 768;\n" " @%p15 bra $L__BB5_18;\n" "\n" "$L__BB5_17:\n" " mul.wide.s32 %rd48, %r158, 16;\n" " add.s64 %rd44, %rd8, %rd48;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f31,%f32,%f33,%f34}, [%rd44];\n" " // end inline asm\n" " shl.b32 %r67, %r158, 4;\n" " mov.u32 %r68, sh_xps;\n" " add.s32 %r69, %r68, %r67;\n" " st.shared.v4.f32 [%r69], {%f31, %f32, %f33, %f34};\n" " add.s64 %rd45, %rd44, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f35,%f36,%f37,%f38}, [%rd45];\n" " // end inline asm\n" " st.shared.v4.f32 [%r69+4096], {%f35, %f36, %f37, %f38};\n" " add.s64 %rd46, %rd44, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f39,%f40,%f41,%f42}, [%rd46];\n" " // end inline asm\n" " st.shared.v4.f32 [%r69+8192], {%f39, %f40, %f41, %f42};\n" " add.s64 %rd47, %rd44, 12288;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f43,%f44,%f45,%f46}, [%rd47];\n" " // end inline asm\n" " st.shared.v4.f32 [%r69+12288], {%f43, %f44, %f45, %f46};\n" " add.s32 %r158, %r158, 1024;\n" " setp.lt.s32 %p16, %r158, %r22;\n" " @%p16 bra $L__BB5_17;\n" " bra.uni $L__BB5_18;\n" "\n" "$L__BB5_3:\n" " setp.ge.s32 %p5, %r1, %r14;\n" " @%p5 bra $L__BB5_18;\n" "\n" " shr.u32 %r60, %r13, 8;\n" " add.s32 %r61, %r60, 1;\n" " and.b32 %r15, %r61, 3;\n" " setp.eq.s32 %p6, %r15, 0;\n" " mov.u32 %r155, %r1;\n" " @%p6 bra $L__BB5_8;\n" "\n" " shl.b64 %rd30, %rd4, 2;\n" " add.s64 %rd29, %rd8, %rd30;\n" " // begin inline asm\n" " ld.global.nc.f32 %f12, [%rd29];\n" " // end inline asm\n" " st.shared.f32 [%r8], %f12;\n" " setp.eq.s32 %p7, %r15, 1;\n" " mov.u32 %r155, %r6;\n" " @%p7 bra $L__BB5_8;\n" "\n" " add.s64 %rd31, %rd29, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f13, [%rd31];\n" " // end inline asm\n" " st.shared.f32 [%r8+1024], %f13;\n" " setp.eq.s32 %p8, %r15, 2;\n" " mov.u32 %r155, %r9;\n" " @%p8 bra $L__BB5_8;\n" "\n" " add.s64 %rd32, %rd29, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f14, [%rd32];\n" " // end inline asm\n" " st.shared.f32 [%r8+2048], %f14;\n" " mov.u32 %r155, %r10;\n" "\n" "$L__BB5_8:\n" " setp.lt.u32 %p9, %r13, 768;\n" " @%p9 bra $L__BB5_18;\n" "\n" " cvt.s64.s32 %rd33, %r155;\n" " add.s64 %rd34, %rd33, %rd7;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd65, %rd21, %rd35;\n" " shl.b32 %r62, %r155, 2;\n" " add.s32 %r64, %r54, %r62;\n" " add.s32 %r156, %r64, 2048;\n" "\n" "$L__BB5_10:\n" " // begin inline asm\n" " ld.global.nc.f32 %f15, [%rd65];\n" " // end inline asm\n" " st.shared.f32 [%r156+-2048], %f15;\n" " add.s64 %rd37, %rd65, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f16, [%rd37];\n" " // end inline asm\n" " st.shared.f32 [%r156+-1024], %f16;\n" " add.s64 %rd38, %rd65, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd38];\n" " // end inline asm\n" " st.shared.f32 [%r156], %f17;\n" " add.s64 %rd39, %rd65, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r156+1024], %f18;\n" " add.s64 %rd65, %rd65, 4096;\n" " add.s32 %r156, %r156, 4096;\n" " add.s32 %r155, %r155, 1024;\n" " setp.lt.s32 %p10, %r155, %r14;\n" " @%p10 bra $L__BB5_10;\n" "\n" "$L__BB5_18:\n" " cvt.u32.u64 %r70, %rd1;\n" " setp.ge.s32 %p17, %r70, %r42;\n" " bar.sync 0;\n" " @%p17 bra $L__BB5_24;\n" "\n" " shr.s32 %r29, %r14, 8;\n" " setp.lt.s32 %p18, %r14, 256;\n" " mov.u32 %r163, 0;\n" " mov.u32 %r162, %r54;\n" " @%p18 bra $L__BB5_22;\n" "\n" " mov.u32 %r162, sh_xps;\n" " mov.u32 %r163, 0;\n" "\n" "$L__BB5_21:\n" " shl.b64 %rd52, %rd2, 2;\n" " add.s64 %rd49, %rd69, %rd52;\n" " // begin inline asm\n" " ld.global.nc.u32 %r78, [%rd49];\n" " // end inline asm\n" " add.s64 %rd50, %rd69, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f48, [%rd50];\n" " // end inline asm\n" " shl.b64 %rd53, %rd3, 3;\n" " add.s64 %rd54, %rd69, %rd53;\n" " add.s64 %rd51, %rd54, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r79,%r80}, [%rd51];\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " shr.u32 %r84, %r78, 16;\n" " and.b32 %r85, %r78, 15;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f50, %r86;\n" " mov.f32 %f52, 0f3F800000;\n" " fma.rn.f32 %f53, %f50, %f52, %f51;\n" " and.b32 %r87, %r79, 255;\n" " cvt.rn.f32.u32 %f54, %r87;\n" " mov.f32 %f55, 0f42000000;\n" " sub.rn.f32 %f56, %f54, %f55;\n" " mul.rn.f32 %f57, %f56, %f53;\n" " shl.b32 %r88, %r2, 2;\n" " add.s32 %r89, %r162, %r88;\n" " ld.shared.f32 %f58, [%r89];\n" " mov.f32 %f59, 0f00000000;\n" " fma.rn.f32 %f60, %f57, %f58, %f59;\n" " and.b32 %r90, %r78, 240;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f61, %r91;\n" " mov.f32 %f62, 0fC9000040;\n" " mov.f32 %f63, 0f3D800000;\n" " fma.rn.f32 %f64, %f61, %f63, %f62;\n" " bfe.u32 %r92, %r79, 8, 8;\n" " cvt.rn.f32.u32 %f65, %r92;\n" " sub.rn.f32 %f66, %f65, %f55;\n" " mul.rn.f32 %f67, %f66, %f64;\n" " ld.shared.f32 %f68, [%r89+128];\n" " fma.rn.f32 %f69, %f67, %f68, %f60;\n" " and.b32 %r93, %r78, 3840;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f70, %r94;\n" " mov.f32 %f71, 0fC7000400;\n" " mov.f32 %f72, 0f3B800000;\n" " fma.rn.f32 %f73, %f70, %f72, %f71;\n" " bfe.u32 %r95, %r79, 16, 8;\n" " cvt.rn.f32.u32 %f74, %r95;\n" " sub.rn.f32 %f75, %f74, %f55;\n" " mul.rn.f32 %f76, %f75, %f73;\n" " ld.shared.f32 %f77, [%r89+256];\n" " fma.rn.f32 %f78, %f76, %f77, %f69;\n" " and.b32 %r96, %r78, 61440;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f79, %r97;\n" " mov.f32 %f80, 0fC5004000;\n" " mov.f32 %f81, 0f39800000;\n" " fma.rn.f32 %f82, %f79, %f81, %f80;\n" " shr.u32 %r98, %r79, 24;\n" " cvt.rn.f32.u32 %f83, %r98;\n" " sub.rn.f32 %f84, %f83, %f55;\n" " mul.rn.f32 %f85, %f84, %f82;\n" " ld.shared.f32 %f86, [%r89+384];\n" " fma.rn.f32 %f87, %f85, %f86, %f78;\n" " and.b32 %r99, %r78, 983040;\n" " or.b32 %r100, %r99, 1258291200;\n" " mov.b32 %f88, %r100;\n" " mov.f32 %f89, 0fC3040000;\n" " mov.f32 %f90, 0f37800000;\n" " fma.rn.f32 %f91, %f88, %f90, %f89;\n" " and.b32 %r101, %r80, 255;\n" " cvt.rn.f32.u32 %f92, %r101;\n" " sub.rn.f32 %f93, %f92, %f55;\n" " mul.rn.f32 %f94, %f93, %f91;\n" " ld.shared.f32 %f95, [%r89+512];\n" " fma.rn.f32 %f96, %f94, %f95, %f87;\n" " and.b32 %r102, %r84, 240;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f97, %r103;\n" " fma.rn.f32 %f98, %f97, %f63, %f62;\n" " bfe.u32 %r104, %r80, 8, 8;\n" " cvt.rn.f32.u32 %f99, %r104;\n" " sub.rn.f32 %f100, %f99, %f55;\n" " mul.rn.f32 %f101, %f100, %f98;\n" " ld.shared.f32 %f102, [%r89+640];\n" " fma.rn.f32 %f103, %f101, %f102, %f96;\n" " and.b32 %r105, %r84, 3840;\n" " or.b32 %r106, %r105, 1258291200;\n" " mov.b32 %f104, %r106;\n" " fma.rn.f32 %f105, %f104, %f72, %f71;\n" " bfe.u32 %r107, %r80, 16, 8;\n" " cvt.rn.f32.u32 %f106, %r107;\n" " sub.rn.f32 %f107, %f106, %f55;\n" " mul.rn.f32 %f108, %f107, %f105;\n" " ld.shared.f32 %f109, [%r89+768];\n" " fma.rn.f32 %f110, %f108, %f109, %f103;\n" " and.b32 %r108, %r84, 61440;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f111, %r109;\n" " fma.rn.f32 %f112, %f111, %f81, %f80;\n" " shr.u32 %r110, %r80, 24;\n" " cvt.rn.f32.u32 %f113, %r110;\n" " sub.rn.f32 %f114, %f113, %f55;\n" " mul.rn.f32 %f115, %f114, %f112;\n" " ld.shared.f32 %f116, [%r89+896];\n" " fma.rn.f32 %f117, %f115, %f116, %f110;\n" " fma.rn.f32 %f201, %f48, %f117, %f201;\n" " add.s64 %rd69, %rd69, 152;\n" " add.s32 %r162, %r162, 1024;\n" " add.s32 %r163, %r163, 1;\n" " setp.lt.s32 %p19, %r163, %r29;\n" " @%p19 bra $L__BB5_21;\n" "\n" "$L__BB5_22:\n" " setp.ge.s32 %p20, %r163, %r29;\n" " @%p20 bra $L__BB5_24;\n" "\n" "$L__BB5_23:\n" " shl.b64 %rd58, %rd2, 2;\n" " add.s64 %rd55, %rd69, %rd58;\n" " // begin inline asm\n" " ld.global.nc.u32 %r111, [%rd55];\n" " // end inline asm\n" " add.s64 %rd56, %rd69, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f118, [%rd56];\n" " // end inline asm\n" " shl.b64 %rd59, %rd3, 3;\n" " add.s64 %rd60, %rd69, %rd59;\n" " add.s64 %rd57, %rd60, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r112,%r113}, [%rd57];\n" " // end inline asm\n" " shr.u32 %r114, %r111, 16;\n" " and.b32 %r115, %r111, 15;\n" " or.b32 %r116, %r115, 1258291200;\n" " mov.b32 %f119, %r116;\n" " mov.f32 %f121, 0f3F800000;\n" " fma.rn.f32 %f122, %f119, %f121, %f51;\n" " and.b32 %r117, %r112, 255;\n" " cvt.rn.f32.u32 %f123, %r117;\n" " mov.f32 %f124, 0f42000000;\n" " sub.rn.f32 %f125, %f123, %f124;\n" " mul.rn.f32 %f126, %f125, %f122;\n" " shl.b32 %r118, %r2, 2;\n" " add.s32 %r119, %r162, %r118;\n" " ld.shared.f32 %f127, [%r119];\n" " mov.f32 %f128, 0f00000000;\n" " fma.rn.f32 %f129, %f126, %f127, %f128;\n" " and.b32 %r120, %r111, 240;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f130, %r121;\n" " mov.f32 %f131, 0fC9000040;\n" " mov.f32 %f132, 0f3D800000;\n" " fma.rn.f32 %f133, %f130, %f132, %f131;\n" " bfe.u32 %r122, %r112, 8, 8;\n" " cvt.rn.f32.u32 %f134, %r122;\n" " sub.rn.f32 %f135, %f134, %f124;\n" " mul.rn.f32 %f136, %f135, %f133;\n" " ld.shared.f32 %f137, [%r119+128];\n" " fma.rn.f32 %f138, %f136, %f137, %f129;\n" " and.b32 %r123, %r111, 3840;\n" " or.b32 %r124, %r123, 1258291200;\n" " mov.b32 %f139, %r124;\n" " mov.f32 %f140, 0fC7000400;\n" " mov.f32 %f141, 0f3B800000;\n" " fma.rn.f32 %f142, %f139, %f141, %f140;\n" " bfe.u32 %r125, %r112, 16, 8;\n" " cvt.rn.f32.u32 %f143, %r125;\n" " sub.rn.f32 %f144, %f143, %f124;\n" " mul.rn.f32 %f145, %f144, %f142;\n" " ld.shared.f32 %f146, [%r119+256];\n" " fma.rn.f32 %f147, %f145, %f146, %f138;\n" " and.b32 %r126, %r111, 61440;\n" " or.b32 %r127, %r126, 1258291200;\n" " mov.b32 %f148, %r127;\n" " mov.f32 %f149, 0fC5004000;\n" " mov.f32 %f150, 0f39800000;\n" " fma.rn.f32 %f151, %f148, %f150, %f149;\n" " shr.u32 %r128, %r112, 24;\n" " cvt.rn.f32.u32 %f152, %r128;\n" " sub.rn.f32 %f153, %f152, %f124;\n" " mul.rn.f32 %f154, %f153, %f151;\n" " ld.shared.f32 %f155, [%r119+384];\n" " fma.rn.f32 %f156, %f154, %f155, %f147;\n" " and.b32 %r129, %r111, 983040;\n" " or.b32 %r130, %r129, 1258291200;\n" " mov.b32 %f157, %r130;\n" " mov.f32 %f158, 0fC3040000;\n" " mov.f32 %f159, 0f37800000;\n" " fma.rn.f32 %f160, %f157, %f159, %f158;\n" " and.b32 %r131, %r113, 255;\n" " cvt.rn.f32.u32 %f161, %r131;\n" " sub.rn.f32 %f162, %f161, %f124;\n" " mul.rn.f32 %f163, %f162, %f160;\n" " ld.shared.f32 %f164, [%r119+512];\n" " fma.rn.f32 %f165, %f163, %f164, %f156;\n" " and.b32 %r132, %r114, 240;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f166, %r133;\n" " fma.rn.f32 %f167, %f166, %f132, %f131;\n" " bfe.u32 %r134, %r113, 8, 8;\n" " cvt.rn.f32.u32 %f168, %r134;\n" " sub.rn.f32 %f169, %f168, %f124;\n" " mul.rn.f32 %f170, %f169, %f167;\n" " ld.shared.f32 %f171, [%r119+640];\n" " fma.rn.f32 %f172, %f170, %f171, %f165;\n" " and.b32 %r135, %r114, 3840;\n" " or.b32 %r136, %r135, 1258291200;\n" " mov.b32 %f173, %r136;\n" " fma.rn.f32 %f174, %f173, %f141, %f140;\n" " bfe.u32 %r137, %r113, 16, 8;\n" " cvt.rn.f32.u32 %f175, %r137;\n" " sub.rn.f32 %f176, %f175, %f124;\n" " mul.rn.f32 %f177, %f176, %f174;\n" " ld.shared.f32 %f178, [%r119+768];\n" " fma.rn.f32 %f179, %f177, %f178, %f172;\n" " and.b32 %r138, %r114, 61440;\n" " or.b32 %r139, %r138, 1258291200;\n" " mov.b32 %f180, %r139;\n" " fma.rn.f32 %f181, %f180, %f150, %f149;\n" " shr.u32 %r140, %r113, 24;\n" " cvt.rn.f32.u32 %f182, %r140;\n" " sub.rn.f32 %f183, %f182, %f124;\n" " mul.rn.f32 %f184, %f183, %f181;\n" " ld.shared.f32 %f185, [%r119+896];\n" " fma.rn.f32 %f186, %f184, %f185, %f179;\n" " fma.rn.f32 %f201, %f118, %f186, %f201;\n" " add.s64 %rd69, %rd69, 152;\n" " add.s32 %r162, %r162, 1024;\n" " add.s32 %r163, %r163, 1;\n" " setp.lt.s32 %p21, %r163, %r29;\n" " @%p21 bra $L__BB5_23;\n" "\n" "$L__BB5_24:\n" " cvt.u32.u64 %r141, %rd7;\n" " add.s32 %r154, %r141, 2048;\n" " setp.lt.s32 %p22, %r154, %r43;\n" " add.s32 %r153, %r153, 2048;\n" " @%p22 bra $L__BB5_2;\n" "\n" "$L__BB5_25:\n" " cvt.u32.u64 %r152, %rd1;\n" " setp.ge.s32 %p23, %r152, %r42;\n" " mov.u32 %r142, 16;\n" " mov.u32 %r151, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f187, %f201, %r142, %r151;\n" " // end inline asm\n" " add.f32 %f190, %f201, %f187;\n" " mov.u32 %r144, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f189, %f190, %r144, %r151;\n" " // end inline asm\n" " add.f32 %f192, %f190, %f189;\n" " mov.u32 %r146, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f191, %f192, %r146, %r151;\n" " // end inline asm\n" " add.f32 %f194, %f192, %f191;\n" " mov.u32 %r148, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f193, %f194, %r148, %r151;\n" " // end inline asm\n" " add.f32 %f196, %f194, %f193;\n" " mov.u32 %r150, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f196, %r150, %r151;\n" " // end inline asm\n" " add.f32 %f9, %f196, %f195;\n" " setp.ne.s32 %p24, %r2, 0;\n" " or.pred %p25, %p24, %p23;\n" " @%p25 bra $L__BB5_27;\n" "\n" " cvta.to.global.u64 %rd61, %rd22;\n" " shl.b64 %rd62, %rd1, 2;\n" " add.s64 %rd63, %rd61, %rd62;\n" " st.global.f32 [%rd63], %f9;\n" "\n" "$L__BB5_27:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_ps_r1p4\n" ".visible .entry fc_matvec_q3k_ps_r1p4(\n" " .param .u64 fc_matvec_q3k_ps_r1p4_param_0,\n" " .param .u64 fc_matvec_q3k_ps_r1p4_param_1,\n" " .param .u64 fc_matvec_q3k_ps_r1p4_param_2,\n" " .param .u32 fc_matvec_q3k_ps_r1p4_param_3,\n" " .param .u32 fc_matvec_q3k_ps_r1p4_param_4,\n" " .param .u32 fc_matvec_q3k_ps_r1p4_param_5\n" ")\n" "{\n" " .reg .pred %p<26>;\n" " .reg .f32 %f<383>;\n" " .reg .b32 %r<269>;\n" " .reg .b64 %rd<79>;\n" "\n" "\n" " ld.param.u64 %rd20, [fc_matvec_q3k_ps_r1p4_param_0];\n" " ld.param.u64 %rd21, [fc_matvec_q3k_ps_r1p4_param_1];\n" " ld.param.u64 %rd22, [fc_matvec_q3k_ps_r1p4_param_2];\n" " ld.param.u32 %r42, [fc_matvec_q3k_ps_r1p4_param_3];\n" " ld.param.u32 %r43, [fc_matvec_q3k_ps_r1p4_param_4];\n" " ld.param.u32 %r44, [fc_matvec_q3k_ps_r1p4_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r45, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r46, %ctaid.x;\n" " shl.b32 %r47, %r46, 3;\n" " add.s32 %r48, %r47, %r45;\n" " cvt.s64.s32 %rd1, %r48;\n" " setp.lt.s32 %p1, %r43, 1;\n" " mov.f32 %f381, 0f00000000;\n" " @%p1 bra $L__BB6_25;\n" "\n" " cvt.u64.u32 %rd2, %r2;\n" " shr.u32 %r51, %r2, 4;\n" " cvt.u64.u32 %rd3, %r51;\n" " mov.u32 %r52, -2;\n" " sub.s32 %r3, %r52, %r1;\n" " not.b32 %r4, %r43;\n" " not.b32 %r5, %r1;\n" " cvt.s64.s32 %rd4, %r1;\n" " shl.b32 %r53, %r1, 4;\n" " mov.u32 %r54, sh_xps;\n" " add.s32 %r6, %r54, %r53;\n" " add.s32 %r7, %r1, 256;\n" " shl.b32 %r55, %r1, 2;\n" " add.s32 %r8, %r54, %r55;\n" " add.s32 %r9, %r1, 512;\n" " add.s32 %r10, %r1, 768;\n" " cvt.s64.s32 %rd23, %r44;\n" " mul.lo.s64 %rd24, %rd23, %rd1;\n" " add.s64 %rd78, %rd20, %rd24;\n" " mov.f32 %f381, 0f00000000;\n" " mov.u32 %r256, 0;\n" " mov.u32 %r257, %r256;\n" "\n" "$L__BB6_2:\n" " add.s32 %r56, %r256, %r4;\n" " max.s32 %r57, %r56, -2049;\n" " sub.s32 %r13, %r3, %r57;\n" " sub.s32 %r58, %r43, %r257;\n" " min.s32 %r14, %r58, 2048;\n" " bar.sync 0;\n" " cvt.s64.s32 %rd7, %r257;\n" " mul.wide.s32 %rd25, %r257, 4;\n" " add.s64 %rd8, %rd21, %rd25;\n" " and.b32 %r59, %r14, 3;\n" " setp.eq.s32 %p2, %r59, 0;\n" " { .reg .b64 %tmp;\n" " cvt.u64.u32 %tmp, %r54;\n" " cvta.shared.u64 %rd26, %tmp; }\n" " or.b64 %rd27, %rd8, %rd26;\n" " and.b64 %rd28, %rd27, 15;\n" " setp.eq.s64 %p3, %rd28, 0;\n" " and.pred %p4, %p2, %p3;\n" " @%p4 bra $L__BB6_11;\n" " bra.uni $L__BB6_3;\n" "\n" "$L__BB6_11:\n" " shr.s32 %r22, %r14, 2;\n" " setp.ge.s32 %p11, %r1, %r22;\n" " @%p11 bra $L__BB6_18;\n" "\n" " add.s32 %r23, %r22, %r5;\n" " shr.u32 %r66, %r23, 8;\n" " add.s32 %r67, %r66, 1;\n" " and.b32 %r24, %r67, 3;\n" " setp.eq.s32 %p12, %r24, 0;\n" " mov.u32 %r261, %r1;\n" " @%p12 bra $L__BB6_16;\n" "\n" " shl.b64 %rd41, %rd4, 4;\n" " add.s64 %rd40, %rd8, %rd41;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f19,%f20,%f21,%f22}, [%rd40];\n" " // end inline asm\n" " st.shared.v4.f32 [%r6], {%f19, %f20, %f21, %f22};\n" " setp.eq.s32 %p13, %r24, 1;\n" " mov.u32 %r261, %r7;\n" " @%p13 bra $L__BB6_16;\n" "\n" " add.s64 %rd42, %rd40, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f23,%f24,%f25,%f26}, [%rd42];\n" " // end inline asm\n" " st.shared.v4.f32 [%r6+4096], {%f23, %f24, %f25, %f26};\n" " setp.eq.s32 %p14, %r24, 2;\n" " mov.u32 %r261, %r9;\n" " @%p14 bra $L__BB6_16;\n" "\n" " add.s64 %rd43, %rd40, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f27,%f28,%f29,%f30}, [%rd43];\n" " // end inline asm\n" " st.shared.v4.f32 [%r6+8192], {%f27, %f28, %f29, %f30};\n" " mov.u32 %r261, %r10;\n" "\n" "$L__BB6_16:\n" " setp.lt.u32 %p15, %r23, 768;\n" " @%p15 bra $L__BB6_18;\n" "\n" "$L__BB6_17:\n" " mul.wide.s32 %rd48, %r261, 16;\n" " add.s64 %rd44, %rd8, %rd48;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f31,%f32,%f33,%f34}, [%rd44];\n" " // end inline asm\n" " shl.b32 %r68, %r261, 4;\n" " mov.u32 %r69, sh_xps;\n" " add.s32 %r70, %r69, %r68;\n" " st.shared.v4.f32 [%r70], {%f31, %f32, %f33, %f34};\n" " add.s64 %rd45, %rd44, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f35,%f36,%f37,%f38}, [%rd45];\n" " // end inline asm\n" " st.shared.v4.f32 [%r70+4096], {%f35, %f36, %f37, %f38};\n" " add.s64 %rd46, %rd44, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f39,%f40,%f41,%f42}, [%rd46];\n" " // end inline asm\n" " st.shared.v4.f32 [%r70+8192], {%f39, %f40, %f41, %f42};\n" " add.s64 %rd47, %rd44, 12288;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f43,%f44,%f45,%f46}, [%rd47];\n" " // end inline asm\n" " st.shared.v4.f32 [%r70+12288], {%f43, %f44, %f45, %f46};\n" " add.s32 %r261, %r261, 1024;\n" " setp.lt.s32 %p16, %r261, %r22;\n" " @%p16 bra $L__BB6_17;\n" " bra.uni $L__BB6_18;\n" "\n" "$L__BB6_3:\n" " setp.ge.s32 %p5, %r1, %r14;\n" " @%p5 bra $L__BB6_18;\n" "\n" " shr.u32 %r61, %r13, 8;\n" " add.s32 %r62, %r61, 1;\n" " and.b32 %r15, %r62, 3;\n" " setp.eq.s32 %p6, %r15, 0;\n" " mov.u32 %r258, %r1;\n" " @%p6 bra $L__BB6_8;\n" "\n" " shl.b64 %rd30, %rd4, 2;\n" " add.s64 %rd29, %rd8, %rd30;\n" " // begin inline asm\n" " ld.global.nc.f32 %f12, [%rd29];\n" " // end inline asm\n" " st.shared.f32 [%r8], %f12;\n" " setp.eq.s32 %p7, %r15, 1;\n" " mov.u32 %r258, %r7;\n" " @%p7 bra $L__BB6_8;\n" "\n" " add.s64 %rd31, %rd29, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f13, [%rd31];\n" " // end inline asm\n" " st.shared.f32 [%r8+1024], %f13;\n" " setp.eq.s32 %p8, %r15, 2;\n" " mov.u32 %r258, %r9;\n" " @%p8 bra $L__BB6_8;\n" "\n" " add.s64 %rd32, %rd29, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f14, [%rd32];\n" " // end inline asm\n" " st.shared.f32 [%r8+2048], %f14;\n" " mov.u32 %r258, %r10;\n" "\n" "$L__BB6_8:\n" " setp.lt.u32 %p9, %r13, 768;\n" " @%p9 bra $L__BB6_18;\n" "\n" " cvt.s64.s32 %rd33, %r258;\n" " add.s64 %rd34, %rd33, %rd7;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd74, %rd21, %rd35;\n" " shl.b32 %r63, %r258, 2;\n" " add.s32 %r65, %r54, %r63;\n" " add.s32 %r259, %r65, 2048;\n" "\n" "$L__BB6_10:\n" " // begin inline asm\n" " ld.global.nc.f32 %f15, [%rd74];\n" " // end inline asm\n" " st.shared.f32 [%r259+-2048], %f15;\n" " add.s64 %rd37, %rd74, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f16, [%rd37];\n" " // end inline asm\n" " st.shared.f32 [%r259+-1024], %f16;\n" " add.s64 %rd38, %rd74, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd38];\n" " // end inline asm\n" " st.shared.f32 [%r259], %f17;\n" " add.s64 %rd39, %rd74, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r259+1024], %f18;\n" " add.s64 %rd74, %rd74, 4096;\n" " add.s32 %r259, %r259, 4096;\n" " add.s32 %r258, %r258, 1024;\n" " setp.lt.s32 %p10, %r258, %r14;\n" " @%p10 bra $L__BB6_10;\n" "\n" "$L__BB6_18:\n" " cvt.u32.u64 %r71, %rd1;\n" " setp.ge.s32 %p17, %r71, %r42;\n" " bar.sync 0;\n" " @%p17 bra $L__BB6_24;\n" "\n" " shr.s32 %r29, %r14, 8;\n" " setp.lt.s32 %p18, %r14, 1024;\n" " mov.u32 %r267, 0;\n" " mov.u32 %r265, %r54;\n" " @%p18 bra $L__BB6_22;\n" "\n" " mov.u32 %r265, sh_xps;\n" " mov.u32 %r263, 4;\n" "\n" "$L__BB6_21:\n" " mov.u32 %r267, %r263;\n" " shl.b64 %rd61, %rd2, 2;\n" " add.s64 %rd49, %rd78, %rd61;\n" " // begin inline asm\n" " ld.global.nc.u32 %r88, [%rd49];\n" " // end inline asm\n" " add.s64 %rd50, %rd78, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f51, [%rd50];\n" " // end inline asm\n" " shl.b64 %rd62, %rd3, 3;\n" " add.s64 %rd63, %rd78, %rd62;\n" " add.s64 %rd51, %rd63, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r89,%r90}, [%rd51];\n" " // end inline asm\n" " add.s64 %rd52, %rd49, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r94, [%rd52];\n" " // end inline asm\n" " add.s64 %rd53, %rd78, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f53, [%rd53];\n" " // end inline asm\n" " add.s64 %rd54, %rd63, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r95,%r96}, [%rd54];\n" " // end inline asm\n" " add.s64 %rd55, %rd49, 304;\n" " // begin inline asm\n" " ld.global.nc.u32 %r100, [%rd55];\n" " // end inline asm\n" " add.s64 %rd56, %rd78, 448;\n" " // begin inline asm\n" " ld.global.nc.f32 %f55, [%rd56];\n" " // end inline asm\n" " add.s64 %rd57, %rd63, 432;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r101,%r102}, [%rd57];\n" " // end inline asm\n" " add.s64 %rd58, %rd49, 456;\n" " // begin inline asm\n" " ld.global.nc.u32 %r106, [%rd58];\n" " // end inline asm\n" " add.s64 %rd59, %rd78, 600;\n" " // begin inline asm\n" " ld.global.nc.f32 %f57, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd63, 584;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r107,%r108}, [%rd60];\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " shr.u32 %r112, %r88, 16;\n" " and.b32 %r113, %r88, 15;\n" " or.b32 %r114, %r113, 1258291200;\n" " mov.b32 %f59, %r114;\n" " mov.f32 %f60, 0fCB000004;\n" " mov.f32 %f61, 0f3F800000;\n" " fma.rn.f32 %f62, %f59, %f61, %f60;\n" " and.b32 %r115, %r89, 255;\n" " cvt.rn.f32.u32 %f63, %r115;\n" " mov.f32 %f64, 0f42000000;\n" " sub.rn.f32 %f65, %f63, %f64;\n" " mul.rn.f32 %f66, %f65, %f62;\n" " shl.b32 %r116, %r2, 2;\n" " add.s32 %r117, %r265, %r116;\n" " ld.shared.f32 %f67, [%r117];\n" " mov.f32 %f68, 0f00000000;\n" " fma.rn.f32 %f69, %f66, %f67, %f68;\n" " and.b32 %r118, %r88, 240;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f70, %r119;\n" " mov.f32 %f71, 0fC9000040;\n" " mov.f32 %f72, 0f3D800000;\n" " fma.rn.f32 %f73, %f70, %f72, %f71;\n" " bfe.u32 %r120, %r89, 8, 8;\n" " cvt.rn.f32.u32 %f74, %r120;\n" " sub.rn.f32 %f75, %f74, %f64;\n" " mul.rn.f32 %f76, %f75, %f73;\n" " ld.shared.f32 %f77, [%r117+128];\n" " fma.rn.f32 %f78, %f76, %f77, %f69;\n" " and.b32 %r121, %r88, 3840;\n" " or.b32 %r122, %r121, 1258291200;\n" " mov.b32 %f79, %r122;\n" " mov.f32 %f80, 0fC7000400;\n" " mov.f32 %f81, 0f3B800000;\n" " fma.rn.f32 %f82, %f79, %f81, %f80;\n" " bfe.u32 %r123, %r89, 16, 8;\n" " cvt.rn.f32.u32 %f83, %r123;\n" " sub.rn.f32 %f84, %f83, %f64;\n" " mul.rn.f32 %f85, %f84, %f82;\n" " ld.shared.f32 %f86, [%r117+256];\n" " fma.rn.f32 %f87, %f85, %f86, %f78;\n" " and.b32 %r124, %r88, 61440;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f88, %r125;\n" " mov.f32 %f89, 0fC5004000;\n" " mov.f32 %f90, 0f39800000;\n" " fma.rn.f32 %f91, %f88, %f90, %f89;\n" " shr.u32 %r126, %r89, 24;\n" " cvt.rn.f32.u32 %f92, %r126;\n" " sub.rn.f32 %f93, %f92, %f64;\n" " mul.rn.f32 %f94, %f93, %f91;\n" " ld.shared.f32 %f95, [%r117+384];\n" " fma.rn.f32 %f96, %f94, %f95, %f87;\n" " and.b32 %r127, %r88, 983040;\n" " or.b32 %r128, %r127, 1258291200;\n" " mov.b32 %f97, %r128;\n" " mov.f32 %f98, 0fC3040000;\n" " mov.f32 %f99, 0f37800000;\n" " fma.rn.f32 %f100, %f97, %f99, %f98;\n" " and.b32 %r129, %r90, 255;\n" " cvt.rn.f32.u32 %f101, %r129;\n" " sub.rn.f32 %f102, %f101, %f64;\n" " mul.rn.f32 %f103, %f102, %f100;\n" " ld.shared.f32 %f104, [%r117+512];\n" " fma.rn.f32 %f105, %f103, %f104, %f96;\n" " and.b32 %r130, %r112, 240;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f106, %r131;\n" " fma.rn.f32 %f107, %f106, %f72, %f71;\n" " bfe.u32 %r132, %r90, 8, 8;\n" " cvt.rn.f32.u32 %f108, %r132;\n" " sub.rn.f32 %f109, %f108, %f64;\n" " mul.rn.f32 %f110, %f109, %f107;\n" " ld.shared.f32 %f111, [%r117+640];\n" " fma.rn.f32 %f112, %f110, %f111, %f105;\n" " and.b32 %r133, %r112, 3840;\n" " or.b32 %r134, %r133, 1258291200;\n" " mov.b32 %f113, %r134;\n" " fma.rn.f32 %f114, %f113, %f81, %f80;\n" " bfe.u32 %r135, %r90, 16, 8;\n" " cvt.rn.f32.u32 %f115, %r135;\n" " sub.rn.f32 %f116, %f115, %f64;\n" " mul.rn.f32 %f117, %f116, %f114;\n" " ld.shared.f32 %f118, [%r117+768];\n" " fma.rn.f32 %f119, %f117, %f118, %f112;\n" " and.b32 %r136, %r112, 61440;\n" " or.b32 %r137, %r136, 1258291200;\n" " mov.b32 %f120, %r137;\n" " fma.rn.f32 %f121, %f120, %f90, %f89;\n" " shr.u32 %r138, %r90, 24;\n" " cvt.rn.f32.u32 %f122, %r138;\n" " sub.rn.f32 %f123, %f122, %f64;\n" " mul.rn.f32 %f124, %f123, %f121;\n" " ld.shared.f32 %f125, [%r117+896];\n" " fma.rn.f32 %f126, %f124, %f125, %f119;\n" " fma.rn.f32 %f127, %f126, %f51, %f381;\n" " shr.u32 %r139, %r94, 16;\n" " and.b32 %r140, %r94, 15;\n" " or.b32 %r141, %r140, 1258291200;\n" " mov.b32 %f128, %r141;\n" " fma.rn.f32 %f129, %f128, %f61, %f60;\n" " and.b32 %r142, %r95, 255;\n" " cvt.rn.f32.u32 %f130, %r142;\n" " sub.rn.f32 %f131, %f130, %f64;\n" " mul.rn.f32 %f132, %f131, %f129;\n" " ld.shared.f32 %f133, [%r117+1024];\n" " fma.rn.f32 %f134, %f132, %f133, %f68;\n" " and.b32 %r143, %r94, 240;\n" " or.b32 %r144, %r143, 1258291200;\n" " mov.b32 %f135, %r144;\n" " fma.rn.f32 %f136, %f135, %f72, %f71;\n" " bfe.u32 %r145, %r95, 8, 8;\n" " cvt.rn.f32.u32 %f137, %r145;\n" " sub.rn.f32 %f138, %f137, %f64;\n" " mul.rn.f32 %f139, %f138, %f136;\n" " ld.shared.f32 %f140, [%r117+1152];\n" " fma.rn.f32 %f141, %f139, %f140, %f134;\n" " and.b32 %r146, %r94, 3840;\n" " or.b32 %r147, %r146, 1258291200;\n" " mov.b32 %f142, %r147;\n" " fma.rn.f32 %f143, %f142, %f81, %f80;\n" " bfe.u32 %r148, %r95, 16, 8;\n" " cvt.rn.f32.u32 %f144, %r148;\n" " sub.rn.f32 %f145, %f144, %f64;\n" " mul.rn.f32 %f146, %f145, %f143;\n" " ld.shared.f32 %f147, [%r117+1280];\n" " fma.rn.f32 %f148, %f146, %f147, %f141;\n" " and.b32 %r149, %r94, 61440;\n" " or.b32 %r150, %r149, 1258291200;\n" " mov.b32 %f149, %r150;\n" " fma.rn.f32 %f150, %f149, %f90, %f89;\n" " shr.u32 %r151, %r95, 24;\n" " cvt.rn.f32.u32 %f151, %r151;\n" " sub.rn.f32 %f152, %f151, %f64;\n" " mul.rn.f32 %f153, %f152, %f150;\n" " ld.shared.f32 %f154, [%r117+1408];\n" " fma.rn.f32 %f155, %f153, %f154, %f148;\n" " and.b32 %r152, %r94, 983040;\n" " or.b32 %r153, %r152, 1258291200;\n" " mov.b32 %f156, %r153;\n" " fma.rn.f32 %f157, %f156, %f99, %f98;\n" " and.b32 %r154, %r96, 255;\n" " cvt.rn.f32.u32 %f158, %r154;\n" " sub.rn.f32 %f159, %f158, %f64;\n" " mul.rn.f32 %f160, %f159, %f157;\n" " ld.shared.f32 %f161, [%r117+1536];\n" " fma.rn.f32 %f162, %f160, %f161, %f155;\n" " and.b32 %r155, %r139, 240;\n" " or.b32 %r156, %r155, 1258291200;\n" " mov.b32 %f163, %r156;\n" " fma.rn.f32 %f164, %f163, %f72, %f71;\n" " bfe.u32 %r157, %r96, 8, 8;\n" " cvt.rn.f32.u32 %f165, %r157;\n" " sub.rn.f32 %f166, %f165, %f64;\n" " mul.rn.f32 %f167, %f166, %f164;\n" " ld.shared.f32 %f168, [%r117+1664];\n" " fma.rn.f32 %f169, %f167, %f168, %f162;\n" " and.b32 %r158, %r139, 3840;\n" " or.b32 %r159, %r158, 1258291200;\n" " mov.b32 %f170, %r159;\n" " fma.rn.f32 %f171, %f170, %f81, %f80;\n" " bfe.u32 %r160, %r96, 16, 8;\n" " cvt.rn.f32.u32 %f172, %r160;\n" " sub.rn.f32 %f173, %f172, %f64;\n" " mul.rn.f32 %f174, %f173, %f171;\n" " ld.shared.f32 %f175, [%r117+1792];\n" " fma.rn.f32 %f176, %f174, %f175, %f169;\n" " and.b32 %r161, %r139, 61440;\n" " or.b32 %r162, %r161, 1258291200;\n" " mov.b32 %f177, %r162;\n" " fma.rn.f32 %f178, %f177, %f90, %f89;\n" " shr.u32 %r163, %r96, 24;\n" " cvt.rn.f32.u32 %f179, %r163;\n" " sub.rn.f32 %f180, %f179, %f64;\n" " mul.rn.f32 %f181, %f180, %f178;\n" " ld.shared.f32 %f182, [%r117+1920];\n" " fma.rn.f32 %f183, %f181, %f182, %f176;\n" " fma.rn.f32 %f184, %f183, %f53, %f127;\n" " shr.u32 %r164, %r100, 16;\n" " and.b32 %r165, %r100, 15;\n" " or.b32 %r166, %r165, 1258291200;\n" " mov.b32 %f185, %r166;\n" " fma.rn.f32 %f186, %f185, %f61, %f60;\n" " and.b32 %r167, %r101, 255;\n" " cvt.rn.f32.u32 %f187, %r167;\n" " sub.rn.f32 %f188, %f187, %f64;\n" " mul.rn.f32 %f189, %f188, %f186;\n" " ld.shared.f32 %f190, [%r117+2048];\n" " fma.rn.f32 %f191, %f189, %f190, %f68;\n" " and.b32 %r168, %r100, 240;\n" " or.b32 %r169, %r168, 1258291200;\n" " mov.b32 %f192, %r169;\n" " fma.rn.f32 %f193, %f192, %f72, %f71;\n" " bfe.u32 %r170, %r101, 8, 8;\n" " cvt.rn.f32.u32 %f194, %r170;\n" " sub.rn.f32 %f195, %f194, %f64;\n" " mul.rn.f32 %f196, %f195, %f193;\n" " ld.shared.f32 %f197, [%r117+2176];\n" " fma.rn.f32 %f198, %f196, %f197, %f191;\n" " and.b32 %r171, %r100, 3840;\n" " or.b32 %r172, %r171, 1258291200;\n" " mov.b32 %f199, %r172;\n" " fma.rn.f32 %f200, %f199, %f81, %f80;\n" " bfe.u32 %r173, %r101, 16, 8;\n" " cvt.rn.f32.u32 %f201, %r173;\n" " sub.rn.f32 %f202, %f201, %f64;\n" " mul.rn.f32 %f203, %f202, %f200;\n" " ld.shared.f32 %f204, [%r117+2304];\n" " fma.rn.f32 %f205, %f203, %f204, %f198;\n" " and.b32 %r174, %r100, 61440;\n" " or.b32 %r175, %r174, 1258291200;\n" " mov.b32 %f206, %r175;\n" " fma.rn.f32 %f207, %f206, %f90, %f89;\n" " shr.u32 %r176, %r101, 24;\n" " cvt.rn.f32.u32 %f208, %r176;\n" " sub.rn.f32 %f209, %f208, %f64;\n" " mul.rn.f32 %f210, %f209, %f207;\n" " ld.shared.f32 %f211, [%r117+2432];\n" " fma.rn.f32 %f212, %f210, %f211, %f205;\n" " and.b32 %r177, %r100, 983040;\n" " or.b32 %r178, %r177, 1258291200;\n" " mov.b32 %f213, %r178;\n" " fma.rn.f32 %f214, %f213, %f99, %f98;\n" " and.b32 %r179, %r102, 255;\n" " cvt.rn.f32.u32 %f215, %r179;\n" " sub.rn.f32 %f216, %f215, %f64;\n" " mul.rn.f32 %f217, %f216, %f214;\n" " ld.shared.f32 %f218, [%r117+2560];\n" " fma.rn.f32 %f219, %f217, %f218, %f212;\n" " and.b32 %r180, %r164, 240;\n" " or.b32 %r181, %r180, 1258291200;\n" " mov.b32 %f220, %r181;\n" " fma.rn.f32 %f221, %f220, %f72, %f71;\n" " bfe.u32 %r182, %r102, 8, 8;\n" " cvt.rn.f32.u32 %f222, %r182;\n" " sub.rn.f32 %f223, %f222, %f64;\n" " mul.rn.f32 %f224, %f223, %f221;\n" " ld.shared.f32 %f225, [%r117+2688];\n" " fma.rn.f32 %f226, %f224, %f225, %f219;\n" " and.b32 %r183, %r164, 3840;\n" " or.b32 %r184, %r183, 1258291200;\n" " mov.b32 %f227, %r184;\n" " fma.rn.f32 %f228, %f227, %f81, %f80;\n" " bfe.u32 %r185, %r102, 16, 8;\n" " cvt.rn.f32.u32 %f229, %r185;\n" " sub.rn.f32 %f230, %f229, %f64;\n" " mul.rn.f32 %f231, %f230, %f228;\n" " ld.shared.f32 %f232, [%r117+2816];\n" " fma.rn.f32 %f233, %f231, %f232, %f226;\n" " and.b32 %r186, %r164, 61440;\n" " or.b32 %r187, %r186, 1258291200;\n" " mov.b32 %f234, %r187;\n" " fma.rn.f32 %f235, %f234, %f90, %f89;\n" " shr.u32 %r188, %r102, 24;\n" " cvt.rn.f32.u32 %f236, %r188;\n" " sub.rn.f32 %f237, %f236, %f64;\n" " mul.rn.f32 %f238, %f237, %f235;\n" " ld.shared.f32 %f239, [%r117+2944];\n" " fma.rn.f32 %f240, %f238, %f239, %f233;\n" " fma.rn.f32 %f241, %f240, %f55, %f184;\n" " shr.u32 %r189, %r106, 16;\n" " and.b32 %r190, %r106, 15;\n" " or.b32 %r191, %r190, 1258291200;\n" " mov.b32 %f242, %r191;\n" " fma.rn.f32 %f243, %f242, %f61, %f60;\n" " and.b32 %r192, %r107, 255;\n" " cvt.rn.f32.u32 %f244, %r192;\n" " sub.rn.f32 %f245, %f244, %f64;\n" " mul.rn.f32 %f246, %f245, %f243;\n" " ld.shared.f32 %f247, [%r117+3072];\n" " fma.rn.f32 %f248, %f246, %f247, %f68;\n" " and.b32 %r193, %r106, 240;\n" " or.b32 %r194, %r193, 1258291200;\n" " mov.b32 %f249, %r194;\n" " fma.rn.f32 %f250, %f249, %f72, %f71;\n" " bfe.u32 %r195, %r107, 8, 8;\n" " cvt.rn.f32.u32 %f251, %r195;\n" " sub.rn.f32 %f252, %f251, %f64;\n" " mul.rn.f32 %f253, %f252, %f250;\n" " ld.shared.f32 %f254, [%r117+3200];\n" " fma.rn.f32 %f255, %f253, %f254, %f248;\n" " and.b32 %r196, %r106, 3840;\n" " or.b32 %r197, %r196, 1258291200;\n" " mov.b32 %f256, %r197;\n" " fma.rn.f32 %f257, %f256, %f81, %f80;\n" " bfe.u32 %r198, %r107, 16, 8;\n" " cvt.rn.f32.u32 %f258, %r198;\n" " sub.rn.f32 %f259, %f258, %f64;\n" " mul.rn.f32 %f260, %f259, %f257;\n" " ld.shared.f32 %f261, [%r117+3328];\n" " fma.rn.f32 %f262, %f260, %f261, %f255;\n" " and.b32 %r199, %r106, 61440;\n" " or.b32 %r200, %r199, 1258291200;\n" " mov.b32 %f263, %r200;\n" " fma.rn.f32 %f264, %f263, %f90, %f89;\n" " shr.u32 %r201, %r107, 24;\n" " cvt.rn.f32.u32 %f265, %r201;\n" " sub.rn.f32 %f266, %f265, %f64;\n" " mul.rn.f32 %f267, %f266, %f264;\n" " ld.shared.f32 %f268, [%r117+3456];\n" " fma.rn.f32 %f269, %f267, %f268, %f262;\n" " and.b32 %r202, %r106, 983040;\n" " or.b32 %r203, %r202, 1258291200;\n" " mov.b32 %f270, %r203;\n" " fma.rn.f32 %f271, %f270, %f99, %f98;\n" " and.b32 %r204, %r108, 255;\n" " cvt.rn.f32.u32 %f272, %r204;\n" " sub.rn.f32 %f273, %f272, %f64;\n" " mul.rn.f32 %f274, %f273, %f271;\n" " ld.shared.f32 %f275, [%r117+3584];\n" " fma.rn.f32 %f276, %f274, %f275, %f269;\n" " and.b32 %r205, %r189, 240;\n" " or.b32 %r206, %r205, 1258291200;\n" " mov.b32 %f277, %r206;\n" " fma.rn.f32 %f278, %f277, %f72, %f71;\n" " bfe.u32 %r207, %r108, 8, 8;\n" " cvt.rn.f32.u32 %f279, %r207;\n" " sub.rn.f32 %f280, %f279, %f64;\n" " mul.rn.f32 %f281, %f280, %f278;\n" " ld.shared.f32 %f282, [%r117+3712];\n" " fma.rn.f32 %f283, %f281, %f282, %f276;\n" " and.b32 %r208, %r189, 3840;\n" " or.b32 %r209, %r208, 1258291200;\n" " mov.b32 %f284, %r209;\n" " fma.rn.f32 %f285, %f284, %f81, %f80;\n" " bfe.u32 %r210, %r108, 16, 8;\n" " cvt.rn.f32.u32 %f286, %r210;\n" " sub.rn.f32 %f287, %f286, %f64;\n" " mul.rn.f32 %f288, %f287, %f285;\n" " ld.shared.f32 %f289, [%r117+3840];\n" " fma.rn.f32 %f290, %f288, %f289, %f283;\n" " and.b32 %r211, %r189, 61440;\n" " or.b32 %r212, %r211, 1258291200;\n" " mov.b32 %f291, %r212;\n" " fma.rn.f32 %f292, %f291, %f90, %f89;\n" " shr.u32 %r213, %r108, 24;\n" " cvt.rn.f32.u32 %f293, %r213;\n" " sub.rn.f32 %f294, %f293, %f64;\n" " mul.rn.f32 %f295, %f294, %f292;\n" " ld.shared.f32 %f296, [%r117+3968];\n" " fma.rn.f32 %f297, %f295, %f296, %f290;\n" " fma.rn.f32 %f381, %f297, %f57, %f241;\n" " add.s64 %rd78, %rd78, 608;\n" " add.s32 %r265, %r265, 4096;\n" " add.s32 %r263, %r267, 4;\n" " setp.le.s32 %p19, %r263, %r29;\n" " @%p19 bra $L__BB6_21;\n" "\n" "$L__BB6_22:\n" " setp.ge.s32 %p20, %r267, %r29;\n" " @%p20 bra $L__BB6_24;\n" "\n" "$L__BB6_23:\n" " shl.b64 %rd67, %rd2, 2;\n" " add.s64 %rd64, %rd78, %rd67;\n" " // begin inline asm\n" " ld.global.nc.u32 %r214, [%rd64];\n" " // end inline asm\n" " add.s64 %rd65, %rd78, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f298, [%rd65];\n" " // end inline asm\n" " shl.b64 %rd68, %rd3, 3;\n" " add.s64 %rd69, %rd78, %rd68;\n" " add.s64 %rd66, %rd69, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r215,%r216}, [%rd66];\n" " // end inline asm\n" " shr.u32 %r217, %r214, 16;\n" " and.b32 %r218, %r214, 15;\n" " or.b32 %r219, %r218, 1258291200;\n" " mov.b32 %f299, %r219;\n" " mov.f32 %f300, 0fCB000004;\n" " mov.f32 %f301, 0f3F800000;\n" " fma.rn.f32 %f302, %f299, %f301, %f300;\n" " and.b32 %r220, %r215, 255;\n" " cvt.rn.f32.u32 %f303, %r220;\n" " mov.f32 %f304, 0f42000000;\n" " sub.rn.f32 %f305, %f303, %f304;\n" " mul.rn.f32 %f306, %f305, %f302;\n" " shl.b32 %r221, %r2, 2;\n" " add.s32 %r222, %r265, %r221;\n" " ld.shared.f32 %f307, [%r222];\n" " mov.f32 %f308, 0f00000000;\n" " fma.rn.f32 %f309, %f306, %f307, %f308;\n" " and.b32 %r223, %r214, 240;\n" " or.b32 %r224, %r223, 1258291200;\n" " mov.b32 %f310, %r224;\n" " mov.f32 %f311, 0fC9000040;\n" " mov.f32 %f312, 0f3D800000;\n" " fma.rn.f32 %f313, %f310, %f312, %f311;\n" " bfe.u32 %r225, %r215, 8, 8;\n" " cvt.rn.f32.u32 %f314, %r225;\n" " sub.rn.f32 %f315, %f314, %f304;\n" " mul.rn.f32 %f316, %f315, %f313;\n" " ld.shared.f32 %f317, [%r222+128];\n" " fma.rn.f32 %f318, %f316, %f317, %f309;\n" " and.b32 %r226, %r214, 3840;\n" " or.b32 %r227, %r226, 1258291200;\n" " mov.b32 %f319, %r227;\n" " mov.f32 %f320, 0fC7000400;\n" " mov.f32 %f321, 0f3B800000;\n" " fma.rn.f32 %f322, %f319, %f321, %f320;\n" " bfe.u32 %r228, %r215, 16, 8;\n" " cvt.rn.f32.u32 %f323, %r228;\n" " sub.rn.f32 %f324, %f323, %f304;\n" " mul.rn.f32 %f325, %f324, %f322;\n" " ld.shared.f32 %f326, [%r222+256];\n" " fma.rn.f32 %f327, %f325, %f326, %f318;\n" " and.b32 %r229, %r214, 61440;\n" " or.b32 %r230, %r229, 1258291200;\n" " mov.b32 %f328, %r230;\n" " mov.f32 %f329, 0fC5004000;\n" " mov.f32 %f330, 0f39800000;\n" " fma.rn.f32 %f331, %f328, %f330, %f329;\n" " shr.u32 %r231, %r215, 24;\n" " cvt.rn.f32.u32 %f332, %r231;\n" " sub.rn.f32 %f333, %f332, %f304;\n" " mul.rn.f32 %f334, %f333, %f331;\n" " ld.shared.f32 %f335, [%r222+384];\n" " fma.rn.f32 %f336, %f334, %f335, %f327;\n" " and.b32 %r232, %r214, 983040;\n" " or.b32 %r233, %r232, 1258291200;\n" " mov.b32 %f337, %r233;\n" " mov.f32 %f338, 0fC3040000;\n" " mov.f32 %f339, 0f37800000;\n" " fma.rn.f32 %f340, %f337, %f339, %f338;\n" " and.b32 %r234, %r216, 255;\n" " cvt.rn.f32.u32 %f341, %r234;\n" " sub.rn.f32 %f342, %f341, %f304;\n" " mul.rn.f32 %f343, %f342, %f340;\n" " ld.shared.f32 %f344, [%r222+512];\n" " fma.rn.f32 %f345, %f343, %f344, %f336;\n" " and.b32 %r235, %r217, 240;\n" " or.b32 %r236, %r235, 1258291200;\n" " mov.b32 %f346, %r236;\n" " fma.rn.f32 %f347, %f346, %f312, %f311;\n" " bfe.u32 %r237, %r216, 8, 8;\n" " cvt.rn.f32.u32 %f348, %r237;\n" " sub.rn.f32 %f349, %f348, %f304;\n" " mul.rn.f32 %f350, %f349, %f347;\n" " ld.shared.f32 %f351, [%r222+640];\n" " fma.rn.f32 %f352, %f350, %f351, %f345;\n" " and.b32 %r238, %r217, 3840;\n" " or.b32 %r239, %r238, 1258291200;\n" " mov.b32 %f353, %r239;\n" " fma.rn.f32 %f354, %f353, %f321, %f320;\n" " bfe.u32 %r240, %r216, 16, 8;\n" " cvt.rn.f32.u32 %f355, %r240;\n" " sub.rn.f32 %f356, %f355, %f304;\n" " mul.rn.f32 %f357, %f356, %f354;\n" " ld.shared.f32 %f358, [%r222+768];\n" " fma.rn.f32 %f359, %f357, %f358, %f352;\n" " and.b32 %r241, %r217, 61440;\n" " or.b32 %r242, %r241, 1258291200;\n" " mov.b32 %f360, %r242;\n" " fma.rn.f32 %f361, %f360, %f330, %f329;\n" " shr.u32 %r243, %r216, 24;\n" " cvt.rn.f32.u32 %f362, %r243;\n" " sub.rn.f32 %f363, %f362, %f304;\n" " mul.rn.f32 %f364, %f363, %f361;\n" " ld.shared.f32 %f365, [%r222+896];\n" " fma.rn.f32 %f366, %f364, %f365, %f359;\n" " fma.rn.f32 %f381, %f298, %f366, %f381;\n" " add.s64 %rd78, %rd78, 152;\n" " add.s32 %r265, %r265, 1024;\n" " add.s32 %r267, %r267, 1;\n" " setp.lt.s32 %p21, %r267, %r29;\n" " @%p21 bra $L__BB6_23;\n" "\n" "$L__BB6_24:\n" " cvt.u32.u64 %r244, %rd7;\n" " add.s32 %r257, %r244, 2048;\n" " setp.lt.s32 %p22, %r257, %r43;\n" " add.s32 %r256, %r256, 2048;\n" " @%p22 bra $L__BB6_2;\n" "\n" "$L__BB6_25:\n" " cvt.u32.u64 %r255, %rd1;\n" " setp.ge.s32 %p23, %r255, %r42;\n" " mov.u32 %r245, 16;\n" " mov.u32 %r254, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f367, %f381, %r245, %r254;\n" " // end inline asm\n" " add.f32 %f370, %f381, %f367;\n" " mov.u32 %r247, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f369, %f370, %r247, %r254;\n" " // end inline asm\n" " add.f32 %f372, %f370, %f369;\n" " mov.u32 %r249, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f371, %f372, %r249, %r254;\n" " // end inline asm\n" " add.f32 %f374, %f372, %f371;\n" " mov.u32 %r251, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f373, %f374, %r251, %r254;\n" " // end inline asm\n" " add.f32 %f376, %f374, %f373;\n" " mov.u32 %r253, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f375, %f376, %r253, %r254;\n" " // end inline asm\n" " add.f32 %f9, %f376, %f375;\n" " setp.ne.s32 %p24, %r2, 0;\n" " or.pred %p25, %p24, %p23;\n" " @%p25 bra $L__BB6_27;\n" "\n" " cvta.to.global.u64 %rd70, %rd22;\n" " shl.b64 %rd71, %rd1, 2;\n" " add.s64 %rd72, %rd70, %rd71;\n" " st.global.f32 [%rd72], %f9;\n" "\n" "$L__BB6_27:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_ps_r1p8\n" ".visible .entry fc_matvec_q3k_ps_r1p8(\n" " .param .u64 fc_matvec_q3k_ps_r1p8_param_0,\n" " .param .u64 fc_matvec_q3k_ps_r1p8_param_1,\n" " .param .u64 fc_matvec_q3k_ps_r1p8_param_2,\n" " .param .u32 fc_matvec_q3k_ps_r1p8_param_3,\n" " .param .u32 fc_matvec_q3k_ps_r1p8_param_4,\n" " .param .u32 fc_matvec_q3k_ps_r1p8_param_5\n" ")\n" "{\n" " .reg .pred %p<26>;\n" " .reg .f32 %f<623>;\n" " .reg .b32 %r<405>;\n" " .reg .b64 %rd<91>;\n" "\n" "\n" " ld.param.u64 %rd20, [fc_matvec_q3k_ps_r1p8_param_0];\n" " ld.param.u64 %rd21, [fc_matvec_q3k_ps_r1p8_param_1];\n" " ld.param.u64 %rd22, [fc_matvec_q3k_ps_r1p8_param_2];\n" " ld.param.u32 %r43, [fc_matvec_q3k_ps_r1p8_param_3];\n" " ld.param.u32 %r44, [fc_matvec_q3k_ps_r1p8_param_4];\n" " ld.param.u32 %r45, [fc_matvec_q3k_ps_r1p8_param_5];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r46, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r47, %ctaid.x;\n" " shl.b32 %r48, %r47, 3;\n" " add.s32 %r49, %r48, %r46;\n" " cvt.s64.s32 %rd1, %r49;\n" " setp.lt.s32 %p1, %r44, 1;\n" " mov.f32 %f621, 0f00000000;\n" " @%p1 bra $L__BB7_25;\n" "\n" " cvt.u64.u32 %rd2, %r2;\n" " shr.u32 %r52, %r2, 4;\n" " cvt.u64.u32 %rd3, %r52;\n" " mov.u32 %r53, -2;\n" " sub.s32 %r3, %r53, %r1;\n" " not.b32 %r4, %r44;\n" " not.b32 %r5, %r1;\n" " cvt.s64.s32 %rd4, %r1;\n" " shl.b32 %r54, %r1, 4;\n" " mov.u32 %r55, sh_xps;\n" " add.s32 %r6, %r55, %r54;\n" " add.s32 %r7, %r1, 256;\n" " shl.b32 %r56, %r1, 2;\n" " add.s32 %r8, %r55, %r56;\n" " add.s32 %r9, %r1, 512;\n" " add.s32 %r10, %r1, 768;\n" " cvt.s64.s32 %rd23, %r45;\n" " mul.lo.s64 %rd24, %rd23, %rd1;\n" " add.s64 %rd90, %rd20, %rd24;\n" " mov.f32 %f621, 0f00000000;\n" " mov.u32 %r392, 0;\n" " mov.u32 %r393, %r392;\n" "\n" "$L__BB7_2:\n" " add.s32 %r57, %r392, %r4;\n" " max.s32 %r58, %r57, -2049;\n" " sub.s32 %r13, %r3, %r58;\n" " sub.s32 %r14, %r44, %r393;\n" " min.s32 %r15, %r14, 2048;\n" " bar.sync 0;\n" " cvt.s64.s32 %rd7, %r393;\n" " mul.wide.s32 %rd25, %r393, 4;\n" " add.s64 %rd8, %rd21, %rd25;\n" " and.b32 %r59, %r15, 3;\n" " setp.eq.s32 %p2, %r59, 0;\n" " { .reg .b64 %tmp;\n" " cvt.u64.u32 %tmp, %r55;\n" " cvta.shared.u64 %rd26, %tmp; }\n" " or.b64 %rd27, %rd8, %rd26;\n" " and.b64 %rd28, %rd27, 15;\n" " setp.eq.s64 %p3, %rd28, 0;\n" " and.pred %p4, %p2, %p3;\n" " @%p4 bra $L__BB7_11;\n" " bra.uni $L__BB7_3;\n" "\n" "$L__BB7_11:\n" " shr.s32 %r23, %r15, 2;\n" " setp.ge.s32 %p11, %r1, %r23;\n" " @%p11 bra $L__BB7_18;\n" "\n" " add.s32 %r24, %r23, %r5;\n" " shr.u32 %r66, %r24, 8;\n" " add.s32 %r67, %r66, 1;\n" " and.b32 %r25, %r67, 3;\n" " setp.eq.s32 %p12, %r25, 0;\n" " mov.u32 %r397, %r1;\n" " @%p12 bra $L__BB7_16;\n" "\n" " shl.b64 %rd41, %rd4, 4;\n" " add.s64 %rd40, %rd8, %rd41;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f19,%f20,%f21,%f22}, [%rd40];\n" " // end inline asm\n" " st.shared.v4.f32 [%r6], {%f19, %f20, %f21, %f22};\n" " setp.eq.s32 %p13, %r25, 1;\n" " mov.u32 %r397, %r7;\n" " @%p13 bra $L__BB7_16;\n" "\n" " add.s64 %rd42, %rd40, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f23,%f24,%f25,%f26}, [%rd42];\n" " // end inline asm\n" " st.shared.v4.f32 [%r6+4096], {%f23, %f24, %f25, %f26};\n" " setp.eq.s32 %p14, %r25, 2;\n" " mov.u32 %r397, %r9;\n" " @%p14 bra $L__BB7_16;\n" "\n" " add.s64 %rd43, %rd40, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f27,%f28,%f29,%f30}, [%rd43];\n" " // end inline asm\n" " st.shared.v4.f32 [%r6+8192], {%f27, %f28, %f29, %f30};\n" " mov.u32 %r397, %r10;\n" "\n" "$L__BB7_16:\n" " setp.lt.u32 %p15, %r24, 768;\n" " @%p15 bra $L__BB7_18;\n" "\n" "$L__BB7_17:\n" " mul.wide.s32 %rd48, %r397, 16;\n" " add.s64 %rd44, %rd8, %rd48;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f31,%f32,%f33,%f34}, [%rd44];\n" " // end inline asm\n" " shl.b32 %r68, %r397, 4;\n" " mov.u32 %r69, sh_xps;\n" " add.s32 %r70, %r69, %r68;\n" " st.shared.v4.f32 [%r70], {%f31, %f32, %f33, %f34};\n" " add.s64 %rd45, %rd44, 4096;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f35,%f36,%f37,%f38}, [%rd45];\n" " // end inline asm\n" " st.shared.v4.f32 [%r70+4096], {%f35, %f36, %f37, %f38};\n" " add.s64 %rd46, %rd44, 8192;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f39,%f40,%f41,%f42}, [%rd46];\n" " // end inline asm\n" " st.shared.v4.f32 [%r70+8192], {%f39, %f40, %f41, %f42};\n" " add.s64 %rd47, %rd44, 12288;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f43,%f44,%f45,%f46}, [%rd47];\n" " // end inline asm\n" " st.shared.v4.f32 [%r70+12288], {%f43, %f44, %f45, %f46};\n" " add.s32 %r397, %r397, 1024;\n" " setp.lt.s32 %p16, %r397, %r23;\n" " @%p16 bra $L__BB7_17;\n" " bra.uni $L__BB7_18;\n" "\n" "$L__BB7_3:\n" " setp.ge.s32 %p5, %r1, %r15;\n" " @%p5 bra $L__BB7_18;\n" "\n" " shr.u32 %r61, %r13, 8;\n" " add.s32 %r62, %r61, 1;\n" " and.b32 %r16, %r62, 3;\n" " setp.eq.s32 %p6, %r16, 0;\n" " mov.u32 %r394, %r1;\n" " @%p6 bra $L__BB7_8;\n" "\n" " shl.b64 %rd30, %rd4, 2;\n" " add.s64 %rd29, %rd8, %rd30;\n" " // begin inline asm\n" " ld.global.nc.f32 %f12, [%rd29];\n" " // end inline asm\n" " st.shared.f32 [%r8], %f12;\n" " setp.eq.s32 %p7, %r16, 1;\n" " mov.u32 %r394, %r7;\n" " @%p7 bra $L__BB7_8;\n" "\n" " add.s64 %rd31, %rd29, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f13, [%rd31];\n" " // end inline asm\n" " st.shared.f32 [%r8+1024], %f13;\n" " setp.eq.s32 %p8, %r16, 2;\n" " mov.u32 %r394, %r9;\n" " @%p8 bra $L__BB7_8;\n" "\n" " add.s64 %rd32, %rd29, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f14, [%rd32];\n" " // end inline asm\n" " st.shared.f32 [%r8+2048], %f14;\n" " mov.u32 %r394, %r10;\n" "\n" "$L__BB7_8:\n" " setp.lt.u32 %p9, %r13, 768;\n" " @%p9 bra $L__BB7_18;\n" "\n" " cvt.s64.s32 %rd33, %r394;\n" " add.s64 %rd34, %rd33, %rd7;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd86, %rd21, %rd35;\n" " shl.b32 %r63, %r394, 2;\n" " add.s32 %r65, %r55, %r63;\n" " add.s32 %r395, %r65, 2048;\n" "\n" "$L__BB7_10:\n" " // begin inline asm\n" " ld.global.nc.f32 %f15, [%rd86];\n" " // end inline asm\n" " st.shared.f32 [%r395+-2048], %f15;\n" " add.s64 %rd37, %rd86, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f16, [%rd37];\n" " // end inline asm\n" " st.shared.f32 [%r395+-1024], %f16;\n" " add.s64 %rd38, %rd86, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd38];\n" " // end inline asm\n" " st.shared.f32 [%r395], %f17;\n" " add.s64 %rd39, %rd86, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r395+1024], %f18;\n" " add.s64 %rd86, %rd86, 4096;\n" " add.s32 %r395, %r395, 4096;\n" " add.s32 %r394, %r394, 1024;\n" " setp.lt.s32 %p10, %r394, %r15;\n" " @%p10 bra $L__BB7_10;\n" "\n" "$L__BB7_18:\n" " cvt.u32.u64 %r71, %rd1;\n" " setp.ge.s32 %p17, %r71, %r43;\n" " bar.sync 0;\n" " @%p17 bra $L__BB7_24;\n" "\n" " setp.lt.s32 %p18, %r14, 2048;\n" " shr.s32 %r30, %r15, 8;\n" " mov.u32 %r403, 0;\n" " mov.u32 %r401, %r55;\n" " @%p18 bra $L__BB7_22;\n" "\n" " mov.u32 %r401, sh_xps;\n" " mov.u32 %r399, 8;\n" "\n" "$L__BB7_21:\n" " mov.u32 %r403, %r399;\n" " shl.b64 %rd73, %rd2, 2;\n" " add.s64 %rd49, %rd90, %rd73;\n" " // begin inline asm\n" " ld.global.nc.u32 %r100, [%rd49];\n" " // end inline asm\n" " add.s64 %rd50, %rd90, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f55, [%rd50];\n" " // end inline asm\n" " shl.b64 %rd74, %rd3, 3;\n" " add.s64 %rd75, %rd90, %rd74;\n" " add.s64 %rd51, %rd75, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r101,%r102}, [%rd51];\n" " // end inline asm\n" " add.s64 %rd52, %rd49, 152;\n" " // begin inline asm\n" " ld.global.nc.u32 %r106, [%rd52];\n" " // end inline asm\n" " add.s64 %rd53, %rd90, 296;\n" " // begin inline asm\n" " ld.global.nc.f32 %f57, [%rd53];\n" " // end inline asm\n" " add.s64 %rd54, %rd75, 280;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r107,%r108}, [%rd54];\n" " // end inline asm\n" " add.s64 %rd55, %rd49, 304;\n" " // begin inline asm\n" " ld.global.nc.u32 %r112, [%rd55];\n" " // end inline asm\n" " add.s64 %rd56, %rd90, 448;\n" " // begin inline asm\n" " ld.global.nc.f32 %f59, [%rd56];\n" " // end inline asm\n" " add.s64 %rd57, %rd75, 432;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r113,%r114}, [%rd57];\n" " // end inline asm\n" " add.s64 %rd58, %rd49, 456;\n" " // begin inline asm\n" " ld.global.nc.u32 %r118, [%rd58];\n" " // end inline asm\n" " add.s64 %rd59, %rd90, 600;\n" " // begin inline asm\n" " ld.global.nc.f32 %f61, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd75, 584;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r119,%r120}, [%rd60];\n" " // end inline asm\n" " add.s64 %rd61, %rd49, 608;\n" " // begin inline asm\n" " ld.global.nc.u32 %r124, [%rd61];\n" " // end inline asm\n" " add.s64 %rd62, %rd90, 752;\n" " // begin inline asm\n" " ld.global.nc.f32 %f63, [%rd62];\n" " // end inline asm\n" " add.s64 %rd63, %rd75, 736;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r125,%r126}, [%rd63];\n" " // end inline asm\n" " add.s64 %rd64, %rd49, 760;\n" " // begin inline asm\n" " ld.global.nc.u32 %r130, [%rd64];\n" " // end inline asm\n" " add.s64 %rd65, %rd90, 904;\n" " // begin inline asm\n" " ld.global.nc.f32 %f65, [%rd65];\n" " // end inline asm\n" " add.s64 %rd66, %rd75, 888;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r131,%r132}, [%rd66];\n" " // end inline asm\n" " add.s64 %rd67, %rd49, 912;\n" " // begin inline asm\n" " ld.global.nc.u32 %r136, [%rd67];\n" " // end inline asm\n" " add.s64 %rd68, %rd90, 1056;\n" " // begin inline asm\n" " ld.global.nc.f32 %f67, [%rd68];\n" " // end inline asm\n" " add.s64 %rd69, %rd75, 1040;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r137,%r138}, [%rd69];\n" " // end inline asm\n" " add.s64 %rd70, %rd49, 1064;\n" " // begin inline asm\n" " ld.global.nc.u32 %r142, [%rd70];\n" " // end inline asm\n" " add.s64 %rd71, %rd90, 1208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f69, [%rd71];\n" " // end inline asm\n" " add.s64 %rd72, %rd75, 1192;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r143,%r144}, [%rd72];\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " // begin inline asm\n" " // end inline asm\n" " shr.u32 %r148, %r100, 16;\n" " and.b32 %r149, %r100, 15;\n" " or.b32 %r150, %r149, 1258291200;\n" " mov.b32 %f71, %r150;\n" " mov.f32 %f72, 0fCB000004;\n" " mov.f32 %f73, 0f3F800000;\n" " fma.rn.f32 %f74, %f71, %f73, %f72;\n" " and.b32 %r151, %r101, 255;\n" " cvt.rn.f32.u32 %f75, %r151;\n" " mov.f32 %f76, 0f42000000;\n" " sub.rn.f32 %f77, %f75, %f76;\n" " mul.rn.f32 %f78, %f77, %f74;\n" " shl.b32 %r152, %r2, 2;\n" " add.s32 %r153, %r401, %r152;\n" " ld.shared.f32 %f79, [%r153];\n" " mov.f32 %f80, 0f00000000;\n" " fma.rn.f32 %f81, %f78, %f79, %f80;\n" " and.b32 %r154, %r100, 240;\n" " or.b32 %r155, %r154, 1258291200;\n" " mov.b32 %f82, %r155;\n" " mov.f32 %f83, 0fC9000040;\n" " mov.f32 %f84, 0f3D800000;\n" " fma.rn.f32 %f85, %f82, %f84, %f83;\n" " bfe.u32 %r156, %r101, 8, 8;\n" " cvt.rn.f32.u32 %f86, %r156;\n" " sub.rn.f32 %f87, %f86, %f76;\n" " mul.rn.f32 %f88, %f87, %f85;\n" " ld.shared.f32 %f89, [%r153+128];\n" " fma.rn.f32 %f90, %f88, %f89, %f81;\n" " and.b32 %r157, %r100, 3840;\n" " or.b32 %r158, %r157, 1258291200;\n" " mov.b32 %f91, %r158;\n" " mov.f32 %f92, 0fC7000400;\n" " mov.f32 %f93, 0f3B800000;\n" " fma.rn.f32 %f94, %f91, %f93, %f92;\n" " bfe.u32 %r159, %r101, 16, 8;\n" " cvt.rn.f32.u32 %f95, %r159;\n" " sub.rn.f32 %f96, %f95, %f76;\n" " mul.rn.f32 %f97, %f96, %f94;\n" " ld.shared.f32 %f98, [%r153+256];\n" " fma.rn.f32 %f99, %f97, %f98, %f90;\n" " and.b32 %r160, %r100, 61440;\n" " or.b32 %r161, %r160, 1258291200;\n" " mov.b32 %f100, %r161;\n" " mov.f32 %f101, 0fC5004000;\n" " mov.f32 %f102, 0f39800000;\n" " fma.rn.f32 %f103, %f100, %f102, %f101;\n" " shr.u32 %r162, %r101, 24;\n" " cvt.rn.f32.u32 %f104, %r162;\n" " sub.rn.f32 %f105, %f104, %f76;\n" " mul.rn.f32 %f106, %f105, %f103;\n" " ld.shared.f32 %f107, [%r153+384];\n" " fma.rn.f32 %f108, %f106, %f107, %f99;\n" " and.b32 %r163, %r100, 983040;\n" " or.b32 %r164, %r163, 1258291200;\n" " mov.b32 %f109, %r164;\n" " mov.f32 %f110, 0fC3040000;\n" " mov.f32 %f111, 0f37800000;\n" " fma.rn.f32 %f112, %f109, %f111, %f110;\n" " and.b32 %r165, %r102, 255;\n" " cvt.rn.f32.u32 %f113, %r165;\n" " sub.rn.f32 %f114, %f113, %f76;\n" " mul.rn.f32 %f115, %f114, %f112;\n" " ld.shared.f32 %f116, [%r153+512];\n" " fma.rn.f32 %f117, %f115, %f116, %f108;\n" " and.b32 %r166, %r148, 240;\n" " or.b32 %r167, %r166, 1258291200;\n" " mov.b32 %f118, %r167;\n" " fma.rn.f32 %f119, %f118, %f84, %f83;\n" " bfe.u32 %r168, %r102, 8, 8;\n" " cvt.rn.f32.u32 %f120, %r168;\n" " sub.rn.f32 %f121, %f120, %f76;\n" " mul.rn.f32 %f122, %f121, %f119;\n" " ld.shared.f32 %f123, [%r153+640];\n" " fma.rn.f32 %f124, %f122, %f123, %f117;\n" " and.b32 %r169, %r148, 3840;\n" " or.b32 %r170, %r169, 1258291200;\n" " mov.b32 %f125, %r170;\n" " fma.rn.f32 %f126, %f125, %f93, %f92;\n" " bfe.u32 %r171, %r102, 16, 8;\n" " cvt.rn.f32.u32 %f127, %r171;\n" " sub.rn.f32 %f128, %f127, %f76;\n" " mul.rn.f32 %f129, %f128, %f126;\n" " ld.shared.f32 %f130, [%r153+768];\n" " fma.rn.f32 %f131, %f129, %f130, %f124;\n" " and.b32 %r172, %r148, 61440;\n" " or.b32 %r173, %r172, 1258291200;\n" " mov.b32 %f132, %r173;\n" " fma.rn.f32 %f133, %f132, %f102, %f101;\n" " shr.u32 %r174, %r102, 24;\n" " cvt.rn.f32.u32 %f134, %r174;\n" " sub.rn.f32 %f135, %f134, %f76;\n" " mul.rn.f32 %f136, %f135, %f133;\n" " ld.shared.f32 %f137, [%r153+896];\n" " fma.rn.f32 %f138, %f136, %f137, %f131;\n" " fma.rn.f32 %f139, %f138, %f55, %f621;\n" " shr.u32 %r175, %r106, 16;\n" " and.b32 %r176, %r106, 15;\n" " or.b32 %r177, %r176, 1258291200;\n" " mov.b32 %f140, %r177;\n" " fma.rn.f32 %f141, %f140, %f73, %f72;\n" " and.b32 %r178, %r107, 255;\n" " cvt.rn.f32.u32 %f142, %r178;\n" " sub.rn.f32 %f143, %f142, %f76;\n" " mul.rn.f32 %f144, %f143, %f141;\n" " ld.shared.f32 %f145, [%r153+1024];\n" " fma.rn.f32 %f146, %f144, %f145, %f80;\n" " and.b32 %r179, %r106, 240;\n" " or.b32 %r180, %r179, 1258291200;\n" " mov.b32 %f147, %r180;\n" " fma.rn.f32 %f148, %f147, %f84, %f83;\n" " bfe.u32 %r181, %r107, 8, 8;\n" " cvt.rn.f32.u32 %f149, %r181;\n" " sub.rn.f32 %f150, %f149, %f76;\n" " mul.rn.f32 %f151, %f150, %f148;\n" " ld.shared.f32 %f152, [%r153+1152];\n" " fma.rn.f32 %f153, %f151, %f152, %f146;\n" " and.b32 %r182, %r106, 3840;\n" " or.b32 %r183, %r182, 1258291200;\n" " mov.b32 %f154, %r183;\n" " fma.rn.f32 %f155, %f154, %f93, %f92;\n" " bfe.u32 %r184, %r107, 16, 8;\n" " cvt.rn.f32.u32 %f156, %r184;\n" " sub.rn.f32 %f157, %f156, %f76;\n" " mul.rn.f32 %f158, %f157, %f155;\n" " ld.shared.f32 %f159, [%r153+1280];\n" " fma.rn.f32 %f160, %f158, %f159, %f153;\n" " and.b32 %r185, %r106, 61440;\n" " or.b32 %r186, %r185, 1258291200;\n" " mov.b32 %f161, %r186;\n" " fma.rn.f32 %f162, %f161, %f102, %f101;\n" " shr.u32 %r187, %r107, 24;\n" " cvt.rn.f32.u32 %f163, %r187;\n" " sub.rn.f32 %f164, %f163, %f76;\n" " mul.rn.f32 %f165, %f164, %f162;\n" " ld.shared.f32 %f166, [%r153+1408];\n" " fma.rn.f32 %f167, %f165, %f166, %f160;\n" " and.b32 %r188, %r106, 983040;\n" " or.b32 %r189, %r188, 1258291200;\n" " mov.b32 %f168, %r189;\n" " fma.rn.f32 %f169, %f168, %f111, %f110;\n" " and.b32 %r190, %r108, 255;\n" " cvt.rn.f32.u32 %f170, %r190;\n" " sub.rn.f32 %f171, %f170, %f76;\n" " mul.rn.f32 %f172, %f171, %f169;\n" " ld.shared.f32 %f173, [%r153+1536];\n" " fma.rn.f32 %f174, %f172, %f173, %f167;\n" " and.b32 %r191, %r175, 240;\n" " or.b32 %r192, %r191, 1258291200;\n" " mov.b32 %f175, %r192;\n" " fma.rn.f32 %f176, %f175, %f84, %f83;\n" " bfe.u32 %r193, %r108, 8, 8;\n" " cvt.rn.f32.u32 %f177, %r193;\n" " sub.rn.f32 %f178, %f177, %f76;\n" " mul.rn.f32 %f179, %f178, %f176;\n" " ld.shared.f32 %f180, [%r153+1664];\n" " fma.rn.f32 %f181, %f179, %f180, %f174;\n" " and.b32 %r194, %r175, 3840;\n" " or.b32 %r195, %r194, 1258291200;\n" " mov.b32 %f182, %r195;\n" " fma.rn.f32 %f183, %f182, %f93, %f92;\n" " bfe.u32 %r196, %r108, 16, 8;\n" " cvt.rn.f32.u32 %f184, %r196;\n" " sub.rn.f32 %f185, %f184, %f76;\n" " mul.rn.f32 %f186, %f185, %f183;\n" " ld.shared.f32 %f187, [%r153+1792];\n" " fma.rn.f32 %f188, %f186, %f187, %f181;\n" " and.b32 %r197, %r175, 61440;\n" " or.b32 %r198, %r197, 1258291200;\n" " mov.b32 %f189, %r198;\n" " fma.rn.f32 %f190, %f189, %f102, %f101;\n" " shr.u32 %r199, %r108, 24;\n" " cvt.rn.f32.u32 %f191, %r199;\n" " sub.rn.f32 %f192, %f191, %f76;\n" " mul.rn.f32 %f193, %f192, %f190;\n" " ld.shared.f32 %f194, [%r153+1920];\n" " fma.rn.f32 %f195, %f193, %f194, %f188;\n" " fma.rn.f32 %f196, %f195, %f57, %f139;\n" " shr.u32 %r200, %r112, 16;\n" " and.b32 %r201, %r112, 15;\n" " or.b32 %r202, %r201, 1258291200;\n" " mov.b32 %f197, %r202;\n" " fma.rn.f32 %f198, %f197, %f73, %f72;\n" " and.b32 %r203, %r113, 255;\n" " cvt.rn.f32.u32 %f199, %r203;\n" " sub.rn.f32 %f200, %f199, %f76;\n" " mul.rn.f32 %f201, %f200, %f198;\n" " ld.shared.f32 %f202, [%r153+2048];\n" " fma.rn.f32 %f203, %f201, %f202, %f80;\n" " and.b32 %r204, %r112, 240;\n" " or.b32 %r205, %r204, 1258291200;\n" " mov.b32 %f204, %r205;\n" " fma.rn.f32 %f205, %f204, %f84, %f83;\n" " bfe.u32 %r206, %r113, 8, 8;\n" " cvt.rn.f32.u32 %f206, %r206;\n" " sub.rn.f32 %f207, %f206, %f76;\n" " mul.rn.f32 %f208, %f207, %f205;\n" " ld.shared.f32 %f209, [%r153+2176];\n" " fma.rn.f32 %f210, %f208, %f209, %f203;\n" " and.b32 %r207, %r112, 3840;\n" " or.b32 %r208, %r207, 1258291200;\n" " mov.b32 %f211, %r208;\n" " fma.rn.f32 %f212, %f211, %f93, %f92;\n" " bfe.u32 %r209, %r113, 16, 8;\n" " cvt.rn.f32.u32 %f213, %r209;\n" " sub.rn.f32 %f214, %f213, %f76;\n" " mul.rn.f32 %f215, %f214, %f212;\n" " ld.shared.f32 %f216, [%r153+2304];\n" " fma.rn.f32 %f217, %f215, %f216, %f210;\n" " and.b32 %r210, %r112, 61440;\n" " or.b32 %r211, %r210, 1258291200;\n" " mov.b32 %f218, %r211;\n" " fma.rn.f32 %f219, %f218, %f102, %f101;\n" " shr.u32 %r212, %r113, 24;\n" " cvt.rn.f32.u32 %f220, %r212;\n" " sub.rn.f32 %f221, %f220, %f76;\n" " mul.rn.f32 %f222, %f221, %f219;\n" " ld.shared.f32 %f223, [%r153+2432];\n" " fma.rn.f32 %f224, %f222, %f223, %f217;\n" " and.b32 %r213, %r112, 983040;\n" " or.b32 %r214, %r213, 1258291200;\n" " mov.b32 %f225, %r214;\n" " fma.rn.f32 %f226, %f225, %f111, %f110;\n" " and.b32 %r215, %r114, 255;\n" " cvt.rn.f32.u32 %f227, %r215;\n" " sub.rn.f32 %f228, %f227, %f76;\n" " mul.rn.f32 %f229, %f228, %f226;\n" " ld.shared.f32 %f230, [%r153+2560];\n" " fma.rn.f32 %f231, %f229, %f230, %f224;\n" " and.b32 %r216, %r200, 240;\n" " or.b32 %r217, %r216, 1258291200;\n" " mov.b32 %f232, %r217;\n" " fma.rn.f32 %f233, %f232, %f84, %f83;\n" " bfe.u32 %r218, %r114, 8, 8;\n" " cvt.rn.f32.u32 %f234, %r218;\n" " sub.rn.f32 %f235, %f234, %f76;\n" " mul.rn.f32 %f236, %f235, %f233;\n" " ld.shared.f32 %f237, [%r153+2688];\n" " fma.rn.f32 %f238, %f236, %f237, %f231;\n" " and.b32 %r219, %r200, 3840;\n" " or.b32 %r220, %r219, 1258291200;\n" " mov.b32 %f239, %r220;\n" " fma.rn.f32 %f240, %f239, %f93, %f92;\n" " bfe.u32 %r221, %r114, 16, 8;\n" " cvt.rn.f32.u32 %f241, %r221;\n" " sub.rn.f32 %f242, %f241, %f76;\n" " mul.rn.f32 %f243, %f242, %f240;\n" " ld.shared.f32 %f244, [%r153+2816];\n" " fma.rn.f32 %f245, %f243, %f244, %f238;\n" " and.b32 %r222, %r200, 61440;\n" " or.b32 %r223, %r222, 1258291200;\n" " mov.b32 %f246, %r223;\n" " fma.rn.f32 %f247, %f246, %f102, %f101;\n" " shr.u32 %r224, %r114, 24;\n" " cvt.rn.f32.u32 %f248, %r224;\n" " sub.rn.f32 %f249, %f248, %f76;\n" " mul.rn.f32 %f250, %f249, %f247;\n" " ld.shared.f32 %f251, [%r153+2944];\n" " fma.rn.f32 %f252, %f250, %f251, %f245;\n" " fma.rn.f32 %f253, %f252, %f59, %f196;\n" " shr.u32 %r225, %r118, 16;\n" " and.b32 %r226, %r118, 15;\n" " or.b32 %r227, %r226, 1258291200;\n" " mov.b32 %f254, %r227;\n" " fma.rn.f32 %f255, %f254, %f73, %f72;\n" " and.b32 %r228, %r119, 255;\n" " cvt.rn.f32.u32 %f256, %r228;\n" " sub.rn.f32 %f257, %f256, %f76;\n" " mul.rn.f32 %f258, %f257, %f255;\n" " ld.shared.f32 %f259, [%r153+3072];\n" " fma.rn.f32 %f260, %f258, %f259, %f80;\n" " and.b32 %r229, %r118, 240;\n" " or.b32 %r230, %r229, 1258291200;\n" " mov.b32 %f261, %r230;\n" " fma.rn.f32 %f262, %f261, %f84, %f83;\n" " bfe.u32 %r231, %r119, 8, 8;\n" " cvt.rn.f32.u32 %f263, %r231;\n" " sub.rn.f32 %f264, %f263, %f76;\n" " mul.rn.f32 %f265, %f264, %f262;\n" " ld.shared.f32 %f266, [%r153+3200];\n" " fma.rn.f32 %f267, %f265, %f266, %f260;\n" " and.b32 %r232, %r118, 3840;\n" " or.b32 %r233, %r232, 1258291200;\n" " mov.b32 %f268, %r233;\n" " fma.rn.f32 %f269, %f268, %f93, %f92;\n" " bfe.u32 %r234, %r119, 16, 8;\n" " cvt.rn.f32.u32 %f270, %r234;\n" " sub.rn.f32 %f271, %f270, %f76;\n" " mul.rn.f32 %f272, %f271, %f269;\n" " ld.shared.f32 %f273, [%r153+3328];\n" " fma.rn.f32 %f274, %f272, %f273, %f267;\n" " and.b32 %r235, %r118, 61440;\n" " or.b32 %r236, %r235, 1258291200;\n" " mov.b32 %f275, %r236;\n" " fma.rn.f32 %f276, %f275, %f102, %f101;\n" " shr.u32 %r237, %r119, 24;\n" " cvt.rn.f32.u32 %f277, %r237;\n" " sub.rn.f32 %f278, %f277, %f76;\n" " mul.rn.f32 %f279, %f278, %f276;\n" " ld.shared.f32 %f280, [%r153+3456];\n" " fma.rn.f32 %f281, %f279, %f280, %f274;\n" " and.b32 %r238, %r118, 983040;\n" " or.b32 %r239, %r238, 1258291200;\n" " mov.b32 %f282, %r239;\n" " fma.rn.f32 %f283, %f282, %f111, %f110;\n" " and.b32 %r240, %r120, 255;\n" " cvt.rn.f32.u32 %f284, %r240;\n" " sub.rn.f32 %f285, %f284, %f76;\n" " mul.rn.f32 %f286, %f285, %f283;\n" " ld.shared.f32 %f287, [%r153+3584];\n" " fma.rn.f32 %f288, %f286, %f287, %f281;\n" " and.b32 %r241, %r225, 240;\n" " or.b32 %r242, %r241, 1258291200;\n" " mov.b32 %f289, %r242;\n" " fma.rn.f32 %f290, %f289, %f84, %f83;\n" " bfe.u32 %r243, %r120, 8, 8;\n" " cvt.rn.f32.u32 %f291, %r243;\n" " sub.rn.f32 %f292, %f291, %f76;\n" " mul.rn.f32 %f293, %f292, %f290;\n" " ld.shared.f32 %f294, [%r153+3712];\n" " fma.rn.f32 %f295, %f293, %f294, %f288;\n" " and.b32 %r244, %r225, 3840;\n" " or.b32 %r245, %r244, 1258291200;\n" " mov.b32 %f296, %r245;\n" " fma.rn.f32 %f297, %f296, %f93, %f92;\n" " bfe.u32 %r246, %r120, 16, 8;\n" " cvt.rn.f32.u32 %f298, %r246;\n" " sub.rn.f32 %f299, %f298, %f76;\n" " mul.rn.f32 %f300, %f299, %f297;\n" " ld.shared.f32 %f301, [%r153+3840];\n" " fma.rn.f32 %f302, %f300, %f301, %f295;\n" " and.b32 %r247, %r225, 61440;\n" " or.b32 %r248, %r247, 1258291200;\n" " mov.b32 %f303, %r248;\n" " fma.rn.f32 %f304, %f303, %f102, %f101;\n" " shr.u32 %r249, %r120, 24;\n" " cvt.rn.f32.u32 %f305, %r249;\n" " sub.rn.f32 %f306, %f305, %f76;\n" " mul.rn.f32 %f307, %f306, %f304;\n" " ld.shared.f32 %f308, [%r153+3968];\n" " fma.rn.f32 %f309, %f307, %f308, %f302;\n" " fma.rn.f32 %f310, %f309, %f61, %f253;\n" " shr.u32 %r250, %r124, 16;\n" " and.b32 %r251, %r124, 15;\n" " or.b32 %r252, %r251, 1258291200;\n" " mov.b32 %f311, %r252;\n" " fma.rn.f32 %f312, %f311, %f73, %f72;\n" " and.b32 %r253, %r125, 255;\n" " cvt.rn.f32.u32 %f313, %r253;\n" " sub.rn.f32 %f314, %f313, %f76;\n" " mul.rn.f32 %f315, %f314, %f312;\n" " ld.shared.f32 %f316, [%r153+4096];\n" " fma.rn.f32 %f317, %f315, %f316, %f80;\n" " and.b32 %r254, %r124, 240;\n" " or.b32 %r255, %r254, 1258291200;\n" " mov.b32 %f318, %r255;\n" " fma.rn.f32 %f319, %f318, %f84, %f83;\n" " bfe.u32 %r256, %r125, 8, 8;\n" " cvt.rn.f32.u32 %f320, %r256;\n" " sub.rn.f32 %f321, %f320, %f76;\n" " mul.rn.f32 %f322, %f321, %f319;\n" " ld.shared.f32 %f323, [%r153+4224];\n" " fma.rn.f32 %f324, %f322, %f323, %f317;\n" " and.b32 %r257, %r124, 3840;\n" " or.b32 %r258, %r257, 1258291200;\n" " mov.b32 %f325, %r258;\n" " fma.rn.f32 %f326, %f325, %f93, %f92;\n" " bfe.u32 %r259, %r125, 16, 8;\n" " cvt.rn.f32.u32 %f327, %r259;\n" " sub.rn.f32 %f328, %f327, %f76;\n" " mul.rn.f32 %f329, %f328, %f326;\n" " ld.shared.f32 %f330, [%r153+4352];\n" " fma.rn.f32 %f331, %f329, %f330, %f324;\n" " and.b32 %r260, %r124, 61440;\n" " or.b32 %r261, %r260, 1258291200;\n" " mov.b32 %f332, %r261;\n" " fma.rn.f32 %f333, %f332, %f102, %f101;\n" " shr.u32 %r262, %r125, 24;\n" " cvt.rn.f32.u32 %f334, %r262;\n" " sub.rn.f32 %f335, %f334, %f76;\n" " mul.rn.f32 %f336, %f335, %f333;\n" " ld.shared.f32 %f337, [%r153+4480];\n" " fma.rn.f32 %f338, %f336, %f337, %f331;\n" " and.b32 %r263, %r124, 983040;\n" " or.b32 %r264, %r263, 1258291200;\n" " mov.b32 %f339, %r264;\n" " fma.rn.f32 %f340, %f339, %f111, %f110;\n" " and.b32 %r265, %r126, 255;\n" " cvt.rn.f32.u32 %f341, %r265;\n" " sub.rn.f32 %f342, %f341, %f76;\n" " mul.rn.f32 %f343, %f342, %f340;\n" " ld.shared.f32 %f344, [%r153+4608];\n" " fma.rn.f32 %f345, %f343, %f344, %f338;\n" " and.b32 %r266, %r250, 240;\n" " or.b32 %r267, %r266, 1258291200;\n" " mov.b32 %f346, %r267;\n" " fma.rn.f32 %f347, %f346, %f84, %f83;\n" " bfe.u32 %r268, %r126, 8, 8;\n" " cvt.rn.f32.u32 %f348, %r268;\n" " sub.rn.f32 %f349, %f348, %f76;\n" " mul.rn.f32 %f350, %f349, %f347;\n" " ld.shared.f32 %f351, [%r153+4736];\n" " fma.rn.f32 %f352, %f350, %f351, %f345;\n" " and.b32 %r269, %r250, 3840;\n" " or.b32 %r270, %r269, 1258291200;\n" " mov.b32 %f353, %r270;\n" " fma.rn.f32 %f354, %f353, %f93, %f92;\n" " bfe.u32 %r271, %r126, 16, 8;\n" " cvt.rn.f32.u32 %f355, %r271;\n" " sub.rn.f32 %f356, %f355, %f76;\n" " mul.rn.f32 %f357, %f356, %f354;\n" " ld.shared.f32 %f358, [%r153+4864];\n" " fma.rn.f32 %f359, %f357, %f358, %f352;\n" " and.b32 %r272, %r250, 61440;\n" " or.b32 %r273, %r272, 1258291200;\n" " mov.b32 %f360, %r273;\n" " fma.rn.f32 %f361, %f360, %f102, %f101;\n" " shr.u32 %r274, %r126, 24;\n" " cvt.rn.f32.u32 %f362, %r274;\n" " sub.rn.f32 %f363, %f362, %f76;\n" " mul.rn.f32 %f364, %f363, %f361;\n" " ld.shared.f32 %f365, [%r153+4992];\n" " fma.rn.f32 %f366, %f364, %f365, %f359;\n" " fma.rn.f32 %f367, %f366, %f63, %f310;\n" " shr.u32 %r275, %r130, 16;\n" " and.b32 %r276, %r130, 15;\n" " or.b32 %r277, %r276, 1258291200;\n" " mov.b32 %f368, %r277;\n" " fma.rn.f32 %f369, %f368, %f73, %f72;\n" " and.b32 %r278, %r131, 255;\n" " cvt.rn.f32.u32 %f370, %r278;\n" " sub.rn.f32 %f371, %f370, %f76;\n" " mul.rn.f32 %f372, %f371, %f369;\n" " ld.shared.f32 %f373, [%r153+5120];\n" " fma.rn.f32 %f374, %f372, %f373, %f80;\n" " and.b32 %r279, %r130, 240;\n" " or.b32 %r280, %r279, 1258291200;\n" " mov.b32 %f375, %r280;\n" " fma.rn.f32 %f376, %f375, %f84, %f83;\n" " bfe.u32 %r281, %r131, 8, 8;\n" " cvt.rn.f32.u32 %f377, %r281;\n" " sub.rn.f32 %f378, %f377, %f76;\n" " mul.rn.f32 %f379, %f378, %f376;\n" " ld.shared.f32 %f380, [%r153+5248];\n" " fma.rn.f32 %f381, %f379, %f380, %f374;\n" " and.b32 %r282, %r130, 3840;\n" " or.b32 %r283, %r282, 1258291200;\n" " mov.b32 %f382, %r283;\n" " fma.rn.f32 %f383, %f382, %f93, %f92;\n" " bfe.u32 %r284, %r131, 16, 8;\n" " cvt.rn.f32.u32 %f384, %r284;\n" " sub.rn.f32 %f385, %f384, %f76;\n" " mul.rn.f32 %f386, %f385, %f383;\n" " ld.shared.f32 %f387, [%r153+5376];\n" " fma.rn.f32 %f388, %f386, %f387, %f381;\n" " and.b32 %r285, %r130, 61440;\n" " or.b32 %r286, %r285, 1258291200;\n" " mov.b32 %f389, %r286;\n" " fma.rn.f32 %f390, %f389, %f102, %f101;\n" " shr.u32 %r287, %r131, 24;\n" " cvt.rn.f32.u32 %f391, %r287;\n" " sub.rn.f32 %f392, %f391, %f76;\n" " mul.rn.f32 %f393, %f392, %f390;\n" " ld.shared.f32 %f394, [%r153+5504];\n" " fma.rn.f32 %f395, %f393, %f394, %f388;\n" " and.b32 %r288, %r130, 983040;\n" " or.b32 %r289, %r288, 1258291200;\n" " mov.b32 %f396, %r289;\n" " fma.rn.f32 %f397, %f396, %f111, %f110;\n" " and.b32 %r290, %r132, 255;\n" " cvt.rn.f32.u32 %f398, %r290;\n" " sub.rn.f32 %f399, %f398, %f76;\n" " mul.rn.f32 %f400, %f399, %f397;\n" " ld.shared.f32 %f401, [%r153+5632];\n" " fma.rn.f32 %f402, %f400, %f401, %f395;\n" " and.b32 %r291, %r275, 240;\n" " or.b32 %r292, %r291, 1258291200;\n" " mov.b32 %f403, %r292;\n" " fma.rn.f32 %f404, %f403, %f84, %f83;\n" " bfe.u32 %r293, %r132, 8, 8;\n" " cvt.rn.f32.u32 %f405, %r293;\n" " sub.rn.f32 %f406, %f405, %f76;\n" " mul.rn.f32 %f407, %f406, %f404;\n" " ld.shared.f32 %f408, [%r153+5760];\n" " fma.rn.f32 %f409, %f407, %f408, %f402;\n" " and.b32 %r294, %r275, 3840;\n" " or.b32 %r295, %r294, 1258291200;\n" " mov.b32 %f410, %r295;\n" " fma.rn.f32 %f411, %f410, %f93, %f92;\n" " bfe.u32 %r296, %r132, 16, 8;\n" " cvt.rn.f32.u32 %f412, %r296;\n" " sub.rn.f32 %f413, %f412, %f76;\n" " mul.rn.f32 %f414, %f413, %f411;\n" " ld.shared.f32 %f415, [%r153+5888];\n" " fma.rn.f32 %f416, %f414, %f415, %f409;\n" " and.b32 %r297, %r275, 61440;\n" " or.b32 %r298, %r297, 1258291200;\n" " mov.b32 %f417, %r298;\n" " fma.rn.f32 %f418, %f417, %f102, %f101;\n" " shr.u32 %r299, %r132, 24;\n" " cvt.rn.f32.u32 %f419, %r299;\n" " sub.rn.f32 %f420, %f419, %f76;\n" " mul.rn.f32 %f421, %f420, %f418;\n" " ld.shared.f32 %f422, [%r153+6016];\n" " fma.rn.f32 %f423, %f421, %f422, %f416;\n" " fma.rn.f32 %f424, %f423, %f65, %f367;\n" " shr.u32 %r300, %r136, 16;\n" " and.b32 %r301, %r136, 15;\n" " or.b32 %r302, %r301, 1258291200;\n" " mov.b32 %f425, %r302;\n" " fma.rn.f32 %f426, %f425, %f73, %f72;\n" " and.b32 %r303, %r137, 255;\n" " cvt.rn.f32.u32 %f427, %r303;\n" " sub.rn.f32 %f428, %f427, %f76;\n" " mul.rn.f32 %f429, %f428, %f426;\n" " ld.shared.f32 %f430, [%r153+6144];\n" " fma.rn.f32 %f431, %f429, %f430, %f80;\n" " and.b32 %r304, %r136, 240;\n" " or.b32 %r305, %r304, 1258291200;\n" " mov.b32 %f432, %r305;\n" " fma.rn.f32 %f433, %f432, %f84, %f83;\n" " bfe.u32 %r306, %r137, 8, 8;\n" " cvt.rn.f32.u32 %f434, %r306;\n" " sub.rn.f32 %f435, %f434, %f76;\n" " mul.rn.f32 %f436, %f435, %f433;\n" " ld.shared.f32 %f437, [%r153+6272];\n" " fma.rn.f32 %f438, %f436, %f437, %f431;\n" " and.b32 %r307, %r136, 3840;\n" " or.b32 %r308, %r307, 1258291200;\n" " mov.b32 %f439, %r308;\n" " fma.rn.f32 %f440, %f439, %f93, %f92;\n" " bfe.u32 %r309, %r137, 16, 8;\n" " cvt.rn.f32.u32 %f441, %r309;\n" " sub.rn.f32 %f442, %f441, %f76;\n" " mul.rn.f32 %f443, %f442, %f440;\n" " ld.shared.f32 %f444, [%r153+6400];\n" " fma.rn.f32 %f445, %f443, %f444, %f438;\n" " and.b32 %r310, %r136, 61440;\n" " or.b32 %r311, %r310, 1258291200;\n" " mov.b32 %f446, %r311;\n" " fma.rn.f32 %f447, %f446, %f102, %f101;\n" " shr.u32 %r312, %r137, 24;\n" " cvt.rn.f32.u32 %f448, %r312;\n" " sub.rn.f32 %f449, %f448, %f76;\n" " mul.rn.f32 %f450, %f449, %f447;\n" " ld.shared.f32 %f451, [%r153+6528];\n" " fma.rn.f32 %f452, %f450, %f451, %f445;\n" " and.b32 %r313, %r136, 983040;\n" " or.b32 %r314, %r313, 1258291200;\n" " mov.b32 %f453, %r314;\n" " fma.rn.f32 %f454, %f453, %f111, %f110;\n" " and.b32 %r315, %r138, 255;\n" " cvt.rn.f32.u32 %f455, %r315;\n" " sub.rn.f32 %f456, %f455, %f76;\n" " mul.rn.f32 %f457, %f456, %f454;\n" " ld.shared.f32 %f458, [%r153+6656];\n" " fma.rn.f32 %f459, %f457, %f458, %f452;\n" " and.b32 %r316, %r300, 240;\n" " or.b32 %r317, %r316, 1258291200;\n" " mov.b32 %f460, %r317;\n" " fma.rn.f32 %f461, %f460, %f84, %f83;\n" " bfe.u32 %r318, %r138, 8, 8;\n" " cvt.rn.f32.u32 %f462, %r318;\n" " sub.rn.f32 %f463, %f462, %f76;\n" " mul.rn.f32 %f464, %f463, %f461;\n" " ld.shared.f32 %f465, [%r153+6784];\n" " fma.rn.f32 %f466, %f464, %f465, %f459;\n" " and.b32 %r319, %r300, 3840;\n" " or.b32 %r320, %r319, 1258291200;\n" " mov.b32 %f467, %r320;\n" " fma.rn.f32 %f468, %f467, %f93, %f92;\n" " bfe.u32 %r321, %r138, 16, 8;\n" " cvt.rn.f32.u32 %f469, %r321;\n" " sub.rn.f32 %f470, %f469, %f76;\n" " mul.rn.f32 %f471, %f470, %f468;\n" " ld.shared.f32 %f472, [%r153+6912];\n" " fma.rn.f32 %f473, %f471, %f472, %f466;\n" " and.b32 %r322, %r300, 61440;\n" " or.b32 %r323, %r322, 1258291200;\n" " mov.b32 %f474, %r323;\n" " fma.rn.f32 %f475, %f474, %f102, %f101;\n" " shr.u32 %r324, %r138, 24;\n" " cvt.rn.f32.u32 %f476, %r324;\n" " sub.rn.f32 %f477, %f476, %f76;\n" " mul.rn.f32 %f478, %f477, %f475;\n" " ld.shared.f32 %f479, [%r153+7040];\n" " fma.rn.f32 %f480, %f478, %f479, %f473;\n" " fma.rn.f32 %f481, %f480, %f67, %f424;\n" " shr.u32 %r325, %r142, 16;\n" " and.b32 %r326, %r142, 15;\n" " or.b32 %r327, %r326, 1258291200;\n" " mov.b32 %f482, %r327;\n" " fma.rn.f32 %f483, %f482, %f73, %f72;\n" " and.b32 %r328, %r143, 255;\n" " cvt.rn.f32.u32 %f484, %r328;\n" " sub.rn.f32 %f485, %f484, %f76;\n" " mul.rn.f32 %f486, %f485, %f483;\n" " ld.shared.f32 %f487, [%r153+7168];\n" " fma.rn.f32 %f488, %f486, %f487, %f80;\n" " and.b32 %r329, %r142, 240;\n" " or.b32 %r330, %r329, 1258291200;\n" " mov.b32 %f489, %r330;\n" " fma.rn.f32 %f490, %f489, %f84, %f83;\n" " bfe.u32 %r331, %r143, 8, 8;\n" " cvt.rn.f32.u32 %f491, %r331;\n" " sub.rn.f32 %f492, %f491, %f76;\n" " mul.rn.f32 %f493, %f492, %f490;\n" " ld.shared.f32 %f494, [%r153+7296];\n" " fma.rn.f32 %f495, %f493, %f494, %f488;\n" " and.b32 %r332, %r142, 3840;\n" " or.b32 %r333, %r332, 1258291200;\n" " mov.b32 %f496, %r333;\n" " fma.rn.f32 %f497, %f496, %f93, %f92;\n" " bfe.u32 %r334, %r143, 16, 8;\n" " cvt.rn.f32.u32 %f498, %r334;\n" " sub.rn.f32 %f499, %f498, %f76;\n" " mul.rn.f32 %f500, %f499, %f497;\n" " ld.shared.f32 %f501, [%r153+7424];\n" " fma.rn.f32 %f502, %f500, %f501, %f495;\n" " and.b32 %r335, %r142, 61440;\n" " or.b32 %r336, %r335, 1258291200;\n" " mov.b32 %f503, %r336;\n" " fma.rn.f32 %f504, %f503, %f102, %f101;\n" " shr.u32 %r337, %r143, 24;\n" " cvt.rn.f32.u32 %f505, %r337;\n" " sub.rn.f32 %f506, %f505, %f76;\n" " mul.rn.f32 %f507, %f506, %f504;\n" " ld.shared.f32 %f508, [%r153+7552];\n" " fma.rn.f32 %f509, %f507, %f508, %f502;\n" " and.b32 %r338, %r142, 983040;\n" " or.b32 %r339, %r338, 1258291200;\n" " mov.b32 %f510, %r339;\n" " fma.rn.f32 %f511, %f510, %f111, %f110;\n" " and.b32 %r340, %r144, 255;\n" " cvt.rn.f32.u32 %f512, %r340;\n" " sub.rn.f32 %f513, %f512, %f76;\n" " mul.rn.f32 %f514, %f513, %f511;\n" " ld.shared.f32 %f515, [%r153+7680];\n" " fma.rn.f32 %f516, %f514, %f515, %f509;\n" " and.b32 %r341, %r325, 240;\n" " or.b32 %r342, %r341, 1258291200;\n" " mov.b32 %f517, %r342;\n" " fma.rn.f32 %f518, %f517, %f84, %f83;\n" " bfe.u32 %r343, %r144, 8, 8;\n" " cvt.rn.f32.u32 %f519, %r343;\n" " sub.rn.f32 %f520, %f519, %f76;\n" " mul.rn.f32 %f521, %f520, %f518;\n" " ld.shared.f32 %f522, [%r153+7808];\n" " fma.rn.f32 %f523, %f521, %f522, %f516;\n" " and.b32 %r344, %r325, 3840;\n" " or.b32 %r345, %r344, 1258291200;\n" " mov.b32 %f524, %r345;\n" " fma.rn.f32 %f525, %f524, %f93, %f92;\n" " bfe.u32 %r346, %r144, 16, 8;\n" " cvt.rn.f32.u32 %f526, %r346;\n" " sub.rn.f32 %f527, %f526, %f76;\n" " mul.rn.f32 %f528, %f527, %f525;\n" " ld.shared.f32 %f529, [%r153+7936];\n" " fma.rn.f32 %f530, %f528, %f529, %f523;\n" " and.b32 %r347, %r325, 61440;\n" " or.b32 %r348, %r347, 1258291200;\n" " mov.b32 %f531, %r348;\n" " fma.rn.f32 %f532, %f531, %f102, %f101;\n" " shr.u32 %r349, %r144, 24;\n" " cvt.rn.f32.u32 %f533, %r349;\n" " sub.rn.f32 %f534, %f533, %f76;\n" " mul.rn.f32 %f535, %f534, %f532;\n" " ld.shared.f32 %f536, [%r153+8064];\n" " fma.rn.f32 %f537, %f535, %f536, %f530;\n" " fma.rn.f32 %f621, %f537, %f69, %f481;\n" " add.s64 %rd90, %rd90, 1216;\n" " add.s32 %r401, %r401, 8192;\n" " add.s32 %r399, %r403, 8;\n" " setp.le.s32 %p19, %r399, %r30;\n" " @%p19 bra $L__BB7_21;\n" "\n" "$L__BB7_22:\n" " setp.ge.s32 %p20, %r403, %r30;\n" " @%p20 bra $L__BB7_24;\n" "\n" "$L__BB7_23:\n" " shl.b64 %rd79, %rd2, 2;\n" " add.s64 %rd76, %rd90, %rd79;\n" " // begin inline asm\n" " ld.global.nc.u32 %r350, [%rd76];\n" " // end inline asm\n" " add.s64 %rd77, %rd90, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f538, [%rd77];\n" " // end inline asm\n" " shl.b64 %rd80, %rd3, 3;\n" " add.s64 %rd81, %rd90, %rd80;\n" " add.s64 %rd78, %rd81, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r351,%r352}, [%rd78];\n" " // end inline asm\n" " shr.u32 %r353, %r350, 16;\n" " and.b32 %r354, %r350, 15;\n" " or.b32 %r355, %r354, 1258291200;\n" " mov.b32 %f539, %r355;\n" " mov.f32 %f540, 0fCB000004;\n" " mov.f32 %f541, 0f3F800000;\n" " fma.rn.f32 %f542, %f539, %f541, %f540;\n" " and.b32 %r356, %r351, 255;\n" " cvt.rn.f32.u32 %f543, %r356;\n" " mov.f32 %f544, 0f42000000;\n" " sub.rn.f32 %f545, %f543, %f544;\n" " mul.rn.f32 %f546, %f545, %f542;\n" " shl.b32 %r357, %r2, 2;\n" " add.s32 %r358, %r401, %r357;\n" " ld.shared.f32 %f547, [%r358];\n" " mov.f32 %f548, 0f00000000;\n" " fma.rn.f32 %f549, %f546, %f547, %f548;\n" " and.b32 %r359, %r350, 240;\n" " or.b32 %r360, %r359, 1258291200;\n" " mov.b32 %f550, %r360;\n" " mov.f32 %f551, 0fC9000040;\n" " mov.f32 %f552, 0f3D800000;\n" " fma.rn.f32 %f553, %f550, %f552, %f551;\n" " bfe.u32 %r361, %r351, 8, 8;\n" " cvt.rn.f32.u32 %f554, %r361;\n" " sub.rn.f32 %f555, %f554, %f544;\n" " mul.rn.f32 %f556, %f555, %f553;\n" " ld.shared.f32 %f557, [%r358+128];\n" " fma.rn.f32 %f558, %f556, %f557, %f549;\n" " and.b32 %r362, %r350, 3840;\n" " or.b32 %r363, %r362, 1258291200;\n" " mov.b32 %f559, %r363;\n" " mov.f32 %f560, 0fC7000400;\n" " mov.f32 %f561, 0f3B800000;\n" " fma.rn.f32 %f562, %f559, %f561, %f560;\n" " bfe.u32 %r364, %r351, 16, 8;\n" " cvt.rn.f32.u32 %f563, %r364;\n" " sub.rn.f32 %f564, %f563, %f544;\n" " mul.rn.f32 %f565, %f564, %f562;\n" " ld.shared.f32 %f566, [%r358+256];\n" " fma.rn.f32 %f567, %f565, %f566, %f558;\n" " and.b32 %r365, %r350, 61440;\n" " or.b32 %r366, %r365, 1258291200;\n" " mov.b32 %f568, %r366;\n" " mov.f32 %f569, 0fC5004000;\n" " mov.f32 %f570, 0f39800000;\n" " fma.rn.f32 %f571, %f568, %f570, %f569;\n" " shr.u32 %r367, %r351, 24;\n" " cvt.rn.f32.u32 %f572, %r367;\n" " sub.rn.f32 %f573, %f572, %f544;\n" " mul.rn.f32 %f574, %f573, %f571;\n" " ld.shared.f32 %f575, [%r358+384];\n" " fma.rn.f32 %f576, %f574, %f575, %f567;\n" " and.b32 %r368, %r350, 983040;\n" " or.b32 %r369, %r368, 1258291200;\n" " mov.b32 %f577, %r369;\n" " mov.f32 %f578, 0fC3040000;\n" " mov.f32 %f579, 0f37800000;\n" " fma.rn.f32 %f580, %f577, %f579, %f578;\n" " and.b32 %r370, %r352, 255;\n" " cvt.rn.f32.u32 %f581, %r370;\n" " sub.rn.f32 %f582, %f581, %f544;\n" " mul.rn.f32 %f583, %f582, %f580;\n" " ld.shared.f32 %f584, [%r358+512];\n" " fma.rn.f32 %f585, %f583, %f584, %f576;\n" " and.b32 %r371, %r353, 240;\n" " or.b32 %r372, %r371, 1258291200;\n" " mov.b32 %f586, %r372;\n" " fma.rn.f32 %f587, %f586, %f552, %f551;\n" " bfe.u32 %r373, %r352, 8, 8;\n" " cvt.rn.f32.u32 %f588, %r373;\n" " sub.rn.f32 %f589, %f588, %f544;\n" " mul.rn.f32 %f590, %f589, %f587;\n" " ld.shared.f32 %f591, [%r358+640];\n" " fma.rn.f32 %f592, %f590, %f591, %f585;\n" " and.b32 %r374, %r353, 3840;\n" " or.b32 %r375, %r374, 1258291200;\n" " mov.b32 %f593, %r375;\n" " fma.rn.f32 %f594, %f593, %f561, %f560;\n" " bfe.u32 %r376, %r352, 16, 8;\n" " cvt.rn.f32.u32 %f595, %r376;\n" " sub.rn.f32 %f596, %f595, %f544;\n" " mul.rn.f32 %f597, %f596, %f594;\n" " ld.shared.f32 %f598, [%r358+768];\n" " fma.rn.f32 %f599, %f597, %f598, %f592;\n" " and.b32 %r377, %r353, 61440;\n" " or.b32 %r378, %r377, 1258291200;\n" " mov.b32 %f600, %r378;\n" " fma.rn.f32 %f601, %f600, %f570, %f569;\n" " shr.u32 %r379, %r352, 24;\n" " cvt.rn.f32.u32 %f602, %r379;\n" " sub.rn.f32 %f603, %f602, %f544;\n" " mul.rn.f32 %f604, %f603, %f601;\n" " ld.shared.f32 %f605, [%r358+896];\n" " fma.rn.f32 %f606, %f604, %f605, %f599;\n" " fma.rn.f32 %f621, %f538, %f606, %f621;\n" " add.s64 %rd90, %rd90, 152;\n" " add.s32 %r401, %r401, 1024;\n" " add.s32 %r403, %r403, 1;\n" " setp.lt.s32 %p21, %r403, %r30;\n" " @%p21 bra $L__BB7_23;\n" "\n" "$L__BB7_24:\n" " cvt.u32.u64 %r380, %rd7;\n" " add.s32 %r393, %r380, 2048;\n" " setp.lt.s32 %p22, %r393, %r44;\n" " add.s32 %r392, %r392, 2048;\n" " @%p22 bra $L__BB7_2;\n" "\n" "$L__BB7_25:\n" " cvt.u32.u64 %r391, %rd1;\n" " setp.ge.s32 %p23, %r391, %r43;\n" " mov.u32 %r381, 16;\n" " mov.u32 %r390, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f607, %f621, %r381, %r390;\n" " // end inline asm\n" " add.f32 %f610, %f621, %f607;\n" " mov.u32 %r383, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f609, %f610, %r383, %r390;\n" " // end inline asm\n" " add.f32 %f612, %f610, %f609;\n" " mov.u32 %r385, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f611, %f612, %r385, %r390;\n" " // end inline asm\n" " add.f32 %f614, %f612, %f611;\n" " mov.u32 %r387, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f613, %f614, %r387, %r390;\n" " // end inline asm\n" " add.f32 %f616, %f614, %f613;\n" " mov.u32 %r389, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f615, %f616, %r389, %r390;\n" " // end inline asm\n" " add.f32 %f9, %f616, %f615;\n" " setp.ne.s32 %p24, %r2, 0;\n" " or.pred %p25, %p24, %p23;\n" " @%p25 bra $L__BB7_27;\n" "\n" " cvta.to.global.u64 %rd82, %rd22;\n" " shl.b64 %rd83, %rd1, 2;\n" " add.s64 %rd84, %rd82, %rd83;\n" " st.global.f32 [%rd84], %f9;\n" "\n" "$L__BB7_27:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_nx\n" ".visible .entry fc_matvec_q3k_nx(\n" " .param .u64 fc_matvec_q3k_nx_param_0,\n" " .param .u64 fc_matvec_q3k_nx_param_1,\n" " .param .u64 fc_matvec_q3k_nx_param_2,\n" " .param .u32 fc_matvec_q3k_nx_param_3,\n" " .param .u32 fc_matvec_q3k_nx_param_4,\n" " .param .u32 fc_matvec_q3k_nx_param_5,\n" " .param .u32 fc_matvec_q3k_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<33>;\n" " .reg .f32 %f<390>;\n" " .reg .b32 %r<218>;\n" " .reg .b64 %rd<92>;\n" "\n" "\n" " ld.param.u64 %rd33, [fc_matvec_q3k_nx_param_0];\n" " ld.param.u64 %rd32, [fc_matvec_q3k_nx_param_1];\n" " ld.param.u64 %rd34, [fc_matvec_q3k_nx_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q3k_nx_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q3k_nx_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q3k_nx_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q3k_nx_param_6];\n" " cvta.to.global.u64 %rd1, %rd34;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r26, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 3;\n" " add.s32 %r29, %r28, %r26;\n" " shl.b32 %r30, %r29, 1;\n" " mov.u32 %r31, %ctaid.y;\n" " shl.b32 %r3, %r31, 2;\n" " sub.s32 %r4, %r25, %r3;\n" " min.s32 %r5, %r4, 4;\n" " or.b32 %r6, %r30, 1;\n" " setp.ge.s32 %p1, %r6, %r22;\n" " cvt.s64.s32 %rd2, %r30;\n" " cvt.s64.s32 %rd3, %r24;\n" " mul.wide.s32 %rd35, %r24, %r30;\n" " add.s64 %rd90, %rd33, %rd35;\n" " mov.u64 %rd91, %rd90;\n" " @%p1 bra $L__BB8_2;\n" "\n" " add.s64 %rd91, %rd90, %rd3;\n" "\n" "$L__BB8_2:\n" " setp.lt.s32 %p2, %r23, 1;\n" " mov.f32 %f374, 0f00000000;\n" " mov.f32 %f375, %f374;\n" " mov.f32 %f376, %f374;\n" " mov.f32 %f377, %f374;\n" " mov.f32 %f378, %f374;\n" " mov.f32 %f379, %f374;\n" " mov.f32 %f380, %f374;\n" " mov.f32 %f381, %f374;\n" " @%p2 bra $L__BB8_20;\n" "\n" " cvt.u64.u32 %rd7, %r2;\n" " shr.u32 %r33, %r2, 4;\n" " cvt.u64.u32 %rd8, %r33;\n" " add.s32 %r34, %r3, 1;\n" " add.s32 %r35, %r3, 2;\n" " add.s32 %r36, %r3, 3;\n" " mul.wide.s32 %rd36, %r23, %r3;\n" " shl.b64 %rd37, %rd36, 2;\n" " mul.wide.s32 %rd38, %r1, 4;\n" " add.s64 %rd9, %rd37, %rd38;\n" " mul.wide.s32 %rd39, %r23, %r34;\n" " shl.b64 %rd40, %rd39, 2;\n" " add.s64 %rd10, %rd40, %rd38;\n" " mul.wide.s32 %rd41, %r23, %r35;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd11, %rd42, %rd38;\n" " shl.b32 %r37, %r1, 2;\n" " mov.u32 %r38, sh_x;\n" " add.s32 %r39, %r38, %r37;\n" " add.s32 %r7, %r39, 4096;\n" " mul.wide.s32 %rd43, %r23, %r36;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd12, %rd44, %rd38;\n" " mov.u32 %r213, 0;\n" " mov.f32 %f374, 0f00000000;\n" " setp.lt.s32 %p4, %r4, 1;\n" " setp.lt.s32 %p5, %r5, 2;\n" " setp.lt.s32 %p6, %r5, 3;\n" " setp.lt.s32 %p7, %r5, 4;\n" " shl.b64 %rd56, %rd7, 2;\n" " shl.b64 %rd57, %rd8, 3;\n" " mov.f32 %f375, %f374;\n" " mov.f32 %f376, %f374;\n" " mov.f32 %f377, %f374;\n" " mov.f32 %f378, %f374;\n" " mov.f32 %f379, %f374;\n" " mov.f32 %f380, %f374;\n" " mov.f32 %f381, %f374;\n" "\n" "$L__BB8_4:\n" " sub.s32 %r40, %r23, %r213;\n" " min.s32 %r9, %r40, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r9;\n" " @%p3 bra $L__BB8_15;\n" "\n" " mov.u32 %r214, %r7;\n" " mov.u64 %rd87, %rd32;\n" " mov.u32 %r215, %r1;\n" "\n" "$L__BB8_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f362, 0f00000000;\n" " @%p4 bra $L__BB8_8;\n" "\n" " mul.wide.s32 %rd77, %r213, 4;\n" " add.s64 %rd76, %rd9, %rd77;\n" " add.s64 %rd46, %rd87, %rd76;\n" " // begin inline asm\n" " ld.global.nc.f32 %f362, [%rd46];\n" " // end inline asm\n" "\n" "$L__BB8_8:\n" " mov.f32 %f363, 0f00000000;\n" " st.shared.f32 [%r214+-4096], %f362;\n" " @%p5 bra $L__BB8_10;\n" "\n" " mul.wide.s32 %rd81, %r213, 4;\n" " add.s64 %rd80, %rd10, %rd81;\n" " add.s64 %rd47, %rd87, %rd80;\n" " // begin inline asm\n" " ld.global.nc.f32 %f363, [%rd47];\n" " // end inline asm\n" "\n" "$L__BB8_10:\n" " mov.f32 %f364, 0f00000000;\n" " add.s32 %r189, %r214, -4096;\n" " st.shared.f32 [%r189+2048], %f363;\n" " @%p6 bra $L__BB8_12;\n" "\n" " mul.wide.s32 %rd79, %r213, 4;\n" " add.s64 %rd78, %rd11, %rd79;\n" " add.s64 %rd48, %rd87, %rd78;\n" " // begin inline asm\n" " ld.global.nc.f32 %f364, [%rd48];\n" " // end inline asm\n" "\n" "$L__BB8_12:\n" " mov.f32 %f365, 0f00000000;\n" " add.s32 %r190, %r214, -4096;\n" " st.shared.f32 [%r190+4096], %f364;\n" " @%p7 bra $L__BB8_14;\n" "\n" " mul.wide.s32 %rd83, %r213, 4;\n" " add.s64 %rd82, %rd12, %rd83;\n" " add.s64 %rd49, %rd87, %rd82;\n" " // begin inline asm\n" " ld.global.nc.f32 %f365, [%rd49];\n" " // end inline asm\n" "\n" "$L__BB8_14:\n" " sub.s32 %r197, %r23, %r213;\n" " min.s32 %r196, %r197, 512;\n" " add.s32 %r191, %r214, -4096;\n" " st.shared.f32 [%r191+6144], %f365;\n" " add.s64 %rd87, %rd87, 1024;\n" " add.s32 %r214, %r214, 1024;\n" " add.s32 %r215, %r215, 256;\n" " setp.lt.s32 %p8, %r215, %r196;\n" " @%p8 bra $L__BB8_6;\n" "\n" "$L__BB8_15:\n" " cvt.u32.u64 %r41, %rd2;\n" " setp.ge.s32 %p9, %r41, %r22;\n" " bar.sync 0;\n" " @%p9 bra $L__BB8_19;\n" "\n" " sub.s32 %r199, %r23, %r213;\n" " min.s32 %r198, %r199, 512;\n" " setp.lt.s32 %p10, %r198, 256;\n" " @%p10 bra $L__BB8_19;\n" "\n" " mov.u32 %r216, 0;\n" " mov.u32 %r217, %r38;\n" "\n" "$L__BB8_18:\n" " sub.s32 %r212, %r23, %r213;\n" " min.s32 %r211, %r212, 512;\n" " shr.s32 %r210, %r211, 8;\n" " add.s64 %rd50, %rd90, %rd56;\n" " // begin inline asm\n" " ld.global.nc.u32 %r44, [%rd50];\n" " // end inline asm\n" " add.s64 %rd51, %rd91, %rd56;\n" " // begin inline asm\n" " ld.global.nc.u32 %r45, [%rd51];\n" " // end inline asm\n" " add.s64 %rd52, %rd90, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f81, [%rd52];\n" " // end inline asm\n" " add.s64 %rd53, %rd91, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f82, [%rd53];\n" " // end inline asm\n" " add.s64 %rd58, %rd90, %rd57;\n" " add.s64 %rd54, %rd58, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r46,%r47}, [%rd54];\n" " // end inline asm\n" " add.s64 %rd59, %rd91, %rd57;\n" " add.s64 %rd55, %rd59, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r48,%r49}, [%rd55];\n" " // end inline asm\n" " shr.u32 %r50, %r44, 16;\n" " shr.u32 %r51, %r45, 16;\n" " and.b32 %r52, %r44, 15;\n" " or.b32 %r53, %r52, 1258291200;\n" " mov.b32 %f83, %r53;\n" " mov.f32 %f84, 0fCB000004;\n" " mov.f32 %f85, 0f3F800000;\n" " fma.rn.f32 %f86, %f83, %f85, %f84;\n" " and.b32 %r54, %r46, 255;\n" " cvt.rn.f32.u32 %f87, %r54;\n" " mov.f32 %f88, 0f42000000;\n" " sub.rn.f32 %f89, %f87, %f88;\n" " and.b32 %r55, %r45, 15;\n" " or.b32 %r56, %r55, 1258291200;\n" " mov.b32 %f90, %r56;\n" " fma.rn.f32 %f91, %f90, %f85, %f84;\n" " and.b32 %r57, %r48, 255;\n" " cvt.rn.f32.u32 %f92, %r57;\n" " sub.rn.f32 %f93, %f92, %f88;\n" " mul.rn.f32 %f94, %f89, %f86;\n" " mul.rn.f32 %f95, %f93, %f91;\n" " shl.b32 %r58, %r2, 2;\n" " add.s32 %r59, %r217, %r58;\n" " ld.shared.f32 %f96, [%r59];\n" " mov.f32 %f97, 0f00000000;\n" " fma.rn.f32 %f98, %f94, %f96, %f97;\n" " fma.rn.f32 %f99, %f95, %f96, %f97;\n" " ld.shared.f32 %f100, [%r59+2048];\n" " fma.rn.f32 %f101, %f94, %f100, %f97;\n" " fma.rn.f32 %f102, %f95, %f100, %f97;\n" " ld.shared.f32 %f103, [%r59+4096];\n" " fma.rn.f32 %f104, %f94, %f103, %f97;\n" " fma.rn.f32 %f105, %f95, %f103, %f97;\n" " ld.shared.f32 %f106, [%r59+6144];\n" " fma.rn.f32 %f107, %f94, %f106, %f97;\n" " fma.rn.f32 %f108, %f95, %f106, %f97;\n" " and.b32 %r60, %r44, 240;\n" " or.b32 %r61, %r60, 1258291200;\n" " mov.b32 %f109, %r61;\n" " mov.f32 %f110, 0fC9000040;\n" " mov.f32 %f111, 0f3D800000;\n" " fma.rn.f32 %f112, %f109, %f111, %f110;\n" " bfe.u32 %r62, %r46, 8, 8;\n" " cvt.rn.f32.u32 %f113, %r62;\n" " sub.rn.f32 %f114, %f113, %f88;\n" " and.b32 %r63, %r45, 240;\n" " or.b32 %r64, %r63, 1258291200;\n" " mov.b32 %f115, %r64;\n" " fma.rn.f32 %f116, %f115, %f111, %f110;\n" " bfe.u32 %r65, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f117, %r65;\n" " sub.rn.f32 %f118, %f117, %f88;\n" " mul.rn.f32 %f119, %f114, %f112;\n" " mul.rn.f32 %f120, %f118, %f116;\n" " ld.shared.f32 %f121, [%r59+128];\n" " fma.rn.f32 %f122, %f119, %f121, %f98;\n" " fma.rn.f32 %f123, %f120, %f121, %f99;\n" " ld.shared.f32 %f124, [%r59+2176];\n" " fma.rn.f32 %f125, %f119, %f124, %f101;\n" " fma.rn.f32 %f126, %f120, %f124, %f102;\n" " ld.shared.f32 %f127, [%r59+4224];\n" " fma.rn.f32 %f128, %f119, %f127, %f104;\n" " fma.rn.f32 %f129, %f120, %f127, %f105;\n" " ld.shared.f32 %f130, [%r59+6272];\n" " fma.rn.f32 %f131, %f119, %f130, %f107;\n" " fma.rn.f32 %f132, %f120, %f130, %f108;\n" " and.b32 %r66, %r44, 3840;\n" " or.b32 %r67, %r66, 1258291200;\n" " mov.b32 %f133, %r67;\n" " mov.f32 %f134, 0fC7000400;\n" " mov.f32 %f135, 0f3B800000;\n" " fma.rn.f32 %f136, %f133, %f135, %f134;\n" " bfe.u32 %r68, %r46, 16, 8;\n" " cvt.rn.f32.u32 %f137, %r68;\n" " sub.rn.f32 %f138, %f137, %f88;\n" " and.b32 %r69, %r45, 3840;\n" " or.b32 %r70, %r69, 1258291200;\n" " mov.b32 %f139, %r70;\n" " fma.rn.f32 %f140, %f139, %f135, %f134;\n" " bfe.u32 %r71, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f141, %r71;\n" " sub.rn.f32 %f142, %f141, %f88;\n" " mul.rn.f32 %f143, %f138, %f136;\n" " mul.rn.f32 %f144, %f142, %f140;\n" " ld.shared.f32 %f145, [%r59+256];\n" " fma.rn.f32 %f146, %f143, %f145, %f122;\n" " fma.rn.f32 %f147, %f144, %f145, %f123;\n" " ld.shared.f32 %f148, [%r59+2304];\n" " fma.rn.f32 %f149, %f143, %f148, %f125;\n" " fma.rn.f32 %f150, %f144, %f148, %f126;\n" " ld.shared.f32 %f151, [%r59+4352];\n" " fma.rn.f32 %f152, %f143, %f151, %f128;\n" " fma.rn.f32 %f153, %f144, %f151, %f129;\n" " ld.shared.f32 %f154, [%r59+6400];\n" " fma.rn.f32 %f155, %f143, %f154, %f131;\n" " fma.rn.f32 %f156, %f144, %f154, %f132;\n" " and.b32 %r72, %r44, 61440;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f157, %r73;\n" " mov.f32 %f158, 0fC5004000;\n" " mov.f32 %f159, 0f39800000;\n" " fma.rn.f32 %f160, %f157, %f159, %f158;\n" " shr.u32 %r74, %r46, 24;\n" " cvt.rn.f32.u32 %f161, %r74;\n" " sub.rn.f32 %f162, %f161, %f88;\n" " and.b32 %r75, %r45, 61440;\n" " or.b32 %r76, %r75, 1258291200;\n" " mov.b32 %f163, %r76;\n" " fma.rn.f32 %f164, %f163, %f159, %f158;\n" " shr.u32 %r77, %r48, 24;\n" " cvt.rn.f32.u32 %f165, %r77;\n" " sub.rn.f32 %f166, %f165, %f88;\n" " mul.rn.f32 %f167, %f162, %f160;\n" " mul.rn.f32 %f168, %f166, %f164;\n" " ld.shared.f32 %f169, [%r59+384];\n" " fma.rn.f32 %f170, %f167, %f169, %f146;\n" " fma.rn.f32 %f171, %f168, %f169, %f147;\n" " ld.shared.f32 %f172, [%r59+2432];\n" " fma.rn.f32 %f173, %f167, %f172, %f149;\n" " fma.rn.f32 %f174, %f168, %f172, %f150;\n" " ld.shared.f32 %f175, [%r59+4480];\n" " fma.rn.f32 %f176, %f167, %f175, %f152;\n" " fma.rn.f32 %f177, %f168, %f175, %f153;\n" " ld.shared.f32 %f178, [%r59+6528];\n" " fma.rn.f32 %f179, %f167, %f178, %f155;\n" " fma.rn.f32 %f180, %f168, %f178, %f156;\n" " and.b32 %r78, %r44, 983040;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f181, %r79;\n" " mov.f32 %f182, 0fC3040000;\n" " mov.f32 %f183, 0f37800000;\n" " fma.rn.f32 %f184, %f181, %f183, %f182;\n" " and.b32 %r80, %r47, 255;\n" " cvt.rn.f32.u32 %f185, %r80;\n" " sub.rn.f32 %f186, %f185, %f88;\n" " and.b32 %r81, %r45, 983040;\n" " or.b32 %r82, %r81, 1258291200;\n" " mov.b32 %f187, %r82;\n" " fma.rn.f32 %f188, %f187, %f183, %f182;\n" " and.b32 %r83, %r49, 255;\n" " cvt.rn.f32.u32 %f189, %r83;\n" " sub.rn.f32 %f190, %f189, %f88;\n" " mul.rn.f32 %f191, %f186, %f184;\n" " mul.rn.f32 %f192, %f190, %f188;\n" " ld.shared.f32 %f193, [%r59+512];\n" " fma.rn.f32 %f194, %f191, %f193, %f170;\n" " fma.rn.f32 %f195, %f192, %f193, %f171;\n" " ld.shared.f32 %f196, [%r59+2560];\n" " fma.rn.f32 %f197, %f191, %f196, %f173;\n" " fma.rn.f32 %f198, %f192, %f196, %f174;\n" " ld.shared.f32 %f199, [%r59+4608];\n" " fma.rn.f32 %f200, %f191, %f199, %f176;\n" " fma.rn.f32 %f201, %f192, %f199, %f177;\n" " ld.shared.f32 %f202, [%r59+6656];\n" " fma.rn.f32 %f203, %f191, %f202, %f179;\n" " fma.rn.f32 %f204, %f192, %f202, %f180;\n" " and.b32 %r84, %r50, 240;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f205, %r85;\n" " fma.rn.f32 %f206, %f205, %f111, %f110;\n" " bfe.u32 %r86, %r47, 8, 8;\n" " cvt.rn.f32.u32 %f207, %r86;\n" " sub.rn.f32 %f208, %f207, %f88;\n" " and.b32 %r87, %r51, 240;\n" " or.b32 %r88, %r87, 1258291200;\n" " mov.b32 %f209, %r88;\n" " fma.rn.f32 %f210, %f209, %f111, %f110;\n" " bfe.u32 %r89, %r49, 8, 8;\n" " cvt.rn.f32.u32 %f211, %r89;\n" " sub.rn.f32 %f212, %f211, %f88;\n" " mul.rn.f32 %f213, %f208, %f206;\n" " mul.rn.f32 %f214, %f212, %f210;\n" " ld.shared.f32 %f215, [%r59+640];\n" " fma.rn.f32 %f216, %f213, %f215, %f194;\n" " fma.rn.f32 %f217, %f214, %f215, %f195;\n" " ld.shared.f32 %f218, [%r59+2688];\n" " fma.rn.f32 %f219, %f213, %f218, %f197;\n" " fma.rn.f32 %f220, %f214, %f218, %f198;\n" " ld.shared.f32 %f221, [%r59+4736];\n" " fma.rn.f32 %f222, %f213, %f221, %f200;\n" " fma.rn.f32 %f223, %f214, %f221, %f201;\n" " ld.shared.f32 %f224, [%r59+6784];\n" " fma.rn.f32 %f225, %f213, %f224, %f203;\n" " fma.rn.f32 %f226, %f214, %f224, %f204;\n" " and.b32 %r90, %r50, 3840;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f227, %r91;\n" " fma.rn.f32 %f228, %f227, %f135, %f134;\n" " bfe.u32 %r92, %r47, 16, 8;\n" " cvt.rn.f32.u32 %f229, %r92;\n" " sub.rn.f32 %f230, %f229, %f88;\n" " and.b32 %r93, %r51, 3840;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f231, %r94;\n" " fma.rn.f32 %f232, %f231, %f135, %f134;\n" " bfe.u32 %r95, %r49, 16, 8;\n" " cvt.rn.f32.u32 %f233, %r95;\n" " sub.rn.f32 %f234, %f233, %f88;\n" " mul.rn.f32 %f235, %f230, %f228;\n" " mul.rn.f32 %f236, %f234, %f232;\n" " ld.shared.f32 %f237, [%r59+768];\n" " fma.rn.f32 %f238, %f235, %f237, %f216;\n" " fma.rn.f32 %f239, %f236, %f237, %f217;\n" " ld.shared.f32 %f240, [%r59+2816];\n" " fma.rn.f32 %f241, %f235, %f240, %f219;\n" " fma.rn.f32 %f242, %f236, %f240, %f220;\n" " ld.shared.f32 %f243, [%r59+4864];\n" " fma.rn.f32 %f244, %f235, %f243, %f222;\n" " fma.rn.f32 %f245, %f236, %f243, %f223;\n" " ld.shared.f32 %f246, [%r59+6912];\n" " fma.rn.f32 %f247, %f235, %f246, %f225;\n" " fma.rn.f32 %f248, %f236, %f246, %f226;\n" " and.b32 %r96, %r50, 61440;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f249, %r97;\n" " fma.rn.f32 %f250, %f249, %f159, %f158;\n" " shr.u32 %r98, %r47, 24;\n" " cvt.rn.f32.u32 %f251, %r98;\n" " sub.rn.f32 %f252, %f251, %f88;\n" " and.b32 %r99, %r51, 61440;\n" " or.b32 %r100, %r99, 1258291200;\n" " mov.b32 %f253, %r100;\n" " fma.rn.f32 %f254, %f253, %f159, %f158;\n" " shr.u32 %r101, %r49, 24;\n" " cvt.rn.f32.u32 %f255, %r101;\n" " sub.rn.f32 %f256, %f255, %f88;\n" " mul.rn.f32 %f257, %f252, %f250;\n" " mul.rn.f32 %f258, %f256, %f254;\n" " ld.shared.f32 %f259, [%r59+896];\n" " fma.rn.f32 %f260, %f257, %f259, %f238;\n" " fma.rn.f32 %f261, %f258, %f259, %f239;\n" " ld.shared.f32 %f262, [%r59+2944];\n" " fma.rn.f32 %f263, %f257, %f262, %f241;\n" " fma.rn.f32 %f264, %f258, %f262, %f242;\n" " ld.shared.f32 %f265, [%r59+4992];\n" " fma.rn.f32 %f266, %f257, %f265, %f244;\n" " fma.rn.f32 %f267, %f258, %f265, %f245;\n" " ld.shared.f32 %f268, [%r59+7040];\n" " fma.rn.f32 %f269, %f257, %f268, %f247;\n" " fma.rn.f32 %f270, %f258, %f268, %f248;\n" " fma.rn.f32 %f377, %f81, %f260, %f377;\n" " fma.rn.f32 %f381, %f82, %f261, %f381;\n" " fma.rn.f32 %f376, %f81, %f263, %f376;\n" " fma.rn.f32 %f380, %f82, %f264, %f380;\n" " fma.rn.f32 %f375, %f81, %f266, %f375;\n" " fma.rn.f32 %f379, %f82, %f267, %f379;\n" " fma.rn.f32 %f374, %f81, %f269, %f374;\n" " fma.rn.f32 %f378, %f82, %f270, %f378;\n" " add.s64 %rd90, %rd90, 152;\n" " add.s64 %rd91, %rd91, 152;\n" " add.s32 %r217, %r217, 1024;\n" " add.s32 %r216, %r216, 1;\n" " setp.lt.s32 %p11, %r216, %r210;\n" " @%p11 bra $L__BB8_18;\n" "\n" "$L__BB8_19:\n" " add.s32 %r213, %r213, 512;\n" " setp.lt.s32 %p12, %r213, %r23;\n" " @%p12 bra $L__BB8_4;\n" "\n" "$L__BB8_20:\n" " mov.u32 %r195, %ctaid.y;\n" " shl.b32 %r194, %r195, 2;\n" " ld.param.u32 %r193, [fc_matvec_q3k_nx_param_6];\n" " sub.s32 %r192, %r193, %r194;\n" " cvt.s64.s32 %rd27, %r22;\n" " mov.u32 %r112, 16;\n" " mov.u32 %r121, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f271, %f377, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f274, %f377, %f271;\n" " mov.u32 %r114, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f273, %f274, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f276, %f274, %f273;\n" " mov.u32 %r116, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f275, %f276, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f278, %f276, %f275;\n" " mov.u32 %r118, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f277, %f278, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f280, %f278, %f277;\n" " mov.u32 %r120, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f279, %f280, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f49, %f280, %f279;\n" " // begin inline asm\n" " shfl.down.b32 %f281, %f381, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f284, %f381, %f281;\n" " // begin inline asm\n" " shfl.down.b32 %f283, %f284, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f286, %f284, %f283;\n" " // begin inline asm\n" " shfl.down.b32 %f285, %f286, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f288, %f286, %f285;\n" " // begin inline asm\n" " shfl.down.b32 %f287, %f288, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f290, %f288, %f287;\n" " // begin inline asm\n" " shfl.down.b32 %f289, %f290, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f50, %f290, %f289;\n" " setp.lt.s32 %p13, %r192, 1;\n" " setp.ne.s32 %p14, %r2, 0;\n" " or.pred %p15, %p14, %p13;\n" " @%p15 bra $L__BB8_25;\n" "\n" " mov.u32 %r209, %ctaid.y;\n" " shl.b32 %r208, %r209, 2;\n" " cvt.u32.u64 %r122, %rd2;\n" " setp.ge.s32 %p16, %r122, %r22;\n" " cvt.s64.s32 %rd60, %r208;\n" " mul.lo.s64 %rd61, %rd60, %rd27;\n" " add.s64 %rd62, %rd61, %rd2;\n" " shl.b64 %rd63, %rd62, 2;\n" " add.s64 %rd28, %rd1, %rd63;\n" " @%p16 bra $L__BB8_23;\n" "\n" " st.global.f32 [%rd28], %f49;\n" "\n" "$L__BB8_23:\n" " @%p1 bra $L__BB8_25;\n" "\n" " st.global.f32 [%rd28+4], %f50;\n" "\n" "$L__BB8_25:\n" " // begin inline asm\n" " shfl.down.b32 %f291, %f376, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f294, %f376, %f291;\n" " // begin inline asm\n" " shfl.down.b32 %f293, %f294, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f296, %f294, %f293;\n" " // begin inline asm\n" " shfl.down.b32 %f295, %f296, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f298, %f296, %f295;\n" " // begin inline asm\n" " shfl.down.b32 %f297, %f298, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f300, %f298, %f297;\n" " // begin inline asm\n" " shfl.down.b32 %f299, %f300, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f51, %f300, %f299;\n" " // begin inline asm\n" " shfl.down.b32 %f301, %f380, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f304, %f380, %f301;\n" " // begin inline asm\n" " shfl.down.b32 %f303, %f304, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f306, %f304, %f303;\n" " // begin inline asm\n" " shfl.down.b32 %f305, %f306, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f308, %f306, %f305;\n" " // begin inline asm\n" " shfl.down.b32 %f307, %f308, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f310, %f308, %f307;\n" " // begin inline asm\n" " shfl.down.b32 %f309, %f310, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f52, %f310, %f309;\n" " setp.lt.s32 %p19, %r5, 2;\n" " or.pred %p20, %p14, %p19;\n" " @%p20 bra $L__BB8_30;\n" "\n" " mov.u32 %r207, %ctaid.y;\n" " shl.b32 %r206, %r207, 2;\n" " cvt.u32.u64 %r143, %rd2;\n" " setp.ge.s32 %p21, %r143, %r22;\n" " add.s32 %r144, %r206, 1;\n" " cvt.s64.s32 %rd64, %r144;\n" " mul.lo.s64 %rd65, %rd64, %rd27;\n" " add.s64 %rd66, %rd65, %rd2;\n" " shl.b64 %rd67, %rd66, 2;\n" " add.s64 %rd29, %rd1, %rd67;\n" " @%p21 bra $L__BB8_28;\n" "\n" " st.global.f32 [%rd29], %f51;\n" "\n" "$L__BB8_28:\n" " @%p1 bra $L__BB8_30;\n" "\n" " st.global.f32 [%rd29+4], %f52;\n" "\n" "$L__BB8_30:\n" " // begin inline asm\n" " shfl.down.b32 %f311, %f375, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f314, %f375, %f311;\n" " // begin inline asm\n" " shfl.down.b32 %f313, %f314, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f316, %f314, %f313;\n" " // begin inline asm\n" " shfl.down.b32 %f315, %f316, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f318, %f316, %f315;\n" " // begin inline asm\n" " shfl.down.b32 %f317, %f318, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f320, %f318, %f317;\n" " // begin inline asm\n" " shfl.down.b32 %f319, %f320, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f53, %f320, %f319;\n" " // begin inline asm\n" " shfl.down.b32 %f321, %f379, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f324, %f379, %f321;\n" " // begin inline asm\n" " shfl.down.b32 %f323, %f324, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f326, %f324, %f323;\n" " // begin inline asm\n" " shfl.down.b32 %f325, %f326, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f328, %f326, %f325;\n" " // begin inline asm\n" " shfl.down.b32 %f327, %f328, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f330, %f328, %f327;\n" " // begin inline asm\n" " shfl.down.b32 %f329, %f330, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f54, %f330, %f329;\n" " setp.lt.s32 %p24, %r5, 3;\n" " or.pred %p25, %p14, %p24;\n" " @%p25 bra $L__BB8_35;\n" "\n" " mov.u32 %r205, %ctaid.y;\n" " shl.b32 %r204, %r205, 2;\n" " cvt.u32.u64 %r165, %rd2;\n" " setp.ge.s32 %p26, %r165, %r22;\n" " add.s32 %r166, %r204, 2;\n" " cvt.s64.s32 %rd68, %r166;\n" " mul.lo.s64 %rd69, %rd68, %rd27;\n" " add.s64 %rd70, %rd69, %rd2;\n" " shl.b64 %rd71, %rd70, 2;\n" " add.s64 %rd30, %rd1, %rd71;\n" " @%p26 bra $L__BB8_33;\n" "\n" " st.global.f32 [%rd30], %f53;\n" "\n" "$L__BB8_33:\n" " @%p1 bra $L__BB8_35;\n" "\n" " st.global.f32 [%rd30+4], %f54;\n" "\n" "$L__BB8_35:\n" " // begin inline asm\n" " shfl.down.b32 %f331, %f374, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f334, %f374, %f331;\n" " // begin inline asm\n" " shfl.down.b32 %f333, %f334, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f336, %f334, %f333;\n" " // begin inline asm\n" " shfl.down.b32 %f335, %f336, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f338, %f336, %f335;\n" " // begin inline asm\n" " shfl.down.b32 %f337, %f338, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f340, %f338, %f337;\n" " // begin inline asm\n" " shfl.down.b32 %f339, %f340, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f55, %f340, %f339;\n" " // begin inline asm\n" " shfl.down.b32 %f341, %f378, %r112, %r121;\n" " // end inline asm\n" " add.f32 %f344, %f378, %f341;\n" " // begin inline asm\n" " shfl.down.b32 %f343, %f344, %r114, %r121;\n" " // end inline asm\n" " add.f32 %f346, %f344, %f343;\n" " // begin inline asm\n" " shfl.down.b32 %f345, %f346, %r116, %r121;\n" " // end inline asm\n" " add.f32 %f348, %f346, %f345;\n" " // begin inline asm\n" " shfl.down.b32 %f347, %f348, %r118, %r121;\n" " // end inline asm\n" " add.f32 %f350, %f348, %f347;\n" " // begin inline asm\n" " shfl.down.b32 %f349, %f350, %r120, %r121;\n" " // end inline asm\n" " add.f32 %f56, %f350, %f349;\n" " setp.lt.s32 %p29, %r5, 4;\n" " or.pred %p30, %p14, %p29;\n" " @%p30 bra $L__BB8_40;\n" "\n" " mov.u32 %r203, %ctaid.y;\n" " shl.b32 %r202, %r203, 2;\n" " cvt.u32.u64 %r187, %rd2;\n" " setp.ge.s32 %p31, %r187, %r22;\n" " add.s32 %r188, %r202, 3;\n" " cvt.s64.s32 %rd72, %r188;\n" " mul.lo.s64 %rd73, %rd72, %rd27;\n" " add.s64 %rd74, %rd73, %rd2;\n" " shl.b64 %rd75, %rd74, 2;\n" " add.s64 %rd31, %rd1, %rd75;\n" " @%p31 bra $L__BB8_38;\n" "\n" " st.global.f32 [%rd31], %f55;\n" "\n" "$L__BB8_38:\n" " @%p1 bra $L__BB8_40;\n" "\n" " st.global.f32 [%rd31+4], %f56;\n" "\n" "$L__BB8_40:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_nx2\n" ".visible .entry fc_matvec_q3k_nx2(\n" " .param .u64 fc_matvec_q3k_nx2_param_0,\n" " .param .u64 fc_matvec_q3k_nx2_param_1,\n" " .param .u64 fc_matvec_q3k_nx2_param_2,\n" " .param .u32 fc_matvec_q3k_nx2_param_3,\n" " .param .u32 fc_matvec_q3k_nx2_param_4,\n" " .param .u32 fc_matvec_q3k_nx2_param_5,\n" " .param .u32 fc_matvec_q3k_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<21>;\n" " .reg .f32 %f<241>;\n" " .reg .b32 %r<149>;\n" " .reg .b64 %rd<69>;\n" "\n" "\n" " ld.param.u64 %rd29, [fc_matvec_q3k_nx2_param_0];\n" " ld.param.u64 %rd28, [fc_matvec_q3k_nx2_param_1];\n" " ld.param.u64 %rd30, [fc_matvec_q3k_nx2_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q3k_nx2_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q3k_nx2_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q3k_nx2_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q3k_nx2_param_6];\n" " cvta.to.global.u64 %rd1, %rd30;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r26, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 3;\n" " add.s32 %r29, %r28, %r26;\n" " shl.b32 %r30, %r29, 1;\n" " mov.u32 %r31, %ctaid.y;\n" " shl.b32 %r3, %r31, 1;\n" " sub.s32 %r4, %r25, %r3;\n" " min.s32 %r5, %r4, 2;\n" " or.b32 %r6, %r30, 1;\n" " setp.ge.s32 %p1, %r6, %r22;\n" " cvt.s64.s32 %rd2, %r30;\n" " cvt.s64.s32 %rd3, %r24;\n" " mul.wide.s32 %rd31, %r24, %r30;\n" " add.s64 %rd67, %rd29, %rd31;\n" " mov.u64 %rd68, %rd67;\n" " @%p1 bra $L__BB9_2;\n" "\n" " add.s64 %rd68, %rd67, %rd3;\n" "\n" "$L__BB9_2:\n" " setp.lt.s32 %p2, %r23, 1;\n" " mov.f32 %f233, 0f00000000;\n" " mov.f32 %f234, %f233;\n" " mov.f32 %f235, %f233;\n" " mov.f32 %f236, %f233;\n" " @%p2 bra $L__BB9_16;\n" "\n" " cvt.u64.u32 %rd7, %r2;\n" " shr.u32 %r33, %r2, 4;\n" " cvt.u64.u32 %rd8, %r33;\n" " add.s32 %r34, %r3, 1;\n" " mul.wide.s32 %rd32, %r23, %r3;\n" " cvt.s64.s32 %rd33, %r1;\n" " add.s64 %rd34, %rd32, %rd33;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd9, %rd28, %rd35;\n" " shl.b32 %r35, %r1, 2;\n" " mov.u32 %r36, sh_x;\n" " add.s32 %r37, %r36, %r35;\n" " add.s32 %r7, %r37, 2048;\n" " mul.wide.s32 %rd36, %r23, %r34;\n" " add.s64 %rd37, %rd36, %rd33;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd10, %rd28, %rd38;\n" " mov.u32 %r144, 0;\n" " mov.f32 %f233, 0f00000000;\n" " setp.lt.s32 %p4, %r4, 1;\n" " setp.lt.s32 %p5, %r5, 2;\n" " shl.b64 %rd48, %rd7, 2;\n" " shl.b64 %rd49, %rd8, 3;\n" " mov.f32 %f234, %f233;\n" " mov.f32 %f235, %f233;\n" " mov.f32 %f236, %f233;\n" "\n" "$L__BB9_4:\n" " sub.s32 %r38, %r23, %r144;\n" " min.s32 %r9, %r38, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r9;\n" " @%p3 bra $L__BB9_11;\n" "\n" " mul.wide.s32 %rd39, %r144, 4;\n" " add.s64 %rd64, %rd9, %rd39;\n" " add.s64 %rd63, %rd10, %rd39;\n" " mov.u32 %r145, %r7;\n" " mov.u32 %r146, %r1;\n" "\n" "$L__BB9_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f228, 0f00000000;\n" " mov.f32 %f227, %f228;\n" " @%p4 bra $L__BB9_8;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f227, [%rd64];\n" " // end inline asm\n" "\n" "$L__BB9_8:\n" " st.shared.f32 [%r145+-2048], %f227;\n" " @%p5 bra $L__BB9_10;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f228, [%rd63];\n" " // end inline asm\n" "\n" "$L__BB9_10:\n" " add.s32 %r143, %r145, -2048;\n" " st.shared.f32 [%r143+2048], %f228;\n" " add.s64 %rd64, %rd64, 1024;\n" " add.s32 %r145, %r145, 1024;\n" " add.s64 %rd63, %rd63, 1024;\n" " add.s32 %r146, %r146, 256;\n" " setp.lt.s32 %p6, %r146, %r9;\n" " @%p6 bra $L__BB9_6;\n" "\n" "$L__BB9_11:\n" " cvt.u32.u64 %r39, %rd2;\n" " setp.ge.s32 %p7, %r39, %r22;\n" " bar.sync 0;\n" " @%p7 bra $L__BB9_15;\n" "\n" " setp.lt.s32 %p8, %r9, 256;\n" " @%p8 bra $L__BB9_15;\n" "\n" " shr.s32 %r16, %r9, 8;\n" " mov.u32 %r147, 0;\n" " mov.u32 %r148, %r36;\n" "\n" "$L__BB9_14:\n" " add.s64 %rd42, %rd67, %rd48;\n" " // begin inline asm\n" " ld.global.nc.u32 %r42, [%rd42];\n" " // end inline asm\n" " add.s64 %rd43, %rd68, %rd48;\n" " // begin inline asm\n" " ld.global.nc.u32 %r43, [%rd43];\n" " // end inline asm\n" " add.s64 %rd44, %rd67, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f41, [%rd44];\n" " // end inline asm\n" " add.s64 %rd45, %rd68, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f42, [%rd45];\n" " // end inline asm\n" " add.s64 %rd50, %rd67, %rd49;\n" " add.s64 %rd46, %rd50, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r44,%r45}, [%rd46];\n" " // end inline asm\n" " add.s64 %rd51, %rd68, %rd49;\n" " add.s64 %rd47, %rd51, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r46,%r47}, [%rd47];\n" " // end inline asm\n" " shr.u32 %r48, %r42, 16;\n" " shr.u32 %r49, %r43, 16;\n" " and.b32 %r50, %r42, 15;\n" " or.b32 %r51, %r50, 1258291200;\n" " mov.b32 %f43, %r51;\n" " mov.f32 %f44, 0fCB000004;\n" " mov.f32 %f45, 0f3F800000;\n" " fma.rn.f32 %f46, %f43, %f45, %f44;\n" " and.b32 %r52, %r44, 255;\n" " cvt.rn.f32.u32 %f47, %r52;\n" " mov.f32 %f48, 0f42000000;\n" " sub.rn.f32 %f49, %f47, %f48;\n" " and.b32 %r53, %r43, 15;\n" " or.b32 %r54, %r53, 1258291200;\n" " mov.b32 %f50, %r54;\n" " fma.rn.f32 %f51, %f50, %f45, %f44;\n" " and.b32 %r55, %r46, 255;\n" " cvt.rn.f32.u32 %f52, %r55;\n" " sub.rn.f32 %f53, %f52, %f48;\n" " mul.rn.f32 %f54, %f49, %f46;\n" " mul.rn.f32 %f55, %f53, %f51;\n" " shl.b32 %r56, %r2, 2;\n" " add.s32 %r57, %r148, %r56;\n" " ld.shared.f32 %f56, [%r57];\n" " mov.f32 %f57, 0f00000000;\n" " fma.rn.f32 %f58, %f54, %f56, %f57;\n" " fma.rn.f32 %f59, %f55, %f56, %f57;\n" " ld.shared.f32 %f60, [%r57+2048];\n" " fma.rn.f32 %f61, %f54, %f60, %f57;\n" " fma.rn.f32 %f62, %f55, %f60, %f57;\n" " and.b32 %r58, %r42, 240;\n" " or.b32 %r59, %r58, 1258291200;\n" " mov.b32 %f63, %r59;\n" " mov.f32 %f64, 0fC9000040;\n" " mov.f32 %f65, 0f3D800000;\n" " fma.rn.f32 %f66, %f63, %f65, %f64;\n" " bfe.u32 %r60, %r44, 8, 8;\n" " cvt.rn.f32.u32 %f67, %r60;\n" " sub.rn.f32 %f68, %f67, %f48;\n" " and.b32 %r61, %r43, 240;\n" " or.b32 %r62, %r61, 1258291200;\n" " mov.b32 %f69, %r62;\n" " fma.rn.f32 %f70, %f69, %f65, %f64;\n" " bfe.u32 %r63, %r46, 8, 8;\n" " cvt.rn.f32.u32 %f71, %r63;\n" " sub.rn.f32 %f72, %f71, %f48;\n" " mul.rn.f32 %f73, %f68, %f66;\n" " mul.rn.f32 %f74, %f72, %f70;\n" " ld.shared.f32 %f75, [%r57+128];\n" " fma.rn.f32 %f76, %f73, %f75, %f58;\n" " fma.rn.f32 %f77, %f74, %f75, %f59;\n" " ld.shared.f32 %f78, [%r57+2176];\n" " fma.rn.f32 %f79, %f73, %f78, %f61;\n" " fma.rn.f32 %f80, %f74, %f78, %f62;\n" " and.b32 %r64, %r42, 3840;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f81, %r65;\n" " mov.f32 %f82, 0fC7000400;\n" " mov.f32 %f83, 0f3B800000;\n" " fma.rn.f32 %f84, %f81, %f83, %f82;\n" " bfe.u32 %r66, %r44, 16, 8;\n" " cvt.rn.f32.u32 %f85, %r66;\n" " sub.rn.f32 %f86, %f85, %f48;\n" " and.b32 %r67, %r43, 3840;\n" " or.b32 %r68, %r67, 1258291200;\n" " mov.b32 %f87, %r68;\n" " fma.rn.f32 %f88, %f87, %f83, %f82;\n" " bfe.u32 %r69, %r46, 16, 8;\n" " cvt.rn.f32.u32 %f89, %r69;\n" " sub.rn.f32 %f90, %f89, %f48;\n" " mul.rn.f32 %f91, %f86, %f84;\n" " mul.rn.f32 %f92, %f90, %f88;\n" " ld.shared.f32 %f93, [%r57+256];\n" " fma.rn.f32 %f94, %f91, %f93, %f76;\n" " fma.rn.f32 %f95, %f92, %f93, %f77;\n" " ld.shared.f32 %f96, [%r57+2304];\n" " fma.rn.f32 %f97, %f91, %f96, %f79;\n" " fma.rn.f32 %f98, %f92, %f96, %f80;\n" " and.b32 %r70, %r42, 61440;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f99, %r71;\n" " mov.f32 %f100, 0fC5004000;\n" " mov.f32 %f101, 0f39800000;\n" " fma.rn.f32 %f102, %f99, %f101, %f100;\n" " shr.u32 %r72, %r44, 24;\n" " cvt.rn.f32.u32 %f103, %r72;\n" " sub.rn.f32 %f104, %f103, %f48;\n" " and.b32 %r73, %r43, 61440;\n" " or.b32 %r74, %r73, 1258291200;\n" " mov.b32 %f105, %r74;\n" " fma.rn.f32 %f106, %f105, %f101, %f100;\n" " shr.u32 %r75, %r46, 24;\n" " cvt.rn.f32.u32 %f107, %r75;\n" " sub.rn.f32 %f108, %f107, %f48;\n" " mul.rn.f32 %f109, %f104, %f102;\n" " mul.rn.f32 %f110, %f108, %f106;\n" " ld.shared.f32 %f111, [%r57+384];\n" " fma.rn.f32 %f112, %f109, %f111, %f94;\n" " fma.rn.f32 %f113, %f110, %f111, %f95;\n" " ld.shared.f32 %f114, [%r57+2432];\n" " fma.rn.f32 %f115, %f109, %f114, %f97;\n" " fma.rn.f32 %f116, %f110, %f114, %f98;\n" " and.b32 %r76, %r42, 983040;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f117, %r77;\n" " mov.f32 %f118, 0fC3040000;\n" " mov.f32 %f119, 0f37800000;\n" " fma.rn.f32 %f120, %f117, %f119, %f118;\n" " and.b32 %r78, %r45, 255;\n" " cvt.rn.f32.u32 %f121, %r78;\n" " sub.rn.f32 %f122, %f121, %f48;\n" " and.b32 %r79, %r43, 983040;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f123, %r80;\n" " fma.rn.f32 %f124, %f123, %f119, %f118;\n" " and.b32 %r81, %r47, 255;\n" " cvt.rn.f32.u32 %f125, %r81;\n" " sub.rn.f32 %f126, %f125, %f48;\n" " mul.rn.f32 %f127, %f122, %f120;\n" " mul.rn.f32 %f128, %f126, %f124;\n" " ld.shared.f32 %f129, [%r57+512];\n" " fma.rn.f32 %f130, %f127, %f129, %f112;\n" " fma.rn.f32 %f131, %f128, %f129, %f113;\n" " ld.shared.f32 %f132, [%r57+2560];\n" " fma.rn.f32 %f133, %f127, %f132, %f115;\n" " fma.rn.f32 %f134, %f128, %f132, %f116;\n" " and.b32 %r82, %r48, 240;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f135, %r83;\n" " fma.rn.f32 %f136, %f135, %f65, %f64;\n" " bfe.u32 %r84, %r45, 8, 8;\n" " cvt.rn.f32.u32 %f137, %r84;\n" " sub.rn.f32 %f138, %f137, %f48;\n" " and.b32 %r85, %r49, 240;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f139, %r86;\n" " fma.rn.f32 %f140, %f139, %f65, %f64;\n" " bfe.u32 %r87, %r47, 8, 8;\n" " cvt.rn.f32.u32 %f141, %r87;\n" " sub.rn.f32 %f142, %f141, %f48;\n" " mul.rn.f32 %f143, %f138, %f136;\n" " mul.rn.f32 %f144, %f142, %f140;\n" " ld.shared.f32 %f145, [%r57+640];\n" " fma.rn.f32 %f146, %f143, %f145, %f130;\n" " fma.rn.f32 %f147, %f144, %f145, %f131;\n" " ld.shared.f32 %f148, [%r57+2688];\n" " fma.rn.f32 %f149, %f143, %f148, %f133;\n" " fma.rn.f32 %f150, %f144, %f148, %f134;\n" " and.b32 %r88, %r48, 3840;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f151, %r89;\n" " fma.rn.f32 %f152, %f151, %f83, %f82;\n" " bfe.u32 %r90, %r45, 16, 8;\n" " cvt.rn.f32.u32 %f153, %r90;\n" " sub.rn.f32 %f154, %f153, %f48;\n" " and.b32 %r91, %r49, 3840;\n" " or.b32 %r92, %r91, 1258291200;\n" " mov.b32 %f155, %r92;\n" " fma.rn.f32 %f156, %f155, %f83, %f82;\n" " bfe.u32 %r93, %r47, 16, 8;\n" " cvt.rn.f32.u32 %f157, %r93;\n" " sub.rn.f32 %f158, %f157, %f48;\n" " mul.rn.f32 %f159, %f154, %f152;\n" " mul.rn.f32 %f160, %f158, %f156;\n" " ld.shared.f32 %f161, [%r57+768];\n" " fma.rn.f32 %f162, %f159, %f161, %f146;\n" " fma.rn.f32 %f163, %f160, %f161, %f147;\n" " ld.shared.f32 %f164, [%r57+2816];\n" " fma.rn.f32 %f165, %f159, %f164, %f149;\n" " fma.rn.f32 %f166, %f160, %f164, %f150;\n" " and.b32 %r94, %r48, 61440;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f167, %r95;\n" " fma.rn.f32 %f168, %f167, %f101, %f100;\n" " shr.u32 %r96, %r45, 24;\n" " cvt.rn.f32.u32 %f169, %r96;\n" " sub.rn.f32 %f170, %f169, %f48;\n" " and.b32 %r97, %r49, 61440;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f171, %r98;\n" " fma.rn.f32 %f172, %f171, %f101, %f100;\n" " shr.u32 %r99, %r47, 24;\n" " cvt.rn.f32.u32 %f173, %r99;\n" " sub.rn.f32 %f174, %f173, %f48;\n" " mul.rn.f32 %f175, %f170, %f168;\n" " mul.rn.f32 %f176, %f174, %f172;\n" " ld.shared.f32 %f177, [%r57+896];\n" " fma.rn.f32 %f178, %f175, %f177, %f162;\n" " fma.rn.f32 %f179, %f176, %f177, %f163;\n" " ld.shared.f32 %f180, [%r57+2944];\n" " fma.rn.f32 %f181, %f175, %f180, %f165;\n" " fma.rn.f32 %f182, %f176, %f180, %f166;\n" " fma.rn.f32 %f234, %f41, %f178, %f234;\n" " fma.rn.f32 %f236, %f42, %f179, %f236;\n" " fma.rn.f32 %f233, %f41, %f181, %f233;\n" " fma.rn.f32 %f235, %f42, %f182, %f235;\n" " add.s64 %rd67, %rd67, 152;\n" " add.s64 %rd68, %rd68, 152;\n" " add.s32 %r148, %r148, 1024;\n" " add.s32 %r147, %r147, 1;\n" " setp.lt.s32 %p9, %r147, %r16;\n" " @%p9 bra $L__BB9_14;\n" "\n" "$L__BB9_15:\n" " add.s32 %r144, %r144, 512;\n" " setp.lt.s32 %p10, %r144, %r23;\n" " @%p10 bra $L__BB9_4;\n" "\n" "$L__BB9_16:\n" " cvt.s64.s32 %rd25, %r22;\n" " mov.u32 %r110, 16;\n" " mov.u32 %r119, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f183, %f234, %r110, %r119;\n" " // end inline asm\n" " add.f32 %f186, %f234, %f183;\n" " mov.u32 %r112, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f185, %f186, %r112, %r119;\n" " // end inline asm\n" " add.f32 %f188, %f186, %f185;\n" " mov.u32 %r114, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f187, %f188, %r114, %r119;\n" " // end inline asm\n" " add.f32 %f190, %f188, %f187;\n" " mov.u32 %r116, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f189, %f190, %r116, %r119;\n" " // end inline asm\n" " add.f32 %f192, %f190, %f189;\n" " mov.u32 %r118, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f191, %f192, %r118, %r119;\n" " // end inline asm\n" " add.f32 %f25, %f192, %f191;\n" " // begin inline asm\n" " shfl.down.b32 %f193, %f236, %r110, %r119;\n" " // end inline asm\n" " add.f32 %f196, %f236, %f193;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f196, %r112, %r119;\n" " // end inline asm\n" " add.f32 %f198, %f196, %f195;\n" " // begin inline asm\n" " shfl.down.b32 %f197, %f198, %r114, %r119;\n" " // end inline asm\n" " add.f32 %f200, %f198, %f197;\n" " // begin inline asm\n" " shfl.down.b32 %f199, %f200, %r116, %r119;\n" " // end inline asm\n" " add.f32 %f202, %f200, %f199;\n" " // begin inline asm\n" " shfl.down.b32 %f201, %f202, %r118, %r119;\n" " // end inline asm\n" " add.f32 %f26, %f202, %f201;\n" " setp.lt.s32 %p11, %r4, 1;\n" " setp.ne.s32 %p12, %r2, 0;\n" " or.pred %p13, %p12, %p11;\n" " @%p13 bra $L__BB9_21;\n" "\n" " cvt.u32.u64 %r120, %rd2;\n" " setp.ge.s32 %p14, %r120, %r22;\n" " cvt.s64.s32 %rd52, %r3;\n" " mul.lo.s64 %rd53, %rd52, %rd25;\n" " add.s64 %rd54, %rd53, %rd2;\n" " shl.b64 %rd55, %rd54, 2;\n" " add.s64 %rd26, %rd1, %rd55;\n" " @%p14 bra $L__BB9_19;\n" "\n" " st.global.f32 [%rd26], %f25;\n" "\n" "$L__BB9_19:\n" " @%p1 bra $L__BB9_21;\n" "\n" " st.global.f32 [%rd26+4], %f26;\n" "\n" "$L__BB9_21:\n" " // begin inline asm\n" " shfl.down.b32 %f203, %f233, %r110, %r119;\n" " // end inline asm\n" " add.f32 %f206, %f233, %f203;\n" " // begin inline asm\n" " shfl.down.b32 %f205, %f206, %r112, %r119;\n" " // end inline asm\n" " add.f32 %f208, %f206, %f205;\n" " // begin inline asm\n" " shfl.down.b32 %f207, %f208, %r114, %r119;\n" " // end inline asm\n" " add.f32 %f210, %f208, %f207;\n" " // begin inline asm\n" " shfl.down.b32 %f209, %f210, %r116, %r119;\n" " // end inline asm\n" " add.f32 %f212, %f210, %f209;\n" " // begin inline asm\n" " shfl.down.b32 %f211, %f212, %r118, %r119;\n" " // end inline asm\n" " add.f32 %f27, %f212, %f211;\n" " // begin inline asm\n" " shfl.down.b32 %f213, %f235, %r110, %r119;\n" " // end inline asm\n" " add.f32 %f216, %f235, %f213;\n" " // begin inline asm\n" " shfl.down.b32 %f215, %f216, %r112, %r119;\n" " // end inline asm\n" " add.f32 %f218, %f216, %f215;\n" " // begin inline asm\n" " shfl.down.b32 %f217, %f218, %r114, %r119;\n" " // end inline asm\n" " add.f32 %f220, %f218, %f217;\n" " // begin inline asm\n" " shfl.down.b32 %f219, %f220, %r116, %r119;\n" " // end inline asm\n" " add.f32 %f222, %f220, %f219;\n" " // begin inline asm\n" " shfl.down.b32 %f221, %f222, %r118, %r119;\n" " // end inline asm\n" " add.f32 %f28, %f222, %f221;\n" " setp.lt.s32 %p17, %r5, 2;\n" " or.pred %p18, %p12, %p17;\n" " @%p18 bra $L__BB9_26;\n" "\n" " cvt.u32.u64 %r141, %rd2;\n" " setp.ge.s32 %p19, %r141, %r22;\n" " add.s32 %r142, %r3, 1;\n" " cvt.s64.s32 %rd56, %r142;\n" " mul.lo.s64 %rd57, %rd56, %rd25;\n" " add.s64 %rd58, %rd57, %rd2;\n" " shl.b64 %rd59, %rd58, 2;\n" " add.s64 %rd27, %rd1, %rd59;\n" " @%p19 bra $L__BB9_24;\n" "\n" " st.global.f32 [%rd27], %f27;\n" "\n" "$L__BB9_24:\n" " @%p1 bra $L__BB9_26;\n" "\n" " st.global.f32 [%rd27+4], %f28;\n" "\n" "$L__BB9_26:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q3k_nx8\n" ".visible .entry fc_matvec_q3k_nx8(\n" " .param .u64 fc_matvec_q3k_nx8_param_0,\n" " .param .u64 fc_matvec_q3k_nx8_param_1,\n" " .param .u64 fc_matvec_q3k_nx8_param_2,\n" " .param .u32 fc_matvec_q3k_nx8_param_3,\n" " .param .u32 fc_matvec_q3k_nx8_param_4,\n" " .param .u32 fc_matvec_q3k_nx8_param_5,\n" " .param .u32 fc_matvec_q3k_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<57>;\n" " .reg .f32 %f<686>;\n" " .reg .b32 %r<306>;\n" " .reg .b64 %rd<140>;\n" "\n" "\n" " ld.param.u64 %rd45, [fc_matvec_q3k_nx8_param_0];\n" " ld.param.u64 %rd44, [fc_matvec_q3k_nx8_param_1];\n" " ld.param.u64 %rd46, [fc_matvec_q3k_nx8_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q3k_nx8_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q3k_nx8_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q3k_nx8_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q3k_nx8_param_6];\n" " cvta.to.global.u64 %rd1, %rd46;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r26, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 3;\n" " add.s32 %r29, %r28, %r26;\n" " shl.b32 %r30, %r29, 1;\n" " mov.u32 %r31, %ctaid.y;\n" " shl.b32 %r3, %r31, 3;\n" " sub.s32 %r4, %r25, %r3;\n" " min.s32 %r5, %r4, 8;\n" " or.b32 %r6, %r30, 1;\n" " setp.ge.s32 %p1, %r6, %r22;\n" " cvt.s64.s32 %rd2, %r30;\n" " cvt.s64.s32 %rd3, %r24;\n" " mul.wide.s32 %rd47, %r24, %r30;\n" " add.s64 %rd138, %rd45, %rd47;\n" " mov.u64 %rd139, %rd138;\n" " @%p1 bra $L__BB10_2;\n" "\n" " add.s64 %rd139, %rd138, %rd3;\n" "\n" "$L__BB10_2:\n" " setp.lt.s32 %p2, %r23, 1;\n" " mov.f32 %f654, 0f00000000;\n" " mov.f32 %f655, %f654;\n" " mov.f32 %f656, %f654;\n" " mov.f32 %f657, %f654;\n" " mov.f32 %f658, %f654;\n" " mov.f32 %f659, %f654;\n" " mov.f32 %f660, %f654;\n" " mov.f32 %f661, %f654;\n" " mov.f32 %f662, %f654;\n" " mov.f32 %f663, %f654;\n" " mov.f32 %f664, %f654;\n" " mov.f32 %f665, %f654;\n" " mov.f32 %f666, %f654;\n" " mov.f32 %f667, %f654;\n" " mov.f32 %f668, %f654;\n" " mov.f32 %f669, %f654;\n" " @%p2 bra $L__BB10_28;\n" "\n" " cvt.u64.u32 %rd7, %r2;\n" " shr.u32 %r33, %r2, 4;\n" " cvt.u64.u32 %rd8, %r33;\n" " add.s32 %r34, %r3, 1;\n" " add.s32 %r35, %r3, 2;\n" " add.s32 %r36, %r3, 3;\n" " add.s32 %r37, %r3, 4;\n" " add.s32 %r38, %r3, 5;\n" " add.s32 %r39, %r3, 6;\n" " add.s32 %r40, %r3, 7;\n" " mul.wide.s32 %rd48, %r23, %r3;\n" " shl.b64 %rd49, %rd48, 2;\n" " mul.wide.s32 %rd50, %r1, 4;\n" " add.s64 %rd9, %rd49, %rd50;\n" " mul.wide.s32 %rd51, %r23, %r34;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd10, %rd52, %rd50;\n" " mul.wide.s32 %rd53, %r23, %r35;\n" " shl.b64 %rd54, %rd53, 2;\n" " add.s64 %rd11, %rd54, %rd50;\n" " mul.wide.s32 %rd55, %r23, %r36;\n" " shl.b64 %rd56, %rd55, 2;\n" " add.s64 %rd12, %rd56, %rd50;\n" " mul.wide.s32 %rd57, %r23, %r37;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd13, %rd58, %rd50;\n" " mul.wide.s32 %rd59, %r23, %r38;\n" " shl.b64 %rd60, %rd59, 2;\n" " add.s64 %rd14, %rd60, %rd50;\n" " mul.wide.s32 %rd61, %r23, %r39;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd15, %rd62, %rd50;\n" " shl.b32 %r41, %r1, 2;\n" " mov.u32 %r42, sh_x;\n" " add.s32 %r43, %r42, %r41;\n" " add.s32 %r7, %r43, 8192;\n" " mul.wide.s32 %rd63, %r23, %r40;\n" " shl.b64 %rd64, %rd63, 2;\n" " add.s64 %rd16, %rd64, %rd50;\n" " mov.u32 %r301, 0;\n" " mov.f32 %f654, 0f00000000;\n" " setp.lt.s32 %p4, %r4, 1;\n" " setp.lt.s32 %p5, %r5, 2;\n" " setp.lt.s32 %p6, %r5, 3;\n" " setp.lt.s32 %p7, %r5, 4;\n" " shl.b64 %rd80, %rd7, 2;\n" " shl.b64 %rd81, %rd8, 3;\n" " mov.f32 %f655, %f654;\n" " mov.f32 %f656, %f654;\n" " mov.f32 %f657, %f654;\n" " mov.f32 %f658, %f654;\n" " mov.f32 %f659, %f654;\n" " mov.f32 %f660, %f654;\n" " mov.f32 %f661, %f654;\n" " mov.f32 %f662, %f654;\n" " mov.f32 %f663, %f654;\n" " mov.f32 %f664, %f654;\n" " mov.f32 %f665, %f654;\n" " mov.f32 %f666, %f654;\n" " mov.f32 %f667, %f654;\n" " mov.f32 %f668, %f654;\n" " mov.f32 %f669, %f654;\n" "\n" "$L__BB10_4:\n" " sub.s32 %r44, %r23, %r301;\n" " min.s32 %r9, %r44, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r9;\n" " @%p3 bra $L__BB10_23;\n" "\n" " mov.u32 %r302, %r7;\n" " mov.u64 %rd135, %rd44;\n" " mov.u32 %r303, %r1;\n" "\n" "$L__BB10_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f630, 0f00000000;\n" " @%p4 bra $L__BB10_8;\n" "\n" " mul.wide.s32 %rd117, %r301, 4;\n" " add.s64 %rd116, %rd9, %rd117;\n" " add.s64 %rd66, %rd135, %rd116;\n" " // begin inline asm\n" " ld.global.nc.f32 %f630, [%rd66];\n" " // end inline asm\n" "\n" "$L__BB10_8:\n" " mov.f32 %f631, 0f00000000;\n" " st.shared.f32 [%r302+-8192], %f630;\n" " @%p5 bra $L__BB10_10;\n" "\n" " mul.wide.s32 %rd129, %r301, 4;\n" " add.s64 %rd128, %rd10, %rd129;\n" " add.s64 %rd67, %rd135, %rd128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f631, [%rd67];\n" " // end inline asm\n" "\n" "$L__BB10_10:\n" " mov.f32 %f632, 0f00000000;\n" " add.s32 %r281, %r302, -8192;\n" " st.shared.f32 [%r281+2048], %f631;\n" " @%p6 bra $L__BB10_12;\n" "\n" " mul.wide.s32 %rd127, %r301, 4;\n" " add.s64 %rd126, %rd11, %rd127;\n" " add.s64 %rd68, %rd135, %rd126;\n" " // begin inline asm\n" " ld.global.nc.f32 %f632, [%rd68];\n" " // end inline asm\n" "\n" "$L__BB10_12:\n" " mov.f32 %f633, 0f00000000;\n" " add.s32 %r282, %r302, -8192;\n" " st.shared.f32 [%r282+4096], %f632;\n" " @%p7 bra $L__BB10_14;\n" "\n" " mul.wide.s32 %rd125, %r301, 4;\n" " add.s64 %rd124, %rd12, %rd125;\n" " add.s64 %rd69, %rd135, %rd124;\n" " // begin inline asm\n" " ld.global.nc.f32 %f633, [%rd69];\n" " // end inline asm\n" "\n" "$L__BB10_14:\n" " mov.f32 %f634, 0f00000000;\n" " add.s32 %r283, %r302, -8192;\n" " st.shared.f32 [%r283+6144], %f633;\n" " setp.lt.s32 %p8, %r5, 5;\n" " @%p8 bra $L__BB10_16;\n" "\n" " mul.wide.s32 %rd123, %r301, 4;\n" " add.s64 %rd122, %rd13, %rd123;\n" " add.s64 %rd70, %rd135, %rd122;\n" " // begin inline asm\n" " ld.global.nc.f32 %f634, [%rd70];\n" " // end inline asm\n" "\n" "$L__BB10_16:\n" " mov.f32 %f635, 0f00000000;\n" " add.s32 %r284, %r302, -8192;\n" " st.shared.f32 [%r284+8192], %f634;\n" " setp.lt.s32 %p9, %r5, 6;\n" " @%p9 bra $L__BB10_18;\n" "\n" " mul.wide.s32 %rd121, %r301, 4;\n" " add.s64 %rd120, %rd14, %rd121;\n" " add.s64 %rd71, %rd135, %rd120;\n" " // begin inline asm\n" " ld.global.nc.f32 %f635, [%rd71];\n" " // end inline asm\n" "\n" "$L__BB10_18:\n" " mov.f32 %f636, 0f00000000;\n" " add.s32 %r285, %r302, -8192;\n" " st.shared.f32 [%r285+10240], %f635;\n" " setp.lt.s32 %p10, %r5, 7;\n" " @%p10 bra $L__BB10_20;\n" "\n" " mul.wide.s32 %rd119, %r301, 4;\n" " add.s64 %rd118, %rd15, %rd119;\n" " add.s64 %rd72, %rd135, %rd118;\n" " // begin inline asm\n" " ld.global.nc.f32 %f636, [%rd72];\n" " // end inline asm\n" "\n" "$L__BB10_20:\n" " mov.f32 %f637, 0f00000000;\n" " add.s32 %r286, %r302, -8192;\n" " st.shared.f32 [%r286+12288], %f636;\n" " setp.lt.s32 %p11, %r5, 8;\n" " @%p11 bra $L__BB10_22;\n" "\n" " mul.wide.s32 %rd131, %r301, 4;\n" " add.s64 %rd130, %rd16, %rd131;\n" " add.s64 %rd73, %rd135, %rd130;\n" " // begin inline asm\n" " ld.global.nc.f32 %f637, [%rd73];\n" " // end inline asm\n" "\n" "$L__BB10_22:\n" " sub.s32 %r293, %r23, %r301;\n" " min.s32 %r292, %r293, 512;\n" " add.s32 %r287, %r302, -8192;\n" " st.shared.f32 [%r287+14336], %f637;\n" " add.s64 %rd135, %rd135, 1024;\n" " add.s32 %r302, %r302, 1024;\n" " add.s32 %r303, %r303, 256;\n" " setp.lt.s32 %p12, %r303, %r292;\n" " @%p12 bra $L__BB10_6;\n" "\n" "$L__BB10_23:\n" " cvt.u32.u64 %r45, %rd2;\n" " setp.ge.s32 %p13, %r45, %r22;\n" " bar.sync 0;\n" " @%p13 bra $L__BB10_27;\n" "\n" " sub.s32 %r295, %r23, %r301;\n" " min.s32 %r294, %r295, 512;\n" " setp.lt.s32 %p14, %r294, 256;\n" " @%p14 bra $L__BB10_27;\n" "\n" " mov.u32 %r304, 0;\n" " mov.u32 %r305, %r42;\n" "\n" "$L__BB10_26:\n" " sub.s32 %r300, %r23, %r301;\n" " min.s32 %r299, %r300, 512;\n" " shr.s32 %r298, %r299, 8;\n" " add.s64 %rd74, %rd138, %rd80;\n" " // begin inline asm\n" " ld.global.nc.u32 %r48, [%rd74];\n" " // end inline asm\n" " add.s64 %rd75, %rd139, %rd80;\n" " // begin inline asm\n" " ld.global.nc.u32 %r49, [%rd75];\n" " // end inline asm\n" " add.s64 %rd76, %rd138, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f161, [%rd76];\n" " // end inline asm\n" " add.s64 %rd77, %rd139, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f162, [%rd77];\n" " // end inline asm\n" " add.s64 %rd82, %rd138, %rd81;\n" " add.s64 %rd78, %rd82, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r50,%r51}, [%rd78];\n" " // end inline asm\n" " add.s64 %rd83, %rd139, %rd81;\n" " add.s64 %rd79, %rd83, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r52,%r53}, [%rd79];\n" " // end inline asm\n" " shr.u32 %r54, %r48, 16;\n" " shr.u32 %r55, %r49, 16;\n" " and.b32 %r56, %r48, 15;\n" " or.b32 %r57, %r56, 1258291200;\n" " mov.b32 %f163, %r57;\n" " mov.f32 %f164, 0fCB000004;\n" " mov.f32 %f165, 0f3F800000;\n" " fma.rn.f32 %f166, %f163, %f165, %f164;\n" " and.b32 %r58, %r50, 255;\n" " cvt.rn.f32.u32 %f167, %r58;\n" " mov.f32 %f168, 0f42000000;\n" " sub.rn.f32 %f169, %f167, %f168;\n" " and.b32 %r59, %r49, 15;\n" " or.b32 %r60, %r59, 1258291200;\n" " mov.b32 %f170, %r60;\n" " fma.rn.f32 %f171, %f170, %f165, %f164;\n" " and.b32 %r61, %r52, 255;\n" " cvt.rn.f32.u32 %f172, %r61;\n" " sub.rn.f32 %f173, %f172, %f168;\n" " mul.rn.f32 %f174, %f169, %f166;\n" " mul.rn.f32 %f175, %f173, %f171;\n" " shl.b32 %r62, %r2, 2;\n" " add.s32 %r63, %r305, %r62;\n" " ld.shared.f32 %f176, [%r63];\n" " mov.f32 %f177, 0f00000000;\n" " fma.rn.f32 %f178, %f174, %f176, %f177;\n" " fma.rn.f32 %f179, %f175, %f176, %f177;\n" " ld.shared.f32 %f180, [%r63+2048];\n" " fma.rn.f32 %f181, %f174, %f180, %f177;\n" " fma.rn.f32 %f182, %f175, %f180, %f177;\n" " ld.shared.f32 %f183, [%r63+4096];\n" " fma.rn.f32 %f184, %f174, %f183, %f177;\n" " fma.rn.f32 %f185, %f175, %f183, %f177;\n" " ld.shared.f32 %f186, [%r63+6144];\n" " fma.rn.f32 %f187, %f174, %f186, %f177;\n" " fma.rn.f32 %f188, %f175, %f186, %f177;\n" " ld.shared.f32 %f189, [%r63+8192];\n" " fma.rn.f32 %f190, %f174, %f189, %f177;\n" " fma.rn.f32 %f191, %f175, %f189, %f177;\n" " ld.shared.f32 %f192, [%r63+10240];\n" " fma.rn.f32 %f193, %f174, %f192, %f177;\n" " fma.rn.f32 %f194, %f175, %f192, %f177;\n" " ld.shared.f32 %f195, [%r63+12288];\n" " fma.rn.f32 %f196, %f174, %f195, %f177;\n" " fma.rn.f32 %f197, %f175, %f195, %f177;\n" " ld.shared.f32 %f198, [%r63+14336];\n" " fma.rn.f32 %f199, %f174, %f198, %f177;\n" " fma.rn.f32 %f200, %f175, %f198, %f177;\n" " and.b32 %r64, %r48, 240;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f201, %r65;\n" " mov.f32 %f202, 0fC9000040;\n" " mov.f32 %f203, 0f3D800000;\n" " fma.rn.f32 %f204, %f201, %f203, %f202;\n" " bfe.u32 %r66, %r50, 8, 8;\n" " cvt.rn.f32.u32 %f205, %r66;\n" " sub.rn.f32 %f206, %f205, %f168;\n" " and.b32 %r67, %r49, 240;\n" " or.b32 %r68, %r67, 1258291200;\n" " mov.b32 %f207, %r68;\n" " fma.rn.f32 %f208, %f207, %f203, %f202;\n" " bfe.u32 %r69, %r52, 8, 8;\n" " cvt.rn.f32.u32 %f209, %r69;\n" " sub.rn.f32 %f210, %f209, %f168;\n" " mul.rn.f32 %f211, %f206, %f204;\n" " mul.rn.f32 %f212, %f210, %f208;\n" " ld.shared.f32 %f213, [%r63+128];\n" " fma.rn.f32 %f214, %f211, %f213, %f178;\n" " fma.rn.f32 %f215, %f212, %f213, %f179;\n" " ld.shared.f32 %f216, [%r63+2176];\n" " fma.rn.f32 %f217, %f211, %f216, %f181;\n" " fma.rn.f32 %f218, %f212, %f216, %f182;\n" " ld.shared.f32 %f219, [%r63+4224];\n" " fma.rn.f32 %f220, %f211, %f219, %f184;\n" " fma.rn.f32 %f221, %f212, %f219, %f185;\n" " ld.shared.f32 %f222, [%r63+6272];\n" " fma.rn.f32 %f223, %f211, %f222, %f187;\n" " fma.rn.f32 %f224, %f212, %f222, %f188;\n" " ld.shared.f32 %f225, [%r63+8320];\n" " fma.rn.f32 %f226, %f211, %f225, %f190;\n" " fma.rn.f32 %f227, %f212, %f225, %f191;\n" " ld.shared.f32 %f228, [%r63+10368];\n" " fma.rn.f32 %f229, %f211, %f228, %f193;\n" " fma.rn.f32 %f230, %f212, %f228, %f194;\n" " ld.shared.f32 %f231, [%r63+12416];\n" " fma.rn.f32 %f232, %f211, %f231, %f196;\n" " fma.rn.f32 %f233, %f212, %f231, %f197;\n" " ld.shared.f32 %f234, [%r63+14464];\n" " fma.rn.f32 %f235, %f211, %f234, %f199;\n" " fma.rn.f32 %f236, %f212, %f234, %f200;\n" " and.b32 %r70, %r48, 3840;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f237, %r71;\n" " mov.f32 %f238, 0fC7000400;\n" " mov.f32 %f239, 0f3B800000;\n" " fma.rn.f32 %f240, %f237, %f239, %f238;\n" " bfe.u32 %r72, %r50, 16, 8;\n" " cvt.rn.f32.u32 %f241, %r72;\n" " sub.rn.f32 %f242, %f241, %f168;\n" " and.b32 %r73, %r49, 3840;\n" " or.b32 %r74, %r73, 1258291200;\n" " mov.b32 %f243, %r74;\n" " fma.rn.f32 %f244, %f243, %f239, %f238;\n" " bfe.u32 %r75, %r52, 16, 8;\n" " cvt.rn.f32.u32 %f245, %r75;\n" " sub.rn.f32 %f246, %f245, %f168;\n" " mul.rn.f32 %f247, %f242, %f240;\n" " mul.rn.f32 %f248, %f246, %f244;\n" " ld.shared.f32 %f249, [%r63+256];\n" " fma.rn.f32 %f250, %f247, %f249, %f214;\n" " fma.rn.f32 %f251, %f248, %f249, %f215;\n" " ld.shared.f32 %f252, [%r63+2304];\n" " fma.rn.f32 %f253, %f247, %f252, %f217;\n" " fma.rn.f32 %f254, %f248, %f252, %f218;\n" " ld.shared.f32 %f255, [%r63+4352];\n" " fma.rn.f32 %f256, %f247, %f255, %f220;\n" " fma.rn.f32 %f257, %f248, %f255, %f221;\n" " ld.shared.f32 %f258, [%r63+6400];\n" " fma.rn.f32 %f259, %f247, %f258, %f223;\n" " fma.rn.f32 %f260, %f248, %f258, %f224;\n" " ld.shared.f32 %f261, [%r63+8448];\n" " fma.rn.f32 %f262, %f247, %f261, %f226;\n" " fma.rn.f32 %f263, %f248, %f261, %f227;\n" " ld.shared.f32 %f264, [%r63+10496];\n" " fma.rn.f32 %f265, %f247, %f264, %f229;\n" " fma.rn.f32 %f266, %f248, %f264, %f230;\n" " ld.shared.f32 %f267, [%r63+12544];\n" " fma.rn.f32 %f268, %f247, %f267, %f232;\n" " fma.rn.f32 %f269, %f248, %f267, %f233;\n" " ld.shared.f32 %f270, [%r63+14592];\n" " fma.rn.f32 %f271, %f247, %f270, %f235;\n" " fma.rn.f32 %f272, %f248, %f270, %f236;\n" " and.b32 %r76, %r48, 61440;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f273, %r77;\n" " mov.f32 %f274, 0fC5004000;\n" " mov.f32 %f275, 0f39800000;\n" " fma.rn.f32 %f276, %f273, %f275, %f274;\n" " shr.u32 %r78, %r50, 24;\n" " cvt.rn.f32.u32 %f277, %r78;\n" " sub.rn.f32 %f278, %f277, %f168;\n" " and.b32 %r79, %r49, 61440;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f279, %r80;\n" " fma.rn.f32 %f280, %f279, %f275, %f274;\n" " shr.u32 %r81, %r52, 24;\n" " cvt.rn.f32.u32 %f281, %r81;\n" " sub.rn.f32 %f282, %f281, %f168;\n" " mul.rn.f32 %f283, %f278, %f276;\n" " mul.rn.f32 %f284, %f282, %f280;\n" " ld.shared.f32 %f285, [%r63+384];\n" " fma.rn.f32 %f286, %f283, %f285, %f250;\n" " fma.rn.f32 %f287, %f284, %f285, %f251;\n" " ld.shared.f32 %f288, [%r63+2432];\n" " fma.rn.f32 %f289, %f283, %f288, %f253;\n" " fma.rn.f32 %f290, %f284, %f288, %f254;\n" " ld.shared.f32 %f291, [%r63+4480];\n" " fma.rn.f32 %f292, %f283, %f291, %f256;\n" " fma.rn.f32 %f293, %f284, %f291, %f257;\n" " ld.shared.f32 %f294, [%r63+6528];\n" " fma.rn.f32 %f295, %f283, %f294, %f259;\n" " fma.rn.f32 %f296, %f284, %f294, %f260;\n" " ld.shared.f32 %f297, [%r63+8576];\n" " fma.rn.f32 %f298, %f283, %f297, %f262;\n" " fma.rn.f32 %f299, %f284, %f297, %f263;\n" " ld.shared.f32 %f300, [%r63+10624];\n" " fma.rn.f32 %f301, %f283, %f300, %f265;\n" " fma.rn.f32 %f302, %f284, %f300, %f266;\n" " ld.shared.f32 %f303, [%r63+12672];\n" " fma.rn.f32 %f304, %f283, %f303, %f268;\n" " fma.rn.f32 %f305, %f284, %f303, %f269;\n" " ld.shared.f32 %f306, [%r63+14720];\n" " fma.rn.f32 %f307, %f283, %f306, %f271;\n" " fma.rn.f32 %f308, %f284, %f306, %f272;\n" " and.b32 %r82, %r48, 983040;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f309, %r83;\n" " mov.f32 %f310, 0fC3040000;\n" " mov.f32 %f311, 0f37800000;\n" " fma.rn.f32 %f312, %f309, %f311, %f310;\n" " and.b32 %r84, %r51, 255;\n" " cvt.rn.f32.u32 %f313, %r84;\n" " sub.rn.f32 %f314, %f313, %f168;\n" " and.b32 %r85, %r49, 983040;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f315, %r86;\n" " fma.rn.f32 %f316, %f315, %f311, %f310;\n" " and.b32 %r87, %r53, 255;\n" " cvt.rn.f32.u32 %f317, %r87;\n" " sub.rn.f32 %f318, %f317, %f168;\n" " mul.rn.f32 %f319, %f314, %f312;\n" " mul.rn.f32 %f320, %f318, %f316;\n" " ld.shared.f32 %f321, [%r63+512];\n" " fma.rn.f32 %f322, %f319, %f321, %f286;\n" " fma.rn.f32 %f323, %f320, %f321, %f287;\n" " ld.shared.f32 %f324, [%r63+2560];\n" " fma.rn.f32 %f325, %f319, %f324, %f289;\n" " fma.rn.f32 %f326, %f320, %f324, %f290;\n" " ld.shared.f32 %f327, [%r63+4608];\n" " fma.rn.f32 %f328, %f319, %f327, %f292;\n" " fma.rn.f32 %f329, %f320, %f327, %f293;\n" " ld.shared.f32 %f330, [%r63+6656];\n" " fma.rn.f32 %f331, %f319, %f330, %f295;\n" " fma.rn.f32 %f332, %f320, %f330, %f296;\n" " ld.shared.f32 %f333, [%r63+8704];\n" " fma.rn.f32 %f334, %f319, %f333, %f298;\n" " fma.rn.f32 %f335, %f320, %f333, %f299;\n" " ld.shared.f32 %f336, [%r63+10752];\n" " fma.rn.f32 %f337, %f319, %f336, %f301;\n" " fma.rn.f32 %f338, %f320, %f336, %f302;\n" " ld.shared.f32 %f339, [%r63+12800];\n" " fma.rn.f32 %f340, %f319, %f339, %f304;\n" " fma.rn.f32 %f341, %f320, %f339, %f305;\n" " ld.shared.f32 %f342, [%r63+14848];\n" " fma.rn.f32 %f343, %f319, %f342, %f307;\n" " fma.rn.f32 %f344, %f320, %f342, %f308;\n" " and.b32 %r88, %r54, 240;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f345, %r89;\n" " fma.rn.f32 %f346, %f345, %f203, %f202;\n" " bfe.u32 %r90, %r51, 8, 8;\n" " cvt.rn.f32.u32 %f347, %r90;\n" " sub.rn.f32 %f348, %f347, %f168;\n" " and.b32 %r91, %r55, 240;\n" " or.b32 %r92, %r91, 1258291200;\n" " mov.b32 %f349, %r92;\n" " fma.rn.f32 %f350, %f349, %f203, %f202;\n" " bfe.u32 %r93, %r53, 8, 8;\n" " cvt.rn.f32.u32 %f351, %r93;\n" " sub.rn.f32 %f352, %f351, %f168;\n" " mul.rn.f32 %f353, %f348, %f346;\n" " mul.rn.f32 %f354, %f352, %f350;\n" " ld.shared.f32 %f355, [%r63+640];\n" " fma.rn.f32 %f356, %f353, %f355, %f322;\n" " fma.rn.f32 %f357, %f354, %f355, %f323;\n" " ld.shared.f32 %f358, [%r63+2688];\n" " fma.rn.f32 %f359, %f353, %f358, %f325;\n" " fma.rn.f32 %f360, %f354, %f358, %f326;\n" " ld.shared.f32 %f361, [%r63+4736];\n" " fma.rn.f32 %f362, %f353, %f361, %f328;\n" " fma.rn.f32 %f363, %f354, %f361, %f329;\n" " ld.shared.f32 %f364, [%r63+6784];\n" " fma.rn.f32 %f365, %f353, %f364, %f331;\n" " fma.rn.f32 %f366, %f354, %f364, %f332;\n" " ld.shared.f32 %f367, [%r63+8832];\n" " fma.rn.f32 %f368, %f353, %f367, %f334;\n" " fma.rn.f32 %f369, %f354, %f367, %f335;\n" " ld.shared.f32 %f370, [%r63+10880];\n" " fma.rn.f32 %f371, %f353, %f370, %f337;\n" " fma.rn.f32 %f372, %f354, %f370, %f338;\n" " ld.shared.f32 %f373, [%r63+12928];\n" " fma.rn.f32 %f374, %f353, %f373, %f340;\n" " fma.rn.f32 %f375, %f354, %f373, %f341;\n" " ld.shared.f32 %f376, [%r63+14976];\n" " fma.rn.f32 %f377, %f353, %f376, %f343;\n" " fma.rn.f32 %f378, %f354, %f376, %f344;\n" " and.b32 %r94, %r54, 3840;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f379, %r95;\n" " fma.rn.f32 %f380, %f379, %f239, %f238;\n" " bfe.u32 %r96, %r51, 16, 8;\n" " cvt.rn.f32.u32 %f381, %r96;\n" " sub.rn.f32 %f382, %f381, %f168;\n" " and.b32 %r97, %r55, 3840;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f383, %r98;\n" " fma.rn.f32 %f384, %f383, %f239, %f238;\n" " bfe.u32 %r99, %r53, 16, 8;\n" " cvt.rn.f32.u32 %f385, %r99;\n" " sub.rn.f32 %f386, %f385, %f168;\n" " mul.rn.f32 %f387, %f382, %f380;\n" " mul.rn.f32 %f388, %f386, %f384;\n" " ld.shared.f32 %f389, [%r63+768];\n" " fma.rn.f32 %f390, %f387, %f389, %f356;\n" " fma.rn.f32 %f391, %f388, %f389, %f357;\n" " ld.shared.f32 %f392, [%r63+2816];\n" " fma.rn.f32 %f393, %f387, %f392, %f359;\n" " fma.rn.f32 %f394, %f388, %f392, %f360;\n" " ld.shared.f32 %f395, [%r63+4864];\n" " fma.rn.f32 %f396, %f387, %f395, %f362;\n" " fma.rn.f32 %f397, %f388, %f395, %f363;\n" " ld.shared.f32 %f398, [%r63+6912];\n" " fma.rn.f32 %f399, %f387, %f398, %f365;\n" " fma.rn.f32 %f400, %f388, %f398, %f366;\n" " ld.shared.f32 %f401, [%r63+8960];\n" " fma.rn.f32 %f402, %f387, %f401, %f368;\n" " fma.rn.f32 %f403, %f388, %f401, %f369;\n" " ld.shared.f32 %f404, [%r63+11008];\n" " fma.rn.f32 %f405, %f387, %f404, %f371;\n" " fma.rn.f32 %f406, %f388, %f404, %f372;\n" " ld.shared.f32 %f407, [%r63+13056];\n" " fma.rn.f32 %f408, %f387, %f407, %f374;\n" " fma.rn.f32 %f409, %f388, %f407, %f375;\n" " ld.shared.f32 %f410, [%r63+15104];\n" " fma.rn.f32 %f411, %f387, %f410, %f377;\n" " fma.rn.f32 %f412, %f388, %f410, %f378;\n" " and.b32 %r100, %r54, 61440;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f413, %r101;\n" " fma.rn.f32 %f414, %f413, %f275, %f274;\n" " shr.u32 %r102, %r51, 24;\n" " cvt.rn.f32.u32 %f415, %r102;\n" " sub.rn.f32 %f416, %f415, %f168;\n" " and.b32 %r103, %r55, 61440;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f417, %r104;\n" " fma.rn.f32 %f418, %f417, %f275, %f274;\n" " shr.u32 %r105, %r53, 24;\n" " cvt.rn.f32.u32 %f419, %r105;\n" " sub.rn.f32 %f420, %f419, %f168;\n" " mul.rn.f32 %f421, %f416, %f414;\n" " mul.rn.f32 %f422, %f420, %f418;\n" " ld.shared.f32 %f423, [%r63+896];\n" " fma.rn.f32 %f424, %f421, %f423, %f390;\n" " fma.rn.f32 %f425, %f422, %f423, %f391;\n" " ld.shared.f32 %f426, [%r63+2944];\n" " fma.rn.f32 %f427, %f421, %f426, %f393;\n" " fma.rn.f32 %f428, %f422, %f426, %f394;\n" " ld.shared.f32 %f429, [%r63+4992];\n" " fma.rn.f32 %f430, %f421, %f429, %f396;\n" " fma.rn.f32 %f431, %f422, %f429, %f397;\n" " ld.shared.f32 %f432, [%r63+7040];\n" " fma.rn.f32 %f433, %f421, %f432, %f399;\n" " fma.rn.f32 %f434, %f422, %f432, %f400;\n" " ld.shared.f32 %f435, [%r63+9088];\n" " fma.rn.f32 %f436, %f421, %f435, %f402;\n" " fma.rn.f32 %f437, %f422, %f435, %f403;\n" " ld.shared.f32 %f438, [%r63+11136];\n" " fma.rn.f32 %f439, %f421, %f438, %f405;\n" " fma.rn.f32 %f440, %f422, %f438, %f406;\n" " ld.shared.f32 %f441, [%r63+13184];\n" " fma.rn.f32 %f442, %f421, %f441, %f408;\n" " fma.rn.f32 %f443, %f422, %f441, %f409;\n" " ld.shared.f32 %f444, [%r63+15232];\n" " fma.rn.f32 %f445, %f421, %f444, %f411;\n" " fma.rn.f32 %f446, %f422, %f444, %f412;\n" " fma.rn.f32 %f661, %f161, %f424, %f661;\n" " fma.rn.f32 %f669, %f162, %f425, %f669;\n" " fma.rn.f32 %f660, %f161, %f427, %f660;\n" " fma.rn.f32 %f668, %f162, %f428, %f668;\n" " fma.rn.f32 %f659, %f161, %f430, %f659;\n" " fma.rn.f32 %f667, %f162, %f431, %f667;\n" " fma.rn.f32 %f658, %f161, %f433, %f658;\n" " fma.rn.f32 %f666, %f162, %f434, %f666;\n" " fma.rn.f32 %f657, %f161, %f436, %f657;\n" " fma.rn.f32 %f665, %f162, %f437, %f665;\n" " fma.rn.f32 %f656, %f161, %f439, %f656;\n" " fma.rn.f32 %f664, %f162, %f440, %f664;\n" " fma.rn.f32 %f655, %f161, %f442, %f655;\n" " fma.rn.f32 %f663, %f162, %f443, %f663;\n" " fma.rn.f32 %f654, %f161, %f445, %f654;\n" " fma.rn.f32 %f662, %f162, %f446, %f662;\n" " add.s64 %rd138, %rd138, 152;\n" " add.s64 %rd139, %rd139, 152;\n" " add.s32 %r305, %r305, 1024;\n" " add.s32 %r304, %r304, 1;\n" " setp.lt.s32 %p15, %r304, %r298;\n" " @%p15 bra $L__BB10_26;\n" "\n" "$L__BB10_27:\n" " add.s32 %r301, %r301, 512;\n" " setp.lt.s32 %p16, %r301, %r23;\n" " @%p16 bra $L__BB10_4;\n" "\n" "$L__BB10_28:\n" " mov.u32 %r291, %ctaid.y;\n" " shl.b32 %r290, %r291, 3;\n" " ld.param.u32 %r289, [fc_matvec_q3k_nx8_param_6];\n" " sub.s32 %r288, %r289, %r290;\n" " cvt.s64.s32 %rd35, %r22;\n" " mov.u32 %r116, 16;\n" " mov.u32 %r125, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f447, %f661, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f450, %f661, %f447;\n" " mov.u32 %r118, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f449, %f450, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f452, %f450, %f449;\n" " mov.u32 %r120, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f451, %f452, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f454, %f452, %f451;\n" " mov.u32 %r122, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f453, %f454, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f456, %f454, %f453;\n" " mov.u32 %r124, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f455, %f456, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f97, %f456, %f455;\n" " // begin inline asm\n" " shfl.down.b32 %f457, %f669, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f460, %f669, %f457;\n" " // begin inline asm\n" " shfl.down.b32 %f459, %f460, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f462, %f460, %f459;\n" " // begin inline asm\n" " shfl.down.b32 %f461, %f462, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f464, %f462, %f461;\n" " // begin inline asm\n" " shfl.down.b32 %f463, %f464, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f466, %f464, %f463;\n" " // begin inline asm\n" " shfl.down.b32 %f465, %f466, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f98, %f466, %f465;\n" " setp.lt.s32 %p17, %r288, 1;\n" " setp.ne.s32 %p18, %r2, 0;\n" " or.pred %p19, %p18, %p17;\n" " @%p19 bra $L__BB10_33;\n" "\n" " cvt.u32.u64 %r126, %rd2;\n" " setp.ge.s32 %p20, %r126, %r22;\n" " cvt.s64.s32 %rd84, %r3;\n" " mul.lo.s64 %rd85, %rd84, %rd35;\n" " add.s64 %rd86, %rd85, %rd2;\n" " shl.b64 %rd87, %rd86, 2;\n" " add.s64 %rd36, %rd1, %rd87;\n" " @%p20 bra $L__BB10_31;\n" "\n" " st.global.f32 [%rd36], %f97;\n" "\n" "$L__BB10_31:\n" " @%p1 bra $L__BB10_33;\n" "\n" " st.global.f32 [%rd36+4], %f98;\n" "\n" "$L__BB10_33:\n" " // begin inline asm\n" " shfl.down.b32 %f467, %f660, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f470, %f660, %f467;\n" " // begin inline asm\n" " shfl.down.b32 %f469, %f470, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f472, %f470, %f469;\n" " // begin inline asm\n" " shfl.down.b32 %f471, %f472, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f474, %f472, %f471;\n" " // begin inline asm\n" " shfl.down.b32 %f473, %f474, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f476, %f474, %f473;\n" " // begin inline asm\n" " shfl.down.b32 %f475, %f476, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f99, %f476, %f475;\n" " // begin inline asm\n" " shfl.down.b32 %f477, %f668, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f480, %f668, %f477;\n" " // begin inline asm\n" " shfl.down.b32 %f479, %f480, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f482, %f480, %f479;\n" " // begin inline asm\n" " shfl.down.b32 %f481, %f482, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f484, %f482, %f481;\n" " // begin inline asm\n" " shfl.down.b32 %f483, %f484, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f486, %f484, %f483;\n" " // begin inline asm\n" " shfl.down.b32 %f485, %f486, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f100, %f486, %f485;\n" " setp.lt.s32 %p23, %r5, 2;\n" " or.pred %p24, %p18, %p23;\n" " @%p24 bra $L__BB10_38;\n" "\n" " cvt.u32.u64 %r147, %rd2;\n" " setp.ge.s32 %p25, %r147, %r22;\n" " add.s32 %r148, %r3, 1;\n" " cvt.s64.s32 %rd88, %r148;\n" " mul.lo.s64 %rd89, %rd88, %rd35;\n" " add.s64 %rd90, %rd89, %rd2;\n" " shl.b64 %rd91, %rd90, 2;\n" " add.s64 %rd37, %rd1, %rd91;\n" " @%p25 bra $L__BB10_36;\n" "\n" " st.global.f32 [%rd37], %f99;\n" "\n" "$L__BB10_36:\n" " @%p1 bra $L__BB10_38;\n" "\n" " st.global.f32 [%rd37+4], %f100;\n" "\n" "$L__BB10_38:\n" " // begin inline asm\n" " shfl.down.b32 %f487, %f659, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f490, %f659, %f487;\n" " // begin inline asm\n" " shfl.down.b32 %f489, %f490, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f492, %f490, %f489;\n" " // begin inline asm\n" " shfl.down.b32 %f491, %f492, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f494, %f492, %f491;\n" " // begin inline asm\n" " shfl.down.b32 %f493, %f494, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f496, %f494, %f493;\n" " // begin inline asm\n" " shfl.down.b32 %f495, %f496, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f101, %f496, %f495;\n" " // begin inline asm\n" " shfl.down.b32 %f497, %f667, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f500, %f667, %f497;\n" " // begin inline asm\n" " shfl.down.b32 %f499, %f500, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f502, %f500, %f499;\n" " // begin inline asm\n" " shfl.down.b32 %f501, %f502, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f504, %f502, %f501;\n" " // begin inline asm\n" " shfl.down.b32 %f503, %f504, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f506, %f504, %f503;\n" " // begin inline asm\n" " shfl.down.b32 %f505, %f506, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f102, %f506, %f505;\n" " setp.lt.s32 %p28, %r5, 3;\n" " or.pred %p29, %p18, %p28;\n" " @%p29 bra $L__BB10_43;\n" "\n" " cvt.u32.u64 %r169, %rd2;\n" " setp.ge.s32 %p30, %r169, %r22;\n" " add.s32 %r170, %r3, 2;\n" " cvt.s64.s32 %rd92, %r170;\n" " mul.lo.s64 %rd93, %rd92, %rd35;\n" " add.s64 %rd94, %rd93, %rd2;\n" " shl.b64 %rd95, %rd94, 2;\n" " add.s64 %rd38, %rd1, %rd95;\n" " @%p30 bra $L__BB10_41;\n" "\n" " st.global.f32 [%rd38], %f101;\n" "\n" "$L__BB10_41:\n" " @%p1 bra $L__BB10_43;\n" "\n" " st.global.f32 [%rd38+4], %f102;\n" "\n" "$L__BB10_43:\n" " // begin inline asm\n" " shfl.down.b32 %f507, %f658, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f510, %f658, %f507;\n" " // begin inline asm\n" " shfl.down.b32 %f509, %f510, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f512, %f510, %f509;\n" " // begin inline asm\n" " shfl.down.b32 %f511, %f512, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f514, %f512, %f511;\n" " // begin inline asm\n" " shfl.down.b32 %f513, %f514, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f516, %f514, %f513;\n" " // begin inline asm\n" " shfl.down.b32 %f515, %f516, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f103, %f516, %f515;\n" " // begin inline asm\n" " shfl.down.b32 %f517, %f666, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f520, %f666, %f517;\n" " // begin inline asm\n" " shfl.down.b32 %f519, %f520, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f522, %f520, %f519;\n" " // begin inline asm\n" " shfl.down.b32 %f521, %f522, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f524, %f522, %f521;\n" " // begin inline asm\n" " shfl.down.b32 %f523, %f524, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f526, %f524, %f523;\n" " // begin inline asm\n" " shfl.down.b32 %f525, %f526, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f104, %f526, %f525;\n" " setp.lt.s32 %p33, %r5, 4;\n" " or.pred %p34, %p18, %p33;\n" " @%p34 bra $L__BB10_48;\n" "\n" " cvt.u32.u64 %r191, %rd2;\n" " setp.ge.s32 %p35, %r191, %r22;\n" " add.s32 %r192, %r3, 3;\n" " cvt.s64.s32 %rd96, %r192;\n" " mul.lo.s64 %rd97, %rd96, %rd35;\n" " add.s64 %rd98, %rd97, %rd2;\n" " shl.b64 %rd99, %rd98, 2;\n" " add.s64 %rd39, %rd1, %rd99;\n" " @%p35 bra $L__BB10_46;\n" "\n" " st.global.f32 [%rd39], %f103;\n" "\n" "$L__BB10_46:\n" " @%p1 bra $L__BB10_48;\n" "\n" " st.global.f32 [%rd39+4], %f104;\n" "\n" "$L__BB10_48:\n" " // begin inline asm\n" " shfl.down.b32 %f527, %f657, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f530, %f657, %f527;\n" " // begin inline asm\n" " shfl.down.b32 %f529, %f530, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f532, %f530, %f529;\n" " // begin inline asm\n" " shfl.down.b32 %f531, %f532, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f534, %f532, %f531;\n" " // begin inline asm\n" " shfl.down.b32 %f533, %f534, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f536, %f534, %f533;\n" " // begin inline asm\n" " shfl.down.b32 %f535, %f536, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f105, %f536, %f535;\n" " // begin inline asm\n" " shfl.down.b32 %f537, %f665, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f540, %f665, %f537;\n" " // begin inline asm\n" " shfl.down.b32 %f539, %f540, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f542, %f540, %f539;\n" " // begin inline asm\n" " shfl.down.b32 %f541, %f542, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f544, %f542, %f541;\n" " // begin inline asm\n" " shfl.down.b32 %f543, %f544, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f546, %f544, %f543;\n" " // begin inline asm\n" " shfl.down.b32 %f545, %f546, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f106, %f546, %f545;\n" " setp.lt.s32 %p38, %r5, 5;\n" " or.pred %p39, %p18, %p38;\n" " @%p39 bra $L__BB10_53;\n" "\n" " cvt.u32.u64 %r213, %rd2;\n" " setp.ge.s32 %p40, %r213, %r22;\n" " add.s32 %r214, %r3, 4;\n" " cvt.s64.s32 %rd100, %r214;\n" " mul.lo.s64 %rd101, %rd100, %rd35;\n" " add.s64 %rd102, %rd101, %rd2;\n" " shl.b64 %rd103, %rd102, 2;\n" " add.s64 %rd40, %rd1, %rd103;\n" " @%p40 bra $L__BB10_51;\n" "\n" " st.global.f32 [%rd40], %f105;\n" "\n" "$L__BB10_51:\n" " @%p1 bra $L__BB10_53;\n" "\n" " st.global.f32 [%rd40+4], %f106;\n" "\n" "$L__BB10_53:\n" " // begin inline asm\n" " shfl.down.b32 %f547, %f656, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f550, %f656, %f547;\n" " // begin inline asm\n" " shfl.down.b32 %f549, %f550, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f552, %f550, %f549;\n" " // begin inline asm\n" " shfl.down.b32 %f551, %f552, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f554, %f552, %f551;\n" " // begin inline asm\n" " shfl.down.b32 %f553, %f554, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f556, %f554, %f553;\n" " // begin inline asm\n" " shfl.down.b32 %f555, %f556, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f107, %f556, %f555;\n" " // begin inline asm\n" " shfl.down.b32 %f557, %f664, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f560, %f664, %f557;\n" " // begin inline asm\n" " shfl.down.b32 %f559, %f560, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f562, %f560, %f559;\n" " // begin inline asm\n" " shfl.down.b32 %f561, %f562, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f564, %f562, %f561;\n" " // begin inline asm\n" " shfl.down.b32 %f563, %f564, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f566, %f564, %f563;\n" " // begin inline asm\n" " shfl.down.b32 %f565, %f566, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f108, %f566, %f565;\n" " setp.lt.s32 %p43, %r5, 6;\n" " or.pred %p44, %p18, %p43;\n" " @%p44 bra $L__BB10_58;\n" "\n" " cvt.u32.u64 %r235, %rd2;\n" " setp.ge.s32 %p45, %r235, %r22;\n" " add.s32 %r236, %r3, 5;\n" " cvt.s64.s32 %rd104, %r236;\n" " mul.lo.s64 %rd105, %rd104, %rd35;\n" " add.s64 %rd106, %rd105, %rd2;\n" " shl.b64 %rd107, %rd106, 2;\n" " add.s64 %rd41, %rd1, %rd107;\n" " @%p45 bra $L__BB10_56;\n" "\n" " st.global.f32 [%rd41], %f107;\n" "\n" "$L__BB10_56:\n" " @%p1 bra $L__BB10_58;\n" "\n" " st.global.f32 [%rd41+4], %f108;\n" "\n" "$L__BB10_58:\n" " // begin inline asm\n" " shfl.down.b32 %f567, %f655, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f570, %f655, %f567;\n" " // begin inline asm\n" " shfl.down.b32 %f569, %f570, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f572, %f570, %f569;\n" " // begin inline asm\n" " shfl.down.b32 %f571, %f572, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f574, %f572, %f571;\n" " // begin inline asm\n" " shfl.down.b32 %f573, %f574, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f576, %f574, %f573;\n" " // begin inline asm\n" " shfl.down.b32 %f575, %f576, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f109, %f576, %f575;\n" " // begin inline asm\n" " shfl.down.b32 %f577, %f663, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f580, %f663, %f577;\n" " // begin inline asm\n" " shfl.down.b32 %f579, %f580, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f582, %f580, %f579;\n" " // begin inline asm\n" " shfl.down.b32 %f581, %f582, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f584, %f582, %f581;\n" " // begin inline asm\n" " shfl.down.b32 %f583, %f584, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f586, %f584, %f583;\n" " // begin inline asm\n" " shfl.down.b32 %f585, %f586, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f110, %f586, %f585;\n" " setp.lt.s32 %p48, %r5, 7;\n" " or.pred %p49, %p18, %p48;\n" " @%p49 bra $L__BB10_63;\n" "\n" " cvt.u32.u64 %r257, %rd2;\n" " setp.ge.s32 %p50, %r257, %r22;\n" " add.s32 %r258, %r3, 6;\n" " cvt.s64.s32 %rd108, %r258;\n" " mul.lo.s64 %rd109, %rd108, %rd35;\n" " add.s64 %rd110, %rd109, %rd2;\n" " shl.b64 %rd111, %rd110, 2;\n" " add.s64 %rd42, %rd1, %rd111;\n" " @%p50 bra $L__BB10_61;\n" "\n" " st.global.f32 [%rd42], %f109;\n" "\n" "$L__BB10_61:\n" " @%p1 bra $L__BB10_63;\n" "\n" " st.global.f32 [%rd42+4], %f110;\n" "\n" "$L__BB10_63:\n" " // begin inline asm\n" " shfl.down.b32 %f587, %f654, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f590, %f654, %f587;\n" " // begin inline asm\n" " shfl.down.b32 %f589, %f590, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f592, %f590, %f589;\n" " // begin inline asm\n" " shfl.down.b32 %f591, %f592, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f594, %f592, %f591;\n" " // begin inline asm\n" " shfl.down.b32 %f593, %f594, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f596, %f594, %f593;\n" " // begin inline asm\n" " shfl.down.b32 %f595, %f596, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f111, %f596, %f595;\n" " // begin inline asm\n" " shfl.down.b32 %f597, %f662, %r116, %r125;\n" " // end inline asm\n" " add.f32 %f600, %f662, %f597;\n" " // begin inline asm\n" " shfl.down.b32 %f599, %f600, %r118, %r125;\n" " // end inline asm\n" " add.f32 %f602, %f600, %f599;\n" " // begin inline asm\n" " shfl.down.b32 %f601, %f602, %r120, %r125;\n" " // end inline asm\n" " add.f32 %f604, %f602, %f601;\n" " // begin inline asm\n" " shfl.down.b32 %f603, %f604, %r122, %r125;\n" " // end inline asm\n" " add.f32 %f606, %f604, %f603;\n" " // begin inline asm\n" " shfl.down.b32 %f605, %f606, %r124, %r125;\n" " // end inline asm\n" " add.f32 %f112, %f606, %f605;\n" " setp.lt.s32 %p53, %r5, 8;\n" " or.pred %p54, %p18, %p53;\n" " @%p54 bra $L__BB10_68;\n" "\n" " cvt.u32.u64 %r279, %rd2;\n" " setp.ge.s32 %p55, %r279, %r22;\n" " add.s32 %r280, %r3, 7;\n" " cvt.s64.s32 %rd112, %r280;\n" " mul.lo.s64 %rd113, %rd112, %rd35;\n" " add.s64 %rd114, %rd113, %rd2;\n" " shl.b64 %rd115, %rd114, 2;\n" " add.s64 %rd43, %rd1, %rd115;\n" " @%p55 bra $L__BB10_66;\n" "\n" " st.global.f32 [%rd43], %f111;\n" "\n" "$L__BB10_66:\n" " @%p1 bra $L__BB10_68;\n" "\n" " st.global.f32 [%rd43+4], %f112;\n" "\n" "$L__BB10_68:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_MATVEC_KQUANTS[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_matvec_q4k\n" ".extern .shared .align 16 .b8 fc_kq_shared[];\n" "\n" ".visible .entry fc_matvec_q4k(\n" " .param .u64 fc_matvec_q4k_param_0,\n" " .param .u64 fc_matvec_q4k_param_1,\n" " .param .u64 fc_matvec_q4k_param_2,\n" " .param .u32 fc_matvec_q4k_param_3,\n" " .param .u32 fc_matvec_q4k_param_4,\n" " .param .u32 fc_matvec_q4k_param_5\n" ")\n" "{\n" " .reg .pred %p<22>;\n" " .reg .b16 %rs<5>;\n" " .reg .f32 %f<204>;\n" " .reg .b32 %r<212>;\n" " .reg .b64 %rd<74>;\n" "\n" "\n" " ld.param.u64 %rd32, [fc_matvec_q4k_param_0];\n" " ld.param.u64 %rd30, [fc_matvec_q4k_param_1];\n" " ld.param.u64 %rd31, [fc_matvec_q4k_param_2];\n" " ld.param.u32 %r41, [fc_matvec_q4k_param_3];\n" " ld.param.u32 %r42, [fc_matvec_q4k_param_4];\n" " ld.param.u32 %r43, [fc_matvec_q4k_param_5];\n" " mov.u32 %r1, %tid.x;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r44, %ctaid.x;\n" " shl.b32 %r45, %r44, 4;\n" " shr.u32 %r46, %r1, 4;\n" " and.b32 %r47, %r46, 268435454;\n" " add.s32 %r48, %r47, %r45;\n" " or.b32 %r3, %r48, 1;\n" " setp.ge.s32 %p1, %r3, %r41;\n" " cvt.s64.s32 %rd1, %r48;\n" " cvt.s64.s32 %rd2, %r43;\n" " mul.wide.s32 %rd33, %r43, %r48;\n" " add.s64 %rd72, %rd32, %rd33;\n" " mov.u64 %rd73, %rd72;\n" " @%p1 bra $L__BB0_2;\n" "\n" " add.s64 %rd73, %rd72, %rd2;\n" "\n" "$L__BB0_2:\n" " setp.lt.s32 %p2, %r42, 1;\n" " mov.f32 %f196, 0f00000000;\n" " mov.f32 %f197, %f196;\n" " mov.f32 %f187, %f196;\n" " mov.f32 %f186, %f196;\n" " @%p2 bra $L__BB0_26;\n" "\n" " and.b32 %r4, %r1, 3;\n" " shr.u32 %r5, %r1, 2;\n" " shl.b32 %r51, %r1, 3;\n" " and.b32 %r6, %r51, 24;\n" " or.b32 %r7, %r6, 64;\n" " cvt.u64.u32 %rd6, %r2;\n" " mov.u32 %r52, -2;\n" " sub.s32 %r8, %r52, %r1;\n" " not.b32 %r9, %r42;\n" " cvt.s64.s32 %rd7, %r1;\n" " shl.b32 %r53, %r1, 2;\n" " mov.u32 %r54, fc_kq_shared;\n" " add.s32 %r10, %r54, %r53;\n" " add.s32 %r11, %r1, 256;\n" " cvt.s64.s32 %rd8, %r11;\n" " add.s32 %r12, %r1, 512;\n" " cvt.s64.s32 %rd9, %r12;\n" " add.s32 %r13, %r1, 768;\n" " mov.f32 %f186, 0f00000000;\n" " mov.u32 %r203, 0;\n" " setp.gt.u32 %p9, %r1, 255;\n" " setp.ne.s32 %p11, %r4, 0;\n" " shl.b64 %rd61, %rd6, 2;\n" " mov.u32 %r204, %r203;\n" " mov.f32 %f187, %f186;\n" " mov.f32 %f197, %f186;\n" " mov.f32 %f196, %f186;\n" "\n" "$L__BB0_4:\n" " add.s32 %r55, %r203, %r9;\n" " max.s32 %r56, %r55, -2049;\n" " sub.s32 %r16, %r8, %r56;\n" " sub.s32 %r57, %r42, %r204;\n" " min.s32 %r17, %r57, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r17;\n" " @%p3 bra $L__BB0_12;\n" "\n" " shr.u32 %r58, %r16, 8;\n" " add.s32 %r59, %r58, 1;\n" " cvt.s64.s32 %rd12, %r204;\n" " and.b32 %r18, %r59, 3;\n" " setp.eq.s32 %p4, %r18, 0;\n" " mov.u32 %r205, %r1;\n" " @%p4 bra $L__BB0_9;\n" "\n" " add.s64 %rd35, %rd7, %rd12;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd34, %rd30, %rd36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd34];\n" " // end inline asm\n" " st.shared.f32 [%r10+256], %f34;\n" " setp.eq.s32 %p5, %r18, 1;\n" " mov.u32 %r205, %r11;\n" " @%p5 bra $L__BB0_9;\n" "\n" " add.s64 %rd38, %rd8, %rd12;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd37, %rd30, %rd39;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd37];\n" " // end inline asm\n" " st.shared.f32 [%r10+1280], %f35;\n" " setp.eq.s32 %p6, %r18, 2;\n" " mov.u32 %r205, %r12;\n" " @%p6 bra $L__BB0_9;\n" "\n" " add.s64 %rd41, %rd9, %rd12;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd40, %rd30, %rd42;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd40];\n" " // end inline asm\n" " st.shared.f32 [%r10+2304], %f36;\n" " mov.u32 %r205, %r13;\n" "\n" "$L__BB0_9:\n" " setp.lt.u32 %p7, %r16, 768;\n" " @%p7 bra $L__BB0_12;\n" "\n" " cvt.s64.s32 %rd43, %r205;\n" " add.s64 %rd44, %rd43, %rd12;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd67, %rd30, %rd45;\n" " shl.b32 %r60, %r205, 2;\n" " add.s32 %r62, %r54, %r60;\n" " add.s32 %r206, %r62, 2304;\n" "\n" "$L__BB0_11:\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd67];\n" " // end inline asm\n" " st.shared.f32 [%r206+-2048], %f37;\n" " add.s64 %rd47, %rd67, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f38, [%rd47];\n" " // end inline asm\n" " st.shared.f32 [%r206+-1024], %f38;\n" " add.s64 %rd48, %rd67, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f39, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r206], %f39;\n" " add.s64 %rd49, %rd67, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f40, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r206+1024], %f40;\n" " add.s64 %rd67, %rd67, 4096;\n" " add.s32 %r206, %r206, 4096;\n" " add.s32 %r205, %r205, 1024;\n" " setp.lt.s32 %p8, %r205, %r17;\n" " @%p8 bra $L__BB0_11;\n" "\n" "$L__BB0_12:\n" " bar.sync 0;\n" " @%p9 bra $L__BB0_18;\n" "\n" " shr.u32 %r25, %r17, 5;\n" " mov.u32 %r208, %r5;\n" "\n" "$L__BB0_14:\n" " mov.u32 %r26, %r208;\n" " and.b32 %r27, %r26, 63;\n" " setp.ge.u32 %p10, %r27, %r25;\n" " @%p10 bra $L__BB0_17;\n" "\n" " shl.b32 %r67, %r26, 5;\n" " and.b32 %r68, %r67, -2048;\n" " shl.b32 %r69, %r27, 5;\n" " or.b32 %r70, %r6, %r69;\n" " add.s32 %r71, %r7, %r69;\n" " or.b32 %r72, %r70, %r68;\n" " add.s32 %r73, %r71, %r68;\n" " and.b32 %r74, %r26, 7;\n" " add.s32 %r75, %r72, %r74;\n" " shl.b32 %r76, %r75, 2;\n" " add.s32 %r78, %r54, %r76;\n" " mov.u32 %r63, 2;\n" " ld.shared.f32 %f45, [%r78+256];\n" " add.f32 %f46, %f45, 0f00000000;\n" " xor.b32 %r79, %r74, 1;\n" " mov.u32 %r65, 1;\n" " add.s32 %r80, %r73, %r79;\n" " shl.b32 %r81, %r80, 2;\n" " add.s32 %r82, %r54, %r81;\n" " ld.shared.f32 %f47, [%r82];\n" " add.f32 %f48, %f46, %f47;\n" " xor.b32 %r83, %r74, 2;\n" " add.s32 %r84, %r73, %r83;\n" " shl.b32 %r85, %r84, 2;\n" " add.s32 %r86, %r54, %r85;\n" " ld.shared.f32 %f49, [%r86];\n" " add.f32 %f50, %f48, %f49;\n" " xor.b32 %r87, %r74, 3;\n" " add.s32 %r88, %r73, %r87;\n" " shl.b32 %r89, %r88, 2;\n" " add.s32 %r90, %r54, %r89;\n" " ld.shared.f32 %f51, [%r90];\n" " add.f32 %f52, %f50, %f51;\n" " xor.b32 %r91, %r74, 4;\n" " add.s32 %r92, %r73, %r91;\n" " shl.b32 %r93, %r92, 2;\n" " add.s32 %r94, %r54, %r93;\n" " ld.shared.f32 %f53, [%r94];\n" " add.f32 %f54, %f52, %f53;\n" " xor.b32 %r95, %r74, 5;\n" " add.s32 %r96, %r73, %r95;\n" " shl.b32 %r97, %r96, 2;\n" " add.s32 %r98, %r54, %r97;\n" " ld.shared.f32 %f55, [%r98];\n" " add.f32 %f56, %f54, %f55;\n" " xor.b32 %r99, %r74, 6;\n" " add.s32 %r100, %r73, %r99;\n" " shl.b32 %r101, %r100, 2;\n" " add.s32 %r102, %r54, %r101;\n" " ld.shared.f32 %f57, [%r102];\n" " add.f32 %f58, %f56, %f57;\n" " xor.b32 %r103, %r74, 7;\n" " add.s32 %r104, %r73, %r103;\n" " shl.b32 %r105, %r104, 2;\n" " add.s32 %r106, %r54, %r105;\n" " ld.shared.f32 %f59, [%r106];\n" " add.f32 %f42, %f58, %f59;\n" " mov.u32 %r66, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f41, %f42, %r63, %r66;\n" " // end inline asm\n" " add.f32 %f44, %f42, %f41;\n" " // begin inline asm\n" " shfl.down.b32 %f43, %f44, %r65, %r66;\n" " // end inline asm\n" " add.f32 %f5, %f44, %f43;\n" " @%p11 bra $L__BB0_17;\n" "\n" " shl.b32 %r107, %r26, 2;\n" " add.s32 %r109, %r54, %r107;\n" " st.shared.f32 [%r109], %f5;\n" "\n" "$L__BB0_17:\n" " add.s32 %r208, %r26, 64;\n" " setp.gt.u32 %p12, %r26, -65;\n" " @%p12 bra $L__BB0_14;\n" "\n" "$L__BB0_18:\n" " cvt.u32.u64 %r110, %rd1;\n" " setp.ge.s32 %p13, %r110, %r41;\n" " bar.sync 0;\n" " @%p13 bra $L__BB0_25;\n" "\n" " shr.s32 %r29, %r17, 8;\n" " shr.u32 %r111, %r204, 5;\n" " sub.s32 %r30, %r2, %r111;\n" " and.b32 %r209, %r30, 31;\n" " shl.b32 %r32, %r29, 3;\n" " setp.ge.s32 %p14, %r209, %r32;\n" " @%p14 bra $L__BB0_22;\n" "\n" " shr.u32 %r112, %r209, 3;\n" " mul.lo.s32 %r113, %r112, 152;\n" " cvt.u64.u32 %rd50, %r113;\n" " and.b32 %r114, %r30, 7;\n" " or.b32 %r115, %r114, 136;\n" " cvt.u64.u32 %rd16, %r115;\n" " add.s64 %rd68, %rd73, %rd50;\n" " add.s64 %rd69, %rd72, %rd50;\n" "\n" "$L__BB0_21:\n" " .pragma \"nounroll\";\n" " add.s64 %rd51, %rd69, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f60, [%rd51];\n" " // end inline asm\n" " add.s64 %rd52, %rd69, %rd16;\n" " // begin inline asm\n" " ld.global.nc.u8 %r116, [%rd52];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r116;\n" " and.b16 %rs2, %rs1, 255;\n" " cvt.rn.f32.u16 %f62, %rs2;\n" " mul.rn.f32 %f63, %f60, %f62;\n" " add.s64 %rd53, %rd68, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f61, [%rd53];\n" " // end inline asm\n" " add.s64 %rd54, %rd68, %rd16;\n" " // begin inline asm\n" " ld.global.nc.u8 %r117, [%rd54];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r117;\n" " and.b16 %rs4, %rs3, 255;\n" " cvt.rn.f32.u16 %f64, %rs4;\n" " mul.rn.f32 %f65, %f61, %f64;\n" " shl.b32 %r118, %r209, 2;\n" " add.s32 %r120, %r54, %r118;\n" " ld.shared.f32 %f66, [%r120];\n" " fma.rn.f32 %f187, %f63, %f66, %f187;\n" " fma.rn.f32 %f186, %f65, %f66, %f186;\n" " add.s64 %rd69, %rd69, 608;\n" " add.s64 %rd68, %rd68, 608;\n" " add.s32 %r209, %r209, 32;\n" " setp.lt.s32 %p15, %r209, %r32;\n" " @%p15 bra $L__BB0_21;\n" "\n" "$L__BB0_22:\n" " setp.lt.s32 %p16, %r17, 256;\n" " @%p16 bra $L__BB0_25;\n" "\n" " add.s32 %r211, %r54, 256;\n" " mov.u32 %r210, 0;\n" "\n" "$L__BB0_24:\n" " add.s64 %rd55, %rd72, %rd61;\n" " // begin inline asm\n" " ld.global.nc.u32 %r124, [%rd55];\n" " // end inline asm\n" " add.s64 %rd56, %rd73, %rd61;\n" " // begin inline asm\n" " ld.global.nc.u32 %r125, [%rd56];\n" " // end inline asm\n" " add.s64 %rd57, %rd72, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r126,%r127}, [%rd57];\n" " // end inline asm\n" " add.s64 %rd58, %rd73, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r128,%r129}, [%rd58];\n" " // end inline asm\n" " add.s64 %rd59, %rd72, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f67, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd73, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f68, [%rd60];\n" " // end inline asm\n" " shr.u32 %r130, %r124, 16;\n" " shr.u32 %r131, %r125, 16;\n" " and.b32 %r132, %r126, 255;\n" " cvt.rn.f32.u32 %f69, %r132;\n" " and.b32 %r133, %r124, 15;\n" " or.b32 %r134, %r133, 1258291200;\n" " mov.b32 %f70, %r134;\n" " mov.f32 %f71, 0fCB000000;\n" " mov.f32 %f72, 0f3F800000;\n" " fma.rn.f32 %f73, %f70, %f72, %f71;\n" " mul.f32 %f74, %f73, %f69;\n" " and.b32 %r135, %r128, 255;\n" " cvt.rn.f32.u32 %f75, %r135;\n" " and.b32 %r136, %r125, 15;\n" " or.b32 %r137, %r136, 1258291200;\n" " mov.b32 %f76, %r137;\n" " fma.rn.f32 %f77, %f76, %f72, %f71;\n" " mul.f32 %f78, %f77, %f75;\n" " shl.b32 %r138, %r2, 2;\n" " add.s32 %r139, %r211, %r138;\n" " ld.shared.f32 %f79, [%r139];\n" " mov.f32 %f80, 0f00000000;\n" " fma.rn.f32 %f81, %f74, %f79, %f80;\n" " fma.rn.f32 %f82, %f78, %f79, %f80;\n" " bfe.u32 %r140, %r126, 8, 8;\n" " cvt.rn.f32.u32 %f83, %r140;\n" " and.b32 %r141, %r124, 240;\n" " or.b32 %r142, %r141, 1258291200;\n" " mov.b32 %f84, %r142;\n" " mov.f32 %f85, 0fC9000000;\n" " mov.f32 %f86, 0f3D800000;\n" " fma.rn.f32 %f87, %f84, %f86, %f85;\n" " mul.f32 %f88, %f87, %f83;\n" " bfe.u32 %r143, %r128, 8, 8;\n" " cvt.rn.f32.u32 %f89, %r143;\n" " and.b32 %r144, %r125, 240;\n" " or.b32 %r145, %r144, 1258291200;\n" " mov.b32 %f90, %r145;\n" " fma.rn.f32 %f91, %f90, %f86, %f85;\n" " mul.f32 %f92, %f91, %f89;\n" " ld.shared.f32 %f93, [%r139+128];\n" " fma.rn.f32 %f94, %f88, %f93, %f81;\n" " fma.rn.f32 %f95, %f92, %f93, %f82;\n" " bfe.u32 %r146, %r126, 16, 8;\n" " cvt.rn.f32.u32 %f96, %r146;\n" " and.b32 %r147, %r124, 3840;\n" " or.b32 %r148, %r147, 1258291200;\n" " mov.b32 %f97, %r148;\n" " mov.f32 %f98, 0fC7000000;\n" " mov.f32 %f99, 0f3B800000;\n" " fma.rn.f32 %f100, %f97, %f99, %f98;\n" " mul.f32 %f101, %f100, %f96;\n" " bfe.u32 %r149, %r128, 16, 8;\n" " cvt.rn.f32.u32 %f102, %r149;\n" " and.b32 %r150, %r125, 3840;\n" " or.b32 %r151, %r150, 1258291200;\n" " mov.b32 %f103, %r151;\n" " fma.rn.f32 %f104, %f103, %f99, %f98;\n" " mul.f32 %f105, %f104, %f102;\n" " ld.shared.f32 %f106, [%r139+256];\n" " fma.rn.f32 %f107, %f101, %f106, %f94;\n" " fma.rn.f32 %f108, %f105, %f106, %f95;\n" " shr.u32 %r152, %r126, 24;\n" " cvt.rn.f32.u32 %f109, %r152;\n" " and.b32 %r153, %r124, 61440;\n" " or.b32 %r154, %r153, 1258291200;\n" " mov.b32 %f110, %r154;\n" " mov.f32 %f111, 0fC5000000;\n" " mov.f32 %f112, 0f39800000;\n" " fma.rn.f32 %f113, %f110, %f112, %f111;\n" " mul.f32 %f114, %f113, %f109;\n" " shr.u32 %r155, %r128, 24;\n" " cvt.rn.f32.u32 %f115, %r155;\n" " and.b32 %r156, %r125, 61440;\n" " or.b32 %r157, %r156, 1258291200;\n" " mov.b32 %f116, %r157;\n" " fma.rn.f32 %f117, %f116, %f112, %f111;\n" " mul.f32 %f118, %f117, %f115;\n" " ld.shared.f32 %f119, [%r139+384];\n" " fma.rn.f32 %f120, %f114, %f119, %f107;\n" " fma.rn.f32 %f121, %f118, %f119, %f108;\n" " and.b32 %r158, %r127, 255;\n" " cvt.rn.f32.u32 %f122, %r158;\n" " bfe.u32 %r159, %r124, 16, 4;\n" " or.b32 %r160, %r159, 1258291200;\n" " mov.b32 %f123, %r160;\n" " fma.rn.f32 %f124, %f123, %f72, %f71;\n" " mul.f32 %f125, %f124, %f122;\n" " and.b32 %r161, %r129, 255;\n" " cvt.rn.f32.u32 %f126, %r161;\n" " bfe.u32 %r162, %r125, 16, 4;\n" " or.b32 %r163, %r162, 1258291200;\n" " mov.b32 %f127, %r163;\n" " fma.rn.f32 %f128, %f127, %f72, %f71;\n" " mul.f32 %f129, %f128, %f126;\n" " ld.shared.f32 %f130, [%r139+512];\n" " fma.rn.f32 %f131, %f125, %f130, %f120;\n" " fma.rn.f32 %f132, %f129, %f130, %f121;\n" " bfe.u32 %r164, %r127, 8, 8;\n" " cvt.rn.f32.u32 %f133, %r164;\n" " and.b32 %r165, %r130, 240;\n" " or.b32 %r166, %r165, 1258291200;\n" " mov.b32 %f134, %r166;\n" " fma.rn.f32 %f135, %f134, %f86, %f85;\n" " mul.f32 %f136, %f135, %f133;\n" " bfe.u32 %r167, %r129, 8, 8;\n" " cvt.rn.f32.u32 %f137, %r167;\n" " and.b32 %r168, %r131, 240;\n" " or.b32 %r169, %r168, 1258291200;\n" " mov.b32 %f138, %r169;\n" " fma.rn.f32 %f139, %f138, %f86, %f85;\n" " mul.f32 %f140, %f139, %f137;\n" " ld.shared.f32 %f141, [%r139+640];\n" " fma.rn.f32 %f142, %f136, %f141, %f131;\n" " fma.rn.f32 %f143, %f140, %f141, %f132;\n" " bfe.u32 %r170, %r127, 16, 8;\n" " cvt.rn.f32.u32 %f144, %r170;\n" " and.b32 %r171, %r130, 3840;\n" " or.b32 %r172, %r171, 1258291200;\n" " mov.b32 %f145, %r172;\n" " fma.rn.f32 %f146, %f145, %f99, %f98;\n" " mul.f32 %f147, %f146, %f144;\n" " bfe.u32 %r173, %r129, 16, 8;\n" " cvt.rn.f32.u32 %f148, %r173;\n" " and.b32 %r174, %r131, 3840;\n" " or.b32 %r175, %r174, 1258291200;\n" " mov.b32 %f149, %r175;\n" " fma.rn.f32 %f150, %f149, %f99, %f98;\n" " mul.f32 %f151, %f150, %f148;\n" " ld.shared.f32 %f152, [%r139+768];\n" " fma.rn.f32 %f153, %f147, %f152, %f142;\n" " fma.rn.f32 %f154, %f151, %f152, %f143;\n" " shr.u32 %r176, %r127, 24;\n" " cvt.rn.f32.u32 %f155, %r176;\n" " and.b32 %r177, %r130, 61440;\n" " or.b32 %r178, %r177, 1258291200;\n" " mov.b32 %f156, %r178;\n" " fma.rn.f32 %f157, %f156, %f112, %f111;\n" " mul.f32 %f158, %f157, %f155;\n" " shr.u32 %r179, %r129, 24;\n" " cvt.rn.f32.u32 %f159, %r179;\n" " and.b32 %r180, %r131, 61440;\n" " or.b32 %r181, %r180, 1258291200;\n" " mov.b32 %f160, %r181;\n" " fma.rn.f32 %f161, %f160, %f112, %f111;\n" " mul.f32 %f162, %f161, %f159;\n" " ld.shared.f32 %f163, [%r139+896];\n" " fma.rn.f32 %f164, %f158, %f163, %f153;\n" " fma.rn.f32 %f165, %f162, %f163, %f154;\n" " fma.rn.f32 %f196, %f67, %f164, %f196;\n" " fma.rn.f32 %f197, %f68, %f165, %f197;\n" " add.s64 %rd72, %rd72, 152;\n" " add.s64 %rd73, %rd73, 152;\n" " add.s32 %r211, %r211, 1024;\n" " add.s32 %r210, %r210, 1;\n" " setp.lt.s32 %p17, %r210, %r29;\n" " @%p17 bra $L__BB0_24;\n" "\n" "$L__BB0_25:\n" " add.s32 %r204, %r204, 2048;\n" " setp.lt.s32 %p18, %r204, %r42;\n" " add.s32 %r203, %r203, 2048;\n" " @%p18 bra $L__BB0_4;\n" "\n" "$L__BB0_26:\n" " sub.rn.f32 %f167, %f196, %f187;\n" " mov.u32 %r200, 1;\n" " mov.u32 %r192, 16;\n" " mov.u32 %r201, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f166, %f167, %r192, %r201;\n" " // end inline asm\n" " add.f32 %f169, %f167, %f166;\n" " mov.u32 %r194, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f168, %f169, %r194, %r201;\n" " // end inline asm\n" " add.f32 %f171, %f169, %f168;\n" " mov.u32 %r196, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f170, %f171, %r196, %r201;\n" " // end inline asm\n" " add.f32 %f173, %f171, %f170;\n" " mov.u32 %r198, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f172, %f173, %r198, %r201;\n" " // end inline asm\n" " add.f32 %f175, %f173, %f172;\n" " // begin inline asm\n" " shfl.down.b32 %f174, %f175, %r200, %r201;\n" " // end inline asm\n" " add.f32 %f24, %f175, %f174;\n" " sub.rn.f32 %f177, %f197, %f186;\n" " // begin inline asm\n" " shfl.down.b32 %f176, %f177, %r192, %r201;\n" " // end inline asm\n" " add.f32 %f179, %f177, %f176;\n" " // begin inline asm\n" " shfl.down.b32 %f178, %f179, %r194, %r201;\n" " // end inline asm\n" " add.f32 %f181, %f179, %f178;\n" " // begin inline asm\n" " shfl.down.b32 %f180, %f181, %r196, %r201;\n" " // end inline asm\n" " add.f32 %f183, %f181, %f180;\n" " // begin inline asm\n" " shfl.down.b32 %f182, %f183, %r198, %r201;\n" " // end inline asm\n" " add.f32 %f185, %f183, %f182;\n" " // begin inline asm\n" " shfl.down.b32 %f184, %f185, %r200, %r201;\n" " // end inline asm\n" " add.f32 %f25, %f185, %f184;\n" " setp.ne.s32 %p19, %r2, 0;\n" " @%p19 bra $L__BB0_31;\n" "\n" " cvt.u32.u64 %r202, %rd1;\n" " setp.ge.s32 %p20, %r202, %r41;\n" " cvta.to.global.u64 %rd62, %rd31;\n" " shl.b64 %rd63, %rd1, 2;\n" " add.s64 %rd29, %rd62, %rd63;\n" " @%p20 bra $L__BB0_29;\n" "\n" " st.global.f32 [%rd29], %f24;\n" "\n" "$L__BB0_29:\n" " @%p1 bra $L__BB0_31;\n" "\n" " st.global.f32 [%rd29+4], %f25;\n" "\n" "$L__BB0_31:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q4k_nx2\n" ".visible .entry fc_matvec_q4k_nx2(\n" " .param .u64 fc_matvec_q4k_nx2_param_0,\n" " .param .u64 fc_matvec_q4k_nx2_param_1,\n" " .param .u64 fc_matvec_q4k_nx2_param_2,\n" " .param .u32 fc_matvec_q4k_nx2_param_3,\n" " .param .u32 fc_matvec_q4k_nx2_param_4,\n" " .param .u32 fc_matvec_q4k_nx2_param_5,\n" " .param .u32 fc_matvec_q4k_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<27>;\n" " .reg .b16 %rs<5>;\n" " .reg .f32 %f<304>;\n" " .reg .b32 %r<264>;\n" " .reg .b64 %rd<83>;\n" "\n" "\n" " ld.param.u64 %rd35, [fc_matvec_q4k_nx2_param_0];\n" " ld.param.u64 %rd34, [fc_matvec_q4k_nx2_param_1];\n" " ld.param.u32 %r38, [fc_matvec_q4k_nx2_param_3];\n" " ld.param.u32 %r39, [fc_matvec_q4k_nx2_param_4];\n" " ld.param.u32 %r40, [fc_matvec_q4k_nx2_param_5];\n" " ld.param.u32 %r41, [fc_matvec_q4k_nx2_param_6];\n" " mov.u32 %r42, %ctaid.y;\n" " shl.b32 %r1, %r42, 1;\n" " sub.s32 %r2, %r41, %r1;\n" " min.s32 %r3, %r2, 2;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r43, %ctaid.x;\n" " shl.b32 %r44, %r43, 4;\n" " shr.u32 %r45, %r4, 4;\n" " and.b32 %r46, %r45, 268435454;\n" " add.s32 %r47, %r46, %r44;\n" " or.b32 %r6, %r47, 1;\n" " setp.ge.s32 %p1, %r6, %r38;\n" " cvt.s64.s32 %rd2, %r47;\n" " cvt.s64.s32 %rd3, %r40;\n" " mul.wide.s32 %rd37, %r40, %r47;\n" " add.s64 %rd81, %rd35, %rd37;\n" " mov.u64 %rd82, %rd81;\n" " @%p1 bra $L__BB1_2;\n" "\n" " add.s64 %rd82, %rd81, %rd3;\n" "\n" "$L__BB1_2:\n" " setp.lt.s32 %p2, %r39, 1;\n" " mov.f32 %f288, 0f00000000;\n" " mov.f32 %f289, %f288;\n" " mov.f32 %f290, %f288;\n" " mov.f32 %f291, %f288;\n" " mov.f32 %f270, %f288;\n" " mov.f32 %f271, %f288;\n" " mov.f32 %f272, %f288;\n" " mov.f32 %f273, %f288;\n" " @%p2 bra $L__BB1_22;\n" "\n" " and.b32 %r7, %r4, 3;\n" " shl.b32 %r49, %r4, 3;\n" " and.b32 %r50, %r49, 24;\n" " cvt.u64.u32 %rd7, %r5;\n" " shr.u32 %r51, %r4, 2;\n" " bfe.u32 %r52, %r4, 2, 3;\n" " bfe.u32 %r8, %r4, 2, 4;\n" " add.s32 %r53, %r1, 1;\n" " and.b32 %r54, %r49, -512;\n" " and.b32 %r55, %r49, 480;\n" " or.b32 %r56, %r50, %r55;\n" " or.b32 %r57, %r56, %r54;\n" " or.b32 %r58, %r57, %r52;\n" " shl.b32 %r59, %r58, 2;\n" " mov.u32 %r60, fc_kq_shared;\n" " add.s32 %r9, %r60, %r59;\n" " xor.b32 %r61, %r52, 1;\n" " or.b32 %r62, %r61, %r54;\n" " or.b32 %r63, %r62, %r55;\n" " or.b32 %r64, %r63, %r50;\n" " shl.b32 %r65, %r64, 2;\n" " add.s32 %r66, %r60, %r65;\n" " xor.b32 %r67, %r52, 2;\n" " or.b32 %r68, %r67, %r54;\n" " or.b32 %r69, %r68, %r55;\n" " or.b32 %r70, %r69, %r50;\n" " shl.b32 %r71, %r70, 2;\n" " add.s32 %r72, %r60, %r71;\n" " xor.b32 %r73, %r52, 3;\n" " or.b32 %r74, %r73, %r54;\n" " or.b32 %r75, %r74, %r55;\n" " or.b32 %r76, %r75, %r50;\n" " shl.b32 %r77, %r76, 2;\n" " add.s32 %r78, %r60, %r77;\n" " xor.b32 %r79, %r52, 4;\n" " or.b32 %r80, %r79, %r54;\n" " or.b32 %r81, %r80, %r55;\n" " or.b32 %r82, %r81, %r50;\n" " shl.b32 %r83, %r82, 2;\n" " add.s32 %r84, %r60, %r83;\n" " xor.b32 %r85, %r52, 5;\n" " or.b32 %r86, %r85, %r54;\n" " or.b32 %r87, %r86, %r55;\n" " or.b32 %r88, %r87, %r50;\n" " shl.b32 %r89, %r88, 2;\n" " add.s32 %r90, %r60, %r89;\n" " xor.b32 %r91, %r52, 6;\n" " or.b32 %r92, %r91, %r54;\n" " or.b32 %r93, %r92, %r55;\n" " or.b32 %r94, %r93, %r50;\n" " shl.b32 %r95, %r94, 2;\n" " add.s32 %r96, %r60, %r95;\n" " xor.b32 %r97, %r52, 7;\n" " or.b32 %r98, %r97, %r54;\n" " or.b32 %r99, %r98, %r55;\n" " or.b32 %r100, %r99, %r50;\n" " shl.b32 %r101, %r100, 2;\n" " add.s32 %r102, %r60, %r101;\n" " shl.b32 %r103, %r51, 2;\n" " add.s32 %r17, %r60, %r103;\n" " shl.b32 %r104, %r4, 2;\n" " add.s32 %r105, %r60, %r104;\n" " add.s32 %r18, %r105, 2176;\n" " mul.wide.s32 %rd38, %r39, %r53;\n" " cvt.s64.s32 %rd39, %r4;\n" " add.s64 %rd40, %rd38, %rd39;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd8, %rd34, %rd41;\n" " mul.wide.s32 %rd42, %r39, %r1;\n" " add.s64 %rd43, %rd42, %rd39;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd9, %rd34, %rd44;\n" " mov.u32 %r258, 0;\n" " mov.f32 %f288, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " setp.gt.u32 %p7, %r4, 127;\n" " setp.ne.s32 %p10, %r7, 0;\n" " shl.b64 %rd59, %rd7, 2;\n" " mov.f32 %f289, %f288;\n" " mov.f32 %f290, %f288;\n" " mov.f32 %f291, %f288;\n" " mov.f32 %f270, %f288;\n" " mov.f32 %f271, %f288;\n" " mov.f32 %f272, %f288;\n" " mov.f32 %f273, %f288;\n" "\n" "$L__BB1_4:\n" " sub.s32 %r106, %r39, %r258;\n" " min.s32 %r20, %r106, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r20;\n" " @%p3 bra $L__BB1_11;\n" "\n" " mul.wide.s32 %rd45, %r258, 4;\n" " add.s64 %rd76, %rd8, %rd45;\n" " add.s64 %rd75, %rd9, %rd45;\n" " mov.u32 %r259, %r18;\n" " mov.u32 %r260, %r4;\n" "\n" "$L__BB1_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f275, 0f00000000;\n" " mov.f32 %f274, %f275;\n" " @%p4 bra $L__BB1_8;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f274, [%rd75];\n" " // end inline asm\n" "\n" "$L__BB1_8:\n" " st.shared.f32 [%r259+-2048], %f274;\n" " @%p5 bra $L__BB1_10;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f275, [%rd76];\n" " // end inline asm\n" "\n" "$L__BB1_10:\n" " sub.s32 %r253, %r39, %r258;\n" " min.s32 %r252, %r253, 512;\n" " st.shared.f32 [%r259], %f275;\n" " add.s32 %r259, %r259, 1024;\n" " add.s64 %rd76, %rd76, 1024;\n" " add.s64 %rd75, %rd75, 1024;\n" " add.s32 %r260, %r260, 256;\n" " setp.lt.s32 %p6, %r260, %r252;\n" " @%p6 bra $L__BB1_6;\n" "\n" "$L__BB1_11:\n" " sub.s32 %r255, %r39, %r258;\n" " min.s32 %r254, %r255, 512;\n" " bar.sync 0;\n" " shr.u32 %r107, %r254, 5;\n" " setp.ge.u32 %p8, %r8, %r107;\n" " or.pred %p9, %p7, %p8;\n" " @%p9 bra $L__BB1_14;\n" "\n" " ld.shared.f32 %f80, [%r9+128];\n" " add.f32 %f81, %f80, 0f00000000;\n" " add.s32 %r227, %r66, 128;\n" " ld.shared.f32 %f82, [%r227];\n" " add.f32 %f83, %f81, %f82;\n" " add.s32 %r228, %r72, 128;\n" " ld.shared.f32 %f84, [%r228];\n" " add.f32 %f85, %f83, %f84;\n" " add.s32 %r229, %r78, 128;\n" " ld.shared.f32 %f86, [%r229];\n" " add.f32 %f87, %f85, %f86;\n" " add.s32 %r230, %r84, 128;\n" " ld.shared.f32 %f88, [%r230];\n" " add.f32 %f89, %f87, %f88;\n" " add.s32 %r231, %r90, 128;\n" " ld.shared.f32 %f90, [%r231];\n" " add.f32 %f91, %f89, %f90;\n" " add.s32 %r232, %r96, 128;\n" " ld.shared.f32 %f92, [%r232];\n" " add.f32 %f93, %f91, %f92;\n" " add.s32 %r233, %r102, 128;\n" " ld.shared.f32 %f94, [%r233];\n" " add.f32 %f77, %f93, %f94;\n" " mov.u32 %r108, 2;\n" " mov.u32 %r111, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f76, %f77, %r108, %r111;\n" " // end inline asm\n" " add.f32 %f79, %f77, %f76;\n" " mov.u32 %r110, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f78, %f79, %r110, %r111;\n" " // end inline asm\n" " add.f32 %f13, %f79, %f78;\n" " @%p10 bra $L__BB1_14;\n" "\n" " st.shared.f32 [%r17], %f13;\n" "\n" "$L__BB1_14:\n" " cvt.u32.u64 %r112, %rd2;\n" " setp.ge.s32 %p11, %r112, %r38;\n" " bar.sync 0;\n" " @%p11 bra $L__BB1_21;\n" "\n" " sub.s32 %r248, %r39, %r258;\n" " min.s32 %r247, %r248, 512;\n" " shr.s32 %r27, %r247, 8;\n" " shr.u32 %r113, %r258, 5;\n" " sub.s32 %r28, %r5, %r113;\n" " and.b32 %r261, %r28, 31;\n" " shl.b32 %r30, %r27, 3;\n" " setp.ge.s32 %p12, %r261, %r30;\n" " @%p12 bra $L__BB1_18;\n" "\n" " shr.u32 %r114, %r261, 3;\n" " mul.lo.s32 %r115, %r114, 152;\n" " cvt.u64.u32 %rd48, %r115;\n" " and.b32 %r116, %r28, 7;\n" " or.b32 %r117, %r116, 136;\n" " cvt.u64.u32 %rd18, %r117;\n" " add.s64 %rd77, %rd82, %rd48;\n" " add.s64 %rd78, %rd81, %rd48;\n" "\n" "$L__BB1_17:\n" " .pragma \"nounroll\";\n" " add.s64 %rd49, %rd78, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f95, [%rd49];\n" " // end inline asm\n" " add.s64 %rd50, %rd78, %rd18;\n" " // begin inline asm\n" " ld.global.nc.u8 %r118, [%rd50];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r118;\n" " and.b16 %rs2, %rs1, 255;\n" " cvt.rn.f32.u16 %f97, %rs2;\n" " mul.rn.f32 %f98, %f95, %f97;\n" " add.s64 %rd51, %rd77, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f96, [%rd51];\n" " // end inline asm\n" " add.s64 %rd52, %rd77, %rd18;\n" " // begin inline asm\n" " ld.global.nc.u8 %r119, [%rd52];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r119;\n" " and.b16 %rs4, %rs3, 255;\n" " cvt.rn.f32.u16 %f99, %rs4;\n" " mul.rn.f32 %f100, %f96, %f99;\n" " shl.b32 %r120, %r261, 2;\n" " add.s32 %r122, %r60, %r120;\n" " ld.shared.f32 %f101, [%r122];\n" " fma.rn.f32 %f271, %f98, %f101, %f271;\n" " fma.rn.f32 %f273, %f100, %f101, %f273;\n" " ld.shared.f32 %f102, [%r122+64];\n" " fma.rn.f32 %f270, %f98, %f102, %f270;\n" " fma.rn.f32 %f272, %f100, %f102, %f272;\n" " add.s64 %rd78, %rd78, 608;\n" " add.s64 %rd77, %rd77, 608;\n" " add.s32 %r261, %r261, 32;\n" " setp.lt.s32 %p13, %r261, %r30;\n" " @%p13 bra $L__BB1_17;\n" "\n" "$L__BB1_18:\n" " sub.s32 %r257, %r39, %r258;\n" " min.s32 %r256, %r257, 512;\n" " setp.lt.s32 %p14, %r256, 256;\n" " @%p14 bra $L__BB1_21;\n" "\n" " add.s32 %r263, %r60, 128;\n" " mov.u32 %r262, 0;\n" "\n" "$L__BB1_20:\n" " sub.s32 %r251, %r39, %r258;\n" " min.s32 %r250, %r251, 512;\n" " shr.s32 %r249, %r247, 8;\n" " add.s64 %rd53, %rd81, %rd59;\n" " // begin inline asm\n" " ld.global.nc.u32 %r126, [%rd53];\n" " // end inline asm\n" " add.s64 %rd54, %rd82, %rd59;\n" " // begin inline asm\n" " ld.global.nc.u32 %r127, [%rd54];\n" " // end inline asm\n" " add.s64 %rd55, %rd81, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r128,%r129}, [%rd55];\n" " // end inline asm\n" " add.s64 %rd56, %rd82, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r130,%r131}, [%rd56];\n" " // end inline asm\n" " add.s64 %rd57, %rd81, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f103, [%rd57];\n" " // end inline asm\n" " add.s64 %rd58, %rd82, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f104, [%rd58];\n" " // end inline asm\n" " shr.u32 %r132, %r126, 16;\n" " shr.u32 %r133, %r127, 16;\n" " and.b32 %r134, %r128, 255;\n" " cvt.rn.f32.u32 %f105, %r134;\n" " and.b32 %r135, %r126, 15;\n" " or.b32 %r136, %r135, 1258291200;\n" " mov.b32 %f106, %r136;\n" " mov.f32 %f107, 0fCB000000;\n" " mov.f32 %f108, 0f3F800000;\n" " fma.rn.f32 %f109, %f106, %f108, %f107;\n" " mul.f32 %f110, %f109, %f105;\n" " and.b32 %r137, %r130, 255;\n" " cvt.rn.f32.u32 %f111, %r137;\n" " and.b32 %r138, %r127, 15;\n" " or.b32 %r139, %r138, 1258291200;\n" " mov.b32 %f112, %r139;\n" " fma.rn.f32 %f113, %f112, %f108, %f107;\n" " mul.f32 %f114, %f113, %f111;\n" " shl.b32 %r140, %r5, 2;\n" " add.s32 %r141, %r263, %r140;\n" " ld.shared.f32 %f115, [%r141];\n" " mov.f32 %f116, 0f00000000;\n" " fma.rn.f32 %f117, %f110, %f115, %f116;\n" " fma.rn.f32 %f118, %f114, %f115, %f116;\n" " ld.shared.f32 %f119, [%r141+2048];\n" " fma.rn.f32 %f120, %f110, %f119, %f116;\n" " fma.rn.f32 %f121, %f114, %f119, %f116;\n" " bfe.u32 %r142, %r128, 8, 8;\n" " cvt.rn.f32.u32 %f122, %r142;\n" " and.b32 %r143, %r126, 240;\n" " or.b32 %r144, %r143, 1258291200;\n" " mov.b32 %f123, %r144;\n" " mov.f32 %f124, 0fC9000000;\n" " mov.f32 %f125, 0f3D800000;\n" " fma.rn.f32 %f126, %f123, %f125, %f124;\n" " mul.f32 %f127, %f126, %f122;\n" " bfe.u32 %r145, %r130, 8, 8;\n" " cvt.rn.f32.u32 %f128, %r145;\n" " and.b32 %r146, %r127, 240;\n" " or.b32 %r147, %r146, 1258291200;\n" " mov.b32 %f129, %r147;\n" " fma.rn.f32 %f130, %f129, %f125, %f124;\n" " mul.f32 %f131, %f130, %f128;\n" " ld.shared.f32 %f132, [%r141+128];\n" " fma.rn.f32 %f133, %f127, %f132, %f117;\n" " fma.rn.f32 %f134, %f131, %f132, %f118;\n" " ld.shared.f32 %f135, [%r141+2176];\n" " fma.rn.f32 %f136, %f127, %f135, %f120;\n" " fma.rn.f32 %f137, %f131, %f135, %f121;\n" " bfe.u32 %r148, %r128, 16, 8;\n" " cvt.rn.f32.u32 %f138, %r148;\n" " and.b32 %r149, %r126, 3840;\n" " or.b32 %r150, %r149, 1258291200;\n" " mov.b32 %f139, %r150;\n" " mov.f32 %f140, 0fC7000000;\n" " mov.f32 %f141, 0f3B800000;\n" " fma.rn.f32 %f142, %f139, %f141, %f140;\n" " mul.f32 %f143, %f142, %f138;\n" " bfe.u32 %r151, %r130, 16, 8;\n" " cvt.rn.f32.u32 %f144, %r151;\n" " and.b32 %r152, %r127, 3840;\n" " or.b32 %r153, %r152, 1258291200;\n" " mov.b32 %f145, %r153;\n" " fma.rn.f32 %f146, %f145, %f141, %f140;\n" " mul.f32 %f147, %f146, %f144;\n" " ld.shared.f32 %f148, [%r141+256];\n" " fma.rn.f32 %f149, %f143, %f148, %f133;\n" " fma.rn.f32 %f150, %f147, %f148, %f134;\n" " ld.shared.f32 %f151, [%r141+2304];\n" " fma.rn.f32 %f152, %f143, %f151, %f136;\n" " fma.rn.f32 %f153, %f147, %f151, %f137;\n" " shr.u32 %r154, %r128, 24;\n" " cvt.rn.f32.u32 %f154, %r154;\n" " and.b32 %r155, %r126, 61440;\n" " or.b32 %r156, %r155, 1258291200;\n" " mov.b32 %f155, %r156;\n" " mov.f32 %f156, 0fC5000000;\n" " mov.f32 %f157, 0f39800000;\n" " fma.rn.f32 %f158, %f155, %f157, %f156;\n" " mul.f32 %f159, %f158, %f154;\n" " shr.u32 %r157, %r130, 24;\n" " cvt.rn.f32.u32 %f160, %r157;\n" " and.b32 %r158, %r127, 61440;\n" " or.b32 %r159, %r158, 1258291200;\n" " mov.b32 %f161, %r159;\n" " fma.rn.f32 %f162, %f161, %f157, %f156;\n" " mul.f32 %f163, %f162, %f160;\n" " ld.shared.f32 %f164, [%r141+384];\n" " fma.rn.f32 %f165, %f159, %f164, %f149;\n" " fma.rn.f32 %f166, %f163, %f164, %f150;\n" " ld.shared.f32 %f167, [%r141+2432];\n" " fma.rn.f32 %f168, %f159, %f167, %f152;\n" " fma.rn.f32 %f169, %f163, %f167, %f153;\n" " and.b32 %r160, %r129, 255;\n" " cvt.rn.f32.u32 %f170, %r160;\n" " bfe.u32 %r161, %r126, 16, 4;\n" " or.b32 %r162, %r161, 1258291200;\n" " mov.b32 %f171, %r162;\n" " fma.rn.f32 %f172, %f171, %f108, %f107;\n" " mul.f32 %f173, %f172, %f170;\n" " and.b32 %r163, %r131, 255;\n" " cvt.rn.f32.u32 %f174, %r163;\n" " bfe.u32 %r164, %r127, 16, 4;\n" " or.b32 %r165, %r164, 1258291200;\n" " mov.b32 %f175, %r165;\n" " fma.rn.f32 %f176, %f175, %f108, %f107;\n" " mul.f32 %f177, %f176, %f174;\n" " ld.shared.f32 %f178, [%r141+512];\n" " fma.rn.f32 %f179, %f173, %f178, %f165;\n" " fma.rn.f32 %f180, %f177, %f178, %f166;\n" " ld.shared.f32 %f181, [%r141+2560];\n" " fma.rn.f32 %f182, %f173, %f181, %f168;\n" " fma.rn.f32 %f183, %f177, %f181, %f169;\n" " bfe.u32 %r166, %r129, 8, 8;\n" " cvt.rn.f32.u32 %f184, %r166;\n" " and.b32 %r167, %r132, 240;\n" " or.b32 %r168, %r167, 1258291200;\n" " mov.b32 %f185, %r168;\n" " fma.rn.f32 %f186, %f185, %f125, %f124;\n" " mul.f32 %f187, %f186, %f184;\n" " bfe.u32 %r169, %r131, 8, 8;\n" " cvt.rn.f32.u32 %f188, %r169;\n" " and.b32 %r170, %r133, 240;\n" " or.b32 %r171, %r170, 1258291200;\n" " mov.b32 %f189, %r171;\n" " fma.rn.f32 %f190, %f189, %f125, %f124;\n" " mul.f32 %f191, %f190, %f188;\n" " ld.shared.f32 %f192, [%r141+640];\n" " fma.rn.f32 %f193, %f187, %f192, %f179;\n" " fma.rn.f32 %f194, %f191, %f192, %f180;\n" " ld.shared.f32 %f195, [%r141+2688];\n" " fma.rn.f32 %f196, %f187, %f195, %f182;\n" " fma.rn.f32 %f197, %f191, %f195, %f183;\n" " bfe.u32 %r172, %r129, 16, 8;\n" " cvt.rn.f32.u32 %f198, %r172;\n" " and.b32 %r173, %r132, 3840;\n" " or.b32 %r174, %r173, 1258291200;\n" " mov.b32 %f199, %r174;\n" " fma.rn.f32 %f200, %f199, %f141, %f140;\n" " mul.f32 %f201, %f200, %f198;\n" " bfe.u32 %r175, %r131, 16, 8;\n" " cvt.rn.f32.u32 %f202, %r175;\n" " and.b32 %r176, %r133, 3840;\n" " or.b32 %r177, %r176, 1258291200;\n" " mov.b32 %f203, %r177;\n" " fma.rn.f32 %f204, %f203, %f141, %f140;\n" " mul.f32 %f205, %f204, %f202;\n" " ld.shared.f32 %f206, [%r141+768];\n" " fma.rn.f32 %f207, %f201, %f206, %f193;\n" " fma.rn.f32 %f208, %f205, %f206, %f194;\n" " ld.shared.f32 %f209, [%r141+2816];\n" " fma.rn.f32 %f210, %f201, %f209, %f196;\n" " fma.rn.f32 %f211, %f205, %f209, %f197;\n" " shr.u32 %r178, %r129, 24;\n" " cvt.rn.f32.u32 %f212, %r178;\n" " and.b32 %r179, %r132, 61440;\n" " or.b32 %r180, %r179, 1258291200;\n" " mov.b32 %f213, %r180;\n" " fma.rn.f32 %f214, %f213, %f157, %f156;\n" " mul.f32 %f215, %f214, %f212;\n" " shr.u32 %r181, %r131, 24;\n" " cvt.rn.f32.u32 %f216, %r181;\n" " and.b32 %r182, %r133, 61440;\n" " or.b32 %r183, %r182, 1258291200;\n" " mov.b32 %f217, %r183;\n" " fma.rn.f32 %f218, %f217, %f157, %f156;\n" " mul.f32 %f219, %f218, %f216;\n" " ld.shared.f32 %f220, [%r141+896];\n" " fma.rn.f32 %f221, %f215, %f220, %f207;\n" " fma.rn.f32 %f222, %f219, %f220, %f208;\n" " ld.shared.f32 %f223, [%r141+2944];\n" " fma.rn.f32 %f224, %f215, %f223, %f210;\n" " fma.rn.f32 %f225, %f219, %f223, %f211;\n" " fma.rn.f32 %f289, %f103, %f221, %f289;\n" " fma.rn.f32 %f291, %f104, %f222, %f291;\n" " fma.rn.f32 %f288, %f103, %f224, %f288;\n" " fma.rn.f32 %f290, %f104, %f225, %f290;\n" " add.s64 %rd81, %rd81, 152;\n" " add.s64 %rd82, %rd82, 152;\n" " add.s32 %r263, %r263, 1024;\n" " add.s32 %r262, %r262, 1;\n" " setp.lt.s32 %p15, %r262, %r249;\n" " @%p15 bra $L__BB1_20;\n" "\n" "$L__BB1_21:\n" " add.s32 %r258, %r258, 512;\n" " setp.lt.s32 %p16, %r258, %r39;\n" " @%p16 bra $L__BB1_4;\n" "\n" "$L__BB1_22:\n" " mov.u32 %r237, %ctaid.y;\n" " shl.b32 %r236, %r237, 1;\n" " ld.param.u32 %r235, [fc_matvec_q4k_nx2_param_6];\n" " sub.s32 %r234, %r235, %r236;\n" " sub.rn.f32 %f50, %f288, %f270;\n" " mov.u32 %r202, 1;\n" " sub.rn.f32 %f51, %f290, %f272;\n" " cvt.s64.s32 %rd31, %r38;\n" " sub.rn.f32 %f227, %f289, %f271;\n" " mov.u32 %r194, 16;\n" " mov.u32 %r203, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f226, %f227, %r194, %r203;\n" " // end inline asm\n" " add.f32 %f229, %f227, %f226;\n" " mov.u32 %r196, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f228, %f229, %r196, %r203;\n" " // end inline asm\n" " add.f32 %f231, %f229, %f228;\n" " mov.u32 %r198, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f230, %f231, %r198, %r203;\n" " // end inline asm\n" " add.f32 %f233, %f231, %f230;\n" " mov.u32 %r200, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f232, %f233, %r200, %r203;\n" " // end inline asm\n" " add.f32 %f235, %f233, %f232;\n" " // begin inline asm\n" " shfl.down.b32 %f234, %f235, %r202, %r203;\n" " // end inline asm\n" " add.f32 %f52, %f235, %f234;\n" " sub.rn.f32 %f237, %f291, %f273;\n" " // begin inline asm\n" " shfl.down.b32 %f236, %f237, %r194, %r203;\n" " // end inline asm\n" " add.f32 %f239, %f237, %f236;\n" " // begin inline asm\n" " shfl.down.b32 %f238, %f239, %r196, %r203;\n" " // end inline asm\n" " add.f32 %f241, %f239, %f238;\n" " // begin inline asm\n" " shfl.down.b32 %f240, %f241, %r198, %r203;\n" " // end inline asm\n" " add.f32 %f243, %f241, %f240;\n" " // begin inline asm\n" " shfl.down.b32 %f242, %f243, %r200, %r203;\n" " // end inline asm\n" " add.f32 %f245, %f243, %f242;\n" " // begin inline asm\n" " shfl.down.b32 %f244, %f245, %r202, %r203;\n" " // end inline asm\n" " add.f32 %f53, %f245, %f244;\n" " setp.lt.s32 %p17, %r234, 1;\n" " setp.ne.s32 %p18, %r5, 0;\n" " or.pred %p19, %p18, %p17;\n" " @%p19 bra $L__BB1_27;\n" "\n" " ld.param.u64 %rd71, [fc_matvec_q4k_nx2_param_2];\n" " cvta.to.global.u64 %rd70, %rd71;\n" " mov.u32 %r246, %ctaid.y;\n" " shl.b32 %r245, %r246, 1;\n" " cvt.u32.u64 %r204, %rd2;\n" " setp.ge.s32 %p20, %r204, %r38;\n" " cvt.s64.s32 %rd60, %r245;\n" " mul.lo.s64 %rd61, %rd60, %rd31;\n" " add.s64 %rd62, %rd61, %rd2;\n" " shl.b64 %rd63, %rd62, 2;\n" " add.s64 %rd32, %rd70, %rd63;\n" " @%p20 bra $L__BB1_25;\n" "\n" " st.global.f32 [%rd32], %f52;\n" "\n" "$L__BB1_25:\n" " @%p1 bra $L__BB1_27;\n" "\n" " st.global.f32 [%rd32+4], %f53;\n" "\n" "$L__BB1_27:\n" " mov.u32 %r242, %ctaid.y;\n" " shl.b32 %r241, %r242, 1;\n" " ld.param.u32 %r240, [fc_matvec_q4k_nx2_param_6];\n" " sub.s32 %r239, %r240, %r241;\n" " min.s32 %r238, %r239, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f246, %f50, %r194, %r203;\n" " // end inline asm\n" " add.f32 %f249, %f50, %f246;\n" " // begin inline asm\n" " shfl.down.b32 %f248, %f249, %r196, %r203;\n" " // end inline asm\n" " add.f32 %f251, %f249, %f248;\n" " // begin inline asm\n" " shfl.down.b32 %f250, %f251, %r198, %r203;\n" " // end inline asm\n" " add.f32 %f253, %f251, %f250;\n" " // begin inline asm\n" " shfl.down.b32 %f252, %f253, %r200, %r203;\n" " // end inline asm\n" " add.f32 %f255, %f253, %f252;\n" " // begin inline asm\n" " shfl.down.b32 %f254, %f255, %r202, %r203;\n" " // end inline asm\n" " add.f32 %f54, %f255, %f254;\n" " // begin inline asm\n" " shfl.down.b32 %f256, %f51, %r194, %r203;\n" " // end inline asm\n" " add.f32 %f259, %f51, %f256;\n" " // begin inline asm\n" " shfl.down.b32 %f258, %f259, %r196, %r203;\n" " // end inline asm\n" " add.f32 %f261, %f259, %f258;\n" " // begin inline asm\n" " shfl.down.b32 %f260, %f261, %r198, %r203;\n" " // end inline asm\n" " add.f32 %f263, %f261, %f260;\n" " // begin inline asm\n" " shfl.down.b32 %f262, %f263, %r200, %r203;\n" " // end inline asm\n" " add.f32 %f265, %f263, %f262;\n" " // begin inline asm\n" " shfl.down.b32 %f264, %f265, %r202, %r203;\n" " // end inline asm\n" " add.f32 %f55, %f265, %f264;\n" " setp.lt.s32 %p23, %r238, 2;\n" " or.pred %p24, %p18, %p23;\n" " @%p24 bra $L__BB1_32;\n" "\n" " ld.param.u64 %rd69, [fc_matvec_q4k_nx2_param_2];\n" " cvta.to.global.u64 %rd68, %rd69;\n" " mov.u32 %r244, %ctaid.y;\n" " shl.b32 %r243, %r244, 1;\n" " cvt.u32.u64 %r225, %rd2;\n" " setp.ge.s32 %p25, %r225, %r38;\n" " add.s32 %r226, %r243, 1;\n" " cvt.s64.s32 %rd64, %r226;\n" " mul.lo.s64 %rd65, %rd64, %rd31;\n" " add.s64 %rd66, %rd65, %rd2;\n" " shl.b64 %rd67, %rd66, 2;\n" " add.s64 %rd33, %rd68, %rd67;\n" " @%p25 bra $L__BB1_30;\n" "\n" " st.global.f32 [%rd33], %f54;\n" "\n" "$L__BB1_30:\n" " @%p1 bra $L__BB1_32;\n" "\n" " st.global.f32 [%rd33+4], %f55;\n" "\n" "$L__BB1_32:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q4k_nx\n" ".visible .entry fc_matvec_q4k_nx(\n" " .param .u64 fc_matvec_q4k_nx_param_0,\n" " .param .u64 fc_matvec_q4k_nx_param_1,\n" " .param .u64 fc_matvec_q4k_nx_param_2,\n" " .param .u32 fc_matvec_q4k_nx_param_3,\n" " .param .u32 fc_matvec_q4k_nx_param_4,\n" " .param .u32 fc_matvec_q4k_nx_param_5,\n" " .param .u32 fc_matvec_q4k_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<39>;\n" " .reg .b16 %rs<5>;\n" " .reg .f32 %f<511>;\n" " .reg .b32 %r<301>;\n" " .reg .b64 %rd<103>;\n" "\n" "\n" " ld.param.u64 %rd39, [fc_matvec_q4k_nx_param_0];\n" " ld.param.u64 %rd38, [fc_matvec_q4k_nx_param_1];\n" " ld.param.u64 %rd40, [fc_matvec_q4k_nx_param_2];\n" " ld.param.u32 %r35, [fc_matvec_q4k_nx_param_3];\n" " ld.param.u32 %r36, [fc_matvec_q4k_nx_param_4];\n" " ld.param.u32 %r37, [fc_matvec_q4k_nx_param_5];\n" " ld.param.u32 %r38, [fc_matvec_q4k_nx_param_6];\n" " cvta.to.global.u64 %rd1, %rd40;\n" " mov.u32 %r39, %ctaid.y;\n" " shl.b32 %r1, %r39, 2;\n" " sub.s32 %r2, %r38, %r1;\n" " min.s32 %r3, %r2, 4;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r40, %ctaid.x;\n" " shl.b32 %r41, %r40, 4;\n" " shr.u32 %r42, %r4, 4;\n" " and.b32 %r43, %r42, 268435454;\n" " add.s32 %r44, %r43, %r41;\n" " or.b32 %r6, %r44, 1;\n" " setp.ge.s32 %p1, %r6, %r35;\n" " cvt.s64.s32 %rd2, %r44;\n" " cvt.s64.s32 %rd3, %r37;\n" " mul.wide.s32 %rd41, %r37, %r44;\n" " add.s64 %rd101, %rd39, %rd41;\n" " mov.u64 %rd102, %rd101;\n" " @%p1 bra $L__BB2_2;\n" "\n" " add.s64 %rd102, %rd101, %rd3;\n" "\n" "$L__BB2_2:\n" " setp.lt.s32 %p2, %r36, 1;\n" " mov.f32 %f479, 0f00000000;\n" " mov.f32 %f480, %f479;\n" " mov.f32 %f481, %f479;\n" " mov.f32 %f482, %f479;\n" " mov.f32 %f483, %f479;\n" " mov.f32 %f484, %f479;\n" " mov.f32 %f485, %f479;\n" " mov.f32 %f486, %f479;\n" " mov.f32 %f443, %f479;\n" " mov.f32 %f444, %f479;\n" " mov.f32 %f445, %f479;\n" " mov.f32 %f446, %f479;\n" " mov.f32 %f447, %f479;\n" " mov.f32 %f448, %f479;\n" " mov.f32 %f449, %f479;\n" " mov.f32 %f450, %f479;\n" " @%p2 bra $L__BB2_29;\n" "\n" " shr.u32 %r7, %r4, 2;\n" " and.b32 %r8, %r4, 3;\n" " shl.b32 %r46, %r4, 3;\n" " and.b32 %r9, %r46, 24;\n" " or.b32 %r10, %r9, 64;\n" " cvt.u64.u32 %rd7, %r5;\n" " add.s32 %r47, %r1, 1;\n" " add.s32 %r48, %r1, 2;\n" " add.s32 %r49, %r1, 3;\n" " mul.wide.s32 %rd42, %r36, %r1;\n" " shl.b64 %rd43, %rd42, 2;\n" " mul.wide.s32 %rd44, %r4, 4;\n" " add.s64 %rd8, %rd43, %rd44;\n" " shl.b32 %r50, %r4, 2;\n" " mov.u32 %r51, fc_kq_shared;\n" " add.s32 %r52, %r51, %r50;\n" " add.s32 %r11, %r52, 4352;\n" " mul.wide.s32 %rd45, %r36, %r47;\n" " shl.b64 %rd46, %rd45, 2;\n" " add.s64 %rd9, %rd46, %rd44;\n" " mul.wide.s32 %rd47, %r36, %r49;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd10, %rd48, %rd44;\n" " mul.wide.s32 %rd49, %r36, %r48;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd11, %rd50, %rd44;\n" " mov.u32 %r294, 0;\n" " mov.f32 %f479, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " setp.lt.s32 %p6, %r3, 3;\n" " setp.lt.s32 %p7, %r3, 4;\n" " shl.b64 %rd67, %rd7, 2;\n" " mov.f32 %f480, %f479;\n" " mov.f32 %f481, %f479;\n" " mov.f32 %f482, %f479;\n" " mov.f32 %f483, %f479;\n" " mov.f32 %f484, %f479;\n" " mov.f32 %f485, %f479;\n" " mov.f32 %f486, %f479;\n" " mov.f32 %f443, %f479;\n" " mov.f32 %f444, %f479;\n" " mov.f32 %f445, %f479;\n" " mov.f32 %f446, %f479;\n" " mov.f32 %f447, %f479;\n" " mov.f32 %f448, %f479;\n" " mov.f32 %f449, %f479;\n" " mov.f32 %f450, %f479;\n" "\n" "$L__BB2_4:\n" " sub.s32 %r53, %r36, %r294;\n" " min.s32 %r13, %r53, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r13;\n" " @%p3 bra $L__BB2_15;\n" "\n" " mov.u32 %r295, %r11;\n" " mov.u64 %rd96, %rd38;\n" " mov.u32 %r296, %r4;\n" "\n" "$L__BB2_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f451, 0f00000000;\n" " @%p4 bra $L__BB2_8;\n" "\n" " mul.wide.s32 %rd85, %r294, 4;\n" " add.s64 %rd84, %rd8, %rd85;\n" " add.s64 %rd52, %rd96, %rd84;\n" " // begin inline asm\n" " ld.global.nc.f32 %f451, [%rd52];\n" " // end inline asm\n" "\n" "$L__BB2_8:\n" " mov.f32 %f452, 0f00000000;\n" " st.shared.f32 [%r295+-4096], %f451;\n" " @%p5 bra $L__BB2_10;\n" "\n" " mul.wide.s32 %rd91, %r294, 4;\n" " add.s64 %rd90, %rd9, %rd91;\n" " add.s64 %rd53, %rd96, %rd90;\n" " // begin inline asm\n" " ld.global.nc.f32 %f452, [%rd53];\n" " // end inline asm\n" "\n" "$L__BB2_10:\n" " mov.f32 %f453, 0f00000000;\n" " st.shared.f32 [%r295+-2048], %f452;\n" " @%p6 bra $L__BB2_12;\n" "\n" " mul.wide.s32 %rd89, %r294, 4;\n" " add.s64 %rd88, %rd11, %rd89;\n" " add.s64 %rd54, %rd96, %rd88;\n" " // begin inline asm\n" " ld.global.nc.f32 %f453, [%rd54];\n" " // end inline asm\n" "\n" "$L__BB2_12:\n" " mov.f32 %f454, 0f00000000;\n" " st.shared.f32 [%r295], %f453;\n" " @%p7 bra $L__BB2_14;\n" "\n" " mul.wide.s32 %rd87, %r294, 4;\n" " add.s64 %rd86, %rd10, %rd87;\n" " add.s64 %rd55, %rd96, %rd86;\n" " // begin inline asm\n" " ld.global.nc.f32 %f454, [%rd55];\n" " // end inline asm\n" "\n" "$L__BB2_14:\n" " sub.s32 %r273, %r36, %r294;\n" " min.s32 %r272, %r273, 512;\n" " st.shared.f32 [%r295+2048], %f454;\n" " add.s64 %rd96, %rd96, 1024;\n" " add.s32 %r295, %r295, 1024;\n" " add.s32 %r296, %r296, 256;\n" " setp.lt.s32 %p8, %r296, %r272;\n" " @%p8 bra $L__BB2_6;\n" "\n" "$L__BB2_15:\n" " setp.gt.u32 %p9, %r4, 255;\n" " bar.sync 0;\n" " @%p9 bra $L__BB2_21;\n" "\n" " mov.u32 %r297, %r7;\n" "\n" "$L__BB2_17:\n" " mov.u32 %r21, %r297;\n" " sub.s32 %r282, %r36, %r294;\n" " min.s32 %r281, %r282, 512;\n" " shr.u32 %r280, %r281, 5;\n" " and.b32 %r22, %r21, 15;\n" " setp.ge.u32 %p10, %r22, %r280;\n" " @%p10 bra $L__BB2_20;\n" "\n" " setp.ne.s32 %p11, %r8, 0;\n" " shl.b32 %r58, %r21, 5;\n" " and.b32 %r59, %r58, -512;\n" " shl.b32 %r60, %r22, 5;\n" " or.b32 %r61, %r9, %r60;\n" " add.s32 %r62, %r10, %r60;\n" " or.b32 %r63, %r61, %r59;\n" " add.s32 %r64, %r62, %r59;\n" " and.b32 %r65, %r21, 7;\n" " add.s32 %r66, %r63, %r65;\n" " shl.b32 %r67, %r66, 2;\n" " add.s32 %r69, %r51, %r67;\n" " mov.u32 %r54, 2;\n" " ld.shared.f32 %f156, [%r69+256];\n" " add.f32 %f157, %f156, 0f00000000;\n" " xor.b32 %r70, %r65, 1;\n" " mov.u32 %r56, 1;\n" " add.s32 %r71, %r64, %r70;\n" " shl.b32 %r72, %r71, 2;\n" " add.s32 %r73, %r51, %r72;\n" " ld.shared.f32 %f158, [%r73];\n" " add.f32 %f159, %f157, %f158;\n" " xor.b32 %r74, %r65, 2;\n" " add.s32 %r75, %r64, %r74;\n" " shl.b32 %r76, %r75, 2;\n" " add.s32 %r77, %r51, %r76;\n" " ld.shared.f32 %f160, [%r77];\n" " add.f32 %f161, %f159, %f160;\n" " xor.b32 %r78, %r65, 3;\n" " add.s32 %r79, %r64, %r78;\n" " shl.b32 %r80, %r79, 2;\n" " add.s32 %r81, %r51, %r80;\n" " ld.shared.f32 %f162, [%r81];\n" " add.f32 %f163, %f161, %f162;\n" " xor.b32 %r82, %r65, 4;\n" " add.s32 %r83, %r64, %r82;\n" " shl.b32 %r84, %r83, 2;\n" " add.s32 %r85, %r51, %r84;\n" " ld.shared.f32 %f164, [%r85];\n" " add.f32 %f165, %f163, %f164;\n" " xor.b32 %r86, %r65, 5;\n" " add.s32 %r87, %r64, %r86;\n" " shl.b32 %r88, %r87, 2;\n" " add.s32 %r89, %r51, %r88;\n" " ld.shared.f32 %f166, [%r89];\n" " add.f32 %f167, %f165, %f166;\n" " xor.b32 %r90, %r65, 6;\n" " add.s32 %r91, %r64, %r90;\n" " shl.b32 %r92, %r91, 2;\n" " add.s32 %r93, %r51, %r92;\n" " ld.shared.f32 %f168, [%r93];\n" " add.f32 %f169, %f167, %f168;\n" " xor.b32 %r94, %r65, 7;\n" " add.s32 %r95, %r64, %r94;\n" " shl.b32 %r96, %r95, 2;\n" " add.s32 %r97, %r51, %r96;\n" " ld.shared.f32 %f170, [%r97];\n" " add.f32 %f153, %f169, %f170;\n" " mov.u32 %r57, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f152, %f153, %r54, %r57;\n" " // end inline asm\n" " add.f32 %f155, %f153, %f152;\n" " // begin inline asm\n" " shfl.down.b32 %f154, %f155, %r56, %r57;\n" " // end inline asm\n" " add.f32 %f25, %f155, %f154;\n" " @%p11 bra $L__BB2_20;\n" "\n" " shl.b32 %r98, %r21, 2;\n" " add.s32 %r100, %r51, %r98;\n" " st.shared.f32 [%r100], %f25;\n" "\n" "$L__BB2_20:\n" " add.s32 %r297, %r21, 64;\n" " setp.gt.u32 %p12, %r21, -65;\n" " @%p12 bra $L__BB2_17;\n" "\n" "$L__BB2_21:\n" " cvt.u32.u64 %r101, %rd2;\n" " setp.ge.s32 %p13, %r101, %r35;\n" " bar.sync 0;\n" " @%p13 bra $L__BB2_28;\n" "\n" " sub.s32 %r275, %r36, %r294;\n" " min.s32 %r274, %r275, 512;\n" " shr.s32 %r24, %r274, 8;\n" " shr.u32 %r102, %r294, 5;\n" " sub.s32 %r25, %r5, %r102;\n" " and.b32 %r298, %r25, 31;\n" " shl.b32 %r27, %r24, 3;\n" " setp.ge.s32 %p14, %r298, %r27;\n" " @%p14 bra $L__BB2_25;\n" "\n" " shr.u32 %r103, %r298, 3;\n" " mul.lo.s32 %r104, %r103, 152;\n" " cvt.u64.u32 %rd56, %r104;\n" " add.s64 %rd97, %rd102, %rd56;\n" " add.s64 %rd98, %rd101, %rd56;\n" "\n" "$L__BB2_24:\n" " .pragma \"nounroll\";\n" " shr.u32 %r293, %r294, 5;\n" " sub.s32 %r292, %r5, %r293;\n" " and.b32 %r291, %r292, 7;\n" " or.b32 %r290, %r291, 136;\n" " cvt.u64.u32 %rd92, %r290;\n" " sub.s32 %r289, %r36, %r294;\n" " min.s32 %r288, %r289, 512;\n" " shr.s32 %r287, %r288, 8;\n" " shl.b32 %r286, %r287, 3;\n" " add.s64 %rd57, %rd98, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f171, [%rd57];\n" " // end inline asm\n" " add.s64 %rd58, %rd98, %rd92;\n" " // begin inline asm\n" " ld.global.nc.u8 %r107, [%rd58];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r107;\n" " and.b16 %rs2, %rs1, 255;\n" " cvt.rn.f32.u16 %f173, %rs2;\n" " mul.rn.f32 %f174, %f171, %f173;\n" " add.s64 %rd59, %rd97, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f172, [%rd59];\n" " // end inline asm\n" " add.s64 %rd60, %rd97, %rd92;\n" " // begin inline asm\n" " ld.global.nc.u8 %r108, [%rd60];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r108;\n" " and.b16 %rs4, %rs3, 255;\n" " cvt.rn.f32.u16 %f175, %rs4;\n" " mul.rn.f32 %f176, %f172, %f175;\n" " shl.b32 %r109, %r298, 2;\n" " add.s32 %r111, %r51, %r109;\n" " ld.shared.f32 %f177, [%r111];\n" " fma.rn.f32 %f446, %f174, %f177, %f446;\n" " fma.rn.f32 %f450, %f176, %f177, %f450;\n" " ld.shared.f32 %f178, [%r111+64];\n" " fma.rn.f32 %f445, %f174, %f178, %f445;\n" " fma.rn.f32 %f449, %f176, %f178, %f449;\n" " ld.shared.f32 %f179, [%r111+128];\n" " fma.rn.f32 %f444, %f174, %f179, %f444;\n" " fma.rn.f32 %f448, %f176, %f179, %f448;\n" " ld.shared.f32 %f180, [%r111+192];\n" " fma.rn.f32 %f443, %f174, %f180, %f443;\n" " fma.rn.f32 %f447, %f176, %f180, %f447;\n" " add.s64 %rd98, %rd98, 608;\n" " add.s64 %rd97, %rd97, 608;\n" " add.s32 %r298, %r298, 32;\n" " setp.lt.s32 %p15, %r298, %r286;\n" " @%p15 bra $L__BB2_24;\n" "\n" "$L__BB2_25:\n" " sub.s32 %r277, %r36, %r294;\n" " min.s32 %r276, %r277, 512;\n" " setp.lt.s32 %p16, %r276, 256;\n" " @%p16 bra $L__BB2_28;\n" "\n" " add.s32 %r300, %r51, 256;\n" " mov.u32 %r299, 0;\n" "\n" "$L__BB2_27:\n" " sub.s32 %r285, %r36, %r294;\n" " min.s32 %r284, %r285, 512;\n" " shr.s32 %r283, %r284, 8;\n" " add.s64 %rd61, %rd101, %rd67;\n" " // begin inline asm\n" " ld.global.nc.u32 %r115, [%rd61];\n" " // end inline asm\n" " add.s64 %rd62, %rd102, %rd67;\n" " // begin inline asm\n" " ld.global.nc.u32 %r116, [%rd62];\n" " // end inline asm\n" " add.s64 %rd63, %rd101, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r117,%r118}, [%rd63];\n" " // end inline asm\n" " add.s64 %rd64, %rd102, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r119,%r120}, [%rd64];\n" " // end inline asm\n" " add.s64 %rd65, %rd101, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f181, [%rd65];\n" " // end inline asm\n" " add.s64 %rd66, %rd102, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f182, [%rd66];\n" " // end inline asm\n" " shr.u32 %r121, %r115, 16;\n" " shr.u32 %r122, %r116, 16;\n" " and.b32 %r123, %r117, 255;\n" " cvt.rn.f32.u32 %f183, %r123;\n" " and.b32 %r124, %r115, 15;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f184, %r125;\n" " mov.f32 %f185, 0fCB000000;\n" " mov.f32 %f186, 0f3F800000;\n" " fma.rn.f32 %f187, %f184, %f186, %f185;\n" " mul.f32 %f188, %f187, %f183;\n" " and.b32 %r126, %r119, 255;\n" " cvt.rn.f32.u32 %f189, %r126;\n" " and.b32 %r127, %r116, 15;\n" " or.b32 %r128, %r127, 1258291200;\n" " mov.b32 %f190, %r128;\n" " fma.rn.f32 %f191, %f190, %f186, %f185;\n" " mul.f32 %f192, %f191, %f189;\n" " shl.b32 %r129, %r5, 2;\n" " add.s32 %r130, %r300, %r129;\n" " ld.shared.f32 %f193, [%r130];\n" " mov.f32 %f194, 0f00000000;\n" " fma.rn.f32 %f195, %f188, %f193, %f194;\n" " fma.rn.f32 %f196, %f192, %f193, %f194;\n" " ld.shared.f32 %f197, [%r130+2048];\n" " fma.rn.f32 %f198, %f188, %f197, %f194;\n" " fma.rn.f32 %f199, %f192, %f197, %f194;\n" " ld.shared.f32 %f200, [%r130+4096];\n" " fma.rn.f32 %f201, %f188, %f200, %f194;\n" " fma.rn.f32 %f202, %f192, %f200, %f194;\n" " ld.shared.f32 %f203, [%r130+6144];\n" " fma.rn.f32 %f204, %f188, %f203, %f194;\n" " fma.rn.f32 %f205, %f192, %f203, %f194;\n" " bfe.u32 %r131, %r117, 8, 8;\n" " cvt.rn.f32.u32 %f206, %r131;\n" " and.b32 %r132, %r115, 240;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f207, %r133;\n" " mov.f32 %f208, 0fC9000000;\n" " mov.f32 %f209, 0f3D800000;\n" " fma.rn.f32 %f210, %f207, %f209, %f208;\n" " mul.f32 %f211, %f210, %f206;\n" " bfe.u32 %r134, %r119, 8, 8;\n" " cvt.rn.f32.u32 %f212, %r134;\n" " and.b32 %r135, %r116, 240;\n" " or.b32 %r136, %r135, 1258291200;\n" " mov.b32 %f213, %r136;\n" " fma.rn.f32 %f214, %f213, %f209, %f208;\n" " mul.f32 %f215, %f214, %f212;\n" " ld.shared.f32 %f216, [%r130+128];\n" " fma.rn.f32 %f217, %f211, %f216, %f195;\n" " fma.rn.f32 %f218, %f215, %f216, %f196;\n" " ld.shared.f32 %f219, [%r130+2176];\n" " fma.rn.f32 %f220, %f211, %f219, %f198;\n" " fma.rn.f32 %f221, %f215, %f219, %f199;\n" " ld.shared.f32 %f222, [%r130+4224];\n" " fma.rn.f32 %f223, %f211, %f222, %f201;\n" " fma.rn.f32 %f224, %f215, %f222, %f202;\n" " ld.shared.f32 %f225, [%r130+6272];\n" " fma.rn.f32 %f226, %f211, %f225, %f204;\n" " fma.rn.f32 %f227, %f215, %f225, %f205;\n" " bfe.u32 %r137, %r117, 16, 8;\n" " cvt.rn.f32.u32 %f228, %r137;\n" " and.b32 %r138, %r115, 3840;\n" " or.b32 %r139, %r138, 1258291200;\n" " mov.b32 %f229, %r139;\n" " mov.f32 %f230, 0fC7000000;\n" " mov.f32 %f231, 0f3B800000;\n" " fma.rn.f32 %f232, %f229, %f231, %f230;\n" " mul.f32 %f233, %f232, %f228;\n" " bfe.u32 %r140, %r119, 16, 8;\n" " cvt.rn.f32.u32 %f234, %r140;\n" " and.b32 %r141, %r116, 3840;\n" " or.b32 %r142, %r141, 1258291200;\n" " mov.b32 %f235, %r142;\n" " fma.rn.f32 %f236, %f235, %f231, %f230;\n" " mul.f32 %f237, %f236, %f234;\n" " ld.shared.f32 %f238, [%r130+256];\n" " fma.rn.f32 %f239, %f233, %f238, %f217;\n" " fma.rn.f32 %f240, %f237, %f238, %f218;\n" " ld.shared.f32 %f241, [%r130+2304];\n" " fma.rn.f32 %f242, %f233, %f241, %f220;\n" " fma.rn.f32 %f243, %f237, %f241, %f221;\n" " ld.shared.f32 %f244, [%r130+4352];\n" " fma.rn.f32 %f245, %f233, %f244, %f223;\n" " fma.rn.f32 %f246, %f237, %f244, %f224;\n" " ld.shared.f32 %f247, [%r130+6400];\n" " fma.rn.f32 %f248, %f233, %f247, %f226;\n" " fma.rn.f32 %f249, %f237, %f247, %f227;\n" " shr.u32 %r143, %r117, 24;\n" " cvt.rn.f32.u32 %f250, %r143;\n" " and.b32 %r144, %r115, 61440;\n" " or.b32 %r145, %r144, 1258291200;\n" " mov.b32 %f251, %r145;\n" " mov.f32 %f252, 0fC5000000;\n" " mov.f32 %f253, 0f39800000;\n" " fma.rn.f32 %f254, %f251, %f253, %f252;\n" " mul.f32 %f255, %f254, %f250;\n" " shr.u32 %r146, %r119, 24;\n" " cvt.rn.f32.u32 %f256, %r146;\n" " and.b32 %r147, %r116, 61440;\n" " or.b32 %r148, %r147, 1258291200;\n" " mov.b32 %f257, %r148;\n" " fma.rn.f32 %f258, %f257, %f253, %f252;\n" " mul.f32 %f259, %f258, %f256;\n" " ld.shared.f32 %f260, [%r130+384];\n" " fma.rn.f32 %f261, %f255, %f260, %f239;\n" " fma.rn.f32 %f262, %f259, %f260, %f240;\n" " ld.shared.f32 %f263, [%r130+2432];\n" " fma.rn.f32 %f264, %f255, %f263, %f242;\n" " fma.rn.f32 %f265, %f259, %f263, %f243;\n" " ld.shared.f32 %f266, [%r130+4480];\n" " fma.rn.f32 %f267, %f255, %f266, %f245;\n" " fma.rn.f32 %f268, %f259, %f266, %f246;\n" " ld.shared.f32 %f269, [%r130+6528];\n" " fma.rn.f32 %f270, %f255, %f269, %f248;\n" " fma.rn.f32 %f271, %f259, %f269, %f249;\n" " and.b32 %r149, %r118, 255;\n" " cvt.rn.f32.u32 %f272, %r149;\n" " bfe.u32 %r150, %r115, 16, 4;\n" " or.b32 %r151, %r150, 1258291200;\n" " mov.b32 %f273, %r151;\n" " fma.rn.f32 %f274, %f273, %f186, %f185;\n" " mul.f32 %f275, %f274, %f272;\n" " and.b32 %r152, %r120, 255;\n" " cvt.rn.f32.u32 %f276, %r152;\n" " bfe.u32 %r153, %r116, 16, 4;\n" " or.b32 %r154, %r153, 1258291200;\n" " mov.b32 %f277, %r154;\n" " fma.rn.f32 %f278, %f277, %f186, %f185;\n" " mul.f32 %f279, %f278, %f276;\n" " ld.shared.f32 %f280, [%r130+512];\n" " fma.rn.f32 %f281, %f275, %f280, %f261;\n" " fma.rn.f32 %f282, %f279, %f280, %f262;\n" " ld.shared.f32 %f283, [%r130+2560];\n" " fma.rn.f32 %f284, %f275, %f283, %f264;\n" " fma.rn.f32 %f285, %f279, %f283, %f265;\n" " ld.shared.f32 %f286, [%r130+4608];\n" " fma.rn.f32 %f287, %f275, %f286, %f267;\n" " fma.rn.f32 %f288, %f279, %f286, %f268;\n" " ld.shared.f32 %f289, [%r130+6656];\n" " fma.rn.f32 %f290, %f275, %f289, %f270;\n" " fma.rn.f32 %f291, %f279, %f289, %f271;\n" " bfe.u32 %r155, %r118, 8, 8;\n" " cvt.rn.f32.u32 %f292, %r155;\n" " and.b32 %r156, %r121, 240;\n" " or.b32 %r157, %r156, 1258291200;\n" " mov.b32 %f293, %r157;\n" " fma.rn.f32 %f294, %f293, %f209, %f208;\n" " mul.f32 %f295, %f294, %f292;\n" " bfe.u32 %r158, %r120, 8, 8;\n" " cvt.rn.f32.u32 %f296, %r158;\n" " and.b32 %r159, %r122, 240;\n" " or.b32 %r160, %r159, 1258291200;\n" " mov.b32 %f297, %r160;\n" " fma.rn.f32 %f298, %f297, %f209, %f208;\n" " mul.f32 %f299, %f298, %f296;\n" " ld.shared.f32 %f300, [%r130+640];\n" " fma.rn.f32 %f301, %f295, %f300, %f281;\n" " fma.rn.f32 %f302, %f299, %f300, %f282;\n" " ld.shared.f32 %f303, [%r130+2688];\n" " fma.rn.f32 %f304, %f295, %f303, %f284;\n" " fma.rn.f32 %f305, %f299, %f303, %f285;\n" " ld.shared.f32 %f306, [%r130+4736];\n" " fma.rn.f32 %f307, %f295, %f306, %f287;\n" " fma.rn.f32 %f308, %f299, %f306, %f288;\n" " ld.shared.f32 %f309, [%r130+6784];\n" " fma.rn.f32 %f310, %f295, %f309, %f290;\n" " fma.rn.f32 %f311, %f299, %f309, %f291;\n" " bfe.u32 %r161, %r118, 16, 8;\n" " cvt.rn.f32.u32 %f312, %r161;\n" " and.b32 %r162, %r121, 3840;\n" " or.b32 %r163, %r162, 1258291200;\n" " mov.b32 %f313, %r163;\n" " fma.rn.f32 %f314, %f313, %f231, %f230;\n" " mul.f32 %f315, %f314, %f312;\n" " bfe.u32 %r164, %r120, 16, 8;\n" " cvt.rn.f32.u32 %f316, %r164;\n" " and.b32 %r165, %r122, 3840;\n" " or.b32 %r166, %r165, 1258291200;\n" " mov.b32 %f317, %r166;\n" " fma.rn.f32 %f318, %f317, %f231, %f230;\n" " mul.f32 %f319, %f318, %f316;\n" " ld.shared.f32 %f320, [%r130+768];\n" " fma.rn.f32 %f321, %f315, %f320, %f301;\n" " fma.rn.f32 %f322, %f319, %f320, %f302;\n" " ld.shared.f32 %f323, [%r130+2816];\n" " fma.rn.f32 %f324, %f315, %f323, %f304;\n" " fma.rn.f32 %f325, %f319, %f323, %f305;\n" " ld.shared.f32 %f326, [%r130+4864];\n" " fma.rn.f32 %f327, %f315, %f326, %f307;\n" " fma.rn.f32 %f328, %f319, %f326, %f308;\n" " ld.shared.f32 %f329, [%r130+6912];\n" " fma.rn.f32 %f330, %f315, %f329, %f310;\n" " fma.rn.f32 %f331, %f319, %f329, %f311;\n" " shr.u32 %r167, %r118, 24;\n" " cvt.rn.f32.u32 %f332, %r167;\n" " and.b32 %r168, %r121, 61440;\n" " or.b32 %r169, %r168, 1258291200;\n" " mov.b32 %f333, %r169;\n" " fma.rn.f32 %f334, %f333, %f253, %f252;\n" " mul.f32 %f335, %f334, %f332;\n" " shr.u32 %r170, %r120, 24;\n" " cvt.rn.f32.u32 %f336, %r170;\n" " and.b32 %r171, %r122, 61440;\n" " or.b32 %r172, %r171, 1258291200;\n" " mov.b32 %f337, %r172;\n" " fma.rn.f32 %f338, %f337, %f253, %f252;\n" " mul.f32 %f339, %f338, %f336;\n" " ld.shared.f32 %f340, [%r130+896];\n" " fma.rn.f32 %f341, %f335, %f340, %f321;\n" " fma.rn.f32 %f342, %f339, %f340, %f322;\n" " ld.shared.f32 %f343, [%r130+2944];\n" " fma.rn.f32 %f344, %f335, %f343, %f324;\n" " fma.rn.f32 %f345, %f339, %f343, %f325;\n" " ld.shared.f32 %f346, [%r130+4992];\n" " fma.rn.f32 %f347, %f335, %f346, %f327;\n" " fma.rn.f32 %f348, %f339, %f346, %f328;\n" " ld.shared.f32 %f349, [%r130+7040];\n" " fma.rn.f32 %f350, %f335, %f349, %f330;\n" " fma.rn.f32 %f351, %f339, %f349, %f331;\n" " fma.rn.f32 %f482, %f181, %f341, %f482;\n" " fma.rn.f32 %f486, %f182, %f342, %f486;\n" " fma.rn.f32 %f481, %f181, %f344, %f481;\n" " fma.rn.f32 %f485, %f182, %f345, %f485;\n" " fma.rn.f32 %f480, %f181, %f347, %f480;\n" " fma.rn.f32 %f484, %f182, %f348, %f484;\n" " fma.rn.f32 %f479, %f181, %f350, %f479;\n" " fma.rn.f32 %f483, %f182, %f351, %f483;\n" " add.s64 %rd101, %rd101, 152;\n" " add.s64 %rd102, %rd102, 152;\n" " add.s32 %r300, %r300, 1024;\n" " add.s32 %r299, %r299, 1;\n" " setp.lt.s32 %p17, %r299, %r283;\n" " @%p17 bra $L__BB2_27;\n" "\n" "$L__BB2_28:\n" " add.s32 %r294, %r294, 512;\n" " setp.lt.s32 %p18, %r294, %r36;\n" " @%p18 bra $L__BB2_4;\n" "\n" "$L__BB2_29:\n" " mov.u32 %r263, %ctaid.y;\n" " shl.b32 %r262, %r263, 2;\n" " ld.param.u32 %r261, [fc_matvec_q4k_nx_param_6];\n" " sub.s32 %r260, %r261, %r262;\n" " sub.rn.f32 %f98, %f481, %f445;\n" " mov.u32 %r191, 1;\n" " sub.rn.f32 %f99, %f485, %f449;\n" " sub.rn.f32 %f100, %f480, %f444;\n" " sub.rn.f32 %f101, %f484, %f448;\n" " sub.rn.f32 %f102, %f479, %f443;\n" " sub.rn.f32 %f103, %f483, %f447;\n" " cvt.s64.s32 %rd33, %r35;\n" " sub.rn.f32 %f353, %f482, %f446;\n" " mov.u32 %r183, 16;\n" " mov.u32 %r192, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f352, %f353, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f355, %f353, %f352;\n" " mov.u32 %r185, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f354, %f355, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f357, %f355, %f354;\n" " mov.u32 %r187, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f356, %f357, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f359, %f357, %f356;\n" " mov.u32 %r189, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f358, %f359, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f361, %f359, %f358;\n" " // begin inline asm\n" " shfl.down.b32 %f360, %f361, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f104, %f361, %f360;\n" " sub.rn.f32 %f363, %f486, %f450;\n" " // begin inline asm\n" " shfl.down.b32 %f362, %f363, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f365, %f363, %f362;\n" " // begin inline asm\n" " shfl.down.b32 %f364, %f365, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f367, %f365, %f364;\n" " // begin inline asm\n" " shfl.down.b32 %f366, %f367, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f369, %f367, %f366;\n" " // begin inline asm\n" " shfl.down.b32 %f368, %f369, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f371, %f369, %f368;\n" " // begin inline asm\n" " shfl.down.b32 %f370, %f371, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f105, %f371, %f370;\n" " setp.lt.s32 %p19, %r260, 1;\n" " setp.ne.s32 %p20, %r5, 0;\n" " or.pred %p21, %p20, %p19;\n" " @%p21 bra $L__BB2_34;\n" "\n" " mov.u32 %r271, %ctaid.y;\n" " shl.b32 %r270, %r271, 2;\n" " cvt.u32.u64 %r193, %rd2;\n" " setp.ge.s32 %p22, %r193, %r35;\n" " cvt.s64.s32 %rd68, %r270;\n" " mul.lo.s64 %rd69, %rd68, %rd33;\n" " add.s64 %rd70, %rd69, %rd2;\n" " shl.b64 %rd71, %rd70, 2;\n" " add.s64 %rd34, %rd1, %rd71;\n" " @%p22 bra $L__BB2_32;\n" "\n" " st.global.f32 [%rd34], %f104;\n" "\n" "$L__BB2_32:\n" " @%p1 bra $L__BB2_34;\n" "\n" " st.global.f32 [%rd34+4], %f105;\n" "\n" "$L__BB2_34:\n" " // begin inline asm\n" " shfl.down.b32 %f372, %f98, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f375, %f98, %f372;\n" " // begin inline asm\n" " shfl.down.b32 %f374, %f375, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f377, %f375, %f374;\n" " // begin inline asm\n" " shfl.down.b32 %f376, %f377, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f379, %f377, %f376;\n" " // begin inline asm\n" " shfl.down.b32 %f378, %f379, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f381, %f379, %f378;\n" " // begin inline asm\n" " shfl.down.b32 %f380, %f381, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f106, %f381, %f380;\n" " // begin inline asm\n" " shfl.down.b32 %f382, %f99, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f385, %f99, %f382;\n" " // begin inline asm\n" " shfl.down.b32 %f384, %f385, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f387, %f385, %f384;\n" " // begin inline asm\n" " shfl.down.b32 %f386, %f387, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f389, %f387, %f386;\n" " // begin inline asm\n" " shfl.down.b32 %f388, %f389, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f391, %f389, %f388;\n" " // begin inline asm\n" " shfl.down.b32 %f390, %f391, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f107, %f391, %f390;\n" " setp.lt.s32 %p25, %r3, 2;\n" " or.pred %p26, %p20, %p25;\n" " @%p26 bra $L__BB2_39;\n" "\n" " mov.u32 %r269, %ctaid.y;\n" " shl.b32 %r268, %r269, 2;\n" " cvt.u32.u64 %r214, %rd2;\n" " setp.ge.s32 %p27, %r214, %r35;\n" " add.s32 %r215, %r268, 1;\n" " cvt.s64.s32 %rd72, %r215;\n" " mul.lo.s64 %rd73, %rd72, %rd33;\n" " add.s64 %rd74, %rd73, %rd2;\n" " shl.b64 %rd75, %rd74, 2;\n" " add.s64 %rd35, %rd1, %rd75;\n" " @%p27 bra $L__BB2_37;\n" "\n" " st.global.f32 [%rd35], %f106;\n" "\n" "$L__BB2_37:\n" " @%p1 bra $L__BB2_39;\n" "\n" " st.global.f32 [%rd35+4], %f107;\n" "\n" "$L__BB2_39:\n" " // begin inline asm\n" " shfl.down.b32 %f392, %f100, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f395, %f100, %f392;\n" " // begin inline asm\n" " shfl.down.b32 %f394, %f395, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f397, %f395, %f394;\n" " // begin inline asm\n" " shfl.down.b32 %f396, %f397, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f399, %f397, %f396;\n" " // begin inline asm\n" " shfl.down.b32 %f398, %f399, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f401, %f399, %f398;\n" " // begin inline asm\n" " shfl.down.b32 %f400, %f401, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f108, %f401, %f400;\n" " // begin inline asm\n" " shfl.down.b32 %f402, %f101, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f405, %f101, %f402;\n" " // begin inline asm\n" " shfl.down.b32 %f404, %f405, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f407, %f405, %f404;\n" " // begin inline asm\n" " shfl.down.b32 %f406, %f407, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f409, %f407, %f406;\n" " // begin inline asm\n" " shfl.down.b32 %f408, %f409, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f411, %f409, %f408;\n" " // begin inline asm\n" " shfl.down.b32 %f410, %f411, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f109, %f411, %f410;\n" " setp.lt.s32 %p30, %r3, 3;\n" " or.pred %p31, %p20, %p30;\n" " @%p31 bra $L__BB2_44;\n" "\n" " mov.u32 %r267, %ctaid.y;\n" " shl.b32 %r266, %r267, 2;\n" " cvt.u32.u64 %r236, %rd2;\n" " setp.ge.s32 %p32, %r236, %r35;\n" " add.s32 %r237, %r266, 2;\n" " cvt.s64.s32 %rd76, %r237;\n" " mul.lo.s64 %rd77, %rd76, %rd33;\n" " add.s64 %rd78, %rd77, %rd2;\n" " shl.b64 %rd79, %rd78, 2;\n" " add.s64 %rd36, %rd1, %rd79;\n" " @%p32 bra $L__BB2_42;\n" "\n" " st.global.f32 [%rd36], %f108;\n" "\n" "$L__BB2_42:\n" " @%p1 bra $L__BB2_44;\n" "\n" " st.global.f32 [%rd36+4], %f109;\n" "\n" "$L__BB2_44:\n" " // begin inline asm\n" " shfl.down.b32 %f412, %f102, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f415, %f102, %f412;\n" " // begin inline asm\n" " shfl.down.b32 %f414, %f415, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f417, %f415, %f414;\n" " // begin inline asm\n" " shfl.down.b32 %f416, %f417, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f419, %f417, %f416;\n" " // begin inline asm\n" " shfl.down.b32 %f418, %f419, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f421, %f419, %f418;\n" " // begin inline asm\n" " shfl.down.b32 %f420, %f421, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f110, %f421, %f420;\n" " // begin inline asm\n" " shfl.down.b32 %f422, %f103, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f425, %f103, %f422;\n" " // begin inline asm\n" " shfl.down.b32 %f424, %f425, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f427, %f425, %f424;\n" " // begin inline asm\n" " shfl.down.b32 %f426, %f427, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f429, %f427, %f426;\n" " // begin inline asm\n" " shfl.down.b32 %f428, %f429, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f431, %f429, %f428;\n" " // begin inline asm\n" " shfl.down.b32 %f430, %f431, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f111, %f431, %f430;\n" " setp.lt.s32 %p35, %r3, 4;\n" " or.pred %p36, %p20, %p35;\n" " @%p36 bra $L__BB2_49;\n" "\n" " mov.u32 %r265, %ctaid.y;\n" " shl.b32 %r264, %r265, 2;\n" " cvt.u32.u64 %r258, %rd2;\n" " setp.ge.s32 %p37, %r258, %r35;\n" " add.s32 %r259, %r264, 3;\n" " cvt.s64.s32 %rd80, %r259;\n" " mul.lo.s64 %rd81, %rd80, %rd33;\n" " add.s64 %rd82, %rd81, %rd2;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd37, %rd1, %rd83;\n" " @%p37 bra $L__BB2_47;\n" "\n" " st.global.f32 [%rd37], %f110;\n" "\n" "$L__BB2_47:\n" " @%p1 bra $L__BB2_49;\n" "\n" " st.global.f32 [%rd37+4], %f111;\n" "\n" "$L__BB2_49:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q4k_nx8\n" ".visible .entry fc_matvec_q4k_nx8(\n" " .param .u64 fc_matvec_q4k_nx8_param_0,\n" " .param .u64 fc_matvec_q4k_nx8_param_1,\n" " .param .u64 fc_matvec_q4k_nx8_param_2,\n" " .param .u32 fc_matvec_q4k_nx8_param_3,\n" " .param .u32 fc_matvec_q4k_nx8_param_4,\n" " .param .u32 fc_matvec_q4k_nx8_param_5,\n" " .param .u32 fc_matvec_q4k_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<68>;\n" " .reg .b16 %rs<5>;\n" " .reg .f32 %f<963>;\n" " .reg .b32 %r<502>;\n" " .reg .b64 %rd<125>;\n" "\n" "\n" " ld.param.u64 %rd43, [fc_matvec_q4k_nx8_param_0];\n" " ld.param.u64 %rd42, [fc_matvec_q4k_nx8_param_1];\n" " ld.param.u64 %rd44, [fc_matvec_q4k_nx8_param_2];\n" " ld.param.u32 %r44, [fc_matvec_q4k_nx8_param_3];\n" " ld.param.u32 %r45, [fc_matvec_q4k_nx8_param_4];\n" " ld.param.u32 %r46, [fc_matvec_q4k_nx8_param_5];\n" " ld.param.u32 %r47, [fc_matvec_q4k_nx8_param_6];\n" " cvta.to.global.u64 %rd1, %rd44;\n" " mov.u32 %r48, %ctaid.y;\n" " shl.b32 %r1, %r48, 3;\n" " sub.s32 %r2, %r47, %r1;\n" " min.s32 %r3, %r2, 8;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r49, %ctaid.x;\n" " shl.b32 %r50, %r49, 4;\n" " shr.u32 %r51, %r4, 4;\n" " and.b32 %r52, %r51, 268435454;\n" " add.s32 %r53, %r52, %r50;\n" " or.b32 %r6, %r53, 1;\n" " setp.ge.s32 %p1, %r6, %r44;\n" " cvt.s64.s32 %rd2, %r53;\n" " cvt.s64.s32 %rd3, %r46;\n" " mul.wide.s32 %rd45, %r46, %r53;\n" " add.s64 %rd123, %rd43, %rd45;\n" " mov.u64 %rd124, %rd123;\n" " @%p1 bra $L__BB3_2;\n" "\n" " add.s64 %rd124, %rd123, %rd3;\n" "\n" "$L__BB3_2:\n" " setp.lt.s32 %p2, %r45, 1;\n" " mov.f32 %f899, 0f00000000;\n" " mov.f32 %f900, %f899;\n" " mov.f32 %f901, %f899;\n" " mov.f32 %f902, %f899;\n" " mov.f32 %f903, %f899;\n" " mov.f32 %f904, %f899;\n" " mov.f32 %f905, %f899;\n" " mov.f32 %f906, %f899;\n" " mov.f32 %f907, %f899;\n" " mov.f32 %f908, %f899;\n" " mov.f32 %f909, %f899;\n" " mov.f32 %f910, %f899;\n" " mov.f32 %f911, %f899;\n" " mov.f32 %f912, %f899;\n" " mov.f32 %f913, %f899;\n" " mov.f32 %f914, %f899;\n" " mov.f32 %f827, %f899;\n" " mov.f32 %f828, %f899;\n" " mov.f32 %f829, %f899;\n" " mov.f32 %f830, %f899;\n" " mov.f32 %f831, %f899;\n" " mov.f32 %f832, %f899;\n" " mov.f32 %f833, %f899;\n" " mov.f32 %f834, %f899;\n" " mov.f32 %f835, %f899;\n" " mov.f32 %f836, %f899;\n" " mov.f32 %f837, %f899;\n" " mov.f32 %f838, %f899;\n" " mov.f32 %f839, %f899;\n" " mov.f32 %f840, %f899;\n" " mov.f32 %f841, %f899;\n" " mov.f32 %f842, %f899;\n" " @%p2 bra $L__BB3_43;\n" "\n" " and.b32 %r7, %r4, 3;\n" " shl.b32 %r55, %r4, 3;\n" " and.b32 %r56, %r55, 24;\n" " or.b32 %r8, %r56, 128;\n" " cvt.u64.u32 %rd7, %r5;\n" " shr.u32 %r9, %r4, 2;\n" " add.s32 %r10, %r9, 64;\n" " max.u32 %r57, %r10, 128;\n" " not.b32 %r58, %r9;\n" " add.s32 %r11, %r57, %r58;\n" " mul.wide.s32 %rd8, %r1, %r45;\n" " add.s32 %r59, %r1, 1;\n" " mul.wide.s32 %rd9, %r59, %r45;\n" " add.s32 %r60, %r1, 2;\n" " mul.wide.s32 %rd10, %r60, %r45;\n" " add.s32 %r61, %r1, 3;\n" " mul.wide.s32 %rd11, %r61, %r45;\n" " add.s32 %r62, %r1, 4;\n" " mul.wide.s32 %rd12, %r62, %r45;\n" " add.s32 %r63, %r1, 5;\n" " mul.wide.s32 %rd13, %r63, %r45;\n" " add.s32 %r64, %r1, 6;\n" " mul.wide.s32 %rd14, %r64, %r45;\n" " add.s32 %r65, %r1, 7;\n" " mul.wide.s32 %rd15, %r65, %r45;\n" " and.b32 %r12, %r11, 64;\n" " bfe.u32 %r13, %r4, 2, 4;\n" " and.b32 %r66, %r55, -512;\n" " and.b32 %r67, %r55, 480;\n" " bfe.u32 %r68, %r4, 2, 3;\n" " or.b32 %r69, %r56, %r67;\n" " or.b32 %r70, %r69, %r66;\n" " or.b32 %r71, %r70, %r68;\n" " shl.b32 %r72, %r71, 2;\n" " mov.u32 %r73, fc_kq_shared;\n" " add.s32 %r14, %r73, %r72;\n" " xor.b32 %r74, %r68, 1;\n" " or.b32 %r75, %r74, %r66;\n" " or.b32 %r76, %r75, %r67;\n" " or.b32 %r77, %r76, %r56;\n" " shl.b32 %r78, %r77, 2;\n" " add.s32 %r79, %r73, %r78;\n" " xor.b32 %r80, %r68, 2;\n" " or.b32 %r81, %r80, %r66;\n" " or.b32 %r82, %r81, %r67;\n" " or.b32 %r83, %r82, %r56;\n" " shl.b32 %r84, %r83, 2;\n" " add.s32 %r85, %r73, %r84;\n" " xor.b32 %r86, %r68, 3;\n" " or.b32 %r87, %r86, %r66;\n" " or.b32 %r88, %r87, %r67;\n" " or.b32 %r89, %r88, %r56;\n" " shl.b32 %r90, %r89, 2;\n" " add.s32 %r91, %r73, %r90;\n" " xor.b32 %r92, %r68, 4;\n" " or.b32 %r93, %r92, %r66;\n" " or.b32 %r94, %r93, %r67;\n" " or.b32 %r95, %r94, %r56;\n" " shl.b32 %r96, %r95, 2;\n" " add.s32 %r97, %r73, %r96;\n" " xor.b32 %r98, %r68, 5;\n" " or.b32 %r99, %r98, %r66;\n" " or.b32 %r100, %r99, %r67;\n" " or.b32 %r101, %r100, %r56;\n" " shl.b32 %r102, %r101, 2;\n" " add.s32 %r103, %r73, %r102;\n" " xor.b32 %r104, %r68, 6;\n" " or.b32 %r105, %r104, %r66;\n" " or.b32 %r106, %r105, %r67;\n" " or.b32 %r107, %r106, %r56;\n" " shl.b32 %r108, %r107, 2;\n" " add.s32 %r109, %r73, %r108;\n" " xor.b32 %r110, %r68, 7;\n" " or.b32 %r111, %r110, %r66;\n" " or.b32 %r112, %r111, %r67;\n" " or.b32 %r113, %r112, %r56;\n" " shl.b32 %r114, %r113, 2;\n" " add.s32 %r115, %r73, %r114;\n" " shl.b32 %r116, %r9, 2;\n" " add.s32 %r22, %r73, %r116;\n" " mov.u32 %r496, 0;\n" " mov.f32 %f899, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " setp.lt.s32 %p6, %r3, 3;\n" " setp.lt.s32 %p7, %r3, 4;\n" " shl.b64 %rd82, %rd7, 2;\n" " mov.f32 %f900, %f899;\n" " mov.f32 %f901, %f899;\n" " mov.f32 %f902, %f899;\n" " mov.f32 %f903, %f899;\n" " mov.f32 %f904, %f899;\n" " mov.f32 %f905, %f899;\n" " mov.f32 %f906, %f899;\n" " mov.f32 %f907, %f899;\n" " mov.f32 %f908, %f899;\n" " mov.f32 %f909, %f899;\n" " mov.f32 %f910, %f899;\n" " mov.f32 %f911, %f899;\n" " mov.f32 %f912, %f899;\n" " mov.f32 %f913, %f899;\n" " mov.f32 %f914, %f899;\n" " mov.f32 %f827, %f899;\n" " mov.f32 %f828, %f899;\n" " mov.f32 %f829, %f899;\n" " mov.f32 %f830, %f899;\n" " mov.f32 %f831, %f899;\n" " mov.f32 %f832, %f899;\n" " mov.f32 %f833, %f899;\n" " mov.f32 %f834, %f899;\n" " mov.f32 %f835, %f899;\n" " mov.f32 %f836, %f899;\n" " mov.f32 %f837, %f899;\n" " mov.f32 %f838, %f899;\n" " mov.f32 %f839, %f899;\n" " mov.f32 %f840, %f899;\n" " mov.f32 %f841, %f899;\n" " mov.f32 %f842, %f899;\n" "\n" "$L__BB3_4:\n" " sub.s32 %r117, %r45, %r496;\n" " min.s32 %r24, %r117, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r24;\n" " @%p3 bra $L__BB3_23;\n" "\n" " mov.u32 %r497, %r4;\n" "\n" "$L__BB3_6:\n" " .pragma \"nounroll\";\n" " cvt.s64.s32 %rd115, %r496;\n" " cvt.s64.s32 %rd46, %r497;\n" " add.s64 %rd19, %rd46, %rd115;\n" " mov.f32 %f843, 0f00000000;\n" " @%p4 bra $L__BB3_8;\n" "\n" " add.s64 %rd48, %rd19, %rd8;\n" " shl.b64 %rd49, %rd48, 2;\n" " add.s64 %rd47, %rd42, %rd49;\n" " // begin inline asm\n" " ld.global.nc.f32 %f843, [%rd47];\n" " // end inline asm\n" "\n" "$L__BB3_8:\n" " mov.f32 %f844, 0f00000000;\n" " shl.b32 %r118, %r497, 2;\n" " add.s32 %r26, %r73, %r118;\n" " st.shared.f32 [%r26+512], %f843;\n" " @%p5 bra $L__BB3_10;\n" "\n" " add.s64 %rd51, %rd19, %rd9;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd50, %rd42, %rd52;\n" " // begin inline asm\n" " ld.global.nc.f32 %f844, [%rd50];\n" " // end inline asm\n" "\n" "$L__BB3_10:\n" " mov.f32 %f845, 0f00000000;\n" " st.shared.f32 [%r26+2560], %f844;\n" " @%p6 bra $L__BB3_12;\n" "\n" " add.s64 %rd54, %rd19, %rd10;\n" " shl.b64 %rd55, %rd54, 2;\n" " add.s64 %rd53, %rd42, %rd55;\n" " // begin inline asm\n" " ld.global.nc.f32 %f845, [%rd53];\n" " // end inline asm\n" "\n" "$L__BB3_12:\n" " mov.f32 %f846, 0f00000000;\n" " st.shared.f32 [%r26+4608], %f845;\n" " @%p7 bra $L__BB3_14;\n" "\n" " add.s64 %rd57, %rd19, %rd11;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd56, %rd42, %rd58;\n" " // begin inline asm\n" " ld.global.nc.f32 %f846, [%rd56];\n" " // end inline asm\n" "\n" "$L__BB3_14:\n" " mov.f32 %f847, 0f00000000;\n" " st.shared.f32 [%r26+6656], %f846;\n" " setp.lt.s32 %p8, %r3, 5;\n" " @%p8 bra $L__BB3_16;\n" "\n" " add.s64 %rd60, %rd19, %rd12;\n" " shl.b64 %rd61, %rd60, 2;\n" " add.s64 %rd59, %rd42, %rd61;\n" " // begin inline asm\n" " ld.global.nc.f32 %f847, [%rd59];\n" " // end inline asm\n" "\n" "$L__BB3_16:\n" " mov.f32 %f848, 0f00000000;\n" " st.shared.f32 [%r26+8704], %f847;\n" " setp.lt.s32 %p9, %r3, 6;\n" " @%p9 bra $L__BB3_18;\n" "\n" " add.s64 %rd63, %rd19, %rd13;\n" " shl.b64 %rd64, %rd63, 2;\n" " add.s64 %rd62, %rd42, %rd64;\n" " // begin inline asm\n" " ld.global.nc.f32 %f848, [%rd62];\n" " // end inline asm\n" "\n" "$L__BB3_18:\n" " mov.f32 %f849, 0f00000000;\n" " st.shared.f32 [%r26+10752], %f848;\n" " setp.lt.s32 %p10, %r3, 7;\n" " @%p10 bra $L__BB3_20;\n" "\n" " add.s64 %rd66, %rd19, %rd14;\n" " shl.b64 %rd67, %rd66, 2;\n" " add.s64 %rd65, %rd42, %rd67;\n" " // begin inline asm\n" " ld.global.nc.f32 %f849, [%rd65];\n" " // end inline asm\n" "\n" "$L__BB3_20:\n" " mov.f32 %f850, 0f00000000;\n" " st.shared.f32 [%r26+12800], %f849;\n" " setp.lt.s32 %p11, %r3, 8;\n" " @%p11 bra $L__BB3_22;\n" "\n" " add.s64 %rd69, %rd19, %rd15;\n" " shl.b64 %rd70, %rd69, 2;\n" " add.s64 %rd68, %rd42, %rd70;\n" " // begin inline asm\n" " ld.global.nc.f32 %f850, [%rd68];\n" " // end inline asm\n" "\n" "$L__BB3_22:\n" " sub.s32 %r472, %r45, %r496;\n" " min.s32 %r471, %r472, 512;\n" " st.shared.f32 [%r26+14848], %f850;\n" " add.s32 %r497, %r497, 256;\n" " setp.lt.s32 %p12, %r497, %r471;\n" " @%p12 bra $L__BB3_6;\n" "\n" "$L__BB3_23:\n" " setp.gt.u32 %p13, %r4, 511;\n" " bar.sync 0;\n" " @%p13 bra $L__BB3_35;\n" "\n" " setp.ne.s32 %p14, %r12, 0;\n" " mov.u32 %r498, %r9;\n" " @%p14 bra $L__BB3_28;\n" "\n" " sub.s32 %r495, %r45, %r496;\n" " min.s32 %r494, %r495, 512;\n" " shr.u32 %r493, %r494, 5;\n" " setp.ge.u32 %p15, %r13, %r493;\n" " mov.u32 %r498, %r10;\n" " @%p15 bra $L__BB3_28;\n" "\n" " setp.ne.s32 %p16, %r7, 0;\n" " ld.shared.f32 %f310, [%r14+512];\n" " add.f32 %f311, %f310, 0f00000000;\n" " add.s32 %r456, %r79, 512;\n" " ld.shared.f32 %f312, [%r456];\n" " add.f32 %f313, %f311, %f312;\n" " add.s32 %r457, %r85, 512;\n" " ld.shared.f32 %f314, [%r457];\n" " add.f32 %f315, %f313, %f314;\n" " add.s32 %r458, %r91, 512;\n" " ld.shared.f32 %f316, [%r458];\n" " add.f32 %f317, %f315, %f316;\n" " add.s32 %r459, %r97, 512;\n" " ld.shared.f32 %f318, [%r459];\n" " add.f32 %f319, %f317, %f318;\n" " add.s32 %r460, %r103, 512;\n" " ld.shared.f32 %f320, [%r460];\n" " add.f32 %f321, %f319, %f320;\n" " add.s32 %r461, %r109, 512;\n" " ld.shared.f32 %f322, [%r461];\n" " add.f32 %f323, %f321, %f322;\n" " add.s32 %r462, %r115, 512;\n" " ld.shared.f32 %f324, [%r462];\n" " add.f32 %f307, %f323, %f324;\n" " mov.u32 %r120, 2;\n" " mov.u32 %r123, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f306, %f307, %r120, %r123;\n" " // end inline asm\n" " add.f32 %f309, %f307, %f306;\n" " mov.u32 %r122, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f308, %f309, %r122, %r123;\n" " // end inline asm\n" " add.f32 %f49, %f309, %f308;\n" " mov.u32 %r498, %r10;\n" " @%p16 bra $L__BB3_28;\n" "\n" " st.shared.f32 [%r22], %f49;\n" " mov.u32 %r498, %r10;\n" "\n" "$L__BB3_28:\n" " setp.lt.u32 %p17, %r11, 64;\n" " @%p17 bra $L__BB3_35;\n" "\n" " sub.s32 %r485, %r45, %r496;\n" " min.s32 %r484, %r485, 512;\n" " shr.u32 %r483, %r484, 5;\n" " and.b32 %r31, %r498, 15;\n" " setp.ge.u32 %p18, %r31, %r483;\n" " @%p18 bra $L__BB3_32;\n" "\n" " and.b32 %r470, %r498, 7;\n" " and.b32 %r469, %r498, 15;\n" " setp.ne.s32 %p19, %r7, 0;\n" " shl.b32 %r130, %r498, 5;\n" " and.b32 %r131, %r130, -512;\n" " shl.b32 %r132, %r469, 5;\n" " add.s32 %r133, %r8, %r132;\n" " add.s32 %r134, %r133, %r131;\n" " add.s32 %r135, %r134, %r470;\n" " shl.b32 %r136, %r135, 2;\n" " add.s32 %r138, %r73, %r136;\n" " mov.u32 %r126, 2;\n" " ld.shared.f32 %f329, [%r138];\n" " add.f32 %f330, %f329, 0f00000000;\n" " xor.b32 %r139, %r470, 1;\n" " mov.u32 %r128, 1;\n" " add.s32 %r140, %r134, %r139;\n" " shl.b32 %r141, %r140, 2;\n" " add.s32 %r142, %r73, %r141;\n" " ld.shared.f32 %f331, [%r142];\n" " add.f32 %f332, %f330, %f331;\n" " xor.b32 %r143, %r470, 2;\n" " add.s32 %r144, %r134, %r143;\n" " shl.b32 %r145, %r144, 2;\n" " add.s32 %r146, %r73, %r145;\n" " ld.shared.f32 %f333, [%r146];\n" " add.f32 %f334, %f332, %f333;\n" " xor.b32 %r147, %r470, 3;\n" " add.s32 %r148, %r134, %r147;\n" " shl.b32 %r149, %r148, 2;\n" " add.s32 %r150, %r73, %r149;\n" " ld.shared.f32 %f335, [%r150];\n" " add.f32 %f336, %f334, %f335;\n" " xor.b32 %r151, %r470, 4;\n" " add.s32 %r152, %r134, %r151;\n" " shl.b32 %r153, %r152, 2;\n" " add.s32 %r154, %r73, %r153;\n" " ld.shared.f32 %f337, [%r154];\n" " add.f32 %f338, %f336, %f337;\n" " xor.b32 %r155, %r470, 5;\n" " add.s32 %r156, %r134, %r155;\n" " shl.b32 %r157, %r156, 2;\n" " add.s32 %r158, %r73, %r157;\n" " ld.shared.f32 %f339, [%r158];\n" " add.f32 %f340, %f338, %f339;\n" " xor.b32 %r159, %r470, 6;\n" " add.s32 %r160, %r134, %r159;\n" " shl.b32 %r161, %r160, 2;\n" " add.s32 %r162, %r73, %r161;\n" " ld.shared.f32 %f341, [%r162];\n" " add.f32 %f342, %f340, %f341;\n" " xor.b32 %r163, %r470, 7;\n" " add.s32 %r164, %r134, %r163;\n" " shl.b32 %r165, %r164, 2;\n" " add.s32 %r166, %r73, %r165;\n" " ld.shared.f32 %f343, [%r166];\n" " add.f32 %f326, %f342, %f343;\n" " mov.u32 %r129, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f325, %f326, %r126, %r129;\n" " // end inline asm\n" " add.f32 %f328, %f326, %f325;\n" " // begin inline asm\n" " shfl.down.b32 %f327, %f328, %r128, %r129;\n" " // end inline asm\n" " add.f32 %f50, %f328, %f327;\n" " @%p19 bra $L__BB3_32;\n" "\n" " shl.b32 %r482, %r498, 2;\n" " add.s32 %r481, %r73, %r482;\n" " st.shared.f32 [%r481], %f50;\n" "\n" "$L__BB3_32:\n" " @%p18 bra $L__BB3_35;\n" "\n" " and.b32 %r478, %r498, 7;\n" " and.b32 %r477, %r498, 15;\n" " setp.ne.s32 %p21, %r7, 0;\n" " shl.b32 %r171, %r498, 5;\n" " add.s32 %r172, %r171, 2048;\n" " and.b32 %r173, %r172, -512;\n" " shl.b32 %r174, %r477, 5;\n" " add.s32 %r175, %r8, %r174;\n" " add.s32 %r176, %r175, %r173;\n" " add.s32 %r177, %r176, %r478;\n" " shl.b32 %r178, %r177, 2;\n" " add.s32 %r180, %r73, %r178;\n" " mov.u32 %r167, 2;\n" " ld.shared.f32 %f348, [%r180];\n" " add.f32 %f349, %f348, 0f00000000;\n" " xor.b32 %r181, %r478, 1;\n" " mov.u32 %r169, 1;\n" " add.s32 %r182, %r176, %r181;\n" " shl.b32 %r183, %r182, 2;\n" " add.s32 %r184, %r73, %r183;\n" " ld.shared.f32 %f350, [%r184];\n" " add.f32 %f351, %f349, %f350;\n" " xor.b32 %r185, %r478, 2;\n" " add.s32 %r186, %r176, %r185;\n" " shl.b32 %r187, %r186, 2;\n" " add.s32 %r188, %r73, %r187;\n" " ld.shared.f32 %f352, [%r188];\n" " add.f32 %f353, %f351, %f352;\n" " xor.b32 %r189, %r478, 3;\n" " add.s32 %r190, %r176, %r189;\n" " shl.b32 %r191, %r190, 2;\n" " add.s32 %r192, %r73, %r191;\n" " ld.shared.f32 %f354, [%r192];\n" " add.f32 %f355, %f353, %f354;\n" " xor.b32 %r193, %r478, 4;\n" " add.s32 %r194, %r176, %r193;\n" " shl.b32 %r195, %r194, 2;\n" " add.s32 %r196, %r73, %r195;\n" " ld.shared.f32 %f356, [%r196];\n" " add.f32 %f357, %f355, %f356;\n" " xor.b32 %r197, %r478, 5;\n" " add.s32 %r198, %r176, %r197;\n" " shl.b32 %r199, %r198, 2;\n" " add.s32 %r200, %r73, %r199;\n" " ld.shared.f32 %f358, [%r200];\n" " add.f32 %f359, %f357, %f358;\n" " xor.b32 %r201, %r478, 6;\n" " add.s32 %r202, %r176, %r201;\n" " shl.b32 %r203, %r202, 2;\n" " add.s32 %r204, %r73, %r203;\n" " ld.shared.f32 %f360, [%r204];\n" " add.f32 %f361, %f359, %f360;\n" " xor.b32 %r205, %r478, 7;\n" " add.s32 %r206, %r176, %r205;\n" " shl.b32 %r207, %r206, 2;\n" " add.s32 %r208, %r73, %r207;\n" " ld.shared.f32 %f362, [%r208];\n" " add.f32 %f345, %f361, %f362;\n" " mov.u32 %r170, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f344, %f345, %r167, %r170;\n" " // end inline asm\n" " add.f32 %f347, %f345, %f344;\n" " // begin inline asm\n" " shfl.down.b32 %f346, %f347, %r169, %r170;\n" " // end inline asm\n" " add.f32 %f51, %f347, %f346;\n" " @%p21 bra $L__BB3_35;\n" "\n" " shl.b32 %r480, %r498, 2;\n" " add.s32 %r479, %r73, %r480;\n" " st.shared.f32 [%r479+256], %f51;\n" "\n" "$L__BB3_35:\n" " cvt.u32.u64 %r209, %rd2;\n" " setp.ge.s32 %p22, %r209, %r44;\n" " bar.sync 0;\n" " @%p22 bra $L__BB3_42;\n" "\n" " sub.s32 %r474, %r45, %r496;\n" " min.s32 %r473, %r474, 512;\n" " shr.s32 %r33, %r473, 8;\n" " shr.u32 %r210, %r496, 5;\n" " sub.s32 %r34, %r5, %r210;\n" " and.b32 %r499, %r34, 31;\n" " shl.b32 %r36, %r33, 3;\n" " setp.ge.s32 %p23, %r499, %r36;\n" " @%p23 bra $L__BB3_39;\n" "\n" " shr.u32 %r211, %r499, 3;\n" " mul.lo.s32 %r212, %r211, 152;\n" " cvt.u64.u32 %rd71, %r212;\n" " and.b32 %r213, %r34, 7;\n" " or.b32 %r214, %r213, 136;\n" " cvt.u64.u32 %rd20, %r214;\n" " add.s64 %rd119, %rd124, %rd71;\n" " add.s64 %rd120, %rd123, %rd71;\n" "\n" "$L__BB3_38:\n" " .pragma \"nounroll\";\n" " sub.s32 %r492, %r45, %r496;\n" " min.s32 %r491, %r492, 512;\n" " shr.s32 %r490, %r491, 8;\n" " shl.b32 %r489, %r490, 3;\n" " add.s64 %rd72, %rd120, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f363, [%rd72];\n" " // end inline asm\n" " add.s64 %rd73, %rd120, %rd20;\n" " // begin inline asm\n" " ld.global.nc.u8 %r215, [%rd73];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r215;\n" " and.b16 %rs2, %rs1, 255;\n" " cvt.rn.f32.u16 %f365, %rs2;\n" " mul.rn.f32 %f366, %f363, %f365;\n" " add.s64 %rd74, %rd119, 148;\n" " // begin inline asm\n" " ld.global.nc.f32 %f364, [%rd74];\n" " // end inline asm\n" " add.s64 %rd75, %rd119, %rd20;\n" " // begin inline asm\n" " ld.global.nc.u8 %r216, [%rd75];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r216;\n" " and.b16 %rs4, %rs3, 255;\n" " cvt.rn.f32.u16 %f367, %rs4;\n" " mul.rn.f32 %f368, %f364, %f367;\n" " shl.b32 %r217, %r499, 2;\n" " add.s32 %r219, %r73, %r217;\n" " ld.shared.f32 %f369, [%r219];\n" " fma.rn.f32 %f834, %f366, %f369, %f834;\n" " fma.rn.f32 %f842, %f368, %f369, %f842;\n" " ld.shared.f32 %f370, [%r219+64];\n" " fma.rn.f32 %f833, %f366, %f370, %f833;\n" " fma.rn.f32 %f841, %f368, %f370, %f841;\n" " ld.shared.f32 %f371, [%r219+128];\n" " fma.rn.f32 %f832, %f366, %f371, %f832;\n" " fma.rn.f32 %f840, %f368, %f371, %f840;\n" " ld.shared.f32 %f372, [%r219+192];\n" " fma.rn.f32 %f831, %f366, %f372, %f831;\n" " fma.rn.f32 %f839, %f368, %f372, %f839;\n" " ld.shared.f32 %f373, [%r219+256];\n" " fma.rn.f32 %f830, %f366, %f373, %f830;\n" " fma.rn.f32 %f838, %f368, %f373, %f838;\n" " ld.shared.f32 %f374, [%r219+320];\n" " fma.rn.f32 %f829, %f366, %f374, %f829;\n" " fma.rn.f32 %f837, %f368, %f374, %f837;\n" " ld.shared.f32 %f375, [%r219+384];\n" " fma.rn.f32 %f828, %f366, %f375, %f828;\n" " fma.rn.f32 %f836, %f368, %f375, %f836;\n" " ld.shared.f32 %f376, [%r219+448];\n" " fma.rn.f32 %f827, %f366, %f376, %f827;\n" " fma.rn.f32 %f835, %f368, %f376, %f835;\n" " add.s64 %rd120, %rd120, 608;\n" " add.s64 %rd119, %rd119, 608;\n" " add.s32 %r499, %r499, 32;\n" " setp.lt.s32 %p24, %r499, %r489;\n" " @%p24 bra $L__BB3_38;\n" "\n" "$L__BB3_39:\n" " sub.s32 %r476, %r45, %r496;\n" " min.s32 %r475, %r476, 512;\n" " setp.lt.s32 %p25, %r475, 256;\n" " @%p25 bra $L__BB3_42;\n" "\n" " add.s32 %r501, %r73, 512;\n" " mov.u32 %r500, 0;\n" "\n" "$L__BB3_41:\n" " sub.s32 %r488, %r45, %r496;\n" " min.s32 %r487, %r488, 512;\n" " shr.s32 %r486, %r487, 8;\n" " add.s64 %rd76, %rd123, %rd82;\n" " // begin inline asm\n" " ld.global.nc.u32 %r223, [%rd76];\n" " // end inline asm\n" " add.s64 %rd77, %rd124, %rd82;\n" " // begin inline asm\n" " ld.global.nc.u32 %r224, [%rd77];\n" " // end inline asm\n" " add.s64 %rd78, %rd123, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r225,%r226}, [%rd78];\n" " // end inline asm\n" " add.s64 %rd79, %rd124, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r227,%r228}, [%rd79];\n" " // end inline asm\n" " add.s64 %rd80, %rd123, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f377, [%rd80];\n" " // end inline asm\n" " add.s64 %rd81, %rd124, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f378, [%rd81];\n" " // end inline asm\n" " shr.u32 %r229, %r223, 16;\n" " shr.u32 %r230, %r224, 16;\n" " and.b32 %r231, %r225, 255;\n" " cvt.rn.f32.u32 %f379, %r231;\n" " and.b32 %r232, %r223, 15;\n" " or.b32 %r233, %r232, 1258291200;\n" " mov.b32 %f380, %r233;\n" " mov.f32 %f381, 0fCB000000;\n" " mov.f32 %f382, 0f3F800000;\n" " fma.rn.f32 %f383, %f380, %f382, %f381;\n" " mul.f32 %f384, %f383, %f379;\n" " and.b32 %r234, %r227, 255;\n" " cvt.rn.f32.u32 %f385, %r234;\n" " and.b32 %r235, %r224, 15;\n" " or.b32 %r236, %r235, 1258291200;\n" " mov.b32 %f386, %r236;\n" " fma.rn.f32 %f387, %f386, %f382, %f381;\n" " mul.f32 %f388, %f387, %f385;\n" " shl.b32 %r237, %r5, 2;\n" " add.s32 %r238, %r501, %r237;\n" " ld.shared.f32 %f389, [%r238];\n" " mov.f32 %f390, 0f00000000;\n" " fma.rn.f32 %f391, %f384, %f389, %f390;\n" " fma.rn.f32 %f392, %f388, %f389, %f390;\n" " ld.shared.f32 %f393, [%r238+2048];\n" " fma.rn.f32 %f394, %f384, %f393, %f390;\n" " fma.rn.f32 %f395, %f388, %f393, %f390;\n" " ld.shared.f32 %f396, [%r238+4096];\n" " fma.rn.f32 %f397, %f384, %f396, %f390;\n" " fma.rn.f32 %f398, %f388, %f396, %f390;\n" " ld.shared.f32 %f399, [%r238+6144];\n" " fma.rn.f32 %f400, %f384, %f399, %f390;\n" " fma.rn.f32 %f401, %f388, %f399, %f390;\n" " ld.shared.f32 %f402, [%r238+8192];\n" " fma.rn.f32 %f403, %f384, %f402, %f390;\n" " fma.rn.f32 %f404, %f388, %f402, %f390;\n" " ld.shared.f32 %f405, [%r238+10240];\n" " fma.rn.f32 %f406, %f384, %f405, %f390;\n" " fma.rn.f32 %f407, %f388, %f405, %f390;\n" " ld.shared.f32 %f408, [%r238+12288];\n" " fma.rn.f32 %f409, %f384, %f408, %f390;\n" " fma.rn.f32 %f410, %f388, %f408, %f390;\n" " ld.shared.f32 %f411, [%r238+14336];\n" " fma.rn.f32 %f412, %f384, %f411, %f390;\n" " fma.rn.f32 %f413, %f388, %f411, %f390;\n" " bfe.u32 %r239, %r225, 8, 8;\n" " cvt.rn.f32.u32 %f414, %r239;\n" " and.b32 %r240, %r223, 240;\n" " or.b32 %r241, %r240, 1258291200;\n" " mov.b32 %f415, %r241;\n" " mov.f32 %f416, 0fC9000000;\n" " mov.f32 %f417, 0f3D800000;\n" " fma.rn.f32 %f418, %f415, %f417, %f416;\n" " mul.f32 %f419, %f418, %f414;\n" " bfe.u32 %r242, %r227, 8, 8;\n" " cvt.rn.f32.u32 %f420, %r242;\n" " and.b32 %r243, %r224, 240;\n" " or.b32 %r244, %r243, 1258291200;\n" " mov.b32 %f421, %r244;\n" " fma.rn.f32 %f422, %f421, %f417, %f416;\n" " mul.f32 %f423, %f422, %f420;\n" " ld.shared.f32 %f424, [%r238+128];\n" " fma.rn.f32 %f425, %f419, %f424, %f391;\n" " fma.rn.f32 %f426, %f423, %f424, %f392;\n" " ld.shared.f32 %f427, [%r238+2176];\n" " fma.rn.f32 %f428, %f419, %f427, %f394;\n" " fma.rn.f32 %f429, %f423, %f427, %f395;\n" " ld.shared.f32 %f430, [%r238+4224];\n" " fma.rn.f32 %f431, %f419, %f430, %f397;\n" " fma.rn.f32 %f432, %f423, %f430, %f398;\n" " ld.shared.f32 %f433, [%r238+6272];\n" " fma.rn.f32 %f434, %f419, %f433, %f400;\n" " fma.rn.f32 %f435, %f423, %f433, %f401;\n" " ld.shared.f32 %f436, [%r238+8320];\n" " fma.rn.f32 %f437, %f419, %f436, %f403;\n" " fma.rn.f32 %f438, %f423, %f436, %f404;\n" " ld.shared.f32 %f439, [%r238+10368];\n" " fma.rn.f32 %f440, %f419, %f439, %f406;\n" " fma.rn.f32 %f441, %f423, %f439, %f407;\n" " ld.shared.f32 %f442, [%r238+12416];\n" " fma.rn.f32 %f443, %f419, %f442, %f409;\n" " fma.rn.f32 %f444, %f423, %f442, %f410;\n" " ld.shared.f32 %f445, [%r238+14464];\n" " fma.rn.f32 %f446, %f419, %f445, %f412;\n" " fma.rn.f32 %f447, %f423, %f445, %f413;\n" " bfe.u32 %r245, %r225, 16, 8;\n" " cvt.rn.f32.u32 %f448, %r245;\n" " and.b32 %r246, %r223, 3840;\n" " or.b32 %r247, %r246, 1258291200;\n" " mov.b32 %f449, %r247;\n" " mov.f32 %f450, 0fC7000000;\n" " mov.f32 %f451, 0f3B800000;\n" " fma.rn.f32 %f452, %f449, %f451, %f450;\n" " mul.f32 %f453, %f452, %f448;\n" " bfe.u32 %r248, %r227, 16, 8;\n" " cvt.rn.f32.u32 %f454, %r248;\n" " and.b32 %r249, %r224, 3840;\n" " or.b32 %r250, %r249, 1258291200;\n" " mov.b32 %f455, %r250;\n" " fma.rn.f32 %f456, %f455, %f451, %f450;\n" " mul.f32 %f457, %f456, %f454;\n" " ld.shared.f32 %f458, [%r238+256];\n" " fma.rn.f32 %f459, %f453, %f458, %f425;\n" " fma.rn.f32 %f460, %f457, %f458, %f426;\n" " ld.shared.f32 %f461, [%r238+2304];\n" " fma.rn.f32 %f462, %f453, %f461, %f428;\n" " fma.rn.f32 %f463, %f457, %f461, %f429;\n" " ld.shared.f32 %f464, [%r238+4352];\n" " fma.rn.f32 %f465, %f453, %f464, %f431;\n" " fma.rn.f32 %f466, %f457, %f464, %f432;\n" " ld.shared.f32 %f467, [%r238+6400];\n" " fma.rn.f32 %f468, %f453, %f467, %f434;\n" " fma.rn.f32 %f469, %f457, %f467, %f435;\n" " ld.shared.f32 %f470, [%r238+8448];\n" " fma.rn.f32 %f471, %f453, %f470, %f437;\n" " fma.rn.f32 %f472, %f457, %f470, %f438;\n" " ld.shared.f32 %f473, [%r238+10496];\n" " fma.rn.f32 %f474, %f453, %f473, %f440;\n" " fma.rn.f32 %f475, %f457, %f473, %f441;\n" " ld.shared.f32 %f476, [%r238+12544];\n" " fma.rn.f32 %f477, %f453, %f476, %f443;\n" " fma.rn.f32 %f478, %f457, %f476, %f444;\n" " ld.shared.f32 %f479, [%r238+14592];\n" " fma.rn.f32 %f480, %f453, %f479, %f446;\n" " fma.rn.f32 %f481, %f457, %f479, %f447;\n" " shr.u32 %r251, %r225, 24;\n" " cvt.rn.f32.u32 %f482, %r251;\n" " and.b32 %r252, %r223, 61440;\n" " or.b32 %r253, %r252, 1258291200;\n" " mov.b32 %f483, %r253;\n" " mov.f32 %f484, 0fC5000000;\n" " mov.f32 %f485, 0f39800000;\n" " fma.rn.f32 %f486, %f483, %f485, %f484;\n" " mul.f32 %f487, %f486, %f482;\n" " shr.u32 %r254, %r227, 24;\n" " cvt.rn.f32.u32 %f488, %r254;\n" " and.b32 %r255, %r224, 61440;\n" " or.b32 %r256, %r255, 1258291200;\n" " mov.b32 %f489, %r256;\n" " fma.rn.f32 %f490, %f489, %f485, %f484;\n" " mul.f32 %f491, %f490, %f488;\n" " ld.shared.f32 %f492, [%r238+384];\n" " fma.rn.f32 %f493, %f487, %f492, %f459;\n" " fma.rn.f32 %f494, %f491, %f492, %f460;\n" " ld.shared.f32 %f495, [%r238+2432];\n" " fma.rn.f32 %f496, %f487, %f495, %f462;\n" " fma.rn.f32 %f497, %f491, %f495, %f463;\n" " ld.shared.f32 %f498, [%r238+4480];\n" " fma.rn.f32 %f499, %f487, %f498, %f465;\n" " fma.rn.f32 %f500, %f491, %f498, %f466;\n" " ld.shared.f32 %f501, [%r238+6528];\n" " fma.rn.f32 %f502, %f487, %f501, %f468;\n" " fma.rn.f32 %f503, %f491, %f501, %f469;\n" " ld.shared.f32 %f504, [%r238+8576];\n" " fma.rn.f32 %f505, %f487, %f504, %f471;\n" " fma.rn.f32 %f506, %f491, %f504, %f472;\n" " ld.shared.f32 %f507, [%r238+10624];\n" " fma.rn.f32 %f508, %f487, %f507, %f474;\n" " fma.rn.f32 %f509, %f491, %f507, %f475;\n" " ld.shared.f32 %f510, [%r238+12672];\n" " fma.rn.f32 %f511, %f487, %f510, %f477;\n" " fma.rn.f32 %f512, %f491, %f510, %f478;\n" " ld.shared.f32 %f513, [%r238+14720];\n" " fma.rn.f32 %f514, %f487, %f513, %f480;\n" " fma.rn.f32 %f515, %f491, %f513, %f481;\n" " and.b32 %r257, %r226, 255;\n" " cvt.rn.f32.u32 %f516, %r257;\n" " bfe.u32 %r258, %r223, 16, 4;\n" " or.b32 %r259, %r258, 1258291200;\n" " mov.b32 %f517, %r259;\n" " fma.rn.f32 %f518, %f517, %f382, %f381;\n" " mul.f32 %f519, %f518, %f516;\n" " and.b32 %r260, %r228, 255;\n" " cvt.rn.f32.u32 %f520, %r260;\n" " bfe.u32 %r261, %r224, 16, 4;\n" " or.b32 %r262, %r261, 1258291200;\n" " mov.b32 %f521, %r262;\n" " fma.rn.f32 %f522, %f521, %f382, %f381;\n" " mul.f32 %f523, %f522, %f520;\n" " ld.shared.f32 %f524, [%r238+512];\n" " fma.rn.f32 %f525, %f519, %f524, %f493;\n" " fma.rn.f32 %f526, %f523, %f524, %f494;\n" " ld.shared.f32 %f527, [%r238+2560];\n" " fma.rn.f32 %f528, %f519, %f527, %f496;\n" " fma.rn.f32 %f529, %f523, %f527, %f497;\n" " ld.shared.f32 %f530, [%r238+4608];\n" " fma.rn.f32 %f531, %f519, %f530, %f499;\n" " fma.rn.f32 %f532, %f523, %f530, %f500;\n" " ld.shared.f32 %f533, [%r238+6656];\n" " fma.rn.f32 %f534, %f519, %f533, %f502;\n" " fma.rn.f32 %f535, %f523, %f533, %f503;\n" " ld.shared.f32 %f536, [%r238+8704];\n" " fma.rn.f32 %f537, %f519, %f536, %f505;\n" " fma.rn.f32 %f538, %f523, %f536, %f506;\n" " ld.shared.f32 %f539, [%r238+10752];\n" " fma.rn.f32 %f540, %f519, %f539, %f508;\n" " fma.rn.f32 %f541, %f523, %f539, %f509;\n" " ld.shared.f32 %f542, [%r238+12800];\n" " fma.rn.f32 %f543, %f519, %f542, %f511;\n" " fma.rn.f32 %f544, %f523, %f542, %f512;\n" " ld.shared.f32 %f545, [%r238+14848];\n" " fma.rn.f32 %f546, %f519, %f545, %f514;\n" " fma.rn.f32 %f547, %f523, %f545, %f515;\n" " bfe.u32 %r263, %r226, 8, 8;\n" " cvt.rn.f32.u32 %f548, %r263;\n" " and.b32 %r264, %r229, 240;\n" " or.b32 %r265, %r264, 1258291200;\n" " mov.b32 %f549, %r265;\n" " fma.rn.f32 %f550, %f549, %f417, %f416;\n" " mul.f32 %f551, %f550, %f548;\n" " bfe.u32 %r266, %r228, 8, 8;\n" " cvt.rn.f32.u32 %f552, %r266;\n" " and.b32 %r267, %r230, 240;\n" " or.b32 %r268, %r267, 1258291200;\n" " mov.b32 %f553, %r268;\n" " fma.rn.f32 %f554, %f553, %f417, %f416;\n" " mul.f32 %f555, %f554, %f552;\n" " ld.shared.f32 %f556, [%r238+640];\n" " fma.rn.f32 %f557, %f551, %f556, %f525;\n" " fma.rn.f32 %f558, %f555, %f556, %f526;\n" " ld.shared.f32 %f559, [%r238+2688];\n" " fma.rn.f32 %f560, %f551, %f559, %f528;\n" " fma.rn.f32 %f561, %f555, %f559, %f529;\n" " ld.shared.f32 %f562, [%r238+4736];\n" " fma.rn.f32 %f563, %f551, %f562, %f531;\n" " fma.rn.f32 %f564, %f555, %f562, %f532;\n" " ld.shared.f32 %f565, [%r238+6784];\n" " fma.rn.f32 %f566, %f551, %f565, %f534;\n" " fma.rn.f32 %f567, %f555, %f565, %f535;\n" " ld.shared.f32 %f568, [%r238+8832];\n" " fma.rn.f32 %f569, %f551, %f568, %f537;\n" " fma.rn.f32 %f570, %f555, %f568, %f538;\n" " ld.shared.f32 %f571, [%r238+10880];\n" " fma.rn.f32 %f572, %f551, %f571, %f540;\n" " fma.rn.f32 %f573, %f555, %f571, %f541;\n" " ld.shared.f32 %f574, [%r238+12928];\n" " fma.rn.f32 %f575, %f551, %f574, %f543;\n" " fma.rn.f32 %f576, %f555, %f574, %f544;\n" " ld.shared.f32 %f577, [%r238+14976];\n" " fma.rn.f32 %f578, %f551, %f577, %f546;\n" " fma.rn.f32 %f579, %f555, %f577, %f547;\n" " bfe.u32 %r269, %r226, 16, 8;\n" " cvt.rn.f32.u32 %f580, %r269;\n" " and.b32 %r270, %r229, 3840;\n" " or.b32 %r271, %r270, 1258291200;\n" " mov.b32 %f581, %r271;\n" " fma.rn.f32 %f582, %f581, %f451, %f450;\n" " mul.f32 %f583, %f582, %f580;\n" " bfe.u32 %r272, %r228, 16, 8;\n" " cvt.rn.f32.u32 %f584, %r272;\n" " and.b32 %r273, %r230, 3840;\n" " or.b32 %r274, %r273, 1258291200;\n" " mov.b32 %f585, %r274;\n" " fma.rn.f32 %f586, %f585, %f451, %f450;\n" " mul.f32 %f587, %f586, %f584;\n" " ld.shared.f32 %f588, [%r238+768];\n" " fma.rn.f32 %f589, %f583, %f588, %f557;\n" " fma.rn.f32 %f590, %f587, %f588, %f558;\n" " ld.shared.f32 %f591, [%r238+2816];\n" " fma.rn.f32 %f592, %f583, %f591, %f560;\n" " fma.rn.f32 %f593, %f587, %f591, %f561;\n" " ld.shared.f32 %f594, [%r238+4864];\n" " fma.rn.f32 %f595, %f583, %f594, %f563;\n" " fma.rn.f32 %f596, %f587, %f594, %f564;\n" " ld.shared.f32 %f597, [%r238+6912];\n" " fma.rn.f32 %f598, %f583, %f597, %f566;\n" " fma.rn.f32 %f599, %f587, %f597, %f567;\n" " ld.shared.f32 %f600, [%r238+8960];\n" " fma.rn.f32 %f601, %f583, %f600, %f569;\n" " fma.rn.f32 %f602, %f587, %f600, %f570;\n" " ld.shared.f32 %f603, [%r238+11008];\n" " fma.rn.f32 %f604, %f583, %f603, %f572;\n" " fma.rn.f32 %f605, %f587, %f603, %f573;\n" " ld.shared.f32 %f606, [%r238+13056];\n" " fma.rn.f32 %f607, %f583, %f606, %f575;\n" " fma.rn.f32 %f608, %f587, %f606, %f576;\n" " ld.shared.f32 %f609, [%r238+15104];\n" " fma.rn.f32 %f610, %f583, %f609, %f578;\n" " fma.rn.f32 %f611, %f587, %f609, %f579;\n" " shr.u32 %r275, %r226, 24;\n" " cvt.rn.f32.u32 %f612, %r275;\n" " and.b32 %r276, %r229, 61440;\n" " or.b32 %r277, %r276, 1258291200;\n" " mov.b32 %f613, %r277;\n" " fma.rn.f32 %f614, %f613, %f485, %f484;\n" " mul.f32 %f615, %f614, %f612;\n" " shr.u32 %r278, %r228, 24;\n" " cvt.rn.f32.u32 %f616, %r278;\n" " and.b32 %r279, %r230, 61440;\n" " or.b32 %r280, %r279, 1258291200;\n" " mov.b32 %f617, %r280;\n" " fma.rn.f32 %f618, %f617, %f485, %f484;\n" " mul.f32 %f619, %f618, %f616;\n" " ld.shared.f32 %f620, [%r238+896];\n" " fma.rn.f32 %f621, %f615, %f620, %f589;\n" " fma.rn.f32 %f622, %f619, %f620, %f590;\n" " ld.shared.f32 %f623, [%r238+2944];\n" " fma.rn.f32 %f624, %f615, %f623, %f592;\n" " fma.rn.f32 %f625, %f619, %f623, %f593;\n" " ld.shared.f32 %f626, [%r238+4992];\n" " fma.rn.f32 %f627, %f615, %f626, %f595;\n" " fma.rn.f32 %f628, %f619, %f626, %f596;\n" " ld.shared.f32 %f629, [%r238+7040];\n" " fma.rn.f32 %f630, %f615, %f629, %f598;\n" " fma.rn.f32 %f631, %f619, %f629, %f599;\n" " ld.shared.f32 %f632, [%r238+9088];\n" " fma.rn.f32 %f633, %f615, %f632, %f601;\n" " fma.rn.f32 %f634, %f619, %f632, %f602;\n" " ld.shared.f32 %f635, [%r238+11136];\n" " fma.rn.f32 %f636, %f615, %f635, %f604;\n" " fma.rn.f32 %f637, %f619, %f635, %f605;\n" " ld.shared.f32 %f638, [%r238+13184];\n" " fma.rn.f32 %f639, %f615, %f638, %f607;\n" " fma.rn.f32 %f640, %f619, %f638, %f608;\n" " ld.shared.f32 %f641, [%r238+15232];\n" " fma.rn.f32 %f642, %f615, %f641, %f610;\n" " fma.rn.f32 %f643, %f619, %f641, %f611;\n" " fma.rn.f32 %f906, %f377, %f621, %f906;\n" " fma.rn.f32 %f914, %f378, %f622, %f914;\n" " fma.rn.f32 %f905, %f377, %f624, %f905;\n" " fma.rn.f32 %f913, %f378, %f625, %f913;\n" " fma.rn.f32 %f904, %f377, %f627, %f904;\n" " fma.rn.f32 %f912, %f378, %f628, %f912;\n" " fma.rn.f32 %f903, %f377, %f630, %f903;\n" " fma.rn.f32 %f911, %f378, %f631, %f911;\n" " fma.rn.f32 %f902, %f377, %f633, %f902;\n" " fma.rn.f32 %f910, %f378, %f634, %f910;\n" " fma.rn.f32 %f901, %f377, %f636, %f901;\n" " fma.rn.f32 %f909, %f378, %f637, %f909;\n" " fma.rn.f32 %f900, %f377, %f639, %f900;\n" " fma.rn.f32 %f908, %f378, %f640, %f908;\n" " fma.rn.f32 %f899, %f377, %f642, %f899;\n" " fma.rn.f32 %f907, %f378, %f643, %f907;\n" " add.s64 %rd123, %rd123, 152;\n" " add.s64 %rd124, %rd124, 152;\n" " add.s32 %r501, %r501, 1024;\n" " add.s32 %r500, %r500, 1;\n" " setp.lt.s32 %p26, %r500, %r486;\n" " @%p26 bra $L__BB3_41;\n" "\n" "$L__BB3_42:\n" " add.s32 %r496, %r496, 512;\n" " setp.lt.s32 %p27, %r496, %r45;\n" " @%p27 bra $L__BB3_4;\n" "\n" "$L__BB3_43:\n" " mov.u32 %r466, %ctaid.y;\n" " shl.b32 %r465, %r466, 3;\n" " ld.param.u32 %r464, [fc_matvec_q4k_nx8_param_6];\n" " sub.s32 %r463, %r464, %r465;\n" " sub.rn.f32 %f196, %f905, %f833;\n" " mov.u32 %r299, 1;\n" " sub.rn.f32 %f197, %f913, %f841;\n" " sub.rn.f32 %f198, %f904, %f832;\n" " sub.rn.f32 %f199, %f912, %f840;\n" " sub.rn.f32 %f200, %f903, %f831;\n" " sub.rn.f32 %f201, %f911, %f839;\n" " sub.rn.f32 %f202, %f902, %f830;\n" " sub.rn.f32 %f203, %f910, %f838;\n" " sub.rn.f32 %f204, %f901, %f829;\n" " sub.rn.f32 %f205, %f909, %f837;\n" " sub.rn.f32 %f206, %f900, %f828;\n" " sub.rn.f32 %f207, %f908, %f836;\n" " sub.rn.f32 %f208, %f899, %f827;\n" " sub.rn.f32 %f209, %f907, %f835;\n" " cvt.s64.s32 %rd33, %r44;\n" " sub.rn.f32 %f645, %f906, %f834;\n" " mov.u32 %r291, 16;\n" " mov.u32 %r300, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f644, %f645, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f647, %f645, %f644;\n" " mov.u32 %r293, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f646, %f647, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f649, %f647, %f646;\n" " mov.u32 %r295, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f648, %f649, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f651, %f649, %f648;\n" " mov.u32 %r297, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f650, %f651, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f653, %f651, %f650;\n" " // begin inline asm\n" " shfl.down.b32 %f652, %f653, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f210, %f653, %f652;\n" " sub.rn.f32 %f655, %f914, %f842;\n" " // begin inline asm\n" " shfl.down.b32 %f654, %f655, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f657, %f655, %f654;\n" " // begin inline asm\n" " shfl.down.b32 %f656, %f657, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f659, %f657, %f656;\n" " // begin inline asm\n" " shfl.down.b32 %f658, %f659, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f661, %f659, %f658;\n" " // begin inline asm\n" " shfl.down.b32 %f660, %f661, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f663, %f661, %f660;\n" " // begin inline asm\n" " shfl.down.b32 %f662, %f663, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f211, %f663, %f662;\n" " setp.lt.s32 %p28, %r463, 1;\n" " setp.ne.s32 %p29, %r5, 0;\n" " or.pred %p30, %p29, %p28;\n" " @%p30 bra $L__BB3_48;\n" "\n" " cvt.u32.u64 %r301, %rd2;\n" " setp.ge.s32 %p31, %r301, %r44;\n" " cvt.s64.s32 %rd83, %r1;\n" " mul.lo.s64 %rd84, %rd83, %rd33;\n" " add.s64 %rd85, %rd84, %rd2;\n" " shl.b64 %rd86, %rd85, 2;\n" " add.s64 %rd34, %rd1, %rd86;\n" " @%p31 bra $L__BB3_46;\n" "\n" " st.global.f32 [%rd34], %f210;\n" "\n" "$L__BB3_46:\n" " @%p1 bra $L__BB3_48;\n" "\n" " st.global.f32 [%rd34+4], %f211;\n" "\n" "$L__BB3_48:\n" " // begin inline asm\n" " shfl.down.b32 %f664, %f196, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f667, %f196, %f664;\n" " // begin inline asm\n" " shfl.down.b32 %f666, %f667, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f669, %f667, %f666;\n" " // begin inline asm\n" " shfl.down.b32 %f668, %f669, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f671, %f669, %f668;\n" " // begin inline asm\n" " shfl.down.b32 %f670, %f671, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f673, %f671, %f670;\n" " // begin inline asm\n" " shfl.down.b32 %f672, %f673, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f212, %f673, %f672;\n" " // begin inline asm\n" " shfl.down.b32 %f674, %f197, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f677, %f197, %f674;\n" " // begin inline asm\n" " shfl.down.b32 %f676, %f677, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f679, %f677, %f676;\n" " // begin inline asm\n" " shfl.down.b32 %f678, %f679, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f681, %f679, %f678;\n" " // begin inline asm\n" " shfl.down.b32 %f680, %f681, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f683, %f681, %f680;\n" " // begin inline asm\n" " shfl.down.b32 %f682, %f683, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f213, %f683, %f682;\n" " setp.lt.s32 %p34, %r3, 2;\n" " or.pred %p35, %p29, %p34;\n" " @%p35 bra $L__BB3_53;\n" "\n" " cvt.u32.u64 %r322, %rd2;\n" " setp.ge.s32 %p36, %r322, %r44;\n" " add.s32 %r323, %r1, 1;\n" " cvt.s64.s32 %rd87, %r323;\n" " mul.lo.s64 %rd88, %rd87, %rd33;\n" " add.s64 %rd89, %rd88, %rd2;\n" " shl.b64 %rd90, %rd89, 2;\n" " add.s64 %rd35, %rd1, %rd90;\n" " @%p36 bra $L__BB3_51;\n" "\n" " st.global.f32 [%rd35], %f212;\n" "\n" "$L__BB3_51:\n" " @%p1 bra $L__BB3_53;\n" "\n" " st.global.f32 [%rd35+4], %f213;\n" "\n" "$L__BB3_53:\n" " // begin inline asm\n" " shfl.down.b32 %f684, %f198, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f687, %f198, %f684;\n" " // begin inline asm\n" " shfl.down.b32 %f686, %f687, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f689, %f687, %f686;\n" " // begin inline asm\n" " shfl.down.b32 %f688, %f689, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f691, %f689, %f688;\n" " // begin inline asm\n" " shfl.down.b32 %f690, %f691, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f693, %f691, %f690;\n" " // begin inline asm\n" " shfl.down.b32 %f692, %f693, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f214, %f693, %f692;\n" " // begin inline asm\n" " shfl.down.b32 %f694, %f199, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f697, %f199, %f694;\n" " // begin inline asm\n" " shfl.down.b32 %f696, %f697, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f699, %f697, %f696;\n" " // begin inline asm\n" " shfl.down.b32 %f698, %f699, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f701, %f699, %f698;\n" " // begin inline asm\n" " shfl.down.b32 %f700, %f701, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f703, %f701, %f700;\n" " // begin inline asm\n" " shfl.down.b32 %f702, %f703, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f215, %f703, %f702;\n" " setp.lt.s32 %p39, %r3, 3;\n" " or.pred %p40, %p29, %p39;\n" " @%p40 bra $L__BB3_58;\n" "\n" " cvt.u32.u64 %r344, %rd2;\n" " setp.ge.s32 %p41, %r344, %r44;\n" " add.s32 %r345, %r1, 2;\n" " cvt.s64.s32 %rd91, %r345;\n" " mul.lo.s64 %rd92, %rd91, %rd33;\n" " add.s64 %rd93, %rd92, %rd2;\n" " shl.b64 %rd94, %rd93, 2;\n" " add.s64 %rd36, %rd1, %rd94;\n" " @%p41 bra $L__BB3_56;\n" "\n" " st.global.f32 [%rd36], %f214;\n" "\n" "$L__BB3_56:\n" " @%p1 bra $L__BB3_58;\n" "\n" " st.global.f32 [%rd36+4], %f215;\n" "\n" "$L__BB3_58:\n" " // begin inline asm\n" " shfl.down.b32 %f704, %f200, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f707, %f200, %f704;\n" " // begin inline asm\n" " shfl.down.b32 %f706, %f707, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f709, %f707, %f706;\n" " // begin inline asm\n" " shfl.down.b32 %f708, %f709, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f711, %f709, %f708;\n" " // begin inline asm\n" " shfl.down.b32 %f710, %f711, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f713, %f711, %f710;\n" " // begin inline asm\n" " shfl.down.b32 %f712, %f713, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f216, %f713, %f712;\n" " // begin inline asm\n" " shfl.down.b32 %f714, %f201, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f717, %f201, %f714;\n" " // begin inline asm\n" " shfl.down.b32 %f716, %f717, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f719, %f717, %f716;\n" " // begin inline asm\n" " shfl.down.b32 %f718, %f719, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f721, %f719, %f718;\n" " // begin inline asm\n" " shfl.down.b32 %f720, %f721, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f723, %f721, %f720;\n" " // begin inline asm\n" " shfl.down.b32 %f722, %f723, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f217, %f723, %f722;\n" " setp.lt.s32 %p44, %r3, 4;\n" " or.pred %p45, %p29, %p44;\n" " @%p45 bra $L__BB3_63;\n" "\n" " cvt.u32.u64 %r366, %rd2;\n" " setp.ge.s32 %p46, %r366, %r44;\n" " add.s32 %r367, %r1, 3;\n" " cvt.s64.s32 %rd95, %r367;\n" " mul.lo.s64 %rd96, %rd95, %rd33;\n" " add.s64 %rd97, %rd96, %rd2;\n" " shl.b64 %rd98, %rd97, 2;\n" " add.s64 %rd37, %rd1, %rd98;\n" " @%p46 bra $L__BB3_61;\n" "\n" " st.global.f32 [%rd37], %f216;\n" "\n" "$L__BB3_61:\n" " @%p1 bra $L__BB3_63;\n" "\n" " st.global.f32 [%rd37+4], %f217;\n" "\n" "$L__BB3_63:\n" " // begin inline asm\n" " shfl.down.b32 %f724, %f202, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f727, %f202, %f724;\n" " // begin inline asm\n" " shfl.down.b32 %f726, %f727, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f729, %f727, %f726;\n" " // begin inline asm\n" " shfl.down.b32 %f728, %f729, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f731, %f729, %f728;\n" " // begin inline asm\n" " shfl.down.b32 %f730, %f731, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f733, %f731, %f730;\n" " // begin inline asm\n" " shfl.down.b32 %f732, %f733, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f218, %f733, %f732;\n" " // begin inline asm\n" " shfl.down.b32 %f734, %f203, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f737, %f203, %f734;\n" " // begin inline asm\n" " shfl.down.b32 %f736, %f737, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f739, %f737, %f736;\n" " // begin inline asm\n" " shfl.down.b32 %f738, %f739, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f741, %f739, %f738;\n" " // begin inline asm\n" " shfl.down.b32 %f740, %f741, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f743, %f741, %f740;\n" " // begin inline asm\n" " shfl.down.b32 %f742, %f743, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f219, %f743, %f742;\n" " setp.lt.s32 %p49, %r3, 5;\n" " or.pred %p50, %p29, %p49;\n" " @%p50 bra $L__BB3_68;\n" "\n" " cvt.u32.u64 %r388, %rd2;\n" " setp.ge.s32 %p51, %r388, %r44;\n" " add.s32 %r389, %r1, 4;\n" " cvt.s64.s32 %rd99, %r389;\n" " mul.lo.s64 %rd100, %rd99, %rd33;\n" " add.s64 %rd101, %rd100, %rd2;\n" " shl.b64 %rd102, %rd101, 2;\n" " add.s64 %rd38, %rd1, %rd102;\n" " @%p51 bra $L__BB3_66;\n" "\n" " st.global.f32 [%rd38], %f218;\n" "\n" "$L__BB3_66:\n" " @%p1 bra $L__BB3_68;\n" "\n" " st.global.f32 [%rd38+4], %f219;\n" "\n" "$L__BB3_68:\n" " // begin inline asm\n" " shfl.down.b32 %f744, %f204, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f747, %f204, %f744;\n" " // begin inline asm\n" " shfl.down.b32 %f746, %f747, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f749, %f747, %f746;\n" " // begin inline asm\n" " shfl.down.b32 %f748, %f749, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f751, %f749, %f748;\n" " // begin inline asm\n" " shfl.down.b32 %f750, %f751, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f753, %f751, %f750;\n" " // begin inline asm\n" " shfl.down.b32 %f752, %f753, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f220, %f753, %f752;\n" " // begin inline asm\n" " shfl.down.b32 %f754, %f205, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f757, %f205, %f754;\n" " // begin inline asm\n" " shfl.down.b32 %f756, %f757, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f759, %f757, %f756;\n" " // begin inline asm\n" " shfl.down.b32 %f758, %f759, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f761, %f759, %f758;\n" " // begin inline asm\n" " shfl.down.b32 %f760, %f761, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f763, %f761, %f760;\n" " // begin inline asm\n" " shfl.down.b32 %f762, %f763, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f221, %f763, %f762;\n" " setp.lt.s32 %p54, %r3, 6;\n" " or.pred %p55, %p29, %p54;\n" " @%p55 bra $L__BB3_73;\n" "\n" " cvt.u32.u64 %r410, %rd2;\n" " setp.ge.s32 %p56, %r410, %r44;\n" " add.s32 %r411, %r1, 5;\n" " cvt.s64.s32 %rd103, %r411;\n" " mul.lo.s64 %rd104, %rd103, %rd33;\n" " add.s64 %rd105, %rd104, %rd2;\n" " shl.b64 %rd106, %rd105, 2;\n" " add.s64 %rd39, %rd1, %rd106;\n" " @%p56 bra $L__BB3_71;\n" "\n" " st.global.f32 [%rd39], %f220;\n" "\n" "$L__BB3_71:\n" " @%p1 bra $L__BB3_73;\n" "\n" " st.global.f32 [%rd39+4], %f221;\n" "\n" "$L__BB3_73:\n" " // begin inline asm\n" " shfl.down.b32 %f764, %f206, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f767, %f206, %f764;\n" " // begin inline asm\n" " shfl.down.b32 %f766, %f767, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f769, %f767, %f766;\n" " // begin inline asm\n" " shfl.down.b32 %f768, %f769, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f771, %f769, %f768;\n" " // begin inline asm\n" " shfl.down.b32 %f770, %f771, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f773, %f771, %f770;\n" " // begin inline asm\n" " shfl.down.b32 %f772, %f773, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f222, %f773, %f772;\n" " // begin inline asm\n" " shfl.down.b32 %f774, %f207, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f777, %f207, %f774;\n" " // begin inline asm\n" " shfl.down.b32 %f776, %f777, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f779, %f777, %f776;\n" " // begin inline asm\n" " shfl.down.b32 %f778, %f779, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f781, %f779, %f778;\n" " // begin inline asm\n" " shfl.down.b32 %f780, %f781, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f783, %f781, %f780;\n" " // begin inline asm\n" " shfl.down.b32 %f782, %f783, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f223, %f783, %f782;\n" " setp.lt.s32 %p59, %r3, 7;\n" " or.pred %p60, %p29, %p59;\n" " @%p60 bra $L__BB3_78;\n" "\n" " cvt.u32.u64 %r432, %rd2;\n" " setp.ge.s32 %p61, %r432, %r44;\n" " add.s32 %r433, %r1, 6;\n" " cvt.s64.s32 %rd107, %r433;\n" " mul.lo.s64 %rd108, %rd107, %rd33;\n" " add.s64 %rd109, %rd108, %rd2;\n" " shl.b64 %rd110, %rd109, 2;\n" " add.s64 %rd40, %rd1, %rd110;\n" " @%p61 bra $L__BB3_76;\n" "\n" " st.global.f32 [%rd40], %f222;\n" "\n" "$L__BB3_76:\n" " @%p1 bra $L__BB3_78;\n" "\n" " st.global.f32 [%rd40+4], %f223;\n" "\n" "$L__BB3_78:\n" " // begin inline asm\n" " shfl.down.b32 %f784, %f208, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f787, %f208, %f784;\n" " // begin inline asm\n" " shfl.down.b32 %f786, %f787, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f789, %f787, %f786;\n" " // begin inline asm\n" " shfl.down.b32 %f788, %f789, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f791, %f789, %f788;\n" " // begin inline asm\n" " shfl.down.b32 %f790, %f791, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f793, %f791, %f790;\n" " // begin inline asm\n" " shfl.down.b32 %f792, %f793, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f224, %f793, %f792;\n" " // begin inline asm\n" " shfl.down.b32 %f794, %f209, %r291, %r300;\n" " // end inline asm\n" " add.f32 %f797, %f209, %f794;\n" " // begin inline asm\n" " shfl.down.b32 %f796, %f797, %r293, %r300;\n" " // end inline asm\n" " add.f32 %f799, %f797, %f796;\n" " // begin inline asm\n" " shfl.down.b32 %f798, %f799, %r295, %r300;\n" " // end inline asm\n" " add.f32 %f801, %f799, %f798;\n" " // begin inline asm\n" " shfl.down.b32 %f800, %f801, %r297, %r300;\n" " // end inline asm\n" " add.f32 %f803, %f801, %f800;\n" " // begin inline asm\n" " shfl.down.b32 %f802, %f803, %r299, %r300;\n" " // end inline asm\n" " add.f32 %f225, %f803, %f802;\n" " setp.lt.s32 %p64, %r3, 8;\n" " or.pred %p65, %p29, %p64;\n" " @%p65 bra $L__BB3_83;\n" "\n" " cvt.u32.u64 %r454, %rd2;\n" " setp.ge.s32 %p66, %r454, %r44;\n" " add.s32 %r455, %r1, 7;\n" " cvt.s64.s32 %rd111, %r455;\n" " mul.lo.s64 %rd112, %rd111, %rd33;\n" " add.s64 %rd113, %rd112, %rd2;\n" " shl.b64 %rd114, %rd113, 2;\n" " add.s64 %rd41, %rd1, %rd114;\n" " @%p66 bra $L__BB3_81;\n" "\n" " st.global.f32 [%rd41], %f224;\n" "\n" "$L__BB3_81:\n" " @%p1 bra $L__BB3_83;\n" "\n" " st.global.f32 [%rd41+4], %f225;\n" "\n" "$L__BB3_83:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q5k\n" ".visible .entry fc_matvec_q5k(\n" " .param .u64 fc_matvec_q5k_param_0,\n" " .param .u64 fc_matvec_q5k_param_1,\n" " .param .u64 fc_matvec_q5k_param_2,\n" " .param .u32 fc_matvec_q5k_param_3,\n" " .param .u32 fc_matvec_q5k_param_4,\n" " .param .u32 fc_matvec_q5k_param_5\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<285>;\n" " .reg .b32 %r<201>;\n" " .reg .b64 %rd<76>;\n" "\n" "\n" " ld.param.u64 %rd32, [fc_matvec_q5k_param_0];\n" " ld.param.u64 %rd33, [fc_matvec_q5k_param_1];\n" " ld.param.u64 %rd34, [fc_matvec_q5k_param_2];\n" " ld.param.u32 %r28, [fc_matvec_q5k_param_3];\n" " ld.param.u32 %r29, [fc_matvec_q5k_param_4];\n" " ld.param.u32 %r30, [fc_matvec_q5k_param_5];\n" " mov.u32 %r1, %tid.x;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r31, %ctaid.x;\n" " shl.b32 %r32, %r31, 4;\n" " shr.u32 %r33, %r1, 4;\n" " and.b32 %r34, %r33, 268435454;\n" " add.s32 %r35, %r34, %r32;\n" " or.b32 %r3, %r35, 1;\n" " cvt.s64.s32 %rd1, %r35;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f281, 0f00000000;\n" " mov.f32 %f282, %f281;\n" " @%p1 bra $L__BB4_15;\n" "\n" " setp.lt.s32 %p2, %r3, %r28;\n" " cvt.s64.s32 %rd35, %r30;\n" " mul.lo.s64 %rd36, %rd35, %rd1;\n" " cvta.to.global.u64 %rd37, %rd32;\n" " add.s64 %rd72, %rd37, %rd36;\n" " add.s64 %rd73, %rd32, %rd36;\n" " cvt.u64.u32 %rd4, %r2;\n" " or.b32 %r38, %r2, 128;\n" " cvt.u64.u32 %rd5, %r38;\n" " mov.u32 %r39, -2;\n" " sub.s32 %r4, %r39, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd6, %r1;\n" " shl.b32 %r40, %r1, 2;\n" " mov.u32 %r41, fc_kq_shared;\n" " add.s32 %r6, %r41, %r40;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd7, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd8, %r8;\n" " add.s32 %r9, %r1, 768;\n" " add.s64 %rd38, %rd73, %rd35;\n" " selp.b64 %rd75, %rd38, %rd73, %p2;\n" " add.s64 %rd39, %rd72, %rd35;\n" " selp.b64 %rd74, %rd39, %rd72, %p2;\n" " mov.f32 %f282, 0f00000000;\n" " mov.u32 %r194, 0;\n" " shl.b64 %rd60, %rd4, 2;\n" " mov.u32 %r195, %r194;\n" " mov.f32 %f281, %f282;\n" "\n" "$L__BB4_2:\n" " add.s32 %r42, %r194, %r5;\n" " max.s32 %r43, %r42, -2049;\n" " sub.s32 %r12, %r4, %r43;\n" " sub.s32 %r44, %r29, %r195;\n" " min.s32 %r13, %r44, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r13;\n" " @%p3 bra $L__BB4_10;\n" "\n" " shr.u32 %r45, %r12, 8;\n" " add.s32 %r46, %r45, 1;\n" " cvt.s64.s32 %rd15, %r195;\n" " and.b32 %r14, %r46, 3;\n" " setp.eq.s32 %p4, %r14, 0;\n" " mov.u32 %r196, %r1;\n" " @%p4 bra $L__BB4_7;\n" "\n" " add.s64 %rd41, %rd6, %rd15;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd40, %rd33, %rd42;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd40];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p5, %r14, 1;\n" " mov.u32 %r196, %r7;\n" " @%p5 bra $L__BB4_7;\n" "\n" " add.s64 %rd44, %rd7, %rd15;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd43, %rd33, %rd45;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd43];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p6, %r14, 2;\n" " mov.u32 %r196, %r8;\n" " @%p6 bra $L__BB4_7;\n" "\n" " add.s64 %rd47, %rd8, %rd15;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd46, %rd33, %rd48;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd46];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r196, %r9;\n" "\n" "$L__BB4_7:\n" " setp.lt.u32 %p7, %r12, 768;\n" " @%p7 bra $L__BB4_10;\n" "\n" " cvt.s64.s32 %rd49, %r196;\n" " add.s64 %rd50, %rd49, %rd15;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd67, %rd33, %rd51;\n" " shl.b32 %r47, %r196, 2;\n" " mov.u32 %r48, fc_kq_shared;\n" " add.s32 %r49, %r48, %r47;\n" " add.s32 %r197, %r49, 2048;\n" "\n" "$L__BB4_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd67];\n" " // end inline asm\n" " st.shared.f32 [%r197+-2048], %f20;\n" " add.s64 %rd53, %rd67, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd53];\n" " // end inline asm\n" " st.shared.f32 [%r197+-1024], %f21;\n" " add.s64 %rd54, %rd67, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd54];\n" " // end inline asm\n" " st.shared.f32 [%r197], %f22;\n" " add.s64 %rd55, %rd67, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd55];\n" " // end inline asm\n" " st.shared.f32 [%r197+1024], %f23;\n" " add.s64 %rd67, %rd67, 4096;\n" " add.s32 %r197, %r197, 4096;\n" " add.s32 %r196, %r196, 1024;\n" " setp.lt.s32 %p8, %r196, %r13;\n" " @%p8 bra $L__BB4_9;\n" "\n" "$L__BB4_10:\n" " cvt.u32.u64 %r50, %rd1;\n" " setp.ge.s32 %p9, %r50, %r28;\n" " bar.sync 0;\n" " @%p9 bra $L__BB4_14;\n" "\n" " setp.lt.s32 %p10, %r13, 256;\n" " @%p10 bra $L__BB4_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r199, 0;\n" " mov.u32 %r200, %r41;\n" "\n" "$L__BB4_13:\n" " add.s64 %rd56, %rd73, %rd60;\n" " // begin inline asm\n" " ld.global.nc.u32 %r53, [%rd56];\n" " // end inline asm\n" " add.s64 %rd57, %rd75, %rd60;\n" " // begin inline asm\n" " ld.global.nc.u32 %r54, [%rd57];\n" " // end inline asm\n" " add.s64 %rd58, %rd73, %rd5;\n" " // begin inline asm\n" " ld.global.nc.u8 %r55, [%rd58];\n" " // end inline asm\n" " add.s64 %rd59, %rd75, %rd5;\n" " // begin inline asm\n" " ld.global.nc.u8 %r56, [%rd59];\n" " // end inline asm\n" " ld.global.v2.u32 {%r57, %r58}, [%rd72+160];\n" " ld.global.v2.u32 {%r61, %r62}, [%rd74+160];\n" " ld.global.v2.u32 {%r65, %r66}, [%rd72+168];\n" " ld.global.v2.u32 {%r69, %r70}, [%rd74+168];\n" " ld.global.v2.f32 {%f24, %f25}, [%rd72+176];\n" " ld.global.v2.f32 {%f28, %f29}, [%rd74+176];\n" " shr.u32 %r73, %r53, 16;\n" " shr.u32 %r74, %r54, 16;\n" " and.b32 %r75, %r53, 15;\n" " or.b32 %r76, %r75, 1258291200;\n" " mov.b32 %f32, %r76;\n" " mov.f32 %f33, 0fCB000000;\n" " mov.f32 %f34, 0f3F800000;\n" " fma.rn.f32 %f35, %f32, %f34, %f33;\n" " and.b32 %r77, %r55, 1;\n" " or.b32 %r78, %r77, 1258291200;\n" " mov.b32 %f36, %r78;\n" " mov.f32 %f37, 0fCD000000;\n" " mov.f32 %f38, 0f41800000;\n" " fma.rn.f32 %f39, %f36, %f38, %f37;\n" " add.f32 %f40, %f35, %f39;\n" " and.b32 %r79, %r54, 15;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f41, %r80;\n" " fma.rn.f32 %f42, %f41, %f34, %f33;\n" " and.b32 %r81, %r56, 1;\n" " or.b32 %r82, %r81, 1258291200;\n" " mov.b32 %f43, %r82;\n" " fma.rn.f32 %f44, %f43, %f38, %f37;\n" " add.f32 %f45, %f42, %f44;\n" " and.b32 %r83, %r57, 255;\n" " cvt.rn.f32.u32 %f46, %r83;\n" " mul.rn.f32 %f47, %f24, %f46;\n" " mul.rn.f32 %f48, %f47, %f40;\n" " and.b32 %r84, %r65, 255;\n" " cvt.rn.f32.u32 %f49, %r84;\n" " mul.rn.f32 %f50, %f25, %f49;\n" " sub.rn.f32 %f51, %f48, %f50;\n" " and.b32 %r85, %r61, 255;\n" " cvt.rn.f32.u32 %f52, %r85;\n" " mul.rn.f32 %f53, %f28, %f52;\n" " mul.rn.f32 %f54, %f53, %f45;\n" " and.b32 %r86, %r69, 255;\n" " cvt.rn.f32.u32 %f55, %r86;\n" " mul.rn.f32 %f56, %f29, %f55;\n" " sub.rn.f32 %f57, %f54, %f56;\n" " shl.b32 %r87, %r2, 2;\n" " add.s32 %r88, %r200, %r87;\n" " ld.shared.f32 %f58, [%r88];\n" " mov.f32 %f59, 0f00000000;\n" " fma.rn.f32 %f60, %f51, %f58, %f59;\n" " fma.rn.f32 %f61, %f57, %f58, %f59;\n" " and.b32 %r89, %r53, 240;\n" " or.b32 %r90, %r89, 1258291200;\n" " mov.b32 %f62, %r90;\n" " mov.f32 %f63, 0fC9000000;\n" " mov.f32 %f64, 0f3D800000;\n" " fma.rn.f32 %f65, %f62, %f64, %f63;\n" " and.b32 %r91, %r55, 2;\n" " or.b32 %r92, %r91, 1258291200;\n" " mov.b32 %f66, %r92;\n" " mov.f32 %f67, 0fCC800000;\n" " mov.f32 %f68, 0f41000000;\n" " fma.rn.f32 %f69, %f66, %f68, %f67;\n" " add.f32 %f70, %f65, %f69;\n" " and.b32 %r93, %r54, 240;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f71, %r94;\n" " fma.rn.f32 %f72, %f71, %f64, %f63;\n" " and.b32 %r95, %r56, 2;\n" " or.b32 %r96, %r95, 1258291200;\n" " mov.b32 %f73, %r96;\n" " fma.rn.f32 %f74, %f73, %f68, %f67;\n" " add.f32 %f75, %f72, %f74;\n" " bfe.u32 %r97, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f76, %r97;\n" " mul.rn.f32 %f77, %f24, %f76;\n" " mul.rn.f32 %f78, %f77, %f70;\n" " bfe.u32 %r98, %r65, 8, 8;\n" " cvt.rn.f32.u32 %f79, %r98;\n" " mul.rn.f32 %f80, %f25, %f79;\n" " sub.rn.f32 %f81, %f78, %f80;\n" " bfe.u32 %r99, %r61, 8, 8;\n" " cvt.rn.f32.u32 %f82, %r99;\n" " mul.rn.f32 %f83, %f28, %f82;\n" " mul.rn.f32 %f84, %f83, %f75;\n" " bfe.u32 %r100, %r69, 8, 8;\n" " cvt.rn.f32.u32 %f85, %r100;\n" " mul.rn.f32 %f86, %f29, %f85;\n" " sub.rn.f32 %f87, %f84, %f86;\n" " ld.shared.f32 %f88, [%r88+128];\n" " fma.rn.f32 %f89, %f81, %f88, %f60;\n" " fma.rn.f32 %f90, %f87, %f88, %f61;\n" " and.b32 %r101, %r53, 3840;\n" " or.b32 %r102, %r101, 1258291200;\n" " mov.b32 %f91, %r102;\n" " mov.f32 %f92, 0fC7000000;\n" " mov.f32 %f93, 0f3B800000;\n" " fma.rn.f32 %f94, %f91, %f93, %f92;\n" " and.b32 %r103, %r55, 4;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f95, %r104;\n" " mov.f32 %f96, 0fCC000000;\n" " mov.f32 %f97, 0f40800000;\n" " fma.rn.f32 %f98, %f95, %f97, %f96;\n" " add.f32 %f99, %f94, %f98;\n" " and.b32 %r105, %r54, 3840;\n" " or.b32 %r106, %r105, 1258291200;\n" " mov.b32 %f100, %r106;\n" " fma.rn.f32 %f101, %f100, %f93, %f92;\n" " and.b32 %r107, %r56, 4;\n" " or.b32 %r108, %r107, 1258291200;\n" " mov.b32 %f102, %r108;\n" " fma.rn.f32 %f103, %f102, %f97, %f96;\n" " add.f32 %f104, %f101, %f103;\n" " bfe.u32 %r109, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f105, %r109;\n" " mul.rn.f32 %f106, %f24, %f105;\n" " mul.rn.f32 %f107, %f106, %f99;\n" " bfe.u32 %r110, %r65, 16, 8;\n" " cvt.rn.f32.u32 %f108, %r110;\n" " mul.rn.f32 %f109, %f25, %f108;\n" " sub.rn.f32 %f110, %f107, %f109;\n" " bfe.u32 %r111, %r61, 16, 8;\n" " cvt.rn.f32.u32 %f111, %r111;\n" " mul.rn.f32 %f112, %f28, %f111;\n" " mul.rn.f32 %f113, %f112, %f104;\n" " bfe.u32 %r112, %r69, 16, 8;\n" " cvt.rn.f32.u32 %f114, %r112;\n" " mul.rn.f32 %f115, %f29, %f114;\n" " sub.rn.f32 %f116, %f113, %f115;\n" " ld.shared.f32 %f117, [%r88+256];\n" " fma.rn.f32 %f118, %f110, %f117, %f89;\n" " fma.rn.f32 %f119, %f116, %f117, %f90;\n" " and.b32 %r113, %r53, 61440;\n" " or.b32 %r114, %r113, 1258291200;\n" " mov.b32 %f120, %r114;\n" " mov.f32 %f121, 0fC5000000;\n" " mov.f32 %f122, 0f39800000;\n" " fma.rn.f32 %f123, %f120, %f122, %f121;\n" " and.b32 %r115, %r55, 8;\n" " or.b32 %r116, %r115, 1258291200;\n" " mov.b32 %f124, %r116;\n" " mov.f32 %f125, 0fCB800000;\n" " mov.f32 %f126, 0f40000000;\n" " fma.rn.f32 %f127, %f124, %f126, %f125;\n" " add.f32 %f128, %f123, %f127;\n" " and.b32 %r117, %r54, 61440;\n" " or.b32 %r118, %r117, 1258291200;\n" " mov.b32 %f129, %r118;\n" " fma.rn.f32 %f130, %f129, %f122, %f121;\n" " and.b32 %r119, %r56, 8;\n" " or.b32 %r120, %r119, 1258291200;\n" " mov.b32 %f131, %r120;\n" " fma.rn.f32 %f132, %f131, %f126, %f125;\n" " add.f32 %f133, %f130, %f132;\n" " shr.u32 %r121, %r57, 24;\n" " cvt.rn.f32.u32 %f134, %r121;\n" " mul.rn.f32 %f135, %f24, %f134;\n" " mul.rn.f32 %f136, %f135, %f128;\n" " shr.u32 %r122, %r65, 24;\n" " cvt.rn.f32.u32 %f137, %r122;\n" " mul.rn.f32 %f138, %f25, %f137;\n" " sub.rn.f32 %f139, %f136, %f138;\n" " shr.u32 %r123, %r61, 24;\n" " cvt.rn.f32.u32 %f140, %r123;\n" " mul.rn.f32 %f141, %f28, %f140;\n" " mul.rn.f32 %f142, %f141, %f133;\n" " shr.u32 %r124, %r69, 24;\n" " cvt.rn.f32.u32 %f143, %r124;\n" " mul.rn.f32 %f144, %f29, %f143;\n" " sub.rn.f32 %f145, %f142, %f144;\n" " ld.shared.f32 %f146, [%r88+384];\n" " fma.rn.f32 %f147, %f139, %f146, %f118;\n" " fma.rn.f32 %f148, %f145, %f146, %f119;\n" " and.b32 %r125, %r53, 983040;\n" " or.b32 %r126, %r125, 1258291200;\n" " mov.b32 %f149, %r126;\n" " mov.f32 %f150, 0fC3000000;\n" " mov.f32 %f151, 0f37800000;\n" " fma.rn.f32 %f152, %f149, %f151, %f150;\n" " and.b32 %r127, %r55, 16;\n" " or.b32 %r128, %r127, 1258291200;\n" " mov.b32 %f153, %r128;\n" " fma.rn.f32 %f154, %f153, %f34, %f33;\n" " add.f32 %f155, %f152, %f154;\n" " and.b32 %r129, %r54, 983040;\n" " or.b32 %r130, %r129, 1258291200;\n" " mov.b32 %f156, %r130;\n" " fma.rn.f32 %f157, %f156, %f151, %f150;\n" " and.b32 %r131, %r56, 16;\n" " or.b32 %r132, %r131, 1258291200;\n" " mov.b32 %f158, %r132;\n" " fma.rn.f32 %f159, %f158, %f34, %f33;\n" " add.f32 %f160, %f157, %f159;\n" " and.b32 %r133, %r58, 255;\n" " cvt.rn.f32.u32 %f161, %r133;\n" " mul.rn.f32 %f162, %f24, %f161;\n" " mul.rn.f32 %f163, %f162, %f155;\n" " and.b32 %r134, %r66, 255;\n" " cvt.rn.f32.u32 %f164, %r134;\n" " mul.rn.f32 %f165, %f25, %f164;\n" " sub.rn.f32 %f166, %f163, %f165;\n" " and.b32 %r135, %r62, 255;\n" " cvt.rn.f32.u32 %f167, %r135;\n" " mul.rn.f32 %f168, %f28, %f167;\n" " mul.rn.f32 %f169, %f168, %f160;\n" " and.b32 %r136, %r70, 255;\n" " cvt.rn.f32.u32 %f170, %r136;\n" " mul.rn.f32 %f171, %f29, %f170;\n" " sub.rn.f32 %f172, %f169, %f171;\n" " ld.shared.f32 %f173, [%r88+512];\n" " fma.rn.f32 %f174, %f166, %f173, %f147;\n" " fma.rn.f32 %f175, %f172, %f173, %f148;\n" " and.b32 %r137, %r73, 240;\n" " or.b32 %r138, %r137, 1258291200;\n" " mov.b32 %f176, %r138;\n" " fma.rn.f32 %f177, %f176, %f64, %f63;\n" " and.b32 %r139, %r55, 32;\n" " or.b32 %r140, %r139, 1258291200;\n" " mov.b32 %f178, %r140;\n" " mov.f32 %f179, 0fCA800000;\n" " mov.f32 %f180, 0f3F000000;\n" " fma.rn.f32 %f181, %f178, %f180, %f179;\n" " add.f32 %f182, %f177, %f181;\n" " and.b32 %r141, %r74, 240;\n" " or.b32 %r142, %r141, 1258291200;\n" " mov.b32 %f183, %r142;\n" " fma.rn.f32 %f184, %f183, %f64, %f63;\n" " and.b32 %r143, %r56, 32;\n" " or.b32 %r144, %r143, 1258291200;\n" " mov.b32 %f185, %r144;\n" " fma.rn.f32 %f186, %f185, %f180, %f179;\n" " add.f32 %f187, %f184, %f186;\n" " bfe.u32 %r145, %r58, 8, 8;\n" " cvt.rn.f32.u32 %f188, %r145;\n" " mul.rn.f32 %f189, %f24, %f188;\n" " mul.rn.f32 %f190, %f189, %f182;\n" " bfe.u32 %r146, %r66, 8, 8;\n" " cvt.rn.f32.u32 %f191, %r146;\n" " mul.rn.f32 %f192, %f25, %f191;\n" " sub.rn.f32 %f193, %f190, %f192;\n" " bfe.u32 %r147, %r62, 8, 8;\n" " cvt.rn.f32.u32 %f194, %r147;\n" " mul.rn.f32 %f195, %f28, %f194;\n" " mul.rn.f32 %f196, %f195, %f187;\n" " bfe.u32 %r148, %r70, 8, 8;\n" " cvt.rn.f32.u32 %f197, %r148;\n" " mul.rn.f32 %f198, %f29, %f197;\n" " sub.rn.f32 %f199, %f196, %f198;\n" " ld.shared.f32 %f200, [%r88+640];\n" " fma.rn.f32 %f201, %f193, %f200, %f174;\n" " fma.rn.f32 %f202, %f199, %f200, %f175;\n" " and.b32 %r149, %r73, 3840;\n" " or.b32 %r150, %r149, 1258291200;\n" " mov.b32 %f203, %r150;\n" " fma.rn.f32 %f204, %f203, %f93, %f92;\n" " and.b32 %r151, %r55, 64;\n" " or.b32 %r152, %r151, 1258291200;\n" " mov.b32 %f205, %r152;\n" " mov.f32 %f206, 0fCA000000;\n" " mov.f32 %f207, 0f3E800000;\n" " fma.rn.f32 %f208, %f205, %f207, %f206;\n" " add.f32 %f209, %f204, %f208;\n" " and.b32 %r153, %r74, 3840;\n" " or.b32 %r154, %r153, 1258291200;\n" " mov.b32 %f210, %r154;\n" " fma.rn.f32 %f211, %f210, %f93, %f92;\n" " and.b32 %r155, %r56, 64;\n" " or.b32 %r156, %r155, 1258291200;\n" " mov.b32 %f212, %r156;\n" " fma.rn.f32 %f213, %f212, %f207, %f206;\n" " add.f32 %f214, %f211, %f213;\n" " bfe.u32 %r157, %r58, 16, 8;\n" " cvt.rn.f32.u32 %f215, %r157;\n" " mul.rn.f32 %f216, %f24, %f215;\n" " mul.rn.f32 %f217, %f216, %f209;\n" " bfe.u32 %r158, %r66, 16, 8;\n" " cvt.rn.f32.u32 %f218, %r158;\n" " mul.rn.f32 %f219, %f25, %f218;\n" " sub.rn.f32 %f220, %f217, %f219;\n" " bfe.u32 %r159, %r62, 16, 8;\n" " cvt.rn.f32.u32 %f221, %r159;\n" " mul.rn.f32 %f222, %f28, %f221;\n" " mul.rn.f32 %f223, %f222, %f214;\n" " bfe.u32 %r160, %r70, 16, 8;\n" " cvt.rn.f32.u32 %f224, %r160;\n" " mul.rn.f32 %f225, %f29, %f224;\n" " sub.rn.f32 %f226, %f223, %f225;\n" " ld.shared.f32 %f227, [%r88+768];\n" " fma.rn.f32 %f228, %f220, %f227, %f201;\n" " fma.rn.f32 %f229, %f226, %f227, %f202;\n" " and.b32 %r161, %r73, 61440;\n" " or.b32 %r162, %r161, 1258291200;\n" " mov.b32 %f230, %r162;\n" " fma.rn.f32 %f231, %f230, %f122, %f121;\n" " and.b32 %r163, %r55, 128;\n" " or.b32 %r164, %r163, 1258291200;\n" " mov.b32 %f232, %r164;\n" " mov.f32 %f233, 0fC9800000;\n" " mov.f32 %f234, 0f3E000000;\n" " fma.rn.f32 %f235, %f232, %f234, %f233;\n" " add.f32 %f236, %f231, %f235;\n" " and.b32 %r165, %r74, 61440;\n" " or.b32 %r166, %r165, 1258291200;\n" " mov.b32 %f237, %r166;\n" " fma.rn.f32 %f238, %f237, %f122, %f121;\n" " and.b32 %r167, %r56, 128;\n" " or.b32 %r168, %r167, 1258291200;\n" " mov.b32 %f239, %r168;\n" " fma.rn.f32 %f240, %f239, %f234, %f233;\n" " add.f32 %f241, %f238, %f240;\n" " shr.u32 %r169, %r58, 24;\n" " cvt.rn.f32.u32 %f242, %r169;\n" " mul.rn.f32 %f243, %f24, %f242;\n" " mul.rn.f32 %f244, %f243, %f236;\n" " shr.u32 %r170, %r66, 24;\n" " cvt.rn.f32.u32 %f245, %r170;\n" " mul.rn.f32 %f246, %f25, %f245;\n" " sub.rn.f32 %f247, %f244, %f246;\n" " shr.u32 %r171, %r62, 24;\n" " cvt.rn.f32.u32 %f248, %r171;\n" " mul.rn.f32 %f249, %f28, %f248;\n" " mul.rn.f32 %f250, %f249, %f241;\n" " shr.u32 %r172, %r70, 24;\n" " cvt.rn.f32.u32 %f251, %r172;\n" " mul.rn.f32 %f252, %f29, %f251;\n" " sub.rn.f32 %f253, %f250, %f252;\n" " ld.shared.f32 %f254, [%r88+896];\n" " fma.rn.f32 %f255, %f247, %f254, %f228;\n" " fma.rn.f32 %f256, %f253, %f254, %f229;\n" " add.rn.f32 %f281, %f281, %f255;\n" " add.rn.f32 %f282, %f282, %f256;\n" " add.s64 %rd72, %rd72, 184;\n" " add.s64 %rd73, %rd73, 184;\n" " add.s64 %rd74, %rd74, 184;\n" " add.s64 %rd75, %rd75, 184;\n" " add.s32 %r200, %r200, 1024;\n" " add.s32 %r199, %r199, 1;\n" " setp.lt.s32 %p11, %r199, %r21;\n" " @%p11 bra $L__BB4_13;\n" "\n" "$L__BB4_14:\n" " add.s32 %r195, %r195, 2048;\n" " setp.lt.s32 %p12, %r195, %r29;\n" " add.s32 %r194, %r194, 2048;\n" " @%p12 bra $L__BB4_2;\n" "\n" "$L__BB4_15:\n" " mov.u32 %r183, 16;\n" " mov.u32 %r192, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f257, %f281, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f260, %f281, %f257;\n" " mov.u32 %r185, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f259, %f260, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f262, %f260, %f259;\n" " mov.u32 %r187, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f261, %f262, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f264, %f262, %f261;\n" " mov.u32 %r189, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f263, %f264, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f266, %f264, %f263;\n" " mov.u32 %r191, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f265, %f266, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f11, %f266, %f265;\n" " // begin inline asm\n" " shfl.down.b32 %f267, %f282, %r183, %r192;\n" " // end inline asm\n" " add.f32 %f270, %f282, %f267;\n" " // begin inline asm\n" " shfl.down.b32 %f269, %f270, %r185, %r192;\n" " // end inline asm\n" " add.f32 %f272, %f270, %f269;\n" " // begin inline asm\n" " shfl.down.b32 %f271, %f272, %r187, %r192;\n" " // end inline asm\n" " add.f32 %f274, %f272, %f271;\n" " // begin inline asm\n" " shfl.down.b32 %f273, %f274, %r189, %r192;\n" " // end inline asm\n" " add.f32 %f276, %f274, %f273;\n" " // begin inline asm\n" " shfl.down.b32 %f275, %f276, %r191, %r192;\n" " // end inline asm\n" " add.f32 %f12, %f276, %f275;\n" " setp.ne.s32 %p13, %r2, 0;\n" " @%p13 bra $L__BB4_20;\n" "\n" " cvt.u32.u64 %r193, %rd1;\n" " setp.ge.s32 %p14, %r193, %r28;\n" " cvta.to.global.u64 %rd61, %rd34;\n" " shl.b64 %rd62, %rd1, 2;\n" " add.s64 %rd31, %rd61, %rd62;\n" " @%p14 bra $L__BB4_18;\n" "\n" " st.global.f32 [%rd31], %f11;\n" "\n" "$L__BB4_18:\n" " setp.ge.s32 %p15, %r3, %r28;\n" " @%p15 bra $L__BB4_20;\n" "\n" " st.global.f32 [%rd31+4], %f12;\n" "\n" "$L__BB4_20:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q5k_nx2\n" ".visible .entry fc_matvec_q5k_nx2(\n" " .param .u64 fc_matvec_q5k_nx2_param_0,\n" " .param .u64 fc_matvec_q5k_nx2_param_1,\n" " .param .u64 fc_matvec_q5k_nx2_param_2,\n" " .param .u32 fc_matvec_q5k_nx2_param_3,\n" " .param .u32 fc_matvec_q5k_nx2_param_4,\n" " .param .u32 fc_matvec_q5k_nx2_param_5,\n" " .param .u32 fc_matvec_q5k_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<21>;\n" " .reg .f32 %f<356>;\n" " .reg .b32 %r<211>;\n" " .reg .b64 %rd<81>;\n" "\n" "\n" " ld.param.u64 %rd36, [fc_matvec_q5k_nx2_param_0];\n" " ld.param.u64 %rd37, [fc_matvec_q5k_nx2_param_1];\n" " ld.param.u64 %rd38, [fc_matvec_q5k_nx2_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q5k_nx2_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q5k_nx2_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q5k_nx2_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q5k_nx2_param_6];\n" " cvta.to.global.u64 %rd1, %rd38;\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r1, %r26, 1;\n" " sub.s32 %r2, %r25, %r1;\n" " min.s32 %r3, %r2, 2;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 4;\n" " shr.u32 %r29, %r4, 4;\n" " and.b32 %r30, %r29, 268435454;\n" " add.s32 %r31, %r30, %r28;\n" " or.b32 %r6, %r31, 1;\n" " cvt.s64.s32 %rd2, %r31;\n" " setp.lt.s32 %p1, %r23, 1;\n" " mov.f32 %f348, 0f00000000;\n" " mov.f32 %f349, %f348;\n" " mov.f32 %f350, %f348;\n" " mov.f32 %f351, %f348;\n" " @%p1 bra $L__BB5_14;\n" "\n" " setp.lt.s32 %p2, %r6, %r22;\n" " cvt.s64.s32 %rd39, %r24;\n" " mul.lo.s64 %rd40, %rd39, %rd2;\n" " cvta.to.global.u64 %rd41, %rd36;\n" " add.s64 %rd77, %rd41, %rd40;\n" " add.s64 %rd78, %rd36, %rd40;\n" " cvt.u64.u32 %rd5, %r5;\n" " or.b32 %r33, %r5, 128;\n" " cvt.u64.u32 %rd6, %r33;\n" " add.s32 %r34, %r1, 1;\n" " mul.wide.s32 %rd42, %r23, %r1;\n" " cvt.s64.s32 %rd43, %r4;\n" " add.s64 %rd44, %rd42, %rd43;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd7, %rd37, %rd45;\n" " shl.b32 %r35, %r4, 2;\n" " mov.u32 %r36, fc_kq_shared;\n" " add.s32 %r37, %r36, %r35;\n" " add.s32 %r7, %r37, 2048;\n" " mul.wide.s32 %rd46, %r23, %r34;\n" " add.s64 %rd47, %rd46, %rd43;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd8, %rd37, %rd48;\n" " add.s64 %rd49, %rd78, %rd39;\n" " selp.b64 %rd80, %rd49, %rd78, %p2;\n" " add.s64 %rd50, %rd77, %rd39;\n" " selp.b64 %rd79, %rd50, %rd77, %p2;\n" " mov.u32 %r206, 0;\n" " mov.f32 %f348, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " shl.b64 %rd58, %rd5, 2;\n" " mov.f32 %f349, %f348;\n" " mov.f32 %f350, %f348;\n" " mov.f32 %f351, %f348;\n" "\n" "$L__BB5_2:\n" " sub.s32 %r38, %r23, %r206;\n" " min.s32 %r9, %r38, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r9;\n" " @%p3 bra $L__BB5_9;\n" "\n" " mul.wide.s32 %rd51, %r206, 4;\n" " add.s64 %rd72, %rd7, %rd51;\n" " add.s64 %rd71, %rd8, %rd51;\n" " mov.u32 %r207, %r7;\n" " mov.u32 %r208, %r4;\n" "\n" "$L__BB5_4:\n" " .pragma \"nounroll\";\n" " mov.f32 %f343, 0f00000000;\n" " mov.f32 %f342, %f343;\n" " @%p4 bra $L__BB5_6;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f342, [%rd72];\n" " // end inline asm\n" "\n" "$L__BB5_6:\n" " st.shared.f32 [%r207+-2048], %f342;\n" " @%p5 bra $L__BB5_8;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f343, [%rd71];\n" " // end inline asm\n" "\n" "$L__BB5_8:\n" " add.s32 %r205, %r207, -2048;\n" " st.shared.f32 [%r205+2048], %f343;\n" " add.s64 %rd72, %rd72, 1024;\n" " add.s32 %r207, %r207, 1024;\n" " add.s64 %rd71, %rd71, 1024;\n" " add.s32 %r208, %r208, 256;\n" " setp.lt.s32 %p6, %r208, %r9;\n" " @%p6 bra $L__BB5_4;\n" "\n" "$L__BB5_9:\n" " cvt.u32.u64 %r39, %rd2;\n" " setp.ge.s32 %p7, %r39, %r22;\n" " bar.sync 0;\n" " @%p7 bra $L__BB5_13;\n" "\n" " setp.lt.s32 %p8, %r9, 256;\n" " @%p8 bra $L__BB5_13;\n" "\n" " shr.s32 %r16, %r9, 8;\n" " mov.u32 %r209, 0;\n" " mov.u32 %r210, %r36;\n" "\n" "$L__BB5_12:\n" " add.s64 %rd54, %rd78, %rd58;\n" " // begin inline asm\n" " ld.global.nc.u32 %r42, [%rd54];\n" " // end inline asm\n" " add.s64 %rd55, %rd80, %rd58;\n" " // begin inline asm\n" " ld.global.nc.u32 %r43, [%rd55];\n" " // end inline asm\n" " add.s64 %rd56, %rd78, %rd6;\n" " // begin inline asm\n" " ld.global.nc.u8 %r44, [%rd56];\n" " // end inline asm\n" " add.s64 %rd57, %rd80, %rd6;\n" " // begin inline asm\n" " ld.global.nc.u8 %r45, [%rd57];\n" " // end inline asm\n" " ld.global.v2.u32 {%r46, %r47}, [%rd77+160];\n" " ld.global.v2.u32 {%r50, %r51}, [%rd79+160];\n" " ld.global.v2.u32 {%r54, %r55}, [%rd77+168];\n" " ld.global.v2.u32 {%r58, %r59}, [%rd79+168];\n" " ld.global.v2.f32 {%f41, %f42}, [%rd77+176];\n" " ld.global.v2.f32 {%f45, %f46}, [%rd79+176];\n" " shr.u32 %r62, %r42, 16;\n" " shr.u32 %r63, %r43, 16;\n" " and.b32 %r64, %r42, 15;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f49, %r65;\n" " mov.f32 %f50, 0fCB000000;\n" " mov.f32 %f51, 0f3F800000;\n" " fma.rn.f32 %f52, %f49, %f51, %f50;\n" " and.b32 %r66, %r44, 1;\n" " or.b32 %r67, %r66, 1258291200;\n" " mov.b32 %f53, %r67;\n" " mov.f32 %f54, 0fCD000000;\n" " mov.f32 %f55, 0f41800000;\n" " fma.rn.f32 %f56, %f53, %f55, %f54;\n" " add.f32 %f57, %f52, %f56;\n" " and.b32 %r68, %r43, 15;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f58, %r69;\n" " fma.rn.f32 %f59, %f58, %f51, %f50;\n" " and.b32 %r70, %r45, 1;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f60, %r71;\n" " fma.rn.f32 %f61, %f60, %f55, %f54;\n" " add.f32 %f62, %f59, %f61;\n" " and.b32 %r72, %r46, 255;\n" " cvt.rn.f32.u32 %f63, %r72;\n" " mul.rn.f32 %f64, %f41, %f63;\n" " mul.rn.f32 %f65, %f64, %f57;\n" " and.b32 %r73, %r54, 255;\n" " cvt.rn.f32.u32 %f66, %r73;\n" " mul.rn.f32 %f67, %f42, %f66;\n" " sub.rn.f32 %f68, %f65, %f67;\n" " and.b32 %r74, %r50, 255;\n" " cvt.rn.f32.u32 %f69, %r74;\n" " mul.rn.f32 %f70, %f45, %f69;\n" " mul.rn.f32 %f71, %f70, %f62;\n" " and.b32 %r75, %r58, 255;\n" " cvt.rn.f32.u32 %f72, %r75;\n" " mul.rn.f32 %f73, %f46, %f72;\n" " sub.rn.f32 %f74, %f71, %f73;\n" " shl.b32 %r76, %r5, 2;\n" " add.s32 %r77, %r210, %r76;\n" " ld.shared.f32 %f75, [%r77];\n" " mov.f32 %f76, 0f00000000;\n" " fma.rn.f32 %f77, %f68, %f75, %f76;\n" " fma.rn.f32 %f78, %f74, %f75, %f76;\n" " ld.shared.f32 %f79, [%r77+2048];\n" " fma.rn.f32 %f80, %f68, %f79, %f76;\n" " fma.rn.f32 %f81, %f74, %f79, %f76;\n" " and.b32 %r78, %r42, 240;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f82, %r79;\n" " mov.f32 %f83, 0fC9000000;\n" " mov.f32 %f84, 0f3D800000;\n" " fma.rn.f32 %f85, %f82, %f84, %f83;\n" " and.b32 %r80, %r44, 2;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f86, %r81;\n" " mov.f32 %f87, 0fCC800000;\n" " mov.f32 %f88, 0f41000000;\n" " fma.rn.f32 %f89, %f86, %f88, %f87;\n" " add.f32 %f90, %f85, %f89;\n" " and.b32 %r82, %r43, 240;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f91, %r83;\n" " fma.rn.f32 %f92, %f91, %f84, %f83;\n" " and.b32 %r84, %r45, 2;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f93, %r85;\n" " fma.rn.f32 %f94, %f93, %f88, %f87;\n" " add.f32 %f95, %f92, %f94;\n" " bfe.u32 %r86, %r46, 8, 8;\n" " cvt.rn.f32.u32 %f96, %r86;\n" " mul.rn.f32 %f97, %f41, %f96;\n" " mul.rn.f32 %f98, %f97, %f90;\n" " bfe.u32 %r87, %r54, 8, 8;\n" " cvt.rn.f32.u32 %f99, %r87;\n" " mul.rn.f32 %f100, %f42, %f99;\n" " sub.rn.f32 %f101, %f98, %f100;\n" " bfe.u32 %r88, %r50, 8, 8;\n" " cvt.rn.f32.u32 %f102, %r88;\n" " mul.rn.f32 %f103, %f45, %f102;\n" " mul.rn.f32 %f104, %f103, %f95;\n" " bfe.u32 %r89, %r58, 8, 8;\n" " cvt.rn.f32.u32 %f105, %r89;\n" " mul.rn.f32 %f106, %f46, %f105;\n" " sub.rn.f32 %f107, %f104, %f106;\n" " ld.shared.f32 %f108, [%r77+128];\n" " fma.rn.f32 %f109, %f101, %f108, %f77;\n" " fma.rn.f32 %f110, %f107, %f108, %f78;\n" " ld.shared.f32 %f111, [%r77+2176];\n" " fma.rn.f32 %f112, %f101, %f111, %f80;\n" " fma.rn.f32 %f113, %f107, %f111, %f81;\n" " and.b32 %r90, %r42, 3840;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f114, %r91;\n" " mov.f32 %f115, 0fC7000000;\n" " mov.f32 %f116, 0f3B800000;\n" " fma.rn.f32 %f117, %f114, %f116, %f115;\n" " and.b32 %r92, %r44, 4;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f118, %r93;\n" " mov.f32 %f119, 0fCC000000;\n" " mov.f32 %f120, 0f40800000;\n" " fma.rn.f32 %f121, %f118, %f120, %f119;\n" " add.f32 %f122, %f117, %f121;\n" " and.b32 %r94, %r43, 3840;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f123, %r95;\n" " fma.rn.f32 %f124, %f123, %f116, %f115;\n" " and.b32 %r96, %r45, 4;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f125, %r97;\n" " fma.rn.f32 %f126, %f125, %f120, %f119;\n" " add.f32 %f127, %f124, %f126;\n" " bfe.u32 %r98, %r46, 16, 8;\n" " cvt.rn.f32.u32 %f128, %r98;\n" " mul.rn.f32 %f129, %f41, %f128;\n" " mul.rn.f32 %f130, %f129, %f122;\n" " bfe.u32 %r99, %r54, 16, 8;\n" " cvt.rn.f32.u32 %f131, %r99;\n" " mul.rn.f32 %f132, %f42, %f131;\n" " sub.rn.f32 %f133, %f130, %f132;\n" " bfe.u32 %r100, %r50, 16, 8;\n" " cvt.rn.f32.u32 %f134, %r100;\n" " mul.rn.f32 %f135, %f45, %f134;\n" " mul.rn.f32 %f136, %f135, %f127;\n" " bfe.u32 %r101, %r58, 16, 8;\n" " cvt.rn.f32.u32 %f137, %r101;\n" " mul.rn.f32 %f138, %f46, %f137;\n" " sub.rn.f32 %f139, %f136, %f138;\n" " ld.shared.f32 %f140, [%r77+256];\n" " fma.rn.f32 %f141, %f133, %f140, %f109;\n" " fma.rn.f32 %f142, %f139, %f140, %f110;\n" " ld.shared.f32 %f143, [%r77+2304];\n" " fma.rn.f32 %f144, %f133, %f143, %f112;\n" " fma.rn.f32 %f145, %f139, %f143, %f113;\n" " and.b32 %r102, %r42, 61440;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f146, %r103;\n" " mov.f32 %f147, 0fC5000000;\n" " mov.f32 %f148, 0f39800000;\n" " fma.rn.f32 %f149, %f146, %f148, %f147;\n" " and.b32 %r104, %r44, 8;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f150, %r105;\n" " mov.f32 %f151, 0fCB800000;\n" " mov.f32 %f152, 0f40000000;\n" " fma.rn.f32 %f153, %f150, %f152, %f151;\n" " add.f32 %f154, %f149, %f153;\n" " and.b32 %r106, %r43, 61440;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f155, %r107;\n" " fma.rn.f32 %f156, %f155, %f148, %f147;\n" " and.b32 %r108, %r45, 8;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f157, %r109;\n" " fma.rn.f32 %f158, %f157, %f152, %f151;\n" " add.f32 %f159, %f156, %f158;\n" " shr.u32 %r110, %r46, 24;\n" " cvt.rn.f32.u32 %f160, %r110;\n" " mul.rn.f32 %f161, %f41, %f160;\n" " mul.rn.f32 %f162, %f161, %f154;\n" " shr.u32 %r111, %r54, 24;\n" " cvt.rn.f32.u32 %f163, %r111;\n" " mul.rn.f32 %f164, %f42, %f163;\n" " sub.rn.f32 %f165, %f162, %f164;\n" " shr.u32 %r112, %r50, 24;\n" " cvt.rn.f32.u32 %f166, %r112;\n" " mul.rn.f32 %f167, %f45, %f166;\n" " mul.rn.f32 %f168, %f167, %f159;\n" " shr.u32 %r113, %r58, 24;\n" " cvt.rn.f32.u32 %f169, %r113;\n" " mul.rn.f32 %f170, %f46, %f169;\n" " sub.rn.f32 %f171, %f168, %f170;\n" " ld.shared.f32 %f172, [%r77+384];\n" " fma.rn.f32 %f173, %f165, %f172, %f141;\n" " fma.rn.f32 %f174, %f171, %f172, %f142;\n" " ld.shared.f32 %f175, [%r77+2432];\n" " fma.rn.f32 %f176, %f165, %f175, %f144;\n" " fma.rn.f32 %f177, %f171, %f175, %f145;\n" " and.b32 %r114, %r42, 983040;\n" " or.b32 %r115, %r114, 1258291200;\n" " mov.b32 %f178, %r115;\n" " mov.f32 %f179, 0fC3000000;\n" " mov.f32 %f180, 0f37800000;\n" " fma.rn.f32 %f181, %f178, %f180, %f179;\n" " and.b32 %r116, %r44, 16;\n" " or.b32 %r117, %r116, 1258291200;\n" " mov.b32 %f182, %r117;\n" " fma.rn.f32 %f183, %f182, %f51, %f50;\n" " add.f32 %f184, %f181, %f183;\n" " and.b32 %r118, %r43, 983040;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f185, %r119;\n" " fma.rn.f32 %f186, %f185, %f180, %f179;\n" " and.b32 %r120, %r45, 16;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f187, %r121;\n" " fma.rn.f32 %f188, %f187, %f51, %f50;\n" " add.f32 %f189, %f186, %f188;\n" " and.b32 %r122, %r47, 255;\n" " cvt.rn.f32.u32 %f190, %r122;\n" " mul.rn.f32 %f191, %f41, %f190;\n" " mul.rn.f32 %f192, %f191, %f184;\n" " and.b32 %r123, %r55, 255;\n" " cvt.rn.f32.u32 %f193, %r123;\n" " mul.rn.f32 %f194, %f42, %f193;\n" " sub.rn.f32 %f195, %f192, %f194;\n" " and.b32 %r124, %r51, 255;\n" " cvt.rn.f32.u32 %f196, %r124;\n" " mul.rn.f32 %f197, %f45, %f196;\n" " mul.rn.f32 %f198, %f197, %f189;\n" " and.b32 %r125, %r59, 255;\n" " cvt.rn.f32.u32 %f199, %r125;\n" " mul.rn.f32 %f200, %f46, %f199;\n" " sub.rn.f32 %f201, %f198, %f200;\n" " ld.shared.f32 %f202, [%r77+512];\n" " fma.rn.f32 %f203, %f195, %f202, %f173;\n" " fma.rn.f32 %f204, %f201, %f202, %f174;\n" " ld.shared.f32 %f205, [%r77+2560];\n" " fma.rn.f32 %f206, %f195, %f205, %f176;\n" " fma.rn.f32 %f207, %f201, %f205, %f177;\n" " and.b32 %r126, %r62, 240;\n" " or.b32 %r127, %r126, 1258291200;\n" " mov.b32 %f208, %r127;\n" " fma.rn.f32 %f209, %f208, %f84, %f83;\n" " and.b32 %r128, %r44, 32;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f210, %r129;\n" " mov.f32 %f211, 0fCA800000;\n" " mov.f32 %f212, 0f3F000000;\n" " fma.rn.f32 %f213, %f210, %f212, %f211;\n" " add.f32 %f214, %f209, %f213;\n" " and.b32 %r130, %r63, 240;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f215, %r131;\n" " fma.rn.f32 %f216, %f215, %f84, %f83;\n" " and.b32 %r132, %r45, 32;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f217, %r133;\n" " fma.rn.f32 %f218, %f217, %f212, %f211;\n" " add.f32 %f219, %f216, %f218;\n" " bfe.u32 %r134, %r47, 8, 8;\n" " cvt.rn.f32.u32 %f220, %r134;\n" " mul.rn.f32 %f221, %f41, %f220;\n" " mul.rn.f32 %f222, %f221, %f214;\n" " bfe.u32 %r135, %r55, 8, 8;\n" " cvt.rn.f32.u32 %f223, %r135;\n" " mul.rn.f32 %f224, %f42, %f223;\n" " sub.rn.f32 %f225, %f222, %f224;\n" " bfe.u32 %r136, %r51, 8, 8;\n" " cvt.rn.f32.u32 %f226, %r136;\n" " mul.rn.f32 %f227, %f45, %f226;\n" " mul.rn.f32 %f228, %f227, %f219;\n" " bfe.u32 %r137, %r59, 8, 8;\n" " cvt.rn.f32.u32 %f229, %r137;\n" " mul.rn.f32 %f230, %f46, %f229;\n" " sub.rn.f32 %f231, %f228, %f230;\n" " ld.shared.f32 %f232, [%r77+640];\n" " fma.rn.f32 %f233, %f225, %f232, %f203;\n" " fma.rn.f32 %f234, %f231, %f232, %f204;\n" " ld.shared.f32 %f235, [%r77+2688];\n" " fma.rn.f32 %f236, %f225, %f235, %f206;\n" " fma.rn.f32 %f237, %f231, %f235, %f207;\n" " and.b32 %r138, %r62, 3840;\n" " or.b32 %r139, %r138, 1258291200;\n" " mov.b32 %f238, %r139;\n" " fma.rn.f32 %f239, %f238, %f116, %f115;\n" " and.b32 %r140, %r44, 64;\n" " or.b32 %r141, %r140, 1258291200;\n" " mov.b32 %f240, %r141;\n" " mov.f32 %f241, 0fCA000000;\n" " mov.f32 %f242, 0f3E800000;\n" " fma.rn.f32 %f243, %f240, %f242, %f241;\n" " add.f32 %f244, %f239, %f243;\n" " and.b32 %r142, %r63, 3840;\n" " or.b32 %r143, %r142, 1258291200;\n" " mov.b32 %f245, %r143;\n" " fma.rn.f32 %f246, %f245, %f116, %f115;\n" " and.b32 %r144, %r45, 64;\n" " or.b32 %r145, %r144, 1258291200;\n" " mov.b32 %f247, %r145;\n" " fma.rn.f32 %f248, %f247, %f242, %f241;\n" " add.f32 %f249, %f246, %f248;\n" " bfe.u32 %r146, %r47, 16, 8;\n" " cvt.rn.f32.u32 %f250, %r146;\n" " mul.rn.f32 %f251, %f41, %f250;\n" " mul.rn.f32 %f252, %f251, %f244;\n" " bfe.u32 %r147, %r55, 16, 8;\n" " cvt.rn.f32.u32 %f253, %r147;\n" " mul.rn.f32 %f254, %f42, %f253;\n" " sub.rn.f32 %f255, %f252, %f254;\n" " bfe.u32 %r148, %r51, 16, 8;\n" " cvt.rn.f32.u32 %f256, %r148;\n" " mul.rn.f32 %f257, %f45, %f256;\n" " mul.rn.f32 %f258, %f257, %f249;\n" " bfe.u32 %r149, %r59, 16, 8;\n" " cvt.rn.f32.u32 %f259, %r149;\n" " mul.rn.f32 %f260, %f46, %f259;\n" " sub.rn.f32 %f261, %f258, %f260;\n" " ld.shared.f32 %f262, [%r77+768];\n" " fma.rn.f32 %f263, %f255, %f262, %f233;\n" " fma.rn.f32 %f264, %f261, %f262, %f234;\n" " ld.shared.f32 %f265, [%r77+2816];\n" " fma.rn.f32 %f266, %f255, %f265, %f236;\n" " fma.rn.f32 %f267, %f261, %f265, %f237;\n" " and.b32 %r150, %r62, 61440;\n" " or.b32 %r151, %r150, 1258291200;\n" " mov.b32 %f268, %r151;\n" " fma.rn.f32 %f269, %f268, %f148, %f147;\n" " and.b32 %r152, %r44, 128;\n" " or.b32 %r153, %r152, 1258291200;\n" " mov.b32 %f270, %r153;\n" " mov.f32 %f271, 0fC9800000;\n" " mov.f32 %f272, 0f3E000000;\n" " fma.rn.f32 %f273, %f270, %f272, %f271;\n" " add.f32 %f274, %f269, %f273;\n" " and.b32 %r154, %r63, 61440;\n" " or.b32 %r155, %r154, 1258291200;\n" " mov.b32 %f275, %r155;\n" " fma.rn.f32 %f276, %f275, %f148, %f147;\n" " and.b32 %r156, %r45, 128;\n" " or.b32 %r157, %r156, 1258291200;\n" " mov.b32 %f277, %r157;\n" " fma.rn.f32 %f278, %f277, %f272, %f271;\n" " add.f32 %f279, %f276, %f278;\n" " shr.u32 %r158, %r47, 24;\n" " cvt.rn.f32.u32 %f280, %r158;\n" " mul.rn.f32 %f281, %f41, %f280;\n" " mul.rn.f32 %f282, %f281, %f274;\n" " shr.u32 %r159, %r55, 24;\n" " cvt.rn.f32.u32 %f283, %r159;\n" " mul.rn.f32 %f284, %f42, %f283;\n" " sub.rn.f32 %f285, %f282, %f284;\n" " shr.u32 %r160, %r51, 24;\n" " cvt.rn.f32.u32 %f286, %r160;\n" " mul.rn.f32 %f287, %f45, %f286;\n" " mul.rn.f32 %f288, %f287, %f279;\n" " shr.u32 %r161, %r59, 24;\n" " cvt.rn.f32.u32 %f289, %r161;\n" " mul.rn.f32 %f290, %f46, %f289;\n" " sub.rn.f32 %f291, %f288, %f290;\n" " ld.shared.f32 %f292, [%r77+896];\n" " fma.rn.f32 %f293, %f285, %f292, %f263;\n" " fma.rn.f32 %f294, %f291, %f292, %f264;\n" " ld.shared.f32 %f295, [%r77+2944];\n" " fma.rn.f32 %f296, %f285, %f295, %f266;\n" " fma.rn.f32 %f297, %f291, %f295, %f267;\n" " add.rn.f32 %f349, %f349, %f293;\n" " add.rn.f32 %f351, %f351, %f294;\n" " add.rn.f32 %f348, %f348, %f296;\n" " add.rn.f32 %f350, %f350, %f297;\n" " add.s64 %rd77, %rd77, 184;\n" " add.s64 %rd78, %rd78, 184;\n" " add.s64 %rd79, %rd79, 184;\n" " add.s64 %rd80, %rd80, 184;\n" " add.s32 %r210, %r210, 1024;\n" " add.s32 %r209, %r209, 1;\n" " setp.lt.s32 %p9, %r209, %r16;\n" " @%p9 bra $L__BB5_12;\n" "\n" "$L__BB5_13:\n" " add.s32 %r206, %r206, 512;\n" " setp.lt.s32 %p10, %r206, %r23;\n" " @%p10 bra $L__BB5_2;\n" "\n" "$L__BB5_14:\n" " cvt.s64.s32 %rd33, %r22;\n" " mov.u32 %r172, 16;\n" " mov.u32 %r181, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f298, %f349, %r172, %r181;\n" " // end inline asm\n" " add.f32 %f301, %f349, %f298;\n" " mov.u32 %r174, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f300, %f301, %r174, %r181;\n" " // end inline asm\n" " add.f32 %f303, %f301, %f300;\n" " mov.u32 %r176, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f302, %f303, %r176, %r181;\n" " // end inline asm\n" " add.f32 %f305, %f303, %f302;\n" " mov.u32 %r178, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f304, %f305, %r178, %r181;\n" " // end inline asm\n" " add.f32 %f307, %f305, %f304;\n" " mov.u32 %r180, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f306, %f307, %r180, %r181;\n" " // end inline asm\n" " add.f32 %f25, %f307, %f306;\n" " // begin inline asm\n" " shfl.down.b32 %f308, %f351, %r172, %r181;\n" " // end inline asm\n" " add.f32 %f311, %f351, %f308;\n" " // begin inline asm\n" " shfl.down.b32 %f310, %f311, %r174, %r181;\n" " // end inline asm\n" " add.f32 %f313, %f311, %f310;\n" " // begin inline asm\n" " shfl.down.b32 %f312, %f313, %r176, %r181;\n" " // end inline asm\n" " add.f32 %f315, %f313, %f312;\n" " // begin inline asm\n" " shfl.down.b32 %f314, %f315, %r178, %r181;\n" " // end inline asm\n" " add.f32 %f317, %f315, %f314;\n" " // begin inline asm\n" " shfl.down.b32 %f316, %f317, %r180, %r181;\n" " // end inline asm\n" " add.f32 %f26, %f317, %f316;\n" " setp.lt.s32 %p11, %r2, 1;\n" " setp.ne.s32 %p12, %r5, 0;\n" " or.pred %p13, %p12, %p11;\n" " @%p13 bra $L__BB5_19;\n" "\n" " cvt.u32.u64 %r182, %rd2;\n" " setp.ge.s32 %p14, %r182, %r22;\n" " cvt.s64.s32 %rd59, %r1;\n" " mul.lo.s64 %rd60, %rd59, %rd33;\n" " add.s64 %rd61, %rd60, %rd2;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd34, %rd1, %rd62;\n" " @%p14 bra $L__BB5_17;\n" "\n" " st.global.f32 [%rd34], %f25;\n" "\n" "$L__BB5_17:\n" " setp.ge.s32 %p15, %r6, %r22;\n" " @%p15 bra $L__BB5_19;\n" "\n" " st.global.f32 [%rd34+4], %f26;\n" "\n" "$L__BB5_19:\n" " // begin inline asm\n" " shfl.down.b32 %f318, %f348, %r172, %r181;\n" " // end inline asm\n" " add.f32 %f321, %f348, %f318;\n" " // begin inline asm\n" " shfl.down.b32 %f320, %f321, %r174, %r181;\n" " // end inline asm\n" " add.f32 %f323, %f321, %f320;\n" " // begin inline asm\n" " shfl.down.b32 %f322, %f323, %r176, %r181;\n" " // end inline asm\n" " add.f32 %f325, %f323, %f322;\n" " // begin inline asm\n" " shfl.down.b32 %f324, %f325, %r178, %r181;\n" " // end inline asm\n" " add.f32 %f327, %f325, %f324;\n" " // begin inline asm\n" " shfl.down.b32 %f326, %f327, %r180, %r181;\n" " // end inline asm\n" " add.f32 %f27, %f327, %f326;\n" " // begin inline asm\n" " shfl.down.b32 %f328, %f350, %r172, %r181;\n" " // end inline asm\n" " add.f32 %f331, %f350, %f328;\n" " // begin inline asm\n" " shfl.down.b32 %f330, %f331, %r174, %r181;\n" " // end inline asm\n" " add.f32 %f333, %f331, %f330;\n" " // begin inline asm\n" " shfl.down.b32 %f332, %f333, %r176, %r181;\n" " // end inline asm\n" " add.f32 %f335, %f333, %f332;\n" " // begin inline asm\n" " shfl.down.b32 %f334, %f335, %r178, %r181;\n" " // end inline asm\n" " add.f32 %f337, %f335, %f334;\n" " // begin inline asm\n" " shfl.down.b32 %f336, %f337, %r180, %r181;\n" " // end inline asm\n" " add.f32 %f28, %f337, %f336;\n" " setp.lt.s32 %p17, %r3, 2;\n" " or.pred %p18, %p12, %p17;\n" " @%p18 bra $L__BB5_24;\n" "\n" " cvt.u32.u64 %r203, %rd2;\n" " setp.ge.s32 %p19, %r203, %r22;\n" " add.s32 %r204, %r1, 1;\n" " cvt.s64.s32 %rd63, %r204;\n" " mul.lo.s64 %rd64, %rd63, %rd33;\n" " add.s64 %rd65, %rd64, %rd2;\n" " shl.b64 %rd66, %rd65, 2;\n" " add.s64 %rd35, %rd1, %rd66;\n" " @%p19 bra $L__BB5_22;\n" "\n" " st.global.f32 [%rd35], %f27;\n" "\n" "$L__BB5_22:\n" " setp.ge.s32 %p20, %r6, %r22;\n" " @%p20 bra $L__BB5_24;\n" "\n" " st.global.f32 [%rd35+4], %f28;\n" "\n" "$L__BB5_24:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q5k_nx\n" ".visible .entry fc_matvec_q5k_nx(\n" " .param .u64 fc_matvec_q5k_nx_param_0,\n" " .param .u64 fc_matvec_q5k_nx_param_1,\n" " .param .u64 fc_matvec_q5k_nx_param_2,\n" " .param .u32 fc_matvec_q5k_nx_param_3,\n" " .param .u32 fc_matvec_q5k_nx_param_4,\n" " .param .u32 fc_matvec_q5k_nx_param_5,\n" " .param .u32 fc_matvec_q5k_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<33>;\n" " .reg .f32 %f<505>;\n" " .reg .b32 %r<280>;\n" " .reg .b64 %rd<104>;\n" "\n" "\n" " ld.param.u64 %rd40, [fc_matvec_q5k_nx_param_0];\n" " ld.param.u64 %rd41, [fc_matvec_q5k_nx_param_1];\n" " ld.param.u64 %rd42, [fc_matvec_q5k_nx_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q5k_nx_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q5k_nx_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q5k_nx_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q5k_nx_param_6];\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r1, %r26, 2;\n" " sub.s32 %r2, %r25, %r1;\n" " mov.u32 %r3, %tid.x;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 4;\n" " shr.u32 %r29, %r3, 4;\n" " and.b32 %r30, %r29, 268435454;\n" " add.s32 %r31, %r30, %r28;\n" " or.b32 %r4, %r31, 1;\n" " cvt.s64.s32 %rd1, %r31;\n" " min.s32 %r5, %r2, 4;\n" " cvta.to.global.u64 %rd2, %rd42;\n" " and.b32 %r6, %r3, 31;\n" " setp.lt.s32 %p1, %r23, 1;\n" " mov.f32 %f489, 0f00000000;\n" " mov.f32 %f490, %f489;\n" " mov.f32 %f491, %f489;\n" " mov.f32 %f492, %f489;\n" " mov.f32 %f493, %f489;\n" " mov.f32 %f494, %f489;\n" " mov.f32 %f495, %f489;\n" " mov.f32 %f496, %f489;\n" " @%p1 bra $L__BB6_18;\n" "\n" " setp.lt.s32 %p2, %r4, %r22;\n" " cvt.s64.s32 %rd43, %r24;\n" " mul.lo.s64 %rd44, %rd43, %rd1;\n" " cvta.to.global.u64 %rd45, %rd40;\n" " add.s64 %rd100, %rd45, %rd44;\n" " add.s64 %rd101, %rd40, %rd44;\n" " cvt.u64.u32 %rd5, %r6;\n" " or.b32 %r33, %r6, 128;\n" " cvt.u64.u32 %rd6, %r33;\n" " add.s32 %r34, %r1, 1;\n" " add.s32 %r35, %r1, 2;\n" " add.s32 %r36, %r1, 3;\n" " mul.wide.s32 %rd46, %r23, %r1;\n" " shl.b64 %rd47, %rd46, 2;\n" " mul.wide.s32 %rd48, %r3, 4;\n" " add.s64 %rd7, %rd47, %rd48;\n" " mul.wide.s32 %rd49, %r23, %r34;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd8, %rd50, %rd48;\n" " mul.wide.s32 %rd51, %r23, %r35;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd9, %rd52, %rd48;\n" " shl.b32 %r37, %r3, 2;\n" " mov.u32 %r38, fc_kq_shared;\n" " add.s32 %r39, %r38, %r37;\n" " add.s32 %r7, %r39, 4096;\n" " mul.wide.s32 %rd53, %r23, %r36;\n" " shl.b64 %rd54, %rd53, 2;\n" " add.s64 %rd10, %rd54, %rd48;\n" " add.s64 %rd55, %rd101, %rd43;\n" " selp.b64 %rd103, %rd55, %rd101, %p2;\n" " add.s64 %rd56, %rd100, %rd43;\n" " selp.b64 %rd102, %rd56, %rd100, %p2;\n" " mov.u32 %r275, 0;\n" " mov.f32 %f489, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r5, 2;\n" " setp.lt.s32 %p6, %r5, 3;\n" " shl.b64 %rd66, %rd5, 2;\n" " mov.f32 %f490, %f489;\n" " mov.f32 %f491, %f489;\n" " mov.f32 %f492, %f489;\n" " mov.f32 %f493, %f489;\n" " mov.f32 %f494, %f489;\n" " mov.f32 %f495, %f489;\n" " mov.f32 %f496, %f489;\n" "\n" "$L__BB6_2:\n" " sub.s32 %r40, %r23, %r275;\n" " min.s32 %r9, %r40, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r3, %r9;\n" " @%p3 bra $L__BB6_13;\n" "\n" " mov.u32 %r276, %r7;\n" " mov.u64 %rd95, %rd41;\n" " mov.u32 %r277, %r3;\n" "\n" "$L__BB6_4:\n" " .pragma \"nounroll\";\n" " mov.f32 %f477, 0f00000000;\n" " @%p4 bra $L__BB6_6;\n" "\n" " mul.wide.s32 %rd84, %r275, 4;\n" " add.s64 %rd83, %rd7, %rd84;\n" " add.s64 %rd58, %rd95, %rd83;\n" " // begin inline asm\n" " ld.global.nc.f32 %f477, [%rd58];\n" " // end inline asm\n" "\n" "$L__BB6_6:\n" " mov.f32 %f478, 0f00000000;\n" " st.shared.f32 [%r276+-4096], %f477;\n" " @%p5 bra $L__BB6_8;\n" "\n" " mul.wide.s32 %rd90, %r275, 4;\n" " add.s64 %rd89, %rd8, %rd90;\n" " add.s64 %rd59, %rd95, %rd89;\n" " // begin inline asm\n" " ld.global.nc.f32 %f478, [%rd59];\n" " // end inline asm\n" "\n" "$L__BB6_8:\n" " mov.f32 %f479, 0f00000000;\n" " add.s32 %r251, %r276, -4096;\n" " st.shared.f32 [%r251+2048], %f478;\n" " @%p6 bra $L__BB6_10;\n" "\n" " mul.wide.s32 %rd88, %r275, 4;\n" " add.s64 %rd87, %rd9, %rd88;\n" " add.s64 %rd60, %rd95, %rd87;\n" " // begin inline asm\n" " ld.global.nc.f32 %f479, [%rd60];\n" " // end inline asm\n" "\n" "$L__BB6_10:\n" " mov.f32 %f480, 0f00000000;\n" " add.s32 %r252, %r276, -4096;\n" " st.shared.f32 [%r252+4096], %f479;\n" " setp.lt.s32 %p7, %r5, 4;\n" " @%p7 bra $L__BB6_12;\n" "\n" " mul.wide.s32 %rd86, %r275, 4;\n" " add.s64 %rd85, %rd10, %rd86;\n" " add.s64 %rd61, %rd95, %rd85;\n" " // begin inline asm\n" " ld.global.nc.f32 %f480, [%rd61];\n" " // end inline asm\n" "\n" "$L__BB6_12:\n" " sub.s32 %r259, %r23, %r275;\n" " min.s32 %r258, %r259, 512;\n" " add.s32 %r253, %r276, -4096;\n" " st.shared.f32 [%r253+6144], %f480;\n" " add.s64 %rd95, %rd95, 1024;\n" " add.s32 %r276, %r276, 1024;\n" " add.s32 %r277, %r277, 256;\n" " setp.lt.s32 %p8, %r277, %r258;\n" " @%p8 bra $L__BB6_4;\n" "\n" "$L__BB6_13:\n" " cvt.u32.u64 %r41, %rd1;\n" " setp.ge.s32 %p9, %r41, %r22;\n" " bar.sync 0;\n" " @%p9 bra $L__BB6_17;\n" "\n" " sub.s32 %r269, %r23, %r275;\n" " min.s32 %r268, %r269, 512;\n" " setp.lt.s32 %p10, %r268, 256;\n" " @%p10 bra $L__BB6_17;\n" "\n" " mov.u32 %r278, 0;\n" " mov.u32 %r279, %r38;\n" "\n" "$L__BB6_16:\n" " sub.s32 %r274, %r23, %r275;\n" " min.s32 %r273, %r274, 512;\n" " shr.s32 %r272, %r273, 8;\n" " add.s64 %rd62, %rd101, %rd66;\n" " // begin inline asm\n" " ld.global.nc.u32 %r44, [%rd62];\n" " // end inline asm\n" " add.s64 %rd63, %rd103, %rd66;\n" " // begin inline asm\n" " ld.global.nc.u32 %r45, [%rd63];\n" " // end inline asm\n" " add.s64 %rd64, %rd101, %rd6;\n" " // begin inline asm\n" " ld.global.nc.u8 %r46, [%rd64];\n" " // end inline asm\n" " add.s64 %rd65, %rd103, %rd6;\n" " // begin inline asm\n" " ld.global.nc.u8 %r47, [%rd65];\n" " // end inline asm\n" " ld.global.v2.u32 {%r48, %r49}, [%rd100+160];\n" " ld.global.v2.u32 {%r52, %r53}, [%rd102+160];\n" " ld.global.v2.u32 {%r56, %r57}, [%rd100+168];\n" " ld.global.v2.u32 {%r60, %r61}, [%rd102+168];\n" " ld.global.v2.f32 {%f81, %f82}, [%rd100+176];\n" " ld.global.v2.f32 {%f85, %f86}, [%rd102+176];\n" " shr.u32 %r64, %r44, 16;\n" " shr.u32 %r65, %r45, 16;\n" " and.b32 %r66, %r44, 15;\n" " or.b32 %r67, %r66, 1258291200;\n" " mov.b32 %f89, %r67;\n" " mov.f32 %f90, 0fCB000000;\n" " mov.f32 %f91, 0f3F800000;\n" " fma.rn.f32 %f92, %f89, %f91, %f90;\n" " and.b32 %r68, %r46, 1;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f93, %r69;\n" " mov.f32 %f94, 0fCD000000;\n" " mov.f32 %f95, 0f41800000;\n" " fma.rn.f32 %f96, %f93, %f95, %f94;\n" " add.f32 %f97, %f92, %f96;\n" " and.b32 %r70, %r45, 15;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f98, %r71;\n" " fma.rn.f32 %f99, %f98, %f91, %f90;\n" " and.b32 %r72, %r47, 1;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f100, %r73;\n" " fma.rn.f32 %f101, %f100, %f95, %f94;\n" " add.f32 %f102, %f99, %f101;\n" " and.b32 %r74, %r48, 255;\n" " cvt.rn.f32.u32 %f103, %r74;\n" " mul.rn.f32 %f104, %f81, %f103;\n" " mul.rn.f32 %f105, %f104, %f97;\n" " and.b32 %r75, %r56, 255;\n" " cvt.rn.f32.u32 %f106, %r75;\n" " mul.rn.f32 %f107, %f82, %f106;\n" " sub.rn.f32 %f108, %f105, %f107;\n" " and.b32 %r76, %r52, 255;\n" " cvt.rn.f32.u32 %f109, %r76;\n" " mul.rn.f32 %f110, %f85, %f109;\n" " mul.rn.f32 %f111, %f110, %f102;\n" " and.b32 %r77, %r60, 255;\n" " cvt.rn.f32.u32 %f112, %r77;\n" " mul.rn.f32 %f113, %f86, %f112;\n" " sub.rn.f32 %f114, %f111, %f113;\n" " shl.b32 %r78, %r6, 2;\n" " add.s32 %r79, %r279, %r78;\n" " ld.shared.f32 %f115, [%r79];\n" " mov.f32 %f116, 0f00000000;\n" " fma.rn.f32 %f117, %f108, %f115, %f116;\n" " fma.rn.f32 %f118, %f114, %f115, %f116;\n" " ld.shared.f32 %f119, [%r79+2048];\n" " fma.rn.f32 %f120, %f108, %f119, %f116;\n" " fma.rn.f32 %f121, %f114, %f119, %f116;\n" " ld.shared.f32 %f122, [%r79+4096];\n" " fma.rn.f32 %f123, %f108, %f122, %f116;\n" " fma.rn.f32 %f124, %f114, %f122, %f116;\n" " ld.shared.f32 %f125, [%r79+6144];\n" " fma.rn.f32 %f126, %f108, %f125, %f116;\n" " fma.rn.f32 %f127, %f114, %f125, %f116;\n" " and.b32 %r80, %r44, 240;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f128, %r81;\n" " mov.f32 %f129, 0fC9000000;\n" " mov.f32 %f130, 0f3D800000;\n" " fma.rn.f32 %f131, %f128, %f130, %f129;\n" " and.b32 %r82, %r46, 2;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f132, %r83;\n" " mov.f32 %f133, 0fCC800000;\n" " mov.f32 %f134, 0f41000000;\n" " fma.rn.f32 %f135, %f132, %f134, %f133;\n" " add.f32 %f136, %f131, %f135;\n" " and.b32 %r84, %r45, 240;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f137, %r85;\n" " fma.rn.f32 %f138, %f137, %f130, %f129;\n" " and.b32 %r86, %r47, 2;\n" " or.b32 %r87, %r86, 1258291200;\n" " mov.b32 %f139, %r87;\n" " fma.rn.f32 %f140, %f139, %f134, %f133;\n" " add.f32 %f141, %f138, %f140;\n" " bfe.u32 %r88, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f142, %r88;\n" " mul.rn.f32 %f143, %f81, %f142;\n" " mul.rn.f32 %f144, %f143, %f136;\n" " bfe.u32 %r89, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f145, %r89;\n" " mul.rn.f32 %f146, %f82, %f145;\n" " sub.rn.f32 %f147, %f144, %f146;\n" " bfe.u32 %r90, %r52, 8, 8;\n" " cvt.rn.f32.u32 %f148, %r90;\n" " mul.rn.f32 %f149, %f85, %f148;\n" " mul.rn.f32 %f150, %f149, %f141;\n" " bfe.u32 %r91, %r60, 8, 8;\n" " cvt.rn.f32.u32 %f151, %r91;\n" " mul.rn.f32 %f152, %f86, %f151;\n" " sub.rn.f32 %f153, %f150, %f152;\n" " ld.shared.f32 %f154, [%r79+128];\n" " fma.rn.f32 %f155, %f147, %f154, %f117;\n" " fma.rn.f32 %f156, %f153, %f154, %f118;\n" " ld.shared.f32 %f157, [%r79+2176];\n" " fma.rn.f32 %f158, %f147, %f157, %f120;\n" " fma.rn.f32 %f159, %f153, %f157, %f121;\n" " ld.shared.f32 %f160, [%r79+4224];\n" " fma.rn.f32 %f161, %f147, %f160, %f123;\n" " fma.rn.f32 %f162, %f153, %f160, %f124;\n" " ld.shared.f32 %f163, [%r79+6272];\n" " fma.rn.f32 %f164, %f147, %f163, %f126;\n" " fma.rn.f32 %f165, %f153, %f163, %f127;\n" " and.b32 %r92, %r44, 3840;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f166, %r93;\n" " mov.f32 %f167, 0fC7000000;\n" " mov.f32 %f168, 0f3B800000;\n" " fma.rn.f32 %f169, %f166, %f168, %f167;\n" " and.b32 %r94, %r46, 4;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f170, %r95;\n" " mov.f32 %f171, 0fCC000000;\n" " mov.f32 %f172, 0f40800000;\n" " fma.rn.f32 %f173, %f170, %f172, %f171;\n" " add.f32 %f174, %f169, %f173;\n" " and.b32 %r96, %r45, 3840;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f175, %r97;\n" " fma.rn.f32 %f176, %f175, %f168, %f167;\n" " and.b32 %r98, %r47, 4;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f177, %r99;\n" " fma.rn.f32 %f178, %f177, %f172, %f171;\n" " add.f32 %f179, %f176, %f178;\n" " bfe.u32 %r100, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f180, %r100;\n" " mul.rn.f32 %f181, %f81, %f180;\n" " mul.rn.f32 %f182, %f181, %f174;\n" " bfe.u32 %r101, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f183, %r101;\n" " mul.rn.f32 %f184, %f82, %f183;\n" " sub.rn.f32 %f185, %f182, %f184;\n" " bfe.u32 %r102, %r52, 16, 8;\n" " cvt.rn.f32.u32 %f186, %r102;\n" " mul.rn.f32 %f187, %f85, %f186;\n" " mul.rn.f32 %f188, %f187, %f179;\n" " bfe.u32 %r103, %r60, 16, 8;\n" " cvt.rn.f32.u32 %f189, %r103;\n" " mul.rn.f32 %f190, %f86, %f189;\n" " sub.rn.f32 %f191, %f188, %f190;\n" " ld.shared.f32 %f192, [%r79+256];\n" " fma.rn.f32 %f193, %f185, %f192, %f155;\n" " fma.rn.f32 %f194, %f191, %f192, %f156;\n" " ld.shared.f32 %f195, [%r79+2304];\n" " fma.rn.f32 %f196, %f185, %f195, %f158;\n" " fma.rn.f32 %f197, %f191, %f195, %f159;\n" " ld.shared.f32 %f198, [%r79+4352];\n" " fma.rn.f32 %f199, %f185, %f198, %f161;\n" " fma.rn.f32 %f200, %f191, %f198, %f162;\n" " ld.shared.f32 %f201, [%r79+6400];\n" " fma.rn.f32 %f202, %f185, %f201, %f164;\n" " fma.rn.f32 %f203, %f191, %f201, %f165;\n" " and.b32 %r104, %r44, 61440;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f204, %r105;\n" " mov.f32 %f205, 0fC5000000;\n" " mov.f32 %f206, 0f39800000;\n" " fma.rn.f32 %f207, %f204, %f206, %f205;\n" " and.b32 %r106, %r46, 8;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f208, %r107;\n" " mov.f32 %f209, 0fCB800000;\n" " mov.f32 %f210, 0f40000000;\n" " fma.rn.f32 %f211, %f208, %f210, %f209;\n" " add.f32 %f212, %f207, %f211;\n" " and.b32 %r108, %r45, 61440;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f213, %r109;\n" " fma.rn.f32 %f214, %f213, %f206, %f205;\n" " and.b32 %r110, %r47, 8;\n" " or.b32 %r111, %r110, 1258291200;\n" " mov.b32 %f215, %r111;\n" " fma.rn.f32 %f216, %f215, %f210, %f209;\n" " add.f32 %f217, %f214, %f216;\n" " shr.u32 %r112, %r48, 24;\n" " cvt.rn.f32.u32 %f218, %r112;\n" " mul.rn.f32 %f219, %f81, %f218;\n" " mul.rn.f32 %f220, %f219, %f212;\n" " shr.u32 %r113, %r56, 24;\n" " cvt.rn.f32.u32 %f221, %r113;\n" " mul.rn.f32 %f222, %f82, %f221;\n" " sub.rn.f32 %f223, %f220, %f222;\n" " shr.u32 %r114, %r52, 24;\n" " cvt.rn.f32.u32 %f224, %r114;\n" " mul.rn.f32 %f225, %f85, %f224;\n" " mul.rn.f32 %f226, %f225, %f217;\n" " shr.u32 %r115, %r60, 24;\n" " cvt.rn.f32.u32 %f227, %r115;\n" " mul.rn.f32 %f228, %f86, %f227;\n" " sub.rn.f32 %f229, %f226, %f228;\n" " ld.shared.f32 %f230, [%r79+384];\n" " fma.rn.f32 %f231, %f223, %f230, %f193;\n" " fma.rn.f32 %f232, %f229, %f230, %f194;\n" " ld.shared.f32 %f233, [%r79+2432];\n" " fma.rn.f32 %f234, %f223, %f233, %f196;\n" " fma.rn.f32 %f235, %f229, %f233, %f197;\n" " ld.shared.f32 %f236, [%r79+4480];\n" " fma.rn.f32 %f237, %f223, %f236, %f199;\n" " fma.rn.f32 %f238, %f229, %f236, %f200;\n" " ld.shared.f32 %f239, [%r79+6528];\n" " fma.rn.f32 %f240, %f223, %f239, %f202;\n" " fma.rn.f32 %f241, %f229, %f239, %f203;\n" " and.b32 %r116, %r44, 983040;\n" " or.b32 %r117, %r116, 1258291200;\n" " mov.b32 %f242, %r117;\n" " mov.f32 %f243, 0fC3000000;\n" " mov.f32 %f244, 0f37800000;\n" " fma.rn.f32 %f245, %f242, %f244, %f243;\n" " and.b32 %r118, %r46, 16;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f246, %r119;\n" " fma.rn.f32 %f247, %f246, %f91, %f90;\n" " add.f32 %f248, %f245, %f247;\n" " and.b32 %r120, %r45, 983040;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f249, %r121;\n" " fma.rn.f32 %f250, %f249, %f244, %f243;\n" " and.b32 %r122, %r47, 16;\n" " or.b32 %r123, %r122, 1258291200;\n" " mov.b32 %f251, %r123;\n" " fma.rn.f32 %f252, %f251, %f91, %f90;\n" " add.f32 %f253, %f250, %f252;\n" " and.b32 %r124, %r49, 255;\n" " cvt.rn.f32.u32 %f254, %r124;\n" " mul.rn.f32 %f255, %f81, %f254;\n" " mul.rn.f32 %f256, %f255, %f248;\n" " and.b32 %r125, %r57, 255;\n" " cvt.rn.f32.u32 %f257, %r125;\n" " mul.rn.f32 %f258, %f82, %f257;\n" " sub.rn.f32 %f259, %f256, %f258;\n" " and.b32 %r126, %r53, 255;\n" " cvt.rn.f32.u32 %f260, %r126;\n" " mul.rn.f32 %f261, %f85, %f260;\n" " mul.rn.f32 %f262, %f261, %f253;\n" " and.b32 %r127, %r61, 255;\n" " cvt.rn.f32.u32 %f263, %r127;\n" " mul.rn.f32 %f264, %f86, %f263;\n" " sub.rn.f32 %f265, %f262, %f264;\n" " ld.shared.f32 %f266, [%r79+512];\n" " fma.rn.f32 %f267, %f259, %f266, %f231;\n" " fma.rn.f32 %f268, %f265, %f266, %f232;\n" " ld.shared.f32 %f269, [%r79+2560];\n" " fma.rn.f32 %f270, %f259, %f269, %f234;\n" " fma.rn.f32 %f271, %f265, %f269, %f235;\n" " ld.shared.f32 %f272, [%r79+4608];\n" " fma.rn.f32 %f273, %f259, %f272, %f237;\n" " fma.rn.f32 %f274, %f265, %f272, %f238;\n" " ld.shared.f32 %f275, [%r79+6656];\n" " fma.rn.f32 %f276, %f259, %f275, %f240;\n" " fma.rn.f32 %f277, %f265, %f275, %f241;\n" " and.b32 %r128, %r64, 240;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f278, %r129;\n" " fma.rn.f32 %f279, %f278, %f130, %f129;\n" " and.b32 %r130, %r46, 32;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f280, %r131;\n" " mov.f32 %f281, 0fCA800000;\n" " mov.f32 %f282, 0f3F000000;\n" " fma.rn.f32 %f283, %f280, %f282, %f281;\n" " add.f32 %f284, %f279, %f283;\n" " and.b32 %r132, %r65, 240;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f285, %r133;\n" " fma.rn.f32 %f286, %f285, %f130, %f129;\n" " and.b32 %r134, %r47, 32;\n" " or.b32 %r135, %r134, 1258291200;\n" " mov.b32 %f287, %r135;\n" " fma.rn.f32 %f288, %f287, %f282, %f281;\n" " add.f32 %f289, %f286, %f288;\n" " bfe.u32 %r136, %r49, 8, 8;\n" " cvt.rn.f32.u32 %f290, %r136;\n" " mul.rn.f32 %f291, %f81, %f290;\n" " mul.rn.f32 %f292, %f291, %f284;\n" " bfe.u32 %r137, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f293, %r137;\n" " mul.rn.f32 %f294, %f82, %f293;\n" " sub.rn.f32 %f295, %f292, %f294;\n" " bfe.u32 %r138, %r53, 8, 8;\n" " cvt.rn.f32.u32 %f296, %r138;\n" " mul.rn.f32 %f297, %f85, %f296;\n" " mul.rn.f32 %f298, %f297, %f289;\n" " bfe.u32 %r139, %r61, 8, 8;\n" " cvt.rn.f32.u32 %f299, %r139;\n" " mul.rn.f32 %f300, %f86, %f299;\n" " sub.rn.f32 %f301, %f298, %f300;\n" " ld.shared.f32 %f302, [%r79+640];\n" " fma.rn.f32 %f303, %f295, %f302, %f267;\n" " fma.rn.f32 %f304, %f301, %f302, %f268;\n" " ld.shared.f32 %f305, [%r79+2688];\n" " fma.rn.f32 %f306, %f295, %f305, %f270;\n" " fma.rn.f32 %f307, %f301, %f305, %f271;\n" " ld.shared.f32 %f308, [%r79+4736];\n" " fma.rn.f32 %f309, %f295, %f308, %f273;\n" " fma.rn.f32 %f310, %f301, %f308, %f274;\n" " ld.shared.f32 %f311, [%r79+6784];\n" " fma.rn.f32 %f312, %f295, %f311, %f276;\n" " fma.rn.f32 %f313, %f301, %f311, %f277;\n" " and.b32 %r140, %r64, 3840;\n" " or.b32 %r141, %r140, 1258291200;\n" " mov.b32 %f314, %r141;\n" " fma.rn.f32 %f315, %f314, %f168, %f167;\n" " and.b32 %r142, %r46, 64;\n" " or.b32 %r143, %r142, 1258291200;\n" " mov.b32 %f316, %r143;\n" " mov.f32 %f317, 0fCA000000;\n" " mov.f32 %f318, 0f3E800000;\n" " fma.rn.f32 %f319, %f316, %f318, %f317;\n" " add.f32 %f320, %f315, %f319;\n" " and.b32 %r144, %r65, 3840;\n" " or.b32 %r145, %r144, 1258291200;\n" " mov.b32 %f321, %r145;\n" " fma.rn.f32 %f322, %f321, %f168, %f167;\n" " and.b32 %r146, %r47, 64;\n" " or.b32 %r147, %r146, 1258291200;\n" " mov.b32 %f323, %r147;\n" " fma.rn.f32 %f324, %f323, %f318, %f317;\n" " add.f32 %f325, %f322, %f324;\n" " bfe.u32 %r148, %r49, 16, 8;\n" " cvt.rn.f32.u32 %f326, %r148;\n" " mul.rn.f32 %f327, %f81, %f326;\n" " mul.rn.f32 %f328, %f327, %f320;\n" " bfe.u32 %r149, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f329, %r149;\n" " mul.rn.f32 %f330, %f82, %f329;\n" " sub.rn.f32 %f331, %f328, %f330;\n" " bfe.u32 %r150, %r53, 16, 8;\n" " cvt.rn.f32.u32 %f332, %r150;\n" " mul.rn.f32 %f333, %f85, %f332;\n" " mul.rn.f32 %f334, %f333, %f325;\n" " bfe.u32 %r151, %r61, 16, 8;\n" " cvt.rn.f32.u32 %f335, %r151;\n" " mul.rn.f32 %f336, %f86, %f335;\n" " sub.rn.f32 %f337, %f334, %f336;\n" " ld.shared.f32 %f338, [%r79+768];\n" " fma.rn.f32 %f339, %f331, %f338, %f303;\n" " fma.rn.f32 %f340, %f337, %f338, %f304;\n" " ld.shared.f32 %f341, [%r79+2816];\n" " fma.rn.f32 %f342, %f331, %f341, %f306;\n" " fma.rn.f32 %f343, %f337, %f341, %f307;\n" " ld.shared.f32 %f344, [%r79+4864];\n" " fma.rn.f32 %f345, %f331, %f344, %f309;\n" " fma.rn.f32 %f346, %f337, %f344, %f310;\n" " ld.shared.f32 %f347, [%r79+6912];\n" " fma.rn.f32 %f348, %f331, %f347, %f312;\n" " fma.rn.f32 %f349, %f337, %f347, %f313;\n" " and.b32 %r152, %r64, 61440;\n" " or.b32 %r153, %r152, 1258291200;\n" " mov.b32 %f350, %r153;\n" " fma.rn.f32 %f351, %f350, %f206, %f205;\n" " and.b32 %r154, %r46, 128;\n" " or.b32 %r155, %r154, 1258291200;\n" " mov.b32 %f352, %r155;\n" " mov.f32 %f353, 0fC9800000;\n" " mov.f32 %f354, 0f3E000000;\n" " fma.rn.f32 %f355, %f352, %f354, %f353;\n" " add.f32 %f356, %f351, %f355;\n" " and.b32 %r156, %r65, 61440;\n" " or.b32 %r157, %r156, 1258291200;\n" " mov.b32 %f357, %r157;\n" " fma.rn.f32 %f358, %f357, %f206, %f205;\n" " and.b32 %r158, %r47, 128;\n" " or.b32 %r159, %r158, 1258291200;\n" " mov.b32 %f359, %r159;\n" " fma.rn.f32 %f360, %f359, %f354, %f353;\n" " add.f32 %f361, %f358, %f360;\n" " shr.u32 %r160, %r49, 24;\n" " cvt.rn.f32.u32 %f362, %r160;\n" " mul.rn.f32 %f363, %f81, %f362;\n" " mul.rn.f32 %f364, %f363, %f356;\n" " shr.u32 %r161, %r57, 24;\n" " cvt.rn.f32.u32 %f365, %r161;\n" " mul.rn.f32 %f366, %f82, %f365;\n" " sub.rn.f32 %f367, %f364, %f366;\n" " shr.u32 %r162, %r53, 24;\n" " cvt.rn.f32.u32 %f368, %r162;\n" " mul.rn.f32 %f369, %f85, %f368;\n" " mul.rn.f32 %f370, %f369, %f361;\n" " shr.u32 %r163, %r61, 24;\n" " cvt.rn.f32.u32 %f371, %r163;\n" " mul.rn.f32 %f372, %f86, %f371;\n" " sub.rn.f32 %f373, %f370, %f372;\n" " ld.shared.f32 %f374, [%r79+896];\n" " fma.rn.f32 %f375, %f367, %f374, %f339;\n" " fma.rn.f32 %f376, %f373, %f374, %f340;\n" " ld.shared.f32 %f377, [%r79+2944];\n" " fma.rn.f32 %f378, %f367, %f377, %f342;\n" " fma.rn.f32 %f379, %f373, %f377, %f343;\n" " ld.shared.f32 %f380, [%r79+4992];\n" " fma.rn.f32 %f381, %f367, %f380, %f345;\n" " fma.rn.f32 %f382, %f373, %f380, %f346;\n" " ld.shared.f32 %f383, [%r79+7040];\n" " fma.rn.f32 %f384, %f367, %f383, %f348;\n" " fma.rn.f32 %f385, %f373, %f383, %f349;\n" " add.rn.f32 %f492, %f492, %f375;\n" " add.rn.f32 %f496, %f496, %f376;\n" " add.rn.f32 %f491, %f491, %f378;\n" " add.rn.f32 %f495, %f495, %f379;\n" " add.rn.f32 %f490, %f490, %f381;\n" " add.rn.f32 %f494, %f494, %f382;\n" " add.rn.f32 %f489, %f489, %f384;\n" " add.rn.f32 %f493, %f493, %f385;\n" " add.s64 %rd100, %rd100, 184;\n" " add.s64 %rd101, %rd101, 184;\n" " add.s64 %rd102, %rd102, 184;\n" " add.s64 %rd103, %rd103, 184;\n" " add.s32 %r279, %r279, 1024;\n" " add.s32 %r278, %r278, 1;\n" " setp.lt.s32 %p11, %r278, %r272;\n" " @%p11 bra $L__BB6_16;\n" "\n" "$L__BB6_17:\n" " add.s32 %r275, %r275, 512;\n" " setp.lt.s32 %p12, %r275, %r23;\n" " @%p12 bra $L__BB6_2;\n" "\n" "$L__BB6_18:\n" " mov.u32 %r257, %ctaid.y;\n" " shl.b32 %r256, %r257, 2;\n" " ld.param.u32 %r255, [fc_matvec_q5k_nx_param_6];\n" " sub.s32 %r254, %r255, %r256;\n" " cvt.s64.s32 %rd35, %r22;\n" " mov.u32 %r174, 16;\n" " mov.u32 %r183, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f386, %f492, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f389, %f492, %f386;\n" " mov.u32 %r176, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f388, %f389, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f391, %f389, %f388;\n" " mov.u32 %r178, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f390, %f391, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f393, %f391, %f390;\n" " mov.u32 %r180, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f392, %f393, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f395, %f393, %f392;\n" " mov.u32 %r182, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f394, %f395, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f49, %f395, %f394;\n" " // begin inline asm\n" " shfl.down.b32 %f396, %f496, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f399, %f496, %f396;\n" " // begin inline asm\n" " shfl.down.b32 %f398, %f399, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f401, %f399, %f398;\n" " // begin inline asm\n" " shfl.down.b32 %f400, %f401, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f403, %f401, %f400;\n" " // begin inline asm\n" " shfl.down.b32 %f402, %f403, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f405, %f403, %f402;\n" " // begin inline asm\n" " shfl.down.b32 %f404, %f405, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f50, %f405, %f404;\n" " setp.lt.s32 %p13, %r254, 1;\n" " setp.ne.s32 %p14, %r6, 0;\n" " or.pred %p15, %p14, %p13;\n" " @%p15 bra $L__BB6_23;\n" "\n" " mov.u32 %r267, %ctaid.y;\n" " shl.b32 %r266, %r267, 2;\n" " cvt.u32.u64 %r184, %rd1;\n" " setp.ge.s32 %p16, %r184, %r22;\n" " cvt.s64.s32 %rd67, %r266;\n" " mul.lo.s64 %rd68, %rd67, %rd35;\n" " add.s64 %rd69, %rd68, %rd1;\n" " shl.b64 %rd70, %rd69, 2;\n" " add.s64 %rd36, %rd2, %rd70;\n" " @%p16 bra $L__BB6_21;\n" "\n" " st.global.f32 [%rd36], %f49;\n" "\n" "$L__BB6_21:\n" " setp.ge.s32 %p17, %r4, %r22;\n" " @%p17 bra $L__BB6_23;\n" "\n" " st.global.f32 [%rd36+4], %f50;\n" "\n" "$L__BB6_23:\n" " // begin inline asm\n" " shfl.down.b32 %f406, %f491, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f409, %f491, %f406;\n" " // begin inline asm\n" " shfl.down.b32 %f408, %f409, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f411, %f409, %f408;\n" " // begin inline asm\n" " shfl.down.b32 %f410, %f411, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f413, %f411, %f410;\n" " // begin inline asm\n" " shfl.down.b32 %f412, %f413, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f415, %f413, %f412;\n" " // begin inline asm\n" " shfl.down.b32 %f414, %f415, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f51, %f415, %f414;\n" " // begin inline asm\n" " shfl.down.b32 %f416, %f495, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f419, %f495, %f416;\n" " // begin inline asm\n" " shfl.down.b32 %f418, %f419, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f421, %f419, %f418;\n" " // begin inline asm\n" " shfl.down.b32 %f420, %f421, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f423, %f421, %f420;\n" " // begin inline asm\n" " shfl.down.b32 %f422, %f423, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f425, %f423, %f422;\n" " // begin inline asm\n" " shfl.down.b32 %f424, %f425, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f52, %f425, %f424;\n" " setp.lt.s32 %p19, %r5, 2;\n" " or.pred %p20, %p14, %p19;\n" " @%p20 bra $L__BB6_28;\n" "\n" " mov.u32 %r265, %ctaid.y;\n" " shl.b32 %r264, %r265, 2;\n" " cvt.u32.u64 %r205, %rd1;\n" " setp.ge.s32 %p21, %r205, %r22;\n" " add.s32 %r206, %r264, 1;\n" " cvt.s64.s32 %rd71, %r206;\n" " mul.lo.s64 %rd72, %rd71, %rd35;\n" " add.s64 %rd73, %rd72, %rd1;\n" " shl.b64 %rd74, %rd73, 2;\n" " add.s64 %rd37, %rd2, %rd74;\n" " @%p21 bra $L__BB6_26;\n" "\n" " st.global.f32 [%rd37], %f51;\n" "\n" "$L__BB6_26:\n" " setp.ge.s32 %p22, %r4, %r22;\n" " @%p22 bra $L__BB6_28;\n" "\n" " st.global.f32 [%rd37+4], %f52;\n" "\n" "$L__BB6_28:\n" " // begin inline asm\n" " shfl.down.b32 %f426, %f490, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f429, %f490, %f426;\n" " // begin inline asm\n" " shfl.down.b32 %f428, %f429, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f431, %f429, %f428;\n" " // begin inline asm\n" " shfl.down.b32 %f430, %f431, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f433, %f431, %f430;\n" " // begin inline asm\n" " shfl.down.b32 %f432, %f433, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f435, %f433, %f432;\n" " // begin inline asm\n" " shfl.down.b32 %f434, %f435, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f53, %f435, %f434;\n" " // begin inline asm\n" " shfl.down.b32 %f436, %f494, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f439, %f494, %f436;\n" " // begin inline asm\n" " shfl.down.b32 %f438, %f439, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f441, %f439, %f438;\n" " // begin inline asm\n" " shfl.down.b32 %f440, %f441, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f443, %f441, %f440;\n" " // begin inline asm\n" " shfl.down.b32 %f442, %f443, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f445, %f443, %f442;\n" " // begin inline asm\n" " shfl.down.b32 %f444, %f445, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f54, %f445, %f444;\n" " setp.lt.s32 %p24, %r5, 3;\n" " or.pred %p25, %p14, %p24;\n" " @%p25 bra $L__BB6_33;\n" "\n" " mov.u32 %r263, %ctaid.y;\n" " shl.b32 %r262, %r263, 2;\n" " cvt.u32.u64 %r227, %rd1;\n" " setp.ge.s32 %p26, %r227, %r22;\n" " add.s32 %r228, %r262, 2;\n" " cvt.s64.s32 %rd75, %r228;\n" " mul.lo.s64 %rd76, %rd75, %rd35;\n" " add.s64 %rd77, %rd76, %rd1;\n" " shl.b64 %rd78, %rd77, 2;\n" " add.s64 %rd38, %rd2, %rd78;\n" " @%p26 bra $L__BB6_31;\n" "\n" " st.global.f32 [%rd38], %f53;\n" "\n" "$L__BB6_31:\n" " setp.ge.s32 %p27, %r4, %r22;\n" " @%p27 bra $L__BB6_33;\n" "\n" " st.global.f32 [%rd38+4], %f54;\n" "\n" "$L__BB6_33:\n" " // begin inline asm\n" " shfl.down.b32 %f446, %f489, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f449, %f489, %f446;\n" " // begin inline asm\n" " shfl.down.b32 %f448, %f449, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f451, %f449, %f448;\n" " // begin inline asm\n" " shfl.down.b32 %f450, %f451, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f453, %f451, %f450;\n" " // begin inline asm\n" " shfl.down.b32 %f452, %f453, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f455, %f453, %f452;\n" " // begin inline asm\n" " shfl.down.b32 %f454, %f455, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f55, %f455, %f454;\n" " // begin inline asm\n" " shfl.down.b32 %f456, %f493, %r174, %r183;\n" " // end inline asm\n" " add.f32 %f459, %f493, %f456;\n" " // begin inline asm\n" " shfl.down.b32 %f458, %f459, %r176, %r183;\n" " // end inline asm\n" " add.f32 %f461, %f459, %f458;\n" " // begin inline asm\n" " shfl.down.b32 %f460, %f461, %r178, %r183;\n" " // end inline asm\n" " add.f32 %f463, %f461, %f460;\n" " // begin inline asm\n" " shfl.down.b32 %f462, %f463, %r180, %r183;\n" " // end inline asm\n" " add.f32 %f465, %f463, %f462;\n" " // begin inline asm\n" " shfl.down.b32 %f464, %f465, %r182, %r183;\n" " // end inline asm\n" " add.f32 %f56, %f465, %f464;\n" " setp.lt.s32 %p29, %r5, 4;\n" " or.pred %p30, %p14, %p29;\n" " @%p30 bra $L__BB6_38;\n" "\n" " mov.u32 %r261, %ctaid.y;\n" " shl.b32 %r260, %r261, 2;\n" " cvt.u32.u64 %r249, %rd1;\n" " setp.ge.s32 %p31, %r249, %r22;\n" " add.s32 %r250, %r260, 3;\n" " cvt.s64.s32 %rd79, %r250;\n" " mul.lo.s64 %rd80, %rd79, %rd35;\n" " add.s64 %rd81, %rd80, %rd1;\n" " shl.b64 %rd82, %rd81, 2;\n" " add.s64 %rd39, %rd2, %rd82;\n" " @%p31 bra $L__BB6_36;\n" "\n" " st.global.f32 [%rd39], %f55;\n" "\n" "$L__BB6_36:\n" " setp.ge.s32 %p32, %r4, %r22;\n" " @%p32 bra $L__BB6_38;\n" "\n" " st.global.f32 [%rd39+4], %f56;\n" "\n" "$L__BB6_38:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q5k_nx8\n" ".visible .entry fc_matvec_q5k_nx8(\n" " .param .u64 fc_matvec_q5k_nx8_param_0,\n" " .param .u64 fc_matvec_q5k_nx8_param_1,\n" " .param .u64 fc_matvec_q5k_nx8_param_2,\n" " .param .u32 fc_matvec_q5k_nx8_param_3,\n" " .param .u32 fc_matvec_q5k_nx8_param_4,\n" " .param .u32 fc_matvec_q5k_nx8_param_5,\n" " .param .u32 fc_matvec_q5k_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<57>;\n" " .reg .f32 %f<801>;\n" " .reg .b32 %r<368>;\n" " .reg .b64 %rd<152>;\n" "\n" "\n" " ld.param.u64 %rd52, [fc_matvec_q5k_nx8_param_0];\n" " ld.param.u64 %rd53, [fc_matvec_q5k_nx8_param_1];\n" " ld.param.u64 %rd54, [fc_matvec_q5k_nx8_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q5k_nx8_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q5k_nx8_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q5k_nx8_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q5k_nx8_param_6];\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r1, %r26, 3;\n" " sub.s32 %r2, %r25, %r1;\n" " mov.u32 %r3, %tid.x;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 4;\n" " shr.u32 %r29, %r3, 4;\n" " and.b32 %r30, %r29, 268435454;\n" " add.s32 %r31, %r30, %r28;\n" " or.b32 %r4, %r31, 1;\n" " cvt.s64.s32 %rd1, %r31;\n" " min.s32 %r5, %r2, 8;\n" " cvta.to.global.u64 %rd2, %rd54;\n" " and.b32 %r6, %r3, 31;\n" " setp.lt.s32 %p1, %r23, 1;\n" " mov.f32 %f769, 0f00000000;\n" " mov.f32 %f770, %f769;\n" " mov.f32 %f771, %f769;\n" " mov.f32 %f772, %f769;\n" " mov.f32 %f773, %f769;\n" " mov.f32 %f774, %f769;\n" " mov.f32 %f775, %f769;\n" " mov.f32 %f776, %f769;\n" " mov.f32 %f777, %f769;\n" " mov.f32 %f778, %f769;\n" " mov.f32 %f779, %f769;\n" " mov.f32 %f780, %f769;\n" " mov.f32 %f781, %f769;\n" " mov.f32 %f782, %f769;\n" " mov.f32 %f783, %f769;\n" " mov.f32 %f784, %f769;\n" " @%p1 bra $L__BB7_26;\n" "\n" " setp.lt.s32 %p2, %r4, %r22;\n" " cvt.s64.s32 %rd55, %r24;\n" " mul.lo.s64 %rd56, %rd55, %rd1;\n" " cvta.to.global.u64 %rd57, %rd52;\n" " add.s64 %rd148, %rd57, %rd56;\n" " add.s64 %rd149, %rd52, %rd56;\n" " cvt.u64.u32 %rd5, %r6;\n" " or.b32 %r33, %r6, 128;\n" " cvt.u64.u32 %rd6, %r33;\n" " add.s32 %r34, %r1, 1;\n" " add.s32 %r35, %r1, 2;\n" " add.s32 %r36, %r1, 3;\n" " add.s32 %r37, %r1, 4;\n" " add.s32 %r38, %r1, 5;\n" " add.s32 %r39, %r1, 6;\n" " add.s32 %r40, %r1, 7;\n" " mul.wide.s32 %rd58, %r23, %r1;\n" " shl.b64 %rd59, %rd58, 2;\n" " mul.wide.s32 %rd60, %r3, 4;\n" " add.s64 %rd7, %rd59, %rd60;\n" " mul.wide.s32 %rd61, %r23, %r34;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd8, %rd62, %rd60;\n" " mul.wide.s32 %rd63, %r23, %r35;\n" " shl.b64 %rd64, %rd63, 2;\n" " add.s64 %rd9, %rd64, %rd60;\n" " mul.wide.s32 %rd65, %r23, %r36;\n" " shl.b64 %rd66, %rd65, 2;\n" " add.s64 %rd10, %rd66, %rd60;\n" " mul.wide.s32 %rd67, %r23, %r37;\n" " shl.b64 %rd68, %rd67, 2;\n" " add.s64 %rd11, %rd68, %rd60;\n" " mul.wide.s32 %rd69, %r23, %r38;\n" " shl.b64 %rd70, %rd69, 2;\n" " add.s64 %rd12, %rd70, %rd60;\n" " mul.wide.s32 %rd71, %r23, %r39;\n" " shl.b64 %rd72, %rd71, 2;\n" " add.s64 %rd13, %rd72, %rd60;\n" " shl.b32 %r41, %r3, 2;\n" " mov.u32 %r42, fc_kq_shared;\n" " add.s32 %r43, %r42, %r41;\n" " add.s32 %r7, %r43, 8192;\n" " mul.wide.s32 %rd73, %r23, %r40;\n" " shl.b64 %rd74, %rd73, 2;\n" " add.s64 %rd14, %rd74, %rd60;\n" " add.s64 %rd75, %rd149, %rd55;\n" " selp.b64 %rd151, %rd75, %rd149, %p2;\n" " add.s64 %rd76, %rd148, %rd55;\n" " selp.b64 %rd150, %rd76, %rd148, %p2;\n" " mov.u32 %r363, 0;\n" " mov.f32 %f769, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r5, 2;\n" " setp.lt.s32 %p6, %r5, 3;\n" " shl.b64 %rd90, %rd5, 2;\n" " mov.f32 %f770, %f769;\n" " mov.f32 %f771, %f769;\n" " mov.f32 %f772, %f769;\n" " mov.f32 %f773, %f769;\n" " mov.f32 %f774, %f769;\n" " mov.f32 %f775, %f769;\n" " mov.f32 %f776, %f769;\n" " mov.f32 %f777, %f769;\n" " mov.f32 %f778, %f769;\n" " mov.f32 %f779, %f769;\n" " mov.f32 %f780, %f769;\n" " mov.f32 %f781, %f769;\n" " mov.f32 %f782, %f769;\n" " mov.f32 %f783, %f769;\n" " mov.f32 %f784, %f769;\n" "\n" "$L__BB7_2:\n" " sub.s32 %r44, %r23, %r363;\n" " min.s32 %r9, %r44, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r3, %r9;\n" " @%p3 bra $L__BB7_21;\n" "\n" " mov.u32 %r364, %r7;\n" " mov.u64 %rd143, %rd53;\n" " mov.u32 %r365, %r3;\n" "\n" "$L__BB7_4:\n" " .pragma \"nounroll\";\n" " mov.f32 %f745, 0f00000000;\n" " @%p4 bra $L__BB7_6;\n" "\n" " mul.wide.s32 %rd124, %r363, 4;\n" " add.s64 %rd123, %rd7, %rd124;\n" " add.s64 %rd78, %rd143, %rd123;\n" " // begin inline asm\n" " ld.global.nc.f32 %f745, [%rd78];\n" " // end inline asm\n" "\n" "$L__BB7_6:\n" " mov.f32 %f746, 0f00000000;\n" " st.shared.f32 [%r364+-8192], %f745;\n" " @%p5 bra $L__BB7_8;\n" "\n" " mul.wide.s32 %rd136, %r363, 4;\n" " add.s64 %rd135, %rd8, %rd136;\n" " add.s64 %rd79, %rd143, %rd135;\n" " // begin inline asm\n" " ld.global.nc.f32 %f746, [%rd79];\n" " // end inline asm\n" "\n" "$L__BB7_8:\n" " mov.f32 %f747, 0f00000000;\n" " add.s32 %r343, %r364, -8192;\n" " st.shared.f32 [%r343+2048], %f746;\n" " @%p6 bra $L__BB7_10;\n" "\n" " mul.wide.s32 %rd134, %r363, 4;\n" " add.s64 %rd133, %rd9, %rd134;\n" " add.s64 %rd80, %rd143, %rd133;\n" " // begin inline asm\n" " ld.global.nc.f32 %f747, [%rd80];\n" " // end inline asm\n" "\n" "$L__BB7_10:\n" " mov.f32 %f748, 0f00000000;\n" " add.s32 %r344, %r364, -8192;\n" " st.shared.f32 [%r344+4096], %f747;\n" " setp.lt.s32 %p7, %r5, 4;\n" " @%p7 bra $L__BB7_12;\n" "\n" " mul.wide.s32 %rd132, %r363, 4;\n" " add.s64 %rd131, %rd10, %rd132;\n" " add.s64 %rd81, %rd143, %rd131;\n" " // begin inline asm\n" " ld.global.nc.f32 %f748, [%rd81];\n" " // end inline asm\n" "\n" "$L__BB7_12:\n" " mov.f32 %f749, 0f00000000;\n" " add.s32 %r345, %r364, -8192;\n" " st.shared.f32 [%r345+6144], %f748;\n" " setp.lt.s32 %p8, %r5, 5;\n" " @%p8 bra $L__BB7_14;\n" "\n" " mul.wide.s32 %rd130, %r363, 4;\n" " add.s64 %rd129, %rd11, %rd130;\n" " add.s64 %rd82, %rd143, %rd129;\n" " // begin inline asm\n" " ld.global.nc.f32 %f749, [%rd82];\n" " // end inline asm\n" "\n" "$L__BB7_14:\n" " mov.f32 %f750, 0f00000000;\n" " add.s32 %r346, %r364, -8192;\n" " st.shared.f32 [%r346+8192], %f749;\n" " setp.lt.s32 %p9, %r5, 6;\n" " @%p9 bra $L__BB7_16;\n" "\n" " mul.wide.s32 %rd128, %r363, 4;\n" " add.s64 %rd127, %rd12, %rd128;\n" " add.s64 %rd83, %rd143, %rd127;\n" " // begin inline asm\n" " ld.global.nc.f32 %f750, [%rd83];\n" " // end inline asm\n" "\n" "$L__BB7_16:\n" " mov.f32 %f751, 0f00000000;\n" " add.s32 %r347, %r364, -8192;\n" " st.shared.f32 [%r347+10240], %f750;\n" " setp.lt.s32 %p10, %r5, 7;\n" " @%p10 bra $L__BB7_18;\n" "\n" " mul.wide.s32 %rd126, %r363, 4;\n" " add.s64 %rd125, %rd13, %rd126;\n" " add.s64 %rd84, %rd143, %rd125;\n" " // begin inline asm\n" " ld.global.nc.f32 %f751, [%rd84];\n" " // end inline asm\n" "\n" "$L__BB7_18:\n" " mov.f32 %f752, 0f00000000;\n" " add.s32 %r348, %r364, -8192;\n" " st.shared.f32 [%r348+12288], %f751;\n" " setp.lt.s32 %p11, %r5, 8;\n" " @%p11 bra $L__BB7_20;\n" "\n" " mul.wide.s32 %rd138, %r363, 4;\n" " add.s64 %rd137, %rd14, %rd138;\n" " add.s64 %rd85, %rd143, %rd137;\n" " // begin inline asm\n" " ld.global.nc.f32 %f752, [%rd85];\n" " // end inline asm\n" "\n" "$L__BB7_20:\n" " sub.s32 %r355, %r23, %r363;\n" " min.s32 %r354, %r355, 512;\n" " add.s32 %r349, %r364, -8192;\n" " st.shared.f32 [%r349+14336], %f752;\n" " add.s64 %rd143, %rd143, 1024;\n" " add.s32 %r364, %r364, 1024;\n" " add.s32 %r365, %r365, 256;\n" " setp.lt.s32 %p12, %r365, %r354;\n" " @%p12 bra $L__BB7_4;\n" "\n" "$L__BB7_21:\n" " cvt.u32.u64 %r45, %rd1;\n" " setp.ge.s32 %p13, %r45, %r22;\n" " bar.sync 0;\n" " @%p13 bra $L__BB7_25;\n" "\n" " sub.s32 %r357, %r23, %r363;\n" " min.s32 %r356, %r357, 512;\n" " setp.lt.s32 %p14, %r356, 256;\n" " @%p14 bra $L__BB7_25;\n" "\n" " mov.u32 %r366, 0;\n" " mov.u32 %r367, %r42;\n" "\n" "$L__BB7_24:\n" " sub.s32 %r362, %r23, %r363;\n" " min.s32 %r361, %r362, 512;\n" " shr.s32 %r360, %r361, 8;\n" " add.s64 %rd86, %rd149, %rd90;\n" " // begin inline asm\n" " ld.global.nc.u32 %r48, [%rd86];\n" " // end inline asm\n" " add.s64 %rd87, %rd151, %rd90;\n" " // begin inline asm\n" " ld.global.nc.u32 %r49, [%rd87];\n" " // end inline asm\n" " add.s64 %rd88, %rd149, %rd6;\n" " // begin inline asm\n" " ld.global.nc.u8 %r50, [%rd88];\n" " // end inline asm\n" " add.s64 %rd89, %rd151, %rd6;\n" " // begin inline asm\n" " ld.global.nc.u8 %r51, [%rd89];\n" " // end inline asm\n" " ld.global.v2.u32 {%r52, %r53}, [%rd148+160];\n" " ld.global.v2.u32 {%r56, %r57}, [%rd150+160];\n" " ld.global.v2.u32 {%r60, %r61}, [%rd148+168];\n" " ld.global.v2.u32 {%r64, %r65}, [%rd150+168];\n" " ld.global.v2.f32 {%f161, %f162}, [%rd148+176];\n" " ld.global.v2.f32 {%f165, %f166}, [%rd150+176];\n" " shr.u32 %r68, %r48, 16;\n" " shr.u32 %r69, %r49, 16;\n" " and.b32 %r70, %r48, 15;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f169, %r71;\n" " mov.f32 %f170, 0fCB000000;\n" " mov.f32 %f171, 0f3F800000;\n" " fma.rn.f32 %f172, %f169, %f171, %f170;\n" " and.b32 %r72, %r50, 1;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f173, %r73;\n" " mov.f32 %f174, 0fCD000000;\n" " mov.f32 %f175, 0f41800000;\n" " fma.rn.f32 %f176, %f173, %f175, %f174;\n" " add.f32 %f177, %f172, %f176;\n" " and.b32 %r74, %r49, 15;\n" " or.b32 %r75, %r74, 1258291200;\n" " mov.b32 %f178, %r75;\n" " fma.rn.f32 %f179, %f178, %f171, %f170;\n" " and.b32 %r76, %r51, 1;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f180, %r77;\n" " fma.rn.f32 %f181, %f180, %f175, %f174;\n" " add.f32 %f182, %f179, %f181;\n" " and.b32 %r78, %r52, 255;\n" " cvt.rn.f32.u32 %f183, %r78;\n" " mul.rn.f32 %f184, %f161, %f183;\n" " mul.rn.f32 %f185, %f184, %f177;\n" " and.b32 %r79, %r60, 255;\n" " cvt.rn.f32.u32 %f186, %r79;\n" " mul.rn.f32 %f187, %f162, %f186;\n" " sub.rn.f32 %f188, %f185, %f187;\n" " and.b32 %r80, %r56, 255;\n" " cvt.rn.f32.u32 %f189, %r80;\n" " mul.rn.f32 %f190, %f165, %f189;\n" " mul.rn.f32 %f191, %f190, %f182;\n" " and.b32 %r81, %r64, 255;\n" " cvt.rn.f32.u32 %f192, %r81;\n" " mul.rn.f32 %f193, %f166, %f192;\n" " sub.rn.f32 %f194, %f191, %f193;\n" " shl.b32 %r82, %r6, 2;\n" " add.s32 %r83, %r367, %r82;\n" " ld.shared.f32 %f195, [%r83];\n" " mov.f32 %f196, 0f00000000;\n" " fma.rn.f32 %f197, %f188, %f195, %f196;\n" " fma.rn.f32 %f198, %f194, %f195, %f196;\n" " ld.shared.f32 %f199, [%r83+2048];\n" " fma.rn.f32 %f200, %f188, %f199, %f196;\n" " fma.rn.f32 %f201, %f194, %f199, %f196;\n" " ld.shared.f32 %f202, [%r83+4096];\n" " fma.rn.f32 %f203, %f188, %f202, %f196;\n" " fma.rn.f32 %f204, %f194, %f202, %f196;\n" " ld.shared.f32 %f205, [%r83+6144];\n" " fma.rn.f32 %f206, %f188, %f205, %f196;\n" " fma.rn.f32 %f207, %f194, %f205, %f196;\n" " ld.shared.f32 %f208, [%r83+8192];\n" " fma.rn.f32 %f209, %f188, %f208, %f196;\n" " fma.rn.f32 %f210, %f194, %f208, %f196;\n" " ld.shared.f32 %f211, [%r83+10240];\n" " fma.rn.f32 %f212, %f188, %f211, %f196;\n" " fma.rn.f32 %f213, %f194, %f211, %f196;\n" " ld.shared.f32 %f214, [%r83+12288];\n" " fma.rn.f32 %f215, %f188, %f214, %f196;\n" " fma.rn.f32 %f216, %f194, %f214, %f196;\n" " ld.shared.f32 %f217, [%r83+14336];\n" " fma.rn.f32 %f218, %f188, %f217, %f196;\n" " fma.rn.f32 %f219, %f194, %f217, %f196;\n" " and.b32 %r84, %r48, 240;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f220, %r85;\n" " mov.f32 %f221, 0fC9000000;\n" " mov.f32 %f222, 0f3D800000;\n" " fma.rn.f32 %f223, %f220, %f222, %f221;\n" " and.b32 %r86, %r50, 2;\n" " or.b32 %r87, %r86, 1258291200;\n" " mov.b32 %f224, %r87;\n" " mov.f32 %f225, 0fCC800000;\n" " mov.f32 %f226, 0f41000000;\n" " fma.rn.f32 %f227, %f224, %f226, %f225;\n" " add.f32 %f228, %f223, %f227;\n" " and.b32 %r88, %r49, 240;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f229, %r89;\n" " fma.rn.f32 %f230, %f229, %f222, %f221;\n" " and.b32 %r90, %r51, 2;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f231, %r91;\n" " fma.rn.f32 %f232, %f231, %f226, %f225;\n" " add.f32 %f233, %f230, %f232;\n" " bfe.u32 %r92, %r52, 8, 8;\n" " cvt.rn.f32.u32 %f234, %r92;\n" " mul.rn.f32 %f235, %f161, %f234;\n" " mul.rn.f32 %f236, %f235, %f228;\n" " bfe.u32 %r93, %r60, 8, 8;\n" " cvt.rn.f32.u32 %f237, %r93;\n" " mul.rn.f32 %f238, %f162, %f237;\n" " sub.rn.f32 %f239, %f236, %f238;\n" " bfe.u32 %r94, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f240, %r94;\n" " mul.rn.f32 %f241, %f165, %f240;\n" " mul.rn.f32 %f242, %f241, %f233;\n" " bfe.u32 %r95, %r64, 8, 8;\n" " cvt.rn.f32.u32 %f243, %r95;\n" " mul.rn.f32 %f244, %f166, %f243;\n" " sub.rn.f32 %f245, %f242, %f244;\n" " ld.shared.f32 %f246, [%r83+128];\n" " fma.rn.f32 %f247, %f239, %f246, %f197;\n" " fma.rn.f32 %f248, %f245, %f246, %f198;\n" " ld.shared.f32 %f249, [%r83+2176];\n" " fma.rn.f32 %f250, %f239, %f249, %f200;\n" " fma.rn.f32 %f251, %f245, %f249, %f201;\n" " ld.shared.f32 %f252, [%r83+4224];\n" " fma.rn.f32 %f253, %f239, %f252, %f203;\n" " fma.rn.f32 %f254, %f245, %f252, %f204;\n" " ld.shared.f32 %f255, [%r83+6272];\n" " fma.rn.f32 %f256, %f239, %f255, %f206;\n" " fma.rn.f32 %f257, %f245, %f255, %f207;\n" " ld.shared.f32 %f258, [%r83+8320];\n" " fma.rn.f32 %f259, %f239, %f258, %f209;\n" " fma.rn.f32 %f260, %f245, %f258, %f210;\n" " ld.shared.f32 %f261, [%r83+10368];\n" " fma.rn.f32 %f262, %f239, %f261, %f212;\n" " fma.rn.f32 %f263, %f245, %f261, %f213;\n" " ld.shared.f32 %f264, [%r83+12416];\n" " fma.rn.f32 %f265, %f239, %f264, %f215;\n" " fma.rn.f32 %f266, %f245, %f264, %f216;\n" " ld.shared.f32 %f267, [%r83+14464];\n" " fma.rn.f32 %f268, %f239, %f267, %f218;\n" " fma.rn.f32 %f269, %f245, %f267, %f219;\n" " and.b32 %r96, %r48, 3840;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f270, %r97;\n" " mov.f32 %f271, 0fC7000000;\n" " mov.f32 %f272, 0f3B800000;\n" " fma.rn.f32 %f273, %f270, %f272, %f271;\n" " and.b32 %r98, %r50, 4;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f274, %r99;\n" " mov.f32 %f275, 0fCC000000;\n" " mov.f32 %f276, 0f40800000;\n" " fma.rn.f32 %f277, %f274, %f276, %f275;\n" " add.f32 %f278, %f273, %f277;\n" " and.b32 %r100, %r49, 3840;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f279, %r101;\n" " fma.rn.f32 %f280, %f279, %f272, %f271;\n" " and.b32 %r102, %r51, 4;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f281, %r103;\n" " fma.rn.f32 %f282, %f281, %f276, %f275;\n" " add.f32 %f283, %f280, %f282;\n" " bfe.u32 %r104, %r52, 16, 8;\n" " cvt.rn.f32.u32 %f284, %r104;\n" " mul.rn.f32 %f285, %f161, %f284;\n" " mul.rn.f32 %f286, %f285, %f278;\n" " bfe.u32 %r105, %r60, 16, 8;\n" " cvt.rn.f32.u32 %f287, %r105;\n" " mul.rn.f32 %f288, %f162, %f287;\n" " sub.rn.f32 %f289, %f286, %f288;\n" " bfe.u32 %r106, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f290, %r106;\n" " mul.rn.f32 %f291, %f165, %f290;\n" " mul.rn.f32 %f292, %f291, %f283;\n" " bfe.u32 %r107, %r64, 16, 8;\n" " cvt.rn.f32.u32 %f293, %r107;\n" " mul.rn.f32 %f294, %f166, %f293;\n" " sub.rn.f32 %f295, %f292, %f294;\n" " ld.shared.f32 %f296, [%r83+256];\n" " fma.rn.f32 %f297, %f289, %f296, %f247;\n" " fma.rn.f32 %f298, %f295, %f296, %f248;\n" " ld.shared.f32 %f299, [%r83+2304];\n" " fma.rn.f32 %f300, %f289, %f299, %f250;\n" " fma.rn.f32 %f301, %f295, %f299, %f251;\n" " ld.shared.f32 %f302, [%r83+4352];\n" " fma.rn.f32 %f303, %f289, %f302, %f253;\n" " fma.rn.f32 %f304, %f295, %f302, %f254;\n" " ld.shared.f32 %f305, [%r83+6400];\n" " fma.rn.f32 %f306, %f289, %f305, %f256;\n" " fma.rn.f32 %f307, %f295, %f305, %f257;\n" " ld.shared.f32 %f308, [%r83+8448];\n" " fma.rn.f32 %f309, %f289, %f308, %f259;\n" " fma.rn.f32 %f310, %f295, %f308, %f260;\n" " ld.shared.f32 %f311, [%r83+10496];\n" " fma.rn.f32 %f312, %f289, %f311, %f262;\n" " fma.rn.f32 %f313, %f295, %f311, %f263;\n" " ld.shared.f32 %f314, [%r83+12544];\n" " fma.rn.f32 %f315, %f289, %f314, %f265;\n" " fma.rn.f32 %f316, %f295, %f314, %f266;\n" " ld.shared.f32 %f317, [%r83+14592];\n" " fma.rn.f32 %f318, %f289, %f317, %f268;\n" " fma.rn.f32 %f319, %f295, %f317, %f269;\n" " and.b32 %r108, %r48, 61440;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f320, %r109;\n" " mov.f32 %f321, 0fC5000000;\n" " mov.f32 %f322, 0f39800000;\n" " fma.rn.f32 %f323, %f320, %f322, %f321;\n" " and.b32 %r110, %r50, 8;\n" " or.b32 %r111, %r110, 1258291200;\n" " mov.b32 %f324, %r111;\n" " mov.f32 %f325, 0fCB800000;\n" " mov.f32 %f326, 0f40000000;\n" " fma.rn.f32 %f327, %f324, %f326, %f325;\n" " add.f32 %f328, %f323, %f327;\n" " and.b32 %r112, %r49, 61440;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f329, %r113;\n" " fma.rn.f32 %f330, %f329, %f322, %f321;\n" " and.b32 %r114, %r51, 8;\n" " or.b32 %r115, %r114, 1258291200;\n" " mov.b32 %f331, %r115;\n" " fma.rn.f32 %f332, %f331, %f326, %f325;\n" " add.f32 %f333, %f330, %f332;\n" " shr.u32 %r116, %r52, 24;\n" " cvt.rn.f32.u32 %f334, %r116;\n" " mul.rn.f32 %f335, %f161, %f334;\n" " mul.rn.f32 %f336, %f335, %f328;\n" " shr.u32 %r117, %r60, 24;\n" " cvt.rn.f32.u32 %f337, %r117;\n" " mul.rn.f32 %f338, %f162, %f337;\n" " sub.rn.f32 %f339, %f336, %f338;\n" " shr.u32 %r118, %r56, 24;\n" " cvt.rn.f32.u32 %f340, %r118;\n" " mul.rn.f32 %f341, %f165, %f340;\n" " mul.rn.f32 %f342, %f341, %f333;\n" " shr.u32 %r119, %r64, 24;\n" " cvt.rn.f32.u32 %f343, %r119;\n" " mul.rn.f32 %f344, %f166, %f343;\n" " sub.rn.f32 %f345, %f342, %f344;\n" " ld.shared.f32 %f346, [%r83+384];\n" " fma.rn.f32 %f347, %f339, %f346, %f297;\n" " fma.rn.f32 %f348, %f345, %f346, %f298;\n" " ld.shared.f32 %f349, [%r83+2432];\n" " fma.rn.f32 %f350, %f339, %f349, %f300;\n" " fma.rn.f32 %f351, %f345, %f349, %f301;\n" " ld.shared.f32 %f352, [%r83+4480];\n" " fma.rn.f32 %f353, %f339, %f352, %f303;\n" " fma.rn.f32 %f354, %f345, %f352, %f304;\n" " ld.shared.f32 %f355, [%r83+6528];\n" " fma.rn.f32 %f356, %f339, %f355, %f306;\n" " fma.rn.f32 %f357, %f345, %f355, %f307;\n" " ld.shared.f32 %f358, [%r83+8576];\n" " fma.rn.f32 %f359, %f339, %f358, %f309;\n" " fma.rn.f32 %f360, %f345, %f358, %f310;\n" " ld.shared.f32 %f361, [%r83+10624];\n" " fma.rn.f32 %f362, %f339, %f361, %f312;\n" " fma.rn.f32 %f363, %f345, %f361, %f313;\n" " ld.shared.f32 %f364, [%r83+12672];\n" " fma.rn.f32 %f365, %f339, %f364, %f315;\n" " fma.rn.f32 %f366, %f345, %f364, %f316;\n" " ld.shared.f32 %f367, [%r83+14720];\n" " fma.rn.f32 %f368, %f339, %f367, %f318;\n" " fma.rn.f32 %f369, %f345, %f367, %f319;\n" " and.b32 %r120, %r48, 983040;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f370, %r121;\n" " mov.f32 %f371, 0fC3000000;\n" " mov.f32 %f372, 0f37800000;\n" " fma.rn.f32 %f373, %f370, %f372, %f371;\n" " and.b32 %r122, %r50, 16;\n" " or.b32 %r123, %r122, 1258291200;\n" " mov.b32 %f374, %r123;\n" " fma.rn.f32 %f375, %f374, %f171, %f170;\n" " add.f32 %f376, %f373, %f375;\n" " and.b32 %r124, %r49, 983040;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f377, %r125;\n" " fma.rn.f32 %f378, %f377, %f372, %f371;\n" " and.b32 %r126, %r51, 16;\n" " or.b32 %r127, %r126, 1258291200;\n" " mov.b32 %f379, %r127;\n" " fma.rn.f32 %f380, %f379, %f171, %f170;\n" " add.f32 %f381, %f378, %f380;\n" " and.b32 %r128, %r53, 255;\n" " cvt.rn.f32.u32 %f382, %r128;\n" " mul.rn.f32 %f383, %f161, %f382;\n" " mul.rn.f32 %f384, %f383, %f376;\n" " and.b32 %r129, %r61, 255;\n" " cvt.rn.f32.u32 %f385, %r129;\n" " mul.rn.f32 %f386, %f162, %f385;\n" " sub.rn.f32 %f387, %f384, %f386;\n" " and.b32 %r130, %r57, 255;\n" " cvt.rn.f32.u32 %f388, %r130;\n" " mul.rn.f32 %f389, %f165, %f388;\n" " mul.rn.f32 %f390, %f389, %f381;\n" " and.b32 %r131, %r65, 255;\n" " cvt.rn.f32.u32 %f391, %r131;\n" " mul.rn.f32 %f392, %f166, %f391;\n" " sub.rn.f32 %f393, %f390, %f392;\n" " ld.shared.f32 %f394, [%r83+512];\n" " fma.rn.f32 %f395, %f387, %f394, %f347;\n" " fma.rn.f32 %f396, %f393, %f394, %f348;\n" " ld.shared.f32 %f397, [%r83+2560];\n" " fma.rn.f32 %f398, %f387, %f397, %f350;\n" " fma.rn.f32 %f399, %f393, %f397, %f351;\n" " ld.shared.f32 %f400, [%r83+4608];\n" " fma.rn.f32 %f401, %f387, %f400, %f353;\n" " fma.rn.f32 %f402, %f393, %f400, %f354;\n" " ld.shared.f32 %f403, [%r83+6656];\n" " fma.rn.f32 %f404, %f387, %f403, %f356;\n" " fma.rn.f32 %f405, %f393, %f403, %f357;\n" " ld.shared.f32 %f406, [%r83+8704];\n" " fma.rn.f32 %f407, %f387, %f406, %f359;\n" " fma.rn.f32 %f408, %f393, %f406, %f360;\n" " ld.shared.f32 %f409, [%r83+10752];\n" " fma.rn.f32 %f410, %f387, %f409, %f362;\n" " fma.rn.f32 %f411, %f393, %f409, %f363;\n" " ld.shared.f32 %f412, [%r83+12800];\n" " fma.rn.f32 %f413, %f387, %f412, %f365;\n" " fma.rn.f32 %f414, %f393, %f412, %f366;\n" " ld.shared.f32 %f415, [%r83+14848];\n" " fma.rn.f32 %f416, %f387, %f415, %f368;\n" " fma.rn.f32 %f417, %f393, %f415, %f369;\n" " and.b32 %r132, %r68, 240;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f418, %r133;\n" " fma.rn.f32 %f419, %f418, %f222, %f221;\n" " and.b32 %r134, %r50, 32;\n" " or.b32 %r135, %r134, 1258291200;\n" " mov.b32 %f420, %r135;\n" " mov.f32 %f421, 0fCA800000;\n" " mov.f32 %f422, 0f3F000000;\n" " fma.rn.f32 %f423, %f420, %f422, %f421;\n" " add.f32 %f424, %f419, %f423;\n" " and.b32 %r136, %r69, 240;\n" " or.b32 %r137, %r136, 1258291200;\n" " mov.b32 %f425, %r137;\n" " fma.rn.f32 %f426, %f425, %f222, %f221;\n" " and.b32 %r138, %r51, 32;\n" " or.b32 %r139, %r138, 1258291200;\n" " mov.b32 %f427, %r139;\n" " fma.rn.f32 %f428, %f427, %f422, %f421;\n" " add.f32 %f429, %f426, %f428;\n" " bfe.u32 %r140, %r53, 8, 8;\n" " cvt.rn.f32.u32 %f430, %r140;\n" " mul.rn.f32 %f431, %f161, %f430;\n" " mul.rn.f32 %f432, %f431, %f424;\n" " bfe.u32 %r141, %r61, 8, 8;\n" " cvt.rn.f32.u32 %f433, %r141;\n" " mul.rn.f32 %f434, %f162, %f433;\n" " sub.rn.f32 %f435, %f432, %f434;\n" " bfe.u32 %r142, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f436, %r142;\n" " mul.rn.f32 %f437, %f165, %f436;\n" " mul.rn.f32 %f438, %f437, %f429;\n" " bfe.u32 %r143, %r65, 8, 8;\n" " cvt.rn.f32.u32 %f439, %r143;\n" " mul.rn.f32 %f440, %f166, %f439;\n" " sub.rn.f32 %f441, %f438, %f440;\n" " ld.shared.f32 %f442, [%r83+640];\n" " fma.rn.f32 %f443, %f435, %f442, %f395;\n" " fma.rn.f32 %f444, %f441, %f442, %f396;\n" " ld.shared.f32 %f445, [%r83+2688];\n" " fma.rn.f32 %f446, %f435, %f445, %f398;\n" " fma.rn.f32 %f447, %f441, %f445, %f399;\n" " ld.shared.f32 %f448, [%r83+4736];\n" " fma.rn.f32 %f449, %f435, %f448, %f401;\n" " fma.rn.f32 %f450, %f441, %f448, %f402;\n" " ld.shared.f32 %f451, [%r83+6784];\n" " fma.rn.f32 %f452, %f435, %f451, %f404;\n" " fma.rn.f32 %f453, %f441, %f451, %f405;\n" " ld.shared.f32 %f454, [%r83+8832];\n" " fma.rn.f32 %f455, %f435, %f454, %f407;\n" " fma.rn.f32 %f456, %f441, %f454, %f408;\n" " ld.shared.f32 %f457, [%r83+10880];\n" " fma.rn.f32 %f458, %f435, %f457, %f410;\n" " fma.rn.f32 %f459, %f441, %f457, %f411;\n" " ld.shared.f32 %f460, [%r83+12928];\n" " fma.rn.f32 %f461, %f435, %f460, %f413;\n" " fma.rn.f32 %f462, %f441, %f460, %f414;\n" " ld.shared.f32 %f463, [%r83+14976];\n" " fma.rn.f32 %f464, %f435, %f463, %f416;\n" " fma.rn.f32 %f465, %f441, %f463, %f417;\n" " and.b32 %r144, %r68, 3840;\n" " or.b32 %r145, %r144, 1258291200;\n" " mov.b32 %f466, %r145;\n" " fma.rn.f32 %f467, %f466, %f272, %f271;\n" " and.b32 %r146, %r50, 64;\n" " or.b32 %r147, %r146, 1258291200;\n" " mov.b32 %f468, %r147;\n" " mov.f32 %f469, 0fCA000000;\n" " mov.f32 %f470, 0f3E800000;\n" " fma.rn.f32 %f471, %f468, %f470, %f469;\n" " add.f32 %f472, %f467, %f471;\n" " and.b32 %r148, %r69, 3840;\n" " or.b32 %r149, %r148, 1258291200;\n" " mov.b32 %f473, %r149;\n" " fma.rn.f32 %f474, %f473, %f272, %f271;\n" " and.b32 %r150, %r51, 64;\n" " or.b32 %r151, %r150, 1258291200;\n" " mov.b32 %f475, %r151;\n" " fma.rn.f32 %f476, %f475, %f470, %f469;\n" " add.f32 %f477, %f474, %f476;\n" " bfe.u32 %r152, %r53, 16, 8;\n" " cvt.rn.f32.u32 %f478, %r152;\n" " mul.rn.f32 %f479, %f161, %f478;\n" " mul.rn.f32 %f480, %f479, %f472;\n" " bfe.u32 %r153, %r61, 16, 8;\n" " cvt.rn.f32.u32 %f481, %r153;\n" " mul.rn.f32 %f482, %f162, %f481;\n" " sub.rn.f32 %f483, %f480, %f482;\n" " bfe.u32 %r154, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f484, %r154;\n" " mul.rn.f32 %f485, %f165, %f484;\n" " mul.rn.f32 %f486, %f485, %f477;\n" " bfe.u32 %r155, %r65, 16, 8;\n" " cvt.rn.f32.u32 %f487, %r155;\n" " mul.rn.f32 %f488, %f166, %f487;\n" " sub.rn.f32 %f489, %f486, %f488;\n" " ld.shared.f32 %f490, [%r83+768];\n" " fma.rn.f32 %f491, %f483, %f490, %f443;\n" " fma.rn.f32 %f492, %f489, %f490, %f444;\n" " ld.shared.f32 %f493, [%r83+2816];\n" " fma.rn.f32 %f494, %f483, %f493, %f446;\n" " fma.rn.f32 %f495, %f489, %f493, %f447;\n" " ld.shared.f32 %f496, [%r83+4864];\n" " fma.rn.f32 %f497, %f483, %f496, %f449;\n" " fma.rn.f32 %f498, %f489, %f496, %f450;\n" " ld.shared.f32 %f499, [%r83+6912];\n" " fma.rn.f32 %f500, %f483, %f499, %f452;\n" " fma.rn.f32 %f501, %f489, %f499, %f453;\n" " ld.shared.f32 %f502, [%r83+8960];\n" " fma.rn.f32 %f503, %f483, %f502, %f455;\n" " fma.rn.f32 %f504, %f489, %f502, %f456;\n" " ld.shared.f32 %f505, [%r83+11008];\n" " fma.rn.f32 %f506, %f483, %f505, %f458;\n" " fma.rn.f32 %f507, %f489, %f505, %f459;\n" " ld.shared.f32 %f508, [%r83+13056];\n" " fma.rn.f32 %f509, %f483, %f508, %f461;\n" " fma.rn.f32 %f510, %f489, %f508, %f462;\n" " ld.shared.f32 %f511, [%r83+15104];\n" " fma.rn.f32 %f512, %f483, %f511, %f464;\n" " fma.rn.f32 %f513, %f489, %f511, %f465;\n" " and.b32 %r156, %r68, 61440;\n" " or.b32 %r157, %r156, 1258291200;\n" " mov.b32 %f514, %r157;\n" " fma.rn.f32 %f515, %f514, %f322, %f321;\n" " and.b32 %r158, %r50, 128;\n" " or.b32 %r159, %r158, 1258291200;\n" " mov.b32 %f516, %r159;\n" " mov.f32 %f517, 0fC9800000;\n" " mov.f32 %f518, 0f3E000000;\n" " fma.rn.f32 %f519, %f516, %f518, %f517;\n" " add.f32 %f520, %f515, %f519;\n" " and.b32 %r160, %r69, 61440;\n" " or.b32 %r161, %r160, 1258291200;\n" " mov.b32 %f521, %r161;\n" " fma.rn.f32 %f522, %f521, %f322, %f321;\n" " and.b32 %r162, %r51, 128;\n" " or.b32 %r163, %r162, 1258291200;\n" " mov.b32 %f523, %r163;\n" " fma.rn.f32 %f524, %f523, %f518, %f517;\n" " add.f32 %f525, %f522, %f524;\n" " shr.u32 %r164, %r53, 24;\n" " cvt.rn.f32.u32 %f526, %r164;\n" " mul.rn.f32 %f527, %f161, %f526;\n" " mul.rn.f32 %f528, %f527, %f520;\n" " shr.u32 %r165, %r61, 24;\n" " cvt.rn.f32.u32 %f529, %r165;\n" " mul.rn.f32 %f530, %f162, %f529;\n" " sub.rn.f32 %f531, %f528, %f530;\n" " shr.u32 %r166, %r57, 24;\n" " cvt.rn.f32.u32 %f532, %r166;\n" " mul.rn.f32 %f533, %f165, %f532;\n" " mul.rn.f32 %f534, %f533, %f525;\n" " shr.u32 %r167, %r65, 24;\n" " cvt.rn.f32.u32 %f535, %r167;\n" " mul.rn.f32 %f536, %f166, %f535;\n" " sub.rn.f32 %f537, %f534, %f536;\n" " ld.shared.f32 %f538, [%r83+896];\n" " fma.rn.f32 %f539, %f531, %f538, %f491;\n" " fma.rn.f32 %f540, %f537, %f538, %f492;\n" " ld.shared.f32 %f541, [%r83+2944];\n" " fma.rn.f32 %f542, %f531, %f541, %f494;\n" " fma.rn.f32 %f543, %f537, %f541, %f495;\n" " ld.shared.f32 %f544, [%r83+4992];\n" " fma.rn.f32 %f545, %f531, %f544, %f497;\n" " fma.rn.f32 %f546, %f537, %f544, %f498;\n" " ld.shared.f32 %f547, [%r83+7040];\n" " fma.rn.f32 %f548, %f531, %f547, %f500;\n" " fma.rn.f32 %f549, %f537, %f547, %f501;\n" " ld.shared.f32 %f550, [%r83+9088];\n" " fma.rn.f32 %f551, %f531, %f550, %f503;\n" " fma.rn.f32 %f552, %f537, %f550, %f504;\n" " ld.shared.f32 %f553, [%r83+11136];\n" " fma.rn.f32 %f554, %f531, %f553, %f506;\n" " fma.rn.f32 %f555, %f537, %f553, %f507;\n" " ld.shared.f32 %f556, [%r83+13184];\n" " fma.rn.f32 %f557, %f531, %f556, %f509;\n" " fma.rn.f32 %f558, %f537, %f556, %f510;\n" " ld.shared.f32 %f559, [%r83+15232];\n" " fma.rn.f32 %f560, %f531, %f559, %f512;\n" " fma.rn.f32 %f561, %f537, %f559, %f513;\n" " add.rn.f32 %f776, %f776, %f539;\n" " add.rn.f32 %f784, %f784, %f540;\n" " add.rn.f32 %f775, %f775, %f542;\n" " add.rn.f32 %f783, %f783, %f543;\n" " add.rn.f32 %f774, %f774, %f545;\n" " add.rn.f32 %f782, %f782, %f546;\n" " add.rn.f32 %f773, %f773, %f548;\n" " add.rn.f32 %f781, %f781, %f549;\n" " add.rn.f32 %f772, %f772, %f551;\n" " add.rn.f32 %f780, %f780, %f552;\n" " add.rn.f32 %f771, %f771, %f554;\n" " add.rn.f32 %f779, %f779, %f555;\n" " add.rn.f32 %f770, %f770, %f557;\n" " add.rn.f32 %f778, %f778, %f558;\n" " add.rn.f32 %f769, %f769, %f560;\n" " add.rn.f32 %f777, %f777, %f561;\n" " add.s64 %rd148, %rd148, 184;\n" " add.s64 %rd149, %rd149, 184;\n" " add.s64 %rd150, %rd150, 184;\n" " add.s64 %rd151, %rd151, 184;\n" " add.s32 %r367, %r367, 1024;\n" " add.s32 %r366, %r366, 1;\n" " setp.lt.s32 %p15, %r366, %r360;\n" " @%p15 bra $L__BB7_24;\n" "\n" "$L__BB7_25:\n" " add.s32 %r363, %r363, 512;\n" " setp.lt.s32 %p16, %r363, %r23;\n" " @%p16 bra $L__BB7_2;\n" "\n" "$L__BB7_26:\n" " mov.u32 %r353, %ctaid.y;\n" " shl.b32 %r352, %r353, 3;\n" " ld.param.u32 %r351, [fc_matvec_q5k_nx8_param_6];\n" " sub.s32 %r350, %r351, %r352;\n" " cvt.s64.s32 %rd43, %r22;\n" " mov.u32 %r178, 16;\n" " mov.u32 %r187, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f562, %f776, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f565, %f776, %f562;\n" " mov.u32 %r180, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f564, %f565, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f567, %f565, %f564;\n" " mov.u32 %r182, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f566, %f567, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f569, %f567, %f566;\n" " mov.u32 %r184, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f568, %f569, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f571, %f569, %f568;\n" " mov.u32 %r186, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f570, %f571, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f97, %f571, %f570;\n" " // begin inline asm\n" " shfl.down.b32 %f572, %f784, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f575, %f784, %f572;\n" " // begin inline asm\n" " shfl.down.b32 %f574, %f575, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f577, %f575, %f574;\n" " // begin inline asm\n" " shfl.down.b32 %f576, %f577, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f579, %f577, %f576;\n" " // begin inline asm\n" " shfl.down.b32 %f578, %f579, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f581, %f579, %f578;\n" " // begin inline asm\n" " shfl.down.b32 %f580, %f581, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f98, %f581, %f580;\n" " setp.lt.s32 %p17, %r350, 1;\n" " setp.ne.s32 %p18, %r6, 0;\n" " or.pred %p19, %p18, %p17;\n" " @%p19 bra $L__BB7_31;\n" "\n" " cvt.u32.u64 %r188, %rd1;\n" " setp.ge.s32 %p20, %r188, %r22;\n" " cvt.s64.s32 %rd91, %r1;\n" " mul.lo.s64 %rd92, %rd91, %rd43;\n" " add.s64 %rd93, %rd92, %rd1;\n" " shl.b64 %rd94, %rd93, 2;\n" " add.s64 %rd44, %rd2, %rd94;\n" " @%p20 bra $L__BB7_29;\n" "\n" " st.global.f32 [%rd44], %f97;\n" "\n" "$L__BB7_29:\n" " setp.ge.s32 %p21, %r4, %r22;\n" " @%p21 bra $L__BB7_31;\n" "\n" " st.global.f32 [%rd44+4], %f98;\n" "\n" "$L__BB7_31:\n" " // begin inline asm\n" " shfl.down.b32 %f582, %f775, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f585, %f775, %f582;\n" " // begin inline asm\n" " shfl.down.b32 %f584, %f585, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f587, %f585, %f584;\n" " // begin inline asm\n" " shfl.down.b32 %f586, %f587, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f589, %f587, %f586;\n" " // begin inline asm\n" " shfl.down.b32 %f588, %f589, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f591, %f589, %f588;\n" " // begin inline asm\n" " shfl.down.b32 %f590, %f591, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f99, %f591, %f590;\n" " // begin inline asm\n" " shfl.down.b32 %f592, %f783, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f595, %f783, %f592;\n" " // begin inline asm\n" " shfl.down.b32 %f594, %f595, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f597, %f595, %f594;\n" " // begin inline asm\n" " shfl.down.b32 %f596, %f597, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f599, %f597, %f596;\n" " // begin inline asm\n" " shfl.down.b32 %f598, %f599, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f601, %f599, %f598;\n" " // begin inline asm\n" " shfl.down.b32 %f600, %f601, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f100, %f601, %f600;\n" " setp.lt.s32 %p23, %r5, 2;\n" " or.pred %p24, %p18, %p23;\n" " @%p24 bra $L__BB7_36;\n" "\n" " cvt.u32.u64 %r209, %rd1;\n" " setp.ge.s32 %p25, %r209, %r22;\n" " add.s32 %r210, %r1, 1;\n" " cvt.s64.s32 %rd95, %r210;\n" " mul.lo.s64 %rd96, %rd95, %rd43;\n" " add.s64 %rd97, %rd96, %rd1;\n" " shl.b64 %rd98, %rd97, 2;\n" " add.s64 %rd45, %rd2, %rd98;\n" " @%p25 bra $L__BB7_34;\n" "\n" " st.global.f32 [%rd45], %f99;\n" "\n" "$L__BB7_34:\n" " setp.ge.s32 %p26, %r4, %r22;\n" " @%p26 bra $L__BB7_36;\n" "\n" " st.global.f32 [%rd45+4], %f100;\n" "\n" "$L__BB7_36:\n" " // begin inline asm\n" " shfl.down.b32 %f602, %f774, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f605, %f774, %f602;\n" " // begin inline asm\n" " shfl.down.b32 %f604, %f605, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f607, %f605, %f604;\n" " // begin inline asm\n" " shfl.down.b32 %f606, %f607, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f609, %f607, %f606;\n" " // begin inline asm\n" " shfl.down.b32 %f608, %f609, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f611, %f609, %f608;\n" " // begin inline asm\n" " shfl.down.b32 %f610, %f611, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f101, %f611, %f610;\n" " // begin inline asm\n" " shfl.down.b32 %f612, %f782, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f615, %f782, %f612;\n" " // begin inline asm\n" " shfl.down.b32 %f614, %f615, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f617, %f615, %f614;\n" " // begin inline asm\n" " shfl.down.b32 %f616, %f617, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f619, %f617, %f616;\n" " // begin inline asm\n" " shfl.down.b32 %f618, %f619, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f621, %f619, %f618;\n" " // begin inline asm\n" " shfl.down.b32 %f620, %f621, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f102, %f621, %f620;\n" " setp.lt.s32 %p28, %r5, 3;\n" " or.pred %p29, %p18, %p28;\n" " @%p29 bra $L__BB7_41;\n" "\n" " cvt.u32.u64 %r231, %rd1;\n" " setp.ge.s32 %p30, %r231, %r22;\n" " add.s32 %r232, %r1, 2;\n" " cvt.s64.s32 %rd99, %r232;\n" " mul.lo.s64 %rd100, %rd99, %rd43;\n" " add.s64 %rd101, %rd100, %rd1;\n" " shl.b64 %rd102, %rd101, 2;\n" " add.s64 %rd46, %rd2, %rd102;\n" " @%p30 bra $L__BB7_39;\n" "\n" " st.global.f32 [%rd46], %f101;\n" "\n" "$L__BB7_39:\n" " setp.ge.s32 %p31, %r4, %r22;\n" " @%p31 bra $L__BB7_41;\n" "\n" " st.global.f32 [%rd46+4], %f102;\n" "\n" "$L__BB7_41:\n" " // begin inline asm\n" " shfl.down.b32 %f622, %f773, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f625, %f773, %f622;\n" " // begin inline asm\n" " shfl.down.b32 %f624, %f625, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f627, %f625, %f624;\n" " // begin inline asm\n" " shfl.down.b32 %f626, %f627, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f629, %f627, %f626;\n" " // begin inline asm\n" " shfl.down.b32 %f628, %f629, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f631, %f629, %f628;\n" " // begin inline asm\n" " shfl.down.b32 %f630, %f631, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f103, %f631, %f630;\n" " // begin inline asm\n" " shfl.down.b32 %f632, %f781, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f635, %f781, %f632;\n" " // begin inline asm\n" " shfl.down.b32 %f634, %f635, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f637, %f635, %f634;\n" " // begin inline asm\n" " shfl.down.b32 %f636, %f637, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f639, %f637, %f636;\n" " // begin inline asm\n" " shfl.down.b32 %f638, %f639, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f641, %f639, %f638;\n" " // begin inline asm\n" " shfl.down.b32 %f640, %f641, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f104, %f641, %f640;\n" " setp.lt.s32 %p33, %r5, 4;\n" " or.pred %p34, %p18, %p33;\n" " @%p34 bra $L__BB7_46;\n" "\n" " cvt.u32.u64 %r253, %rd1;\n" " setp.ge.s32 %p35, %r253, %r22;\n" " add.s32 %r254, %r1, 3;\n" " cvt.s64.s32 %rd103, %r254;\n" " mul.lo.s64 %rd104, %rd103, %rd43;\n" " add.s64 %rd105, %rd104, %rd1;\n" " shl.b64 %rd106, %rd105, 2;\n" " add.s64 %rd47, %rd2, %rd106;\n" " @%p35 bra $L__BB7_44;\n" "\n" " st.global.f32 [%rd47], %f103;\n" "\n" "$L__BB7_44:\n" " setp.ge.s32 %p36, %r4, %r22;\n" " @%p36 bra $L__BB7_46;\n" "\n" " st.global.f32 [%rd47+4], %f104;\n" "\n" "$L__BB7_46:\n" " // begin inline asm\n" " shfl.down.b32 %f642, %f772, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f645, %f772, %f642;\n" " // begin inline asm\n" " shfl.down.b32 %f644, %f645, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f647, %f645, %f644;\n" " // begin inline asm\n" " shfl.down.b32 %f646, %f647, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f649, %f647, %f646;\n" " // begin inline asm\n" " shfl.down.b32 %f648, %f649, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f651, %f649, %f648;\n" " // begin inline asm\n" " shfl.down.b32 %f650, %f651, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f105, %f651, %f650;\n" " // begin inline asm\n" " shfl.down.b32 %f652, %f780, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f655, %f780, %f652;\n" " // begin inline asm\n" " shfl.down.b32 %f654, %f655, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f657, %f655, %f654;\n" " // begin inline asm\n" " shfl.down.b32 %f656, %f657, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f659, %f657, %f656;\n" " // begin inline asm\n" " shfl.down.b32 %f658, %f659, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f661, %f659, %f658;\n" " // begin inline asm\n" " shfl.down.b32 %f660, %f661, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f106, %f661, %f660;\n" " setp.lt.s32 %p38, %r5, 5;\n" " or.pred %p39, %p18, %p38;\n" " @%p39 bra $L__BB7_51;\n" "\n" " cvt.u32.u64 %r275, %rd1;\n" " setp.ge.s32 %p40, %r275, %r22;\n" " add.s32 %r276, %r1, 4;\n" " cvt.s64.s32 %rd107, %r276;\n" " mul.lo.s64 %rd108, %rd107, %rd43;\n" " add.s64 %rd109, %rd108, %rd1;\n" " shl.b64 %rd110, %rd109, 2;\n" " add.s64 %rd48, %rd2, %rd110;\n" " @%p40 bra $L__BB7_49;\n" "\n" " st.global.f32 [%rd48], %f105;\n" "\n" "$L__BB7_49:\n" " setp.ge.s32 %p41, %r4, %r22;\n" " @%p41 bra $L__BB7_51;\n" "\n" " st.global.f32 [%rd48+4], %f106;\n" "\n" "$L__BB7_51:\n" " // begin inline asm\n" " shfl.down.b32 %f662, %f771, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f665, %f771, %f662;\n" " // begin inline asm\n" " shfl.down.b32 %f664, %f665, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f667, %f665, %f664;\n" " // begin inline asm\n" " shfl.down.b32 %f666, %f667, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f669, %f667, %f666;\n" " // begin inline asm\n" " shfl.down.b32 %f668, %f669, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f671, %f669, %f668;\n" " // begin inline asm\n" " shfl.down.b32 %f670, %f671, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f107, %f671, %f670;\n" " // begin inline asm\n" " shfl.down.b32 %f672, %f779, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f675, %f779, %f672;\n" " // begin inline asm\n" " shfl.down.b32 %f674, %f675, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f677, %f675, %f674;\n" " // begin inline asm\n" " shfl.down.b32 %f676, %f677, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f679, %f677, %f676;\n" " // begin inline asm\n" " shfl.down.b32 %f678, %f679, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f681, %f679, %f678;\n" " // begin inline asm\n" " shfl.down.b32 %f680, %f681, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f108, %f681, %f680;\n" " setp.lt.s32 %p43, %r5, 6;\n" " or.pred %p44, %p18, %p43;\n" " @%p44 bra $L__BB7_56;\n" "\n" " cvt.u32.u64 %r297, %rd1;\n" " setp.ge.s32 %p45, %r297, %r22;\n" " add.s32 %r298, %r1, 5;\n" " cvt.s64.s32 %rd111, %r298;\n" " mul.lo.s64 %rd112, %rd111, %rd43;\n" " add.s64 %rd113, %rd112, %rd1;\n" " shl.b64 %rd114, %rd113, 2;\n" " add.s64 %rd49, %rd2, %rd114;\n" " @%p45 bra $L__BB7_54;\n" "\n" " st.global.f32 [%rd49], %f107;\n" "\n" "$L__BB7_54:\n" " setp.ge.s32 %p46, %r4, %r22;\n" " @%p46 bra $L__BB7_56;\n" "\n" " st.global.f32 [%rd49+4], %f108;\n" "\n" "$L__BB7_56:\n" " // begin inline asm\n" " shfl.down.b32 %f682, %f770, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f685, %f770, %f682;\n" " // begin inline asm\n" " shfl.down.b32 %f684, %f685, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f687, %f685, %f684;\n" " // begin inline asm\n" " shfl.down.b32 %f686, %f687, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f689, %f687, %f686;\n" " // begin inline asm\n" " shfl.down.b32 %f688, %f689, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f691, %f689, %f688;\n" " // begin inline asm\n" " shfl.down.b32 %f690, %f691, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f109, %f691, %f690;\n" " // begin inline asm\n" " shfl.down.b32 %f692, %f778, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f695, %f778, %f692;\n" " // begin inline asm\n" " shfl.down.b32 %f694, %f695, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f697, %f695, %f694;\n" " // begin inline asm\n" " shfl.down.b32 %f696, %f697, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f699, %f697, %f696;\n" " // begin inline asm\n" " shfl.down.b32 %f698, %f699, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f701, %f699, %f698;\n" " // begin inline asm\n" " shfl.down.b32 %f700, %f701, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f110, %f701, %f700;\n" " setp.lt.s32 %p48, %r5, 7;\n" " or.pred %p49, %p18, %p48;\n" " @%p49 bra $L__BB7_61;\n" "\n" " cvt.u32.u64 %r319, %rd1;\n" " setp.ge.s32 %p50, %r319, %r22;\n" " add.s32 %r320, %r1, 6;\n" " cvt.s64.s32 %rd115, %r320;\n" " mul.lo.s64 %rd116, %rd115, %rd43;\n" " add.s64 %rd117, %rd116, %rd1;\n" " shl.b64 %rd118, %rd117, 2;\n" " add.s64 %rd50, %rd2, %rd118;\n" " @%p50 bra $L__BB7_59;\n" "\n" " st.global.f32 [%rd50], %f109;\n" "\n" "$L__BB7_59:\n" " setp.ge.s32 %p51, %r4, %r22;\n" " @%p51 bra $L__BB7_61;\n" "\n" " st.global.f32 [%rd50+4], %f110;\n" "\n" "$L__BB7_61:\n" " // begin inline asm\n" " shfl.down.b32 %f702, %f769, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f705, %f769, %f702;\n" " // begin inline asm\n" " shfl.down.b32 %f704, %f705, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f707, %f705, %f704;\n" " // begin inline asm\n" " shfl.down.b32 %f706, %f707, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f709, %f707, %f706;\n" " // begin inline asm\n" " shfl.down.b32 %f708, %f709, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f711, %f709, %f708;\n" " // begin inline asm\n" " shfl.down.b32 %f710, %f711, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f111, %f711, %f710;\n" " // begin inline asm\n" " shfl.down.b32 %f712, %f777, %r178, %r187;\n" " // end inline asm\n" " add.f32 %f715, %f777, %f712;\n" " // begin inline asm\n" " shfl.down.b32 %f714, %f715, %r180, %r187;\n" " // end inline asm\n" " add.f32 %f717, %f715, %f714;\n" " // begin inline asm\n" " shfl.down.b32 %f716, %f717, %r182, %r187;\n" " // end inline asm\n" " add.f32 %f719, %f717, %f716;\n" " // begin inline asm\n" " shfl.down.b32 %f718, %f719, %r184, %r187;\n" " // end inline asm\n" " add.f32 %f721, %f719, %f718;\n" " // begin inline asm\n" " shfl.down.b32 %f720, %f721, %r186, %r187;\n" " // end inline asm\n" " add.f32 %f112, %f721, %f720;\n" " setp.lt.s32 %p53, %r5, 8;\n" " or.pred %p54, %p18, %p53;\n" " @%p54 bra $L__BB7_66;\n" "\n" " cvt.u32.u64 %r341, %rd1;\n" " setp.ge.s32 %p55, %r341, %r22;\n" " add.s32 %r342, %r1, 7;\n" " cvt.s64.s32 %rd119, %r342;\n" " mul.lo.s64 %rd120, %rd119, %rd43;\n" " add.s64 %rd121, %rd120, %rd1;\n" " shl.b64 %rd122, %rd121, 2;\n" " add.s64 %rd51, %rd2, %rd122;\n" " @%p55 bra $L__BB7_64;\n" "\n" " st.global.f32 [%rd51], %f111;\n" "\n" "$L__BB7_64:\n" " setp.ge.s32 %p56, %r4, %r22;\n" " @%p56 bra $L__BB7_66;\n" "\n" " st.global.f32 [%rd51+4], %f112;\n" "\n" "$L__BB7_66:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q6k\n" ".visible .entry fc_matvec_q6k(\n" " .param .u64 fc_matvec_q6k_param_0,\n" " .param .u64 fc_matvec_q6k_param_1,\n" " .param .u64 fc_matvec_q6k_param_2,\n" " .param .u32 fc_matvec_q6k_param_3,\n" " .param .u32 fc_matvec_q6k_param_4,\n" " .param .u32 fc_matvec_q6k_param_5\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .b16 %rs<3>;\n" " .reg .f32 %f<228>;\n" " .reg .b32 %r<175>;\n" " .reg .b64 %rd<66>;\n" "\n" "\n" " ld.param.u64 %rd26, [fc_matvec_q6k_param_0];\n" " ld.param.u64 %rd24, [fc_matvec_q6k_param_1];\n" " ld.param.u64 %rd25, [fc_matvec_q6k_param_2];\n" " ld.param.u32 %r28, [fc_matvec_q6k_param_3];\n" " ld.param.u32 %r29, [fc_matvec_q6k_param_4];\n" " ld.param.u32 %r30, [fc_matvec_q6k_param_5];\n" " mov.u32 %r1, %tid.x;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r31, %ctaid.x;\n" " shl.b32 %r32, %r31, 4;\n" " shr.u32 %r33, %r1, 4;\n" " and.b32 %r34, %r33, 268435454;\n" " add.s32 %r35, %r34, %r32;\n" " or.b32 %r3, %r35, 1;\n" " setp.ge.s32 %p1, %r3, %r28;\n" " cvt.s64.s32 %rd1, %r35;\n" " cvt.s64.s32 %rd2, %r30;\n" " mul.wide.s32 %rd27, %r30, %r35;\n" " add.s64 %rd64, %rd26, %rd27;\n" " mov.u64 %rd65, %rd64;\n" " @%p1 bra $L__BB8_2;\n" "\n" " add.s64 %rd65, %rd64, %rd2;\n" "\n" "$L__BB8_2:\n" " setp.lt.s32 %p2, %r29, 1;\n" " mov.f32 %f224, 0f00000000;\n" " mov.f32 %f225, %f224;\n" " @%p2 bra $L__BB8_17;\n" "\n" " cvt.u64.u32 %rd6, %r2;\n" " shr.u32 %r38, %r2, 1;\n" " or.b32 %r39, %r38, 192;\n" " and.b32 %r40, %r39, 200;\n" " cvt.u64.u32 %rd7, %r40;\n" " mov.u32 %r41, -2;\n" " sub.s32 %r4, %r41, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd8, %r1;\n" " shl.b32 %r42, %r1, 2;\n" " mov.u32 %r43, fc_kq_shared;\n" " add.s32 %r6, %r43, %r42;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd9, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd10, %r8;\n" " add.s32 %r9, %r1, 768;\n" " mov.f32 %f225, 0f00000000;\n" " mov.u32 %r168, 0;\n" " mov.u32 %r169, %r168;\n" " mov.f32 %f224, %f225;\n" "\n" "$L__BB8_4:\n" " add.s32 %r44, %r168, %r5;\n" " max.s32 %r45, %r44, -2049;\n" " sub.s32 %r12, %r4, %r45;\n" " sub.s32 %r46, %r29, %r169;\n" " min.s32 %r13, %r46, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r13;\n" " @%p3 bra $L__BB8_12;\n" "\n" " shr.u32 %r47, %r12, 8;\n" " add.s32 %r48, %r47, 1;\n" " cvt.s64.s32 %rd13, %r169;\n" " and.b32 %r14, %r48, 3;\n" " setp.eq.s32 %p4, %r14, 0;\n" " mov.u32 %r170, %r1;\n" " @%p4 bra $L__BB8_9;\n" "\n" " add.s64 %rd29, %rd8, %rd13;\n" " shl.b64 %rd30, %rd29, 2;\n" " add.s64 %rd28, %rd24, %rd30;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd28];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p5, %r14, 1;\n" " mov.u32 %r170, %r7;\n" " @%p5 bra $L__BB8_9;\n" "\n" " add.s64 %rd32, %rd9, %rd13;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd31, %rd24, %rd33;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd31];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p6, %r14, 2;\n" " mov.u32 %r170, %r8;\n" " @%p6 bra $L__BB8_9;\n" "\n" " add.s64 %rd35, %rd10, %rd13;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd34, %rd24, %rd36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd34];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r170, %r9;\n" "\n" "$L__BB8_9:\n" " setp.lt.u32 %p7, %r12, 768;\n" " @%p7 bra $L__BB8_12;\n" "\n" " cvt.s64.s32 %rd37, %r170;\n" " add.s64 %rd38, %rd37, %rd13;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd61, %rd24, %rd39;\n" " shl.b32 %r49, %r170, 2;\n" " mov.u32 %r50, fc_kq_shared;\n" " add.s32 %r51, %r50, %r49;\n" " add.s32 %r171, %r51, 2048;\n" "\n" "$L__BB8_11:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd61];\n" " // end inline asm\n" " st.shared.f32 [%r171+-2048], %f20;\n" " add.s64 %rd41, %rd61, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r171+-1024], %f21;\n" " add.s64 %rd42, %rd61, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r171], %f22;\n" " add.s64 %rd43, %rd61, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd43];\n" " // end inline asm\n" " st.shared.f32 [%r171+1024], %f23;\n" " add.s64 %rd61, %rd61, 4096;\n" " add.s32 %r171, %r171, 4096;\n" " add.s32 %r170, %r170, 1024;\n" " setp.lt.s32 %p8, %r170, %r13;\n" " @%p8 bra $L__BB8_11;\n" "\n" "$L__BB8_12:\n" " cvt.u32.u64 %r52, %rd1;\n" " setp.ge.s32 %p9, %r52, %r28;\n" " bar.sync 0;\n" " @%p9 bra $L__BB8_16;\n" "\n" " setp.lt.s32 %p10, %r13, 256;\n" " @%p10 bra $L__BB8_16;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r173, 0;\n" " mov.u32 %r174, %r43;\n" "\n" "$L__BB8_15:\n" " shl.b64 %rd52, %rd6, 2;\n" " add.s64 %rd44, %rd64, %rd52;\n" " // begin inline asm\n" " ld.global.nc.u32 %r55, [%rd44];\n" " // end inline asm\n" " add.s64 %rd45, %rd65, %rd52;\n" " // begin inline asm\n" " ld.global.nc.u32 %r56, [%rd45];\n" " // end inline asm\n" " shl.b64 %rd53, %rd6, 1;\n" " add.s64 %rd54, %rd64, %rd53;\n" " add.s64 %rd46, %rd54, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs1, [%rd46];\n" " // end inline asm\n" " cvt.u32.u16 %r61, %rs1;\n" " add.s64 %rd55, %rd65, %rd53;\n" " add.s64 %rd47, %rd55, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs2, [%rd47];\n" " // end inline asm\n" " cvt.u32.u16 %r62, %rs2;\n" " add.s64 %rd48, %rd64, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd48];\n" " // end inline asm\n" " add.s64 %rd49, %rd65, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd49];\n" " // end inline asm\n" " add.s64 %rd50, %rd64, %rd7;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r57,%r58}, [%rd50];\n" " // end inline asm\n" " add.s64 %rd51, %rd65, %rd7;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r59,%r60}, [%rd51];\n" " // end inline asm\n" " shr.u32 %r63, %r55, 16;\n" " shr.u32 %r64, %r56, 16;\n" " and.b32 %r65, %r55, 15;\n" " or.b32 %r66, %r65, 1258291200;\n" " mov.b32 %f26, %r66;\n" " mov.f32 %f27, 0fCB000020;\n" " mov.f32 %f28, 0f3F800000;\n" " fma.rn.f32 %f29, %f26, %f28, %f27;\n" " and.b32 %r67, %r61, 3;\n" " or.b32 %r68, %r67, 1258291200;\n" " mov.b32 %f30, %r68;\n" " mov.f32 %f31, 0fCD000000;\n" " mov.f32 %f32, 0f41800000;\n" " fma.rn.f32 %f33, %f30, %f32, %f31;\n" " add.f32 %f34, %f29, %f33;\n" " and.b32 %r69, %r56, 15;\n" " or.b32 %r70, %r69, 1258291200;\n" " mov.b32 %f35, %r70;\n" " fma.rn.f32 %f36, %f35, %f28, %f27;\n" " and.b32 %r71, %r62, 3;\n" " or.b32 %r72, %r71, 1258291200;\n" " mov.b32 %f37, %r72;\n" " fma.rn.f32 %f38, %f37, %f32, %f31;\n" " add.f32 %f39, %f36, %f38;\n" " and.b32 %r73, %r57, 255;\n" " cvt.rn.f32.u32 %f40, %r73;\n" " add.f32 %f41, %f40, 0fC3000000;\n" " mul.rn.f32 %f42, %f41, %f34;\n" " and.b32 %r74, %r59, 255;\n" " cvt.rn.f32.u32 %f43, %r74;\n" " add.f32 %f44, %f43, 0fC3000000;\n" " mul.rn.f32 %f45, %f44, %f39;\n" " shl.b32 %r75, %r2, 2;\n" " add.s32 %r76, %r174, %r75;\n" " ld.shared.f32 %f46, [%r76];\n" " mov.f32 %f47, 0f00000000;\n" " fma.rn.f32 %f48, %f42, %f46, %f47;\n" " fma.rn.f32 %f49, %f45, %f46, %f47;\n" " and.b32 %r77, %r55, 3840;\n" " or.b32 %r78, %r77, 1258291200;\n" " mov.b32 %f50, %r78;\n" " mov.f32 %f51, 0fC7002000;\n" " mov.f32 %f52, 0f3B800000;\n" " fma.rn.f32 %f53, %f50, %f52, %f51;\n" " and.b32 %r79, %r61, 12;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f54, %r80;\n" " mov.f32 %f55, 0fCC000000;\n" " mov.f32 %f56, 0f40800000;\n" " fma.rn.f32 %f57, %f54, %f56, %f55;\n" " add.f32 %f58, %f53, %f57;\n" " and.b32 %r81, %r56, 3840;\n" " or.b32 %r82, %r81, 1258291200;\n" " mov.b32 %f59, %r82;\n" " fma.rn.f32 %f60, %f59, %f52, %f51;\n" " and.b32 %r83, %r62, 12;\n" " or.b32 %r84, %r83, 1258291200;\n" " mov.b32 %f61, %r84;\n" " fma.rn.f32 %f62, %f61, %f56, %f55;\n" " add.f32 %f63, %f60, %f62;\n" " bfe.u32 %r85, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f64, %r85;\n" " add.f32 %f65, %f64, 0fC3000000;\n" " mul.rn.f32 %f66, %f65, %f58;\n" " bfe.u32 %r86, %r59, 8, 8;\n" " cvt.rn.f32.u32 %f67, %r86;\n" " add.f32 %f68, %f67, 0fC3000000;\n" " mul.rn.f32 %f69, %f68, %f63;\n" " ld.shared.f32 %f70, [%r76+128];\n" " fma.rn.f32 %f71, %f66, %f70, %f48;\n" " fma.rn.f32 %f72, %f69, %f70, %f49;\n" " and.b32 %r87, %r55, 240;\n" " or.b32 %r88, %r87, 1258291200;\n" " mov.b32 %f73, %r88;\n" " mov.f32 %f74, 0fC9000200;\n" " mov.f32 %f75, 0f3D800000;\n" " fma.rn.f32 %f76, %f73, %f75, %f74;\n" " and.b32 %r89, %r61, 48;\n" " or.b32 %r90, %r89, 1258291200;\n" " mov.b32 %f77, %r90;\n" " mov.f32 %f78, 0fCB000000;\n" " fma.rn.f32 %f79, %f77, %f28, %f78;\n" " add.f32 %f80, %f76, %f79;\n" " and.b32 %r91, %r56, 240;\n" " or.b32 %r92, %r91, 1258291200;\n" " mov.b32 %f81, %r92;\n" " fma.rn.f32 %f82, %f81, %f75, %f74;\n" " and.b32 %r93, %r62, 48;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f83, %r94;\n" " fma.rn.f32 %f84, %f83, %f28, %f78;\n" " add.f32 %f85, %f82, %f84;\n" " bfe.u32 %r95, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f86, %r95;\n" " add.f32 %f87, %f86, 0fC3000000;\n" " mul.rn.f32 %f88, %f87, %f80;\n" " bfe.u32 %r96, %r59, 16, 8;\n" " cvt.rn.f32.u32 %f89, %r96;\n" " add.f32 %f90, %f89, 0fC3000000;\n" " mul.rn.f32 %f91, %f90, %f85;\n" " ld.shared.f32 %f92, [%r76+256];\n" " fma.rn.f32 %f93, %f88, %f92, %f71;\n" " fma.rn.f32 %f94, %f91, %f92, %f72;\n" " and.b32 %r97, %r55, 61440;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f95, %r98;\n" " mov.f32 %f96, 0fC5020000;\n" " mov.f32 %f97, 0f39800000;\n" " fma.rn.f32 %f98, %f95, %f97, %f96;\n" " and.b32 %r99, %r61, 192;\n" " or.b32 %r100, %r99, 1258291200;\n" " mov.b32 %f99, %r100;\n" " mov.f32 %f100, 0fCA000000;\n" " mov.f32 %f101, 0f3E800000;\n" " fma.rn.f32 %f102, %f99, %f101, %f100;\n" " add.f32 %f103, %f98, %f102;\n" " and.b32 %r101, %r56, 61440;\n" " or.b32 %r102, %r101, 1258291200;\n" " mov.b32 %f104, %r102;\n" " fma.rn.f32 %f105, %f104, %f97, %f96;\n" " and.b32 %r103, %r62, 192;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f106, %r104;\n" " fma.rn.f32 %f107, %f106, %f101, %f100;\n" " add.f32 %f108, %f105, %f107;\n" " shr.u32 %r105, %r57, 24;\n" " cvt.rn.f32.u32 %f109, %r105;\n" " add.f32 %f110, %f109, 0fC3000000;\n" " mul.rn.f32 %f111, %f110, %f103;\n" " shr.u32 %r106, %r59, 24;\n" " cvt.rn.f32.u32 %f112, %r106;\n" " add.f32 %f113, %f112, 0fC3000000;\n" " mul.rn.f32 %f114, %f113, %f108;\n" " ld.shared.f32 %f115, [%r76+384];\n" " fma.rn.f32 %f116, %f111, %f115, %f93;\n" " fma.rn.f32 %f117, %f114, %f115, %f94;\n" " bfe.u32 %r107, %r55, 16, 4;\n" " or.b32 %r108, %r107, 1258291200;\n" " mov.b32 %f118, %r108;\n" " fma.rn.f32 %f119, %f118, %f28, %f27;\n" " and.b32 %r109, %r61, 768;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f120, %r110;\n" " mov.f32 %f121, 0fC9000000;\n" " fma.rn.f32 %f122, %f120, %f75, %f121;\n" " add.f32 %f123, %f119, %f122;\n" " bfe.u32 %r111, %r56, 16, 4;\n" " or.b32 %r112, %r111, 1258291200;\n" " mov.b32 %f124, %r112;\n" " fma.rn.f32 %f125, %f124, %f28, %f27;\n" " and.b32 %r113, %r62, 768;\n" " or.b32 %r114, %r113, 1258291200;\n" " mov.b32 %f126, %r114;\n" " fma.rn.f32 %f127, %f126, %f75, %f121;\n" " add.f32 %f128, %f125, %f127;\n" " and.b32 %r115, %r58, 255;\n" " cvt.rn.f32.u32 %f129, %r115;\n" " add.f32 %f130, %f129, 0fC3000000;\n" " mul.rn.f32 %f131, %f130, %f123;\n" " and.b32 %r116, %r60, 255;\n" " cvt.rn.f32.u32 %f132, %r116;\n" " add.f32 %f133, %f132, 0fC3000000;\n" " mul.rn.f32 %f134, %f133, %f128;\n" " ld.shared.f32 %f135, [%r76+512];\n" " fma.rn.f32 %f136, %f131, %f135, %f116;\n" " fma.rn.f32 %f137, %f134, %f135, %f117;\n" " and.b32 %r117, %r63, 3840;\n" " or.b32 %r118, %r117, 1258291200;\n" " mov.b32 %f138, %r118;\n" " fma.rn.f32 %f139, %f138, %f52, %f51;\n" " and.b32 %r119, %r61, 3072;\n" " or.b32 %r120, %r119, 1258291200;\n" " mov.b32 %f140, %r120;\n" " mov.f32 %f141, 0fC8000000;\n" " mov.f32 %f142, 0f3C800000;\n" " fma.rn.f32 %f143, %f140, %f142, %f141;\n" " add.f32 %f144, %f139, %f143;\n" " and.b32 %r121, %r64, 3840;\n" " or.b32 %r122, %r121, 1258291200;\n" " mov.b32 %f145, %r122;\n" " fma.rn.f32 %f146, %f145, %f52, %f51;\n" " and.b32 %r123, %r62, 3072;\n" " or.b32 %r124, %r123, 1258291200;\n" " mov.b32 %f147, %r124;\n" " fma.rn.f32 %f148, %f147, %f142, %f141;\n" " add.f32 %f149, %f146, %f148;\n" " bfe.u32 %r125, %r58, 8, 8;\n" " cvt.rn.f32.u32 %f150, %r125;\n" " add.f32 %f151, %f150, 0fC3000000;\n" " mul.rn.f32 %f152, %f151, %f144;\n" " bfe.u32 %r126, %r60, 8, 8;\n" " cvt.rn.f32.u32 %f153, %r126;\n" " add.f32 %f154, %f153, 0fC3000000;\n" " mul.rn.f32 %f155, %f154, %f149;\n" " ld.shared.f32 %f156, [%r76+640];\n" " fma.rn.f32 %f157, %f152, %f156, %f136;\n" " fma.rn.f32 %f158, %f155, %f156, %f137;\n" " and.b32 %r127, %r63, 240;\n" " or.b32 %r128, %r127, 1258291200;\n" " mov.b32 %f159, %r128;\n" " fma.rn.f32 %f160, %f159, %f75, %f74;\n" " and.b32 %r129, %r61, 12288;\n" " or.b32 %r130, %r129, 1258291200;\n" " mov.b32 %f161, %r130;\n" " mov.f32 %f162, 0fC7000000;\n" " fma.rn.f32 %f163, %f161, %f52, %f162;\n" " add.f32 %f164, %f160, %f163;\n" " and.b32 %r131, %r64, 240;\n" " or.b32 %r132, %r131, 1258291200;\n" " mov.b32 %f165, %r132;\n" " fma.rn.f32 %f166, %f165, %f75, %f74;\n" " and.b32 %r133, %r62, 12288;\n" " or.b32 %r134, %r133, 1258291200;\n" " mov.b32 %f167, %r134;\n" " fma.rn.f32 %f168, %f167, %f52, %f162;\n" " add.f32 %f169, %f166, %f168;\n" " bfe.u32 %r135, %r58, 16, 8;\n" " cvt.rn.f32.u32 %f170, %r135;\n" " add.f32 %f171, %f170, 0fC3000000;\n" " mul.rn.f32 %f172, %f171, %f164;\n" " bfe.u32 %r136, %r60, 16, 8;\n" " cvt.rn.f32.u32 %f173, %r136;\n" " add.f32 %f174, %f173, 0fC3000000;\n" " mul.rn.f32 %f175, %f174, %f169;\n" " ld.shared.f32 %f176, [%r76+768];\n" " fma.rn.f32 %f177, %f172, %f176, %f157;\n" " fma.rn.f32 %f178, %f175, %f176, %f158;\n" " and.b32 %r137, %r63, 61440;\n" " or.b32 %r138, %r137, 1258291200;\n" " mov.b32 %f179, %r138;\n" " fma.rn.f32 %f180, %f179, %f97, %f96;\n" " and.b32 %r139, %r61, 49152;\n" " or.b32 %r140, %r139, 1258291200;\n" " mov.b32 %f181, %r140;\n" " mov.f32 %f182, 0fC6000000;\n" " mov.f32 %f183, 0f3A800000;\n" " fma.rn.f32 %f184, %f181, %f183, %f182;\n" " add.f32 %f185, %f180, %f184;\n" " and.b32 %r141, %r64, 61440;\n" " or.b32 %r142, %r141, 1258291200;\n" " mov.b32 %f186, %r142;\n" " fma.rn.f32 %f187, %f186, %f97, %f96;\n" " and.b32 %r143, %r62, 49152;\n" " or.b32 %r144, %r143, 1258291200;\n" " mov.b32 %f188, %r144;\n" " fma.rn.f32 %f189, %f188, %f183, %f182;\n" " add.f32 %f190, %f187, %f189;\n" " shr.u32 %r145, %r58, 24;\n" " cvt.rn.f32.u32 %f191, %r145;\n" " add.f32 %f192, %f191, 0fC3000000;\n" " mul.rn.f32 %f193, %f192, %f185;\n" " shr.u32 %r146, %r60, 24;\n" " cvt.rn.f32.u32 %f194, %r146;\n" " add.f32 %f195, %f194, 0fC3000000;\n" " mul.rn.f32 %f196, %f195, %f190;\n" " ld.shared.f32 %f197, [%r76+896];\n" " fma.rn.f32 %f198, %f193, %f197, %f177;\n" " fma.rn.f32 %f199, %f196, %f197, %f178;\n" " fma.rn.f32 %f224, %f24, %f198, %f224;\n" " fma.rn.f32 %f225, %f25, %f199, %f225;\n" " add.s64 %rd64, %rd64, 216;\n" " add.s64 %rd65, %rd65, 216;\n" " add.s32 %r174, %r174, 1024;\n" " add.s32 %r173, %r173, 1;\n" " setp.lt.s32 %p11, %r173, %r21;\n" " @%p11 bra $L__BB8_15;\n" "\n" "$L__BB8_16:\n" " add.s32 %r169, %r169, 2048;\n" " setp.lt.s32 %p12, %r169, %r29;\n" " add.s32 %r168, %r168, 2048;\n" " @%p12 bra $L__BB8_4;\n" "\n" "$L__BB8_17:\n" " mov.u32 %r157, 16;\n" " mov.u32 %r166, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f200, %f224, %r157, %r166;\n" " // end inline asm\n" " add.f32 %f203, %f224, %f200;\n" " mov.u32 %r159, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f202, %f203, %r159, %r166;\n" " // end inline asm\n" " add.f32 %f205, %f203, %f202;\n" " mov.u32 %r161, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f204, %f205, %r161, %r166;\n" " // end inline asm\n" " add.f32 %f207, %f205, %f204;\n" " mov.u32 %r163, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f206, %f207, %r163, %r166;\n" " // end inline asm\n" " add.f32 %f209, %f207, %f206;\n" " mov.u32 %r165, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f208, %f209, %r165, %r166;\n" " // end inline asm\n" " add.f32 %f11, %f209, %f208;\n" " // begin inline asm\n" " shfl.down.b32 %f210, %f225, %r157, %r166;\n" " // end inline asm\n" " add.f32 %f213, %f225, %f210;\n" " // begin inline asm\n" " shfl.down.b32 %f212, %f213, %r159, %r166;\n" " // end inline asm\n" " add.f32 %f215, %f213, %f212;\n" " // begin inline asm\n" " shfl.down.b32 %f214, %f215, %r161, %r166;\n" " // end inline asm\n" " add.f32 %f217, %f215, %f214;\n" " // begin inline asm\n" " shfl.down.b32 %f216, %f217, %r163, %r166;\n" " // end inline asm\n" " add.f32 %f219, %f217, %f216;\n" " // begin inline asm\n" " shfl.down.b32 %f218, %f219, %r165, %r166;\n" " // end inline asm\n" " add.f32 %f12, %f219, %f218;\n" " setp.ne.s32 %p13, %r2, 0;\n" " @%p13 bra $L__BB8_22;\n" "\n" " cvt.u32.u64 %r167, %rd1;\n" " setp.ge.s32 %p14, %r167, %r28;\n" " cvta.to.global.u64 %rd56, %rd25;\n" " shl.b64 %rd57, %rd1, 2;\n" " add.s64 %rd23, %rd56, %rd57;\n" " @%p14 bra $L__BB8_20;\n" "\n" " st.global.f32 [%rd23], %f11;\n" "\n" "$L__BB8_20:\n" " @%p1 bra $L__BB8_22;\n" "\n" " st.global.f32 [%rd23+4], %f12;\n" "\n" "$L__BB8_22:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q6k_nx2\n" ".visible .entry fc_matvec_q6k_nx2(\n" " .param .u64 fc_matvec_q6k_nx2_param_0,\n" " .param .u64 fc_matvec_q6k_nx2_param_1,\n" " .param .u64 fc_matvec_q6k_nx2_param_2,\n" " .param .u32 fc_matvec_q6k_nx2_param_3,\n" " .param .u32 fc_matvec_q6k_nx2_param_4,\n" " .param .u32 fc_matvec_q6k_nx2_param_5,\n" " .param .u32 fc_matvec_q6k_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<21>;\n" " .reg .b16 %rs<3>;\n" " .reg .f32 %f<299>;\n" " .reg .b32 %r<185>;\n" " .reg .b64 %rd<71>;\n" "\n" "\n" " ld.param.u64 %rd29, [fc_matvec_q6k_nx2_param_0];\n" " ld.param.u64 %rd28, [fc_matvec_q6k_nx2_param_1];\n" " ld.param.u64 %rd30, [fc_matvec_q6k_nx2_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q6k_nx2_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q6k_nx2_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q6k_nx2_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q6k_nx2_param_6];\n" " cvta.to.global.u64 %rd1, %rd30;\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r1, %r26, 1;\n" " sub.s32 %r2, %r25, %r1;\n" " min.s32 %r3, %r2, 2;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 4;\n" " shr.u32 %r29, %r4, 4;\n" " and.b32 %r30, %r29, 268435454;\n" " add.s32 %r31, %r30, %r28;\n" " or.b32 %r6, %r31, 1;\n" " setp.ge.s32 %p1, %r6, %r22;\n" " cvt.s64.s32 %rd2, %r31;\n" " cvt.s64.s32 %rd3, %r24;\n" " mul.wide.s32 %rd31, %r24, %r31;\n" " add.s64 %rd69, %rd29, %rd31;\n" " mov.u64 %rd70, %rd69;\n" " @%p1 bra $L__BB9_2;\n" "\n" " add.s64 %rd70, %rd69, %rd3;\n" "\n" "$L__BB9_2:\n" " setp.lt.s32 %p2, %r23, 1;\n" " mov.f32 %f291, 0f00000000;\n" " mov.f32 %f292, %f291;\n" " mov.f32 %f293, %f291;\n" " mov.f32 %f294, %f291;\n" " @%p2 bra $L__BB9_16;\n" "\n" " cvt.u64.u32 %rd7, %r5;\n" " shr.u32 %r33, %r5, 1;\n" " or.b32 %r34, %r33, 192;\n" " and.b32 %r35, %r34, 200;\n" " cvt.u64.u32 %rd8, %r35;\n" " add.s32 %r36, %r1, 1;\n" " mul.wide.s32 %rd32, %r23, %r1;\n" " cvt.s64.s32 %rd33, %r4;\n" " add.s64 %rd34, %rd32, %rd33;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd9, %rd28, %rd35;\n" " shl.b32 %r37, %r4, 2;\n" " mov.u32 %r38, fc_kq_shared;\n" " add.s32 %r39, %r38, %r37;\n" " add.s32 %r7, %r39, 2048;\n" " mul.wide.s32 %rd36, %r23, %r36;\n" " add.s64 %rd37, %rd36, %rd33;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd10, %rd28, %rd38;\n" " mov.u32 %r180, 0;\n" " mov.f32 %f291, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " mov.f32 %f292, %f291;\n" " mov.f32 %f293, %f291;\n" " mov.f32 %f294, %f291;\n" "\n" "$L__BB9_4:\n" " sub.s32 %r40, %r23, %r180;\n" " min.s32 %r9, %r40, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r9;\n" " @%p3 bra $L__BB9_11;\n" "\n" " mul.wide.s32 %rd39, %r180, 4;\n" " add.s64 %rd66, %rd9, %rd39;\n" " add.s64 %rd65, %rd10, %rd39;\n" " mov.u32 %r181, %r7;\n" " mov.u32 %r182, %r4;\n" "\n" "$L__BB9_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f286, 0f00000000;\n" " mov.f32 %f285, %f286;\n" " @%p4 bra $L__BB9_8;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f285, [%rd66];\n" " // end inline asm\n" "\n" "$L__BB9_8:\n" " st.shared.f32 [%r181+-2048], %f285;\n" " @%p5 bra $L__BB9_10;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f286, [%rd65];\n" " // end inline asm\n" "\n" "$L__BB9_10:\n" " add.s32 %r179, %r181, -2048;\n" " st.shared.f32 [%r179+2048], %f286;\n" " add.s64 %rd66, %rd66, 1024;\n" " add.s32 %r181, %r181, 1024;\n" " add.s64 %rd65, %rd65, 1024;\n" " add.s32 %r182, %r182, 256;\n" " setp.lt.s32 %p6, %r182, %r9;\n" " @%p6 bra $L__BB9_6;\n" "\n" "$L__BB9_11:\n" " cvt.u32.u64 %r41, %rd2;\n" " setp.ge.s32 %p7, %r41, %r22;\n" " bar.sync 0;\n" " @%p7 bra $L__BB9_15;\n" "\n" " setp.lt.s32 %p8, %r9, 256;\n" " @%p8 bra $L__BB9_15;\n" "\n" " shr.s32 %r16, %r9, 8;\n" " mov.u32 %r183, 0;\n" " mov.u32 %r184, %r38;\n" "\n" "$L__BB9_14:\n" " shl.b64 %rd50, %rd7, 2;\n" " add.s64 %rd42, %rd69, %rd50;\n" " // begin inline asm\n" " ld.global.nc.u32 %r44, [%rd42];\n" " // end inline asm\n" " add.s64 %rd43, %rd70, %rd50;\n" " // begin inline asm\n" " ld.global.nc.u32 %r45, [%rd43];\n" " // end inline asm\n" " shl.b64 %rd51, %rd7, 1;\n" " add.s64 %rd52, %rd69, %rd51;\n" " add.s64 %rd44, %rd52, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs1, [%rd44];\n" " // end inline asm\n" " cvt.u32.u16 %r50, %rs1;\n" " add.s64 %rd53, %rd70, %rd51;\n" " add.s64 %rd45, %rd53, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs2, [%rd45];\n" " // end inline asm\n" " cvt.u32.u16 %r51, %rs2;\n" " add.s64 %rd46, %rd69, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f41, [%rd46];\n" " // end inline asm\n" " add.s64 %rd47, %rd70, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f42, [%rd47];\n" " // end inline asm\n" " add.s64 %rd48, %rd69, %rd8;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r46,%r47}, [%rd48];\n" " // end inline asm\n" " add.s64 %rd49, %rd70, %rd8;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r48,%r49}, [%rd49];\n" " // end inline asm\n" " shr.u32 %r52, %r44, 16;\n" " shr.u32 %r53, %r45, 16;\n" " and.b32 %r54, %r44, 15;\n" " or.b32 %r55, %r54, 1258291200;\n" " mov.b32 %f43, %r55;\n" " mov.f32 %f44, 0fCB000020;\n" " mov.f32 %f45, 0f3F800000;\n" " fma.rn.f32 %f46, %f43, %f45, %f44;\n" " and.b32 %r56, %r50, 3;\n" " or.b32 %r57, %r56, 1258291200;\n" " mov.b32 %f47, %r57;\n" " mov.f32 %f48, 0fCD000000;\n" " mov.f32 %f49, 0f41800000;\n" " fma.rn.f32 %f50, %f47, %f49, %f48;\n" " add.f32 %f51, %f46, %f50;\n" " and.b32 %r58, %r45, 15;\n" " or.b32 %r59, %r58, 1258291200;\n" " mov.b32 %f52, %r59;\n" " fma.rn.f32 %f53, %f52, %f45, %f44;\n" " and.b32 %r60, %r51, 3;\n" " or.b32 %r61, %r60, 1258291200;\n" " mov.b32 %f54, %r61;\n" " fma.rn.f32 %f55, %f54, %f49, %f48;\n" " add.f32 %f56, %f53, %f55;\n" " and.b32 %r62, %r46, 255;\n" " cvt.rn.f32.u32 %f57, %r62;\n" " add.f32 %f58, %f57, 0fC3000000;\n" " mul.rn.f32 %f59, %f58, %f51;\n" " and.b32 %r63, %r48, 255;\n" " cvt.rn.f32.u32 %f60, %r63;\n" " add.f32 %f61, %f60, 0fC3000000;\n" " mul.rn.f32 %f62, %f61, %f56;\n" " shl.b32 %r64, %r5, 2;\n" " add.s32 %r65, %r184, %r64;\n" " ld.shared.f32 %f63, [%r65];\n" " mov.f32 %f64, 0f00000000;\n" " fma.rn.f32 %f65, %f59, %f63, %f64;\n" " fma.rn.f32 %f66, %f62, %f63, %f64;\n" " ld.shared.f32 %f67, [%r65+2048];\n" " fma.rn.f32 %f68, %f59, %f67, %f64;\n" " fma.rn.f32 %f69, %f62, %f67, %f64;\n" " and.b32 %r66, %r44, 3840;\n" " or.b32 %r67, %r66, 1258291200;\n" " mov.b32 %f70, %r67;\n" " mov.f32 %f71, 0fC7002000;\n" " mov.f32 %f72, 0f3B800000;\n" " fma.rn.f32 %f73, %f70, %f72, %f71;\n" " and.b32 %r68, %r50, 12;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f74, %r69;\n" " mov.f32 %f75, 0fCC000000;\n" " mov.f32 %f76, 0f40800000;\n" " fma.rn.f32 %f77, %f74, %f76, %f75;\n" " add.f32 %f78, %f73, %f77;\n" " and.b32 %r70, %r45, 3840;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f79, %r71;\n" " fma.rn.f32 %f80, %f79, %f72, %f71;\n" " and.b32 %r72, %r51, 12;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f81, %r73;\n" " fma.rn.f32 %f82, %f81, %f76, %f75;\n" " add.f32 %f83, %f80, %f82;\n" " bfe.u32 %r74, %r46, 8, 8;\n" " cvt.rn.f32.u32 %f84, %r74;\n" " add.f32 %f85, %f84, 0fC3000000;\n" " mul.rn.f32 %f86, %f85, %f78;\n" " bfe.u32 %r75, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f87, %r75;\n" " add.f32 %f88, %f87, 0fC3000000;\n" " mul.rn.f32 %f89, %f88, %f83;\n" " ld.shared.f32 %f90, [%r65+128];\n" " fma.rn.f32 %f91, %f86, %f90, %f65;\n" " fma.rn.f32 %f92, %f89, %f90, %f66;\n" " ld.shared.f32 %f93, [%r65+2176];\n" " fma.rn.f32 %f94, %f86, %f93, %f68;\n" " fma.rn.f32 %f95, %f89, %f93, %f69;\n" " and.b32 %r76, %r44, 240;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f96, %r77;\n" " mov.f32 %f97, 0fC9000200;\n" " mov.f32 %f98, 0f3D800000;\n" " fma.rn.f32 %f99, %f96, %f98, %f97;\n" " and.b32 %r78, %r50, 48;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f100, %r79;\n" " mov.f32 %f101, 0fCB000000;\n" " fma.rn.f32 %f102, %f100, %f45, %f101;\n" " add.f32 %f103, %f99, %f102;\n" " and.b32 %r80, %r45, 240;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f104, %r81;\n" " fma.rn.f32 %f105, %f104, %f98, %f97;\n" " and.b32 %r82, %r51, 48;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f106, %r83;\n" " fma.rn.f32 %f107, %f106, %f45, %f101;\n" " add.f32 %f108, %f105, %f107;\n" " bfe.u32 %r84, %r46, 16, 8;\n" " cvt.rn.f32.u32 %f109, %r84;\n" " add.f32 %f110, %f109, 0fC3000000;\n" " mul.rn.f32 %f111, %f110, %f103;\n" " bfe.u32 %r85, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f112, %r85;\n" " add.f32 %f113, %f112, 0fC3000000;\n" " mul.rn.f32 %f114, %f113, %f108;\n" " ld.shared.f32 %f115, [%r65+256];\n" " fma.rn.f32 %f116, %f111, %f115, %f91;\n" " fma.rn.f32 %f117, %f114, %f115, %f92;\n" " ld.shared.f32 %f118, [%r65+2304];\n" " fma.rn.f32 %f119, %f111, %f118, %f94;\n" " fma.rn.f32 %f120, %f114, %f118, %f95;\n" " and.b32 %r86, %r44, 61440;\n" " or.b32 %r87, %r86, 1258291200;\n" " mov.b32 %f121, %r87;\n" " mov.f32 %f122, 0fC5020000;\n" " mov.f32 %f123, 0f39800000;\n" " fma.rn.f32 %f124, %f121, %f123, %f122;\n" " and.b32 %r88, %r50, 192;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f125, %r89;\n" " mov.f32 %f126, 0fCA000000;\n" " mov.f32 %f127, 0f3E800000;\n" " fma.rn.f32 %f128, %f125, %f127, %f126;\n" " add.f32 %f129, %f124, %f128;\n" " and.b32 %r90, %r45, 61440;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f130, %r91;\n" " fma.rn.f32 %f131, %f130, %f123, %f122;\n" " and.b32 %r92, %r51, 192;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f132, %r93;\n" " fma.rn.f32 %f133, %f132, %f127, %f126;\n" " add.f32 %f134, %f131, %f133;\n" " shr.u32 %r94, %r46, 24;\n" " cvt.rn.f32.u32 %f135, %r94;\n" " add.f32 %f136, %f135, 0fC3000000;\n" " mul.rn.f32 %f137, %f136, %f129;\n" " shr.u32 %r95, %r48, 24;\n" " cvt.rn.f32.u32 %f138, %r95;\n" " add.f32 %f139, %f138, 0fC3000000;\n" " mul.rn.f32 %f140, %f139, %f134;\n" " ld.shared.f32 %f141, [%r65+384];\n" " fma.rn.f32 %f142, %f137, %f141, %f116;\n" " fma.rn.f32 %f143, %f140, %f141, %f117;\n" " ld.shared.f32 %f144, [%r65+2432];\n" " fma.rn.f32 %f145, %f137, %f144, %f119;\n" " fma.rn.f32 %f146, %f140, %f144, %f120;\n" " bfe.u32 %r96, %r44, 16, 4;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f147, %r97;\n" " fma.rn.f32 %f148, %f147, %f45, %f44;\n" " and.b32 %r98, %r50, 768;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f149, %r99;\n" " mov.f32 %f150, 0fC9000000;\n" " fma.rn.f32 %f151, %f149, %f98, %f150;\n" " add.f32 %f152, %f148, %f151;\n" " bfe.u32 %r100, %r45, 16, 4;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f153, %r101;\n" " fma.rn.f32 %f154, %f153, %f45, %f44;\n" " and.b32 %r102, %r51, 768;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f155, %r103;\n" " fma.rn.f32 %f156, %f155, %f98, %f150;\n" " add.f32 %f157, %f154, %f156;\n" " and.b32 %r104, %r47, 255;\n" " cvt.rn.f32.u32 %f158, %r104;\n" " add.f32 %f159, %f158, 0fC3000000;\n" " mul.rn.f32 %f160, %f159, %f152;\n" " and.b32 %r105, %r49, 255;\n" " cvt.rn.f32.u32 %f161, %r105;\n" " add.f32 %f162, %f161, 0fC3000000;\n" " mul.rn.f32 %f163, %f162, %f157;\n" " ld.shared.f32 %f164, [%r65+512];\n" " fma.rn.f32 %f165, %f160, %f164, %f142;\n" " fma.rn.f32 %f166, %f163, %f164, %f143;\n" " ld.shared.f32 %f167, [%r65+2560];\n" " fma.rn.f32 %f168, %f160, %f167, %f145;\n" " fma.rn.f32 %f169, %f163, %f167, %f146;\n" " and.b32 %r106, %r52, 3840;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f170, %r107;\n" " fma.rn.f32 %f171, %f170, %f72, %f71;\n" " and.b32 %r108, %r50, 3072;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f172, %r109;\n" " mov.f32 %f173, 0fC8000000;\n" " mov.f32 %f174, 0f3C800000;\n" " fma.rn.f32 %f175, %f172, %f174, %f173;\n" " add.f32 %f176, %f171, %f175;\n" " and.b32 %r110, %r53, 3840;\n" " or.b32 %r111, %r110, 1258291200;\n" " mov.b32 %f177, %r111;\n" " fma.rn.f32 %f178, %f177, %f72, %f71;\n" " and.b32 %r112, %r51, 3072;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f179, %r113;\n" " fma.rn.f32 %f180, %f179, %f174, %f173;\n" " add.f32 %f181, %f178, %f180;\n" " bfe.u32 %r114, %r47, 8, 8;\n" " cvt.rn.f32.u32 %f182, %r114;\n" " add.f32 %f183, %f182, 0fC3000000;\n" " mul.rn.f32 %f184, %f183, %f176;\n" " bfe.u32 %r115, %r49, 8, 8;\n" " cvt.rn.f32.u32 %f185, %r115;\n" " add.f32 %f186, %f185, 0fC3000000;\n" " mul.rn.f32 %f187, %f186, %f181;\n" " ld.shared.f32 %f188, [%r65+640];\n" " fma.rn.f32 %f189, %f184, %f188, %f165;\n" " fma.rn.f32 %f190, %f187, %f188, %f166;\n" " ld.shared.f32 %f191, [%r65+2688];\n" " fma.rn.f32 %f192, %f184, %f191, %f168;\n" " fma.rn.f32 %f193, %f187, %f191, %f169;\n" " and.b32 %r116, %r52, 240;\n" " or.b32 %r117, %r116, 1258291200;\n" " mov.b32 %f194, %r117;\n" " fma.rn.f32 %f195, %f194, %f98, %f97;\n" " and.b32 %r118, %r50, 12288;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f196, %r119;\n" " mov.f32 %f197, 0fC7000000;\n" " fma.rn.f32 %f198, %f196, %f72, %f197;\n" " add.f32 %f199, %f195, %f198;\n" " and.b32 %r120, %r53, 240;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f200, %r121;\n" " fma.rn.f32 %f201, %f200, %f98, %f97;\n" " and.b32 %r122, %r51, 12288;\n" " or.b32 %r123, %r122, 1258291200;\n" " mov.b32 %f202, %r123;\n" " fma.rn.f32 %f203, %f202, %f72, %f197;\n" " add.f32 %f204, %f201, %f203;\n" " bfe.u32 %r124, %r47, 16, 8;\n" " cvt.rn.f32.u32 %f205, %r124;\n" " add.f32 %f206, %f205, 0fC3000000;\n" " mul.rn.f32 %f207, %f206, %f199;\n" " bfe.u32 %r125, %r49, 16, 8;\n" " cvt.rn.f32.u32 %f208, %r125;\n" " add.f32 %f209, %f208, 0fC3000000;\n" " mul.rn.f32 %f210, %f209, %f204;\n" " ld.shared.f32 %f211, [%r65+768];\n" " fma.rn.f32 %f212, %f207, %f211, %f189;\n" " fma.rn.f32 %f213, %f210, %f211, %f190;\n" " ld.shared.f32 %f214, [%r65+2816];\n" " fma.rn.f32 %f215, %f207, %f214, %f192;\n" " fma.rn.f32 %f216, %f210, %f214, %f193;\n" " and.b32 %r126, %r52, 61440;\n" " or.b32 %r127, %r126, 1258291200;\n" " mov.b32 %f217, %r127;\n" " fma.rn.f32 %f218, %f217, %f123, %f122;\n" " and.b32 %r128, %r50, 49152;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f219, %r129;\n" " mov.f32 %f220, 0fC6000000;\n" " mov.f32 %f221, 0f3A800000;\n" " fma.rn.f32 %f222, %f219, %f221, %f220;\n" " add.f32 %f223, %f218, %f222;\n" " and.b32 %r130, %r53, 61440;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f224, %r131;\n" " fma.rn.f32 %f225, %f224, %f123, %f122;\n" " and.b32 %r132, %r51, 49152;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f226, %r133;\n" " fma.rn.f32 %f227, %f226, %f221, %f220;\n" " add.f32 %f228, %f225, %f227;\n" " shr.u32 %r134, %r47, 24;\n" " cvt.rn.f32.u32 %f229, %r134;\n" " add.f32 %f230, %f229, 0fC3000000;\n" " mul.rn.f32 %f231, %f230, %f223;\n" " shr.u32 %r135, %r49, 24;\n" " cvt.rn.f32.u32 %f232, %r135;\n" " add.f32 %f233, %f232, 0fC3000000;\n" " mul.rn.f32 %f234, %f233, %f228;\n" " ld.shared.f32 %f235, [%r65+896];\n" " fma.rn.f32 %f236, %f231, %f235, %f212;\n" " fma.rn.f32 %f237, %f234, %f235, %f213;\n" " ld.shared.f32 %f238, [%r65+2944];\n" " fma.rn.f32 %f239, %f231, %f238, %f215;\n" " fma.rn.f32 %f240, %f234, %f238, %f216;\n" " fma.rn.f32 %f292, %f41, %f236, %f292;\n" " fma.rn.f32 %f294, %f42, %f237, %f294;\n" " fma.rn.f32 %f291, %f41, %f239, %f291;\n" " fma.rn.f32 %f293, %f42, %f240, %f293;\n" " add.s64 %rd69, %rd69, 216;\n" " add.s64 %rd70, %rd70, 216;\n" " add.s32 %r184, %r184, 1024;\n" " add.s32 %r183, %r183, 1;\n" " setp.lt.s32 %p9, %r183, %r16;\n" " @%p9 bra $L__BB9_14;\n" "\n" "$L__BB9_15:\n" " add.s32 %r180, %r180, 512;\n" " setp.lt.s32 %p10, %r180, %r23;\n" " @%p10 bra $L__BB9_4;\n" "\n" "$L__BB9_16:\n" " cvt.s64.s32 %rd25, %r22;\n" " mov.u32 %r146, 16;\n" " mov.u32 %r155, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f241, %f292, %r146, %r155;\n" " // end inline asm\n" " add.f32 %f244, %f292, %f241;\n" " mov.u32 %r148, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f243, %f244, %r148, %r155;\n" " // end inline asm\n" " add.f32 %f246, %f244, %f243;\n" " mov.u32 %r150, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f245, %f246, %r150, %r155;\n" " // end inline asm\n" " add.f32 %f248, %f246, %f245;\n" " mov.u32 %r152, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f247, %f248, %r152, %r155;\n" " // end inline asm\n" " add.f32 %f250, %f248, %f247;\n" " mov.u32 %r154, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f249, %f250, %r154, %r155;\n" " // end inline asm\n" " add.f32 %f25, %f250, %f249;\n" " // begin inline asm\n" " shfl.down.b32 %f251, %f294, %r146, %r155;\n" " // end inline asm\n" " add.f32 %f254, %f294, %f251;\n" " // begin inline asm\n" " shfl.down.b32 %f253, %f254, %r148, %r155;\n" " // end inline asm\n" " add.f32 %f256, %f254, %f253;\n" " // begin inline asm\n" " shfl.down.b32 %f255, %f256, %r150, %r155;\n" " // end inline asm\n" " add.f32 %f258, %f256, %f255;\n" " // begin inline asm\n" " shfl.down.b32 %f257, %f258, %r152, %r155;\n" " // end inline asm\n" " add.f32 %f260, %f258, %f257;\n" " // begin inline asm\n" " shfl.down.b32 %f259, %f260, %r154, %r155;\n" " // end inline asm\n" " add.f32 %f26, %f260, %f259;\n" " setp.lt.s32 %p11, %r2, 1;\n" " setp.ne.s32 %p12, %r5, 0;\n" " or.pred %p13, %p12, %p11;\n" " @%p13 bra $L__BB9_21;\n" "\n" " cvt.u32.u64 %r156, %rd2;\n" " setp.ge.s32 %p14, %r156, %r22;\n" " cvt.s64.s32 %rd54, %r1;\n" " mul.lo.s64 %rd55, %rd54, %rd25;\n" " add.s64 %rd56, %rd55, %rd2;\n" " shl.b64 %rd57, %rd56, 2;\n" " add.s64 %rd26, %rd1, %rd57;\n" " @%p14 bra $L__BB9_19;\n" "\n" " st.global.f32 [%rd26], %f25;\n" "\n" "$L__BB9_19:\n" " @%p1 bra $L__BB9_21;\n" "\n" " st.global.f32 [%rd26+4], %f26;\n" "\n" "$L__BB9_21:\n" " // begin inline asm\n" " shfl.down.b32 %f261, %f291, %r146, %r155;\n" " // end inline asm\n" " add.f32 %f264, %f291, %f261;\n" " // begin inline asm\n" " shfl.down.b32 %f263, %f264, %r148, %r155;\n" " // end inline asm\n" " add.f32 %f266, %f264, %f263;\n" " // begin inline asm\n" " shfl.down.b32 %f265, %f266, %r150, %r155;\n" " // end inline asm\n" " add.f32 %f268, %f266, %f265;\n" " // begin inline asm\n" " shfl.down.b32 %f267, %f268, %r152, %r155;\n" " // end inline asm\n" " add.f32 %f270, %f268, %f267;\n" " // begin inline asm\n" " shfl.down.b32 %f269, %f270, %r154, %r155;\n" " // end inline asm\n" " add.f32 %f27, %f270, %f269;\n" " // begin inline asm\n" " shfl.down.b32 %f271, %f293, %r146, %r155;\n" " // end inline asm\n" " add.f32 %f274, %f293, %f271;\n" " // begin inline asm\n" " shfl.down.b32 %f273, %f274, %r148, %r155;\n" " // end inline asm\n" " add.f32 %f276, %f274, %f273;\n" " // begin inline asm\n" " shfl.down.b32 %f275, %f276, %r150, %r155;\n" " // end inline asm\n" " add.f32 %f278, %f276, %f275;\n" " // begin inline asm\n" " shfl.down.b32 %f277, %f278, %r152, %r155;\n" " // end inline asm\n" " add.f32 %f280, %f278, %f277;\n" " // begin inline asm\n" " shfl.down.b32 %f279, %f280, %r154, %r155;\n" " // end inline asm\n" " add.f32 %f28, %f280, %f279;\n" " setp.lt.s32 %p17, %r3, 2;\n" " or.pred %p18, %p12, %p17;\n" " @%p18 bra $L__BB9_26;\n" "\n" " cvt.u32.u64 %r177, %rd2;\n" " setp.ge.s32 %p19, %r177, %r22;\n" " add.s32 %r178, %r1, 1;\n" " cvt.s64.s32 %rd58, %r178;\n" " mul.lo.s64 %rd59, %rd58, %rd25;\n" " add.s64 %rd60, %rd59, %rd2;\n" " shl.b64 %rd61, %rd60, 2;\n" " add.s64 %rd27, %rd1, %rd61;\n" " @%p19 bra $L__BB9_24;\n" "\n" " st.global.f32 [%rd27], %f27;\n" "\n" "$L__BB9_24:\n" " @%p1 bra $L__BB9_26;\n" "\n" " st.global.f32 [%rd27+4], %f28;\n" "\n" "$L__BB9_26:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q6k_nx\n" ".visible .entry fc_matvec_q6k_nx(\n" " .param .u64 fc_matvec_q6k_nx_param_0,\n" " .param .u64 fc_matvec_q6k_nx_param_1,\n" " .param .u64 fc_matvec_q6k_nx_param_2,\n" " .param .u32 fc_matvec_q6k_nx_param_3,\n" " .param .u32 fc_matvec_q6k_nx_param_4,\n" " .param .u32 fc_matvec_q6k_nx_param_5,\n" " .param .u32 fc_matvec_q6k_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<33>;\n" " .reg .b16 %rs<3>;\n" " .reg .f32 %f<448>;\n" " .reg .b32 %r<254>;\n" " .reg .b64 %rd<94>;\n" "\n" "\n" " ld.param.u64 %rd33, [fc_matvec_q6k_nx_param_0];\n" " ld.param.u64 %rd32, [fc_matvec_q6k_nx_param_1];\n" " ld.param.u64 %rd34, [fc_matvec_q6k_nx_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q6k_nx_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q6k_nx_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q6k_nx_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q6k_nx_param_6];\n" " cvta.to.global.u64 %rd1, %rd34;\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r1, %r26, 2;\n" " sub.s32 %r2, %r25, %r1;\n" " min.s32 %r3, %r2, 4;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 4;\n" " shr.u32 %r29, %r4, 4;\n" " and.b32 %r30, %r29, 268435454;\n" " add.s32 %r31, %r30, %r28;\n" " or.b32 %r6, %r31, 1;\n" " setp.ge.s32 %p1, %r6, %r22;\n" " cvt.s64.s32 %rd2, %r31;\n" " cvt.s64.s32 %rd3, %r24;\n" " mul.wide.s32 %rd35, %r24, %r31;\n" " add.s64 %rd92, %rd33, %rd35;\n" " mov.u64 %rd93, %rd92;\n" " @%p1 bra $L__BB10_2;\n" "\n" " add.s64 %rd93, %rd92, %rd3;\n" "\n" "$L__BB10_2:\n" " setp.lt.s32 %p2, %r23, 1;\n" " mov.f32 %f432, 0f00000000;\n" " mov.f32 %f433, %f432;\n" " mov.f32 %f434, %f432;\n" " mov.f32 %f435, %f432;\n" " mov.f32 %f436, %f432;\n" " mov.f32 %f437, %f432;\n" " mov.f32 %f438, %f432;\n" " mov.f32 %f439, %f432;\n" " @%p2 bra $L__BB10_20;\n" "\n" " cvt.u64.u32 %rd7, %r5;\n" " shr.u32 %r33, %r5, 1;\n" " or.b32 %r34, %r33, 192;\n" " and.b32 %r35, %r34, 200;\n" " cvt.u64.u32 %rd8, %r35;\n" " add.s32 %r36, %r1, 1;\n" " add.s32 %r37, %r1, 2;\n" " add.s32 %r38, %r1, 3;\n" " mul.wide.s32 %rd36, %r23, %r1;\n" " shl.b64 %rd37, %rd36, 2;\n" " mul.wide.s32 %rd38, %r4, 4;\n" " add.s64 %rd9, %rd37, %rd38;\n" " mul.wide.s32 %rd39, %r23, %r36;\n" " shl.b64 %rd40, %rd39, 2;\n" " add.s64 %rd10, %rd40, %rd38;\n" " mul.wide.s32 %rd41, %r23, %r37;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd11, %rd42, %rd38;\n" " shl.b32 %r39, %r4, 2;\n" " mov.u32 %r40, fc_kq_shared;\n" " add.s32 %r41, %r40, %r39;\n" " add.s32 %r7, %r41, 4096;\n" " mul.wide.s32 %rd43, %r23, %r38;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd12, %rd44, %rd38;\n" " mov.u32 %r249, 0;\n" " mov.f32 %f432, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " setp.lt.s32 %p6, %r3, 3;\n" " setp.lt.s32 %p7, %r3, 4;\n" " mov.f32 %f433, %f432;\n" " mov.f32 %f434, %f432;\n" " mov.f32 %f435, %f432;\n" " mov.f32 %f436, %f432;\n" " mov.f32 %f437, %f432;\n" " mov.f32 %f438, %f432;\n" " mov.f32 %f439, %f432;\n" "\n" "$L__BB10_4:\n" " sub.s32 %r42, %r23, %r249;\n" " min.s32 %r9, %r42, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r9;\n" " @%p3 bra $L__BB10_15;\n" "\n" " mov.u32 %r250, %r7;\n" " mov.u64 %rd89, %rd32;\n" " mov.u32 %r251, %r4;\n" "\n" "$L__BB10_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f420, 0f00000000;\n" " @%p4 bra $L__BB10_8;\n" "\n" " mul.wide.s32 %rd79, %r249, 4;\n" " add.s64 %rd78, %rd9, %rd79;\n" " add.s64 %rd46, %rd89, %rd78;\n" " // begin inline asm\n" " ld.global.nc.f32 %f420, [%rd46];\n" " // end inline asm\n" "\n" "$L__BB10_8:\n" " mov.f32 %f421, 0f00000000;\n" " st.shared.f32 [%r250+-4096], %f420;\n" " @%p5 bra $L__BB10_10;\n" "\n" " mul.wide.s32 %rd83, %r249, 4;\n" " add.s64 %rd82, %rd10, %rd83;\n" " add.s64 %rd47, %rd89, %rd82;\n" " // begin inline asm\n" " ld.global.nc.f32 %f421, [%rd47];\n" " // end inline asm\n" "\n" "$L__BB10_10:\n" " mov.f32 %f422, 0f00000000;\n" " add.s32 %r225, %r250, -4096;\n" " st.shared.f32 [%r225+2048], %f421;\n" " @%p6 bra $L__BB10_12;\n" "\n" " mul.wide.s32 %rd81, %r249, 4;\n" " add.s64 %rd80, %rd11, %rd81;\n" " add.s64 %rd48, %rd89, %rd80;\n" " // begin inline asm\n" " ld.global.nc.f32 %f422, [%rd48];\n" " // end inline asm\n" "\n" "$L__BB10_12:\n" " mov.f32 %f423, 0f00000000;\n" " add.s32 %r226, %r250, -4096;\n" " st.shared.f32 [%r226+4096], %f422;\n" " @%p7 bra $L__BB10_14;\n" "\n" " mul.wide.s32 %rd85, %r249, 4;\n" " add.s64 %rd84, %rd12, %rd85;\n" " add.s64 %rd49, %rd89, %rd84;\n" " // begin inline asm\n" " ld.global.nc.f32 %f423, [%rd49];\n" " // end inline asm\n" "\n" "$L__BB10_14:\n" " sub.s32 %r233, %r23, %r249;\n" " min.s32 %r232, %r233, 512;\n" " add.s32 %r227, %r250, -4096;\n" " st.shared.f32 [%r227+6144], %f423;\n" " add.s64 %rd89, %rd89, 1024;\n" " add.s32 %r250, %r250, 1024;\n" " add.s32 %r251, %r251, 256;\n" " setp.lt.s32 %p8, %r251, %r232;\n" " @%p8 bra $L__BB10_6;\n" "\n" "$L__BB10_15:\n" " cvt.u32.u64 %r43, %rd2;\n" " setp.ge.s32 %p9, %r43, %r22;\n" " bar.sync 0;\n" " @%p9 bra $L__BB10_19;\n" "\n" " sub.s32 %r235, %r23, %r249;\n" " min.s32 %r234, %r235, 512;\n" " setp.lt.s32 %p10, %r234, 256;\n" " @%p10 bra $L__BB10_19;\n" "\n" " mov.u32 %r252, 0;\n" " mov.u32 %r253, %r40;\n" "\n" "$L__BB10_18:\n" " sub.s32 %r248, %r23, %r249;\n" " min.s32 %r247, %r248, 512;\n" " shr.s32 %r246, %r247, 8;\n" " shl.b64 %rd58, %rd7, 2;\n" " add.s64 %rd50, %rd92, %rd58;\n" " // begin inline asm\n" " ld.global.nc.u32 %r46, [%rd50];\n" " // end inline asm\n" " add.s64 %rd51, %rd93, %rd58;\n" " // begin inline asm\n" " ld.global.nc.u32 %r47, [%rd51];\n" " // end inline asm\n" " shl.b64 %rd59, %rd7, 1;\n" " add.s64 %rd60, %rd92, %rd59;\n" " add.s64 %rd52, %rd60, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs1, [%rd52];\n" " // end inline asm\n" " add.s64 %rd61, %rd93, %rd59;\n" " add.s64 %rd53, %rd61, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs2, [%rd53];\n" " // end inline asm\n" " add.s64 %rd54, %rd92, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f81, [%rd54];\n" " // end inline asm\n" " add.s64 %rd55, %rd93, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f82, [%rd55];\n" " // end inline asm\n" " add.s64 %rd56, %rd92, %rd8;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r48,%r49}, [%rd56];\n" " // end inline asm\n" " add.s64 %rd57, %rd93, %rd8;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r50,%r51}, [%rd57];\n" " // end inline asm\n" " cvt.u32.u16 %r52, %rs1;\n" " cvt.u32.u16 %r53, %rs2;\n" " shr.u32 %r54, %r46, 16;\n" " shr.u32 %r55, %r47, 16;\n" " and.b32 %r56, %r46, 15;\n" " or.b32 %r57, %r56, 1258291200;\n" " mov.b32 %f83, %r57;\n" " mov.f32 %f84, 0fCB000020;\n" " mov.f32 %f85, 0f3F800000;\n" " fma.rn.f32 %f86, %f83, %f85, %f84;\n" " and.b32 %r58, %r52, 3;\n" " or.b32 %r59, %r58, 1258291200;\n" " mov.b32 %f87, %r59;\n" " mov.f32 %f88, 0fCD000000;\n" " mov.f32 %f89, 0f41800000;\n" " fma.rn.f32 %f90, %f87, %f89, %f88;\n" " add.f32 %f91, %f86, %f90;\n" " and.b32 %r60, %r47, 15;\n" " or.b32 %r61, %r60, 1258291200;\n" " mov.b32 %f92, %r61;\n" " fma.rn.f32 %f93, %f92, %f85, %f84;\n" " and.b32 %r62, %r53, 3;\n" " or.b32 %r63, %r62, 1258291200;\n" " mov.b32 %f94, %r63;\n" " fma.rn.f32 %f95, %f94, %f89, %f88;\n" " add.f32 %f96, %f93, %f95;\n" " and.b32 %r64, %r48, 255;\n" " cvt.rn.f32.u32 %f97, %r64;\n" " add.f32 %f98, %f97, 0fC3000000;\n" " mul.rn.f32 %f99, %f98, %f91;\n" " and.b32 %r65, %r50, 255;\n" " cvt.rn.f32.u32 %f100, %r65;\n" " add.f32 %f101, %f100, 0fC3000000;\n" " mul.rn.f32 %f102, %f101, %f96;\n" " shl.b32 %r66, %r5, 2;\n" " add.s32 %r67, %r253, %r66;\n" " ld.shared.f32 %f103, [%r67];\n" " mov.f32 %f104, 0f00000000;\n" " fma.rn.f32 %f105, %f99, %f103, %f104;\n" " fma.rn.f32 %f106, %f102, %f103, %f104;\n" " ld.shared.f32 %f107, [%r67+2048];\n" " fma.rn.f32 %f108, %f99, %f107, %f104;\n" " fma.rn.f32 %f109, %f102, %f107, %f104;\n" " ld.shared.f32 %f110, [%r67+4096];\n" " fma.rn.f32 %f111, %f99, %f110, %f104;\n" " fma.rn.f32 %f112, %f102, %f110, %f104;\n" " ld.shared.f32 %f113, [%r67+6144];\n" " fma.rn.f32 %f114, %f99, %f113, %f104;\n" " fma.rn.f32 %f115, %f102, %f113, %f104;\n" " and.b32 %r68, %r46, 3840;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f116, %r69;\n" " mov.f32 %f117, 0fC7002000;\n" " mov.f32 %f118, 0f3B800000;\n" " fma.rn.f32 %f119, %f116, %f118, %f117;\n" " and.b32 %r70, %r52, 12;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f120, %r71;\n" " mov.f32 %f121, 0fCC000000;\n" " mov.f32 %f122, 0f40800000;\n" " fma.rn.f32 %f123, %f120, %f122, %f121;\n" " add.f32 %f124, %f119, %f123;\n" " and.b32 %r72, %r47, 3840;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f125, %r73;\n" " fma.rn.f32 %f126, %f125, %f118, %f117;\n" " and.b32 %r74, %r53, 12;\n" " or.b32 %r75, %r74, 1258291200;\n" " mov.b32 %f127, %r75;\n" " fma.rn.f32 %f128, %f127, %f122, %f121;\n" " add.f32 %f129, %f126, %f128;\n" " bfe.u32 %r76, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f130, %r76;\n" " add.f32 %f131, %f130, 0fC3000000;\n" " mul.rn.f32 %f132, %f131, %f124;\n" " bfe.u32 %r77, %r50, 8, 8;\n" " cvt.rn.f32.u32 %f133, %r77;\n" " add.f32 %f134, %f133, 0fC3000000;\n" " mul.rn.f32 %f135, %f134, %f129;\n" " ld.shared.f32 %f136, [%r67+128];\n" " fma.rn.f32 %f137, %f132, %f136, %f105;\n" " fma.rn.f32 %f138, %f135, %f136, %f106;\n" " ld.shared.f32 %f139, [%r67+2176];\n" " fma.rn.f32 %f140, %f132, %f139, %f108;\n" " fma.rn.f32 %f141, %f135, %f139, %f109;\n" " ld.shared.f32 %f142, [%r67+4224];\n" " fma.rn.f32 %f143, %f132, %f142, %f111;\n" " fma.rn.f32 %f144, %f135, %f142, %f112;\n" " ld.shared.f32 %f145, [%r67+6272];\n" " fma.rn.f32 %f146, %f132, %f145, %f114;\n" " fma.rn.f32 %f147, %f135, %f145, %f115;\n" " and.b32 %r78, %r46, 240;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f148, %r79;\n" " mov.f32 %f149, 0fC9000200;\n" " mov.f32 %f150, 0f3D800000;\n" " fma.rn.f32 %f151, %f148, %f150, %f149;\n" " and.b32 %r80, %r52, 48;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f152, %r81;\n" " mov.f32 %f153, 0fCB000000;\n" " fma.rn.f32 %f154, %f152, %f85, %f153;\n" " add.f32 %f155, %f151, %f154;\n" " and.b32 %r82, %r47, 240;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f156, %r83;\n" " fma.rn.f32 %f157, %f156, %f150, %f149;\n" " and.b32 %r84, %r53, 48;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f158, %r85;\n" " fma.rn.f32 %f159, %f158, %f85, %f153;\n" " add.f32 %f160, %f157, %f159;\n" " bfe.u32 %r86, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f161, %r86;\n" " add.f32 %f162, %f161, 0fC3000000;\n" " mul.rn.f32 %f163, %f162, %f155;\n" " bfe.u32 %r87, %r50, 16, 8;\n" " cvt.rn.f32.u32 %f164, %r87;\n" " add.f32 %f165, %f164, 0fC3000000;\n" " mul.rn.f32 %f166, %f165, %f160;\n" " ld.shared.f32 %f167, [%r67+256];\n" " fma.rn.f32 %f168, %f163, %f167, %f137;\n" " fma.rn.f32 %f169, %f166, %f167, %f138;\n" " ld.shared.f32 %f170, [%r67+2304];\n" " fma.rn.f32 %f171, %f163, %f170, %f140;\n" " fma.rn.f32 %f172, %f166, %f170, %f141;\n" " ld.shared.f32 %f173, [%r67+4352];\n" " fma.rn.f32 %f174, %f163, %f173, %f143;\n" " fma.rn.f32 %f175, %f166, %f173, %f144;\n" " ld.shared.f32 %f176, [%r67+6400];\n" " fma.rn.f32 %f177, %f163, %f176, %f146;\n" " fma.rn.f32 %f178, %f166, %f176, %f147;\n" " and.b32 %r88, %r46, 61440;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f179, %r89;\n" " mov.f32 %f180, 0fC5020000;\n" " mov.f32 %f181, 0f39800000;\n" " fma.rn.f32 %f182, %f179, %f181, %f180;\n" " and.b32 %r90, %r52, 192;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f183, %r91;\n" " mov.f32 %f184, 0fCA000000;\n" " mov.f32 %f185, 0f3E800000;\n" " fma.rn.f32 %f186, %f183, %f185, %f184;\n" " add.f32 %f187, %f182, %f186;\n" " and.b32 %r92, %r47, 61440;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f188, %r93;\n" " fma.rn.f32 %f189, %f188, %f181, %f180;\n" " and.b32 %r94, %r53, 192;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f190, %r95;\n" " fma.rn.f32 %f191, %f190, %f185, %f184;\n" " add.f32 %f192, %f189, %f191;\n" " shr.u32 %r96, %r48, 24;\n" " cvt.rn.f32.u32 %f193, %r96;\n" " add.f32 %f194, %f193, 0fC3000000;\n" " mul.rn.f32 %f195, %f194, %f187;\n" " shr.u32 %r97, %r50, 24;\n" " cvt.rn.f32.u32 %f196, %r97;\n" " add.f32 %f197, %f196, 0fC3000000;\n" " mul.rn.f32 %f198, %f197, %f192;\n" " ld.shared.f32 %f199, [%r67+384];\n" " fma.rn.f32 %f200, %f195, %f199, %f168;\n" " fma.rn.f32 %f201, %f198, %f199, %f169;\n" " ld.shared.f32 %f202, [%r67+2432];\n" " fma.rn.f32 %f203, %f195, %f202, %f171;\n" " fma.rn.f32 %f204, %f198, %f202, %f172;\n" " ld.shared.f32 %f205, [%r67+4480];\n" " fma.rn.f32 %f206, %f195, %f205, %f174;\n" " fma.rn.f32 %f207, %f198, %f205, %f175;\n" " ld.shared.f32 %f208, [%r67+6528];\n" " fma.rn.f32 %f209, %f195, %f208, %f177;\n" " fma.rn.f32 %f210, %f198, %f208, %f178;\n" " bfe.u32 %r98, %r46, 16, 4;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f211, %r99;\n" " fma.rn.f32 %f212, %f211, %f85, %f84;\n" " and.b32 %r100, %r52, 768;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f213, %r101;\n" " mov.f32 %f214, 0fC9000000;\n" " fma.rn.f32 %f215, %f213, %f150, %f214;\n" " add.f32 %f216, %f212, %f215;\n" " bfe.u32 %r102, %r47, 16, 4;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f217, %r103;\n" " fma.rn.f32 %f218, %f217, %f85, %f84;\n" " and.b32 %r104, %r53, 768;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f219, %r105;\n" " fma.rn.f32 %f220, %f219, %f150, %f214;\n" " add.f32 %f221, %f218, %f220;\n" " and.b32 %r106, %r49, 255;\n" " cvt.rn.f32.u32 %f222, %r106;\n" " add.f32 %f223, %f222, 0fC3000000;\n" " mul.rn.f32 %f224, %f223, %f216;\n" " and.b32 %r107, %r51, 255;\n" " cvt.rn.f32.u32 %f225, %r107;\n" " add.f32 %f226, %f225, 0fC3000000;\n" " mul.rn.f32 %f227, %f226, %f221;\n" " ld.shared.f32 %f228, [%r67+512];\n" " fma.rn.f32 %f229, %f224, %f228, %f200;\n" " fma.rn.f32 %f230, %f227, %f228, %f201;\n" " ld.shared.f32 %f231, [%r67+2560];\n" " fma.rn.f32 %f232, %f224, %f231, %f203;\n" " fma.rn.f32 %f233, %f227, %f231, %f204;\n" " ld.shared.f32 %f234, [%r67+4608];\n" " fma.rn.f32 %f235, %f224, %f234, %f206;\n" " fma.rn.f32 %f236, %f227, %f234, %f207;\n" " ld.shared.f32 %f237, [%r67+6656];\n" " fma.rn.f32 %f238, %f224, %f237, %f209;\n" " fma.rn.f32 %f239, %f227, %f237, %f210;\n" " and.b32 %r108, %r54, 3840;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f240, %r109;\n" " fma.rn.f32 %f241, %f240, %f118, %f117;\n" " and.b32 %r110, %r52, 3072;\n" " or.b32 %r111, %r110, 1258291200;\n" " mov.b32 %f242, %r111;\n" " mov.f32 %f243, 0fC8000000;\n" " mov.f32 %f244, 0f3C800000;\n" " fma.rn.f32 %f245, %f242, %f244, %f243;\n" " add.f32 %f246, %f241, %f245;\n" " and.b32 %r112, %r55, 3840;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f247, %r113;\n" " fma.rn.f32 %f248, %f247, %f118, %f117;\n" " and.b32 %r114, %r53, 3072;\n" " or.b32 %r115, %r114, 1258291200;\n" " mov.b32 %f249, %r115;\n" " fma.rn.f32 %f250, %f249, %f244, %f243;\n" " add.f32 %f251, %f248, %f250;\n" " bfe.u32 %r116, %r49, 8, 8;\n" " cvt.rn.f32.u32 %f252, %r116;\n" " add.f32 %f253, %f252, 0fC3000000;\n" " mul.rn.f32 %f254, %f253, %f246;\n" " bfe.u32 %r117, %r51, 8, 8;\n" " cvt.rn.f32.u32 %f255, %r117;\n" " add.f32 %f256, %f255, 0fC3000000;\n" " mul.rn.f32 %f257, %f256, %f251;\n" " ld.shared.f32 %f258, [%r67+640];\n" " fma.rn.f32 %f259, %f254, %f258, %f229;\n" " fma.rn.f32 %f260, %f257, %f258, %f230;\n" " ld.shared.f32 %f261, [%r67+2688];\n" " fma.rn.f32 %f262, %f254, %f261, %f232;\n" " fma.rn.f32 %f263, %f257, %f261, %f233;\n" " ld.shared.f32 %f264, [%r67+4736];\n" " fma.rn.f32 %f265, %f254, %f264, %f235;\n" " fma.rn.f32 %f266, %f257, %f264, %f236;\n" " ld.shared.f32 %f267, [%r67+6784];\n" " fma.rn.f32 %f268, %f254, %f267, %f238;\n" " fma.rn.f32 %f269, %f257, %f267, %f239;\n" " and.b32 %r118, %r54, 240;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f270, %r119;\n" " fma.rn.f32 %f271, %f270, %f150, %f149;\n" " and.b32 %r120, %r52, 12288;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f272, %r121;\n" " mov.f32 %f273, 0fC7000000;\n" " fma.rn.f32 %f274, %f272, %f118, %f273;\n" " add.f32 %f275, %f271, %f274;\n" " and.b32 %r122, %r55, 240;\n" " or.b32 %r123, %r122, 1258291200;\n" " mov.b32 %f276, %r123;\n" " fma.rn.f32 %f277, %f276, %f150, %f149;\n" " and.b32 %r124, %r53, 12288;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f278, %r125;\n" " fma.rn.f32 %f279, %f278, %f118, %f273;\n" " add.f32 %f280, %f277, %f279;\n" " bfe.u32 %r126, %r49, 16, 8;\n" " cvt.rn.f32.u32 %f281, %r126;\n" " add.f32 %f282, %f281, 0fC3000000;\n" " mul.rn.f32 %f283, %f282, %f275;\n" " bfe.u32 %r127, %r51, 16, 8;\n" " cvt.rn.f32.u32 %f284, %r127;\n" " add.f32 %f285, %f284, 0fC3000000;\n" " mul.rn.f32 %f286, %f285, %f280;\n" " ld.shared.f32 %f287, [%r67+768];\n" " fma.rn.f32 %f288, %f283, %f287, %f259;\n" " fma.rn.f32 %f289, %f286, %f287, %f260;\n" " ld.shared.f32 %f290, [%r67+2816];\n" " fma.rn.f32 %f291, %f283, %f290, %f262;\n" " fma.rn.f32 %f292, %f286, %f290, %f263;\n" " ld.shared.f32 %f293, [%r67+4864];\n" " fma.rn.f32 %f294, %f283, %f293, %f265;\n" " fma.rn.f32 %f295, %f286, %f293, %f266;\n" " ld.shared.f32 %f296, [%r67+6912];\n" " fma.rn.f32 %f297, %f283, %f296, %f268;\n" " fma.rn.f32 %f298, %f286, %f296, %f269;\n" " and.b32 %r128, %r54, 61440;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f299, %r129;\n" " fma.rn.f32 %f300, %f299, %f181, %f180;\n" " and.b32 %r130, %r52, 49152;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f301, %r131;\n" " mov.f32 %f302, 0fC6000000;\n" " mov.f32 %f303, 0f3A800000;\n" " fma.rn.f32 %f304, %f301, %f303, %f302;\n" " add.f32 %f305, %f300, %f304;\n" " and.b32 %r132, %r55, 61440;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f306, %r133;\n" " fma.rn.f32 %f307, %f306, %f181, %f180;\n" " and.b32 %r134, %r53, 49152;\n" " or.b32 %r135, %r134, 1258291200;\n" " mov.b32 %f308, %r135;\n" " fma.rn.f32 %f309, %f308, %f303, %f302;\n" " add.f32 %f310, %f307, %f309;\n" " shr.u32 %r136, %r49, 24;\n" " cvt.rn.f32.u32 %f311, %r136;\n" " add.f32 %f312, %f311, 0fC3000000;\n" " mul.rn.f32 %f313, %f312, %f305;\n" " shr.u32 %r137, %r51, 24;\n" " cvt.rn.f32.u32 %f314, %r137;\n" " add.f32 %f315, %f314, 0fC3000000;\n" " mul.rn.f32 %f316, %f315, %f310;\n" " ld.shared.f32 %f317, [%r67+896];\n" " fma.rn.f32 %f318, %f313, %f317, %f288;\n" " fma.rn.f32 %f319, %f316, %f317, %f289;\n" " ld.shared.f32 %f320, [%r67+2944];\n" " fma.rn.f32 %f321, %f313, %f320, %f291;\n" " fma.rn.f32 %f322, %f316, %f320, %f292;\n" " ld.shared.f32 %f323, [%r67+4992];\n" " fma.rn.f32 %f324, %f313, %f323, %f294;\n" " fma.rn.f32 %f325, %f316, %f323, %f295;\n" " ld.shared.f32 %f326, [%r67+7040];\n" " fma.rn.f32 %f327, %f313, %f326, %f297;\n" " fma.rn.f32 %f328, %f316, %f326, %f298;\n" " fma.rn.f32 %f435, %f81, %f318, %f435;\n" " fma.rn.f32 %f439, %f82, %f319, %f439;\n" " fma.rn.f32 %f434, %f81, %f321, %f434;\n" " fma.rn.f32 %f438, %f82, %f322, %f438;\n" " fma.rn.f32 %f433, %f81, %f324, %f433;\n" " fma.rn.f32 %f437, %f82, %f325, %f437;\n" " fma.rn.f32 %f432, %f81, %f327, %f432;\n" " fma.rn.f32 %f436, %f82, %f328, %f436;\n" " add.s64 %rd92, %rd92, 216;\n" " add.s64 %rd93, %rd93, 216;\n" " add.s32 %r253, %r253, 1024;\n" " add.s32 %r252, %r252, 1;\n" " setp.lt.s32 %p11, %r252, %r246;\n" " @%p11 bra $L__BB10_18;\n" "\n" "$L__BB10_19:\n" " add.s32 %r249, %r249, 512;\n" " setp.lt.s32 %p12, %r249, %r23;\n" " @%p12 bra $L__BB10_4;\n" "\n" "$L__BB10_20:\n" " mov.u32 %r231, %ctaid.y;\n" " shl.b32 %r230, %r231, 2;\n" " ld.param.u32 %r229, [fc_matvec_q6k_nx_param_6];\n" " sub.s32 %r228, %r229, %r230;\n" " cvt.s64.s32 %rd27, %r22;\n" " mov.u32 %r148, 16;\n" " mov.u32 %r157, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f329, %f435, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f332, %f435, %f329;\n" " mov.u32 %r150, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f331, %f332, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f334, %f332, %f331;\n" " mov.u32 %r152, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f333, %f334, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f336, %f334, %f333;\n" " mov.u32 %r154, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f335, %f336, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f338, %f336, %f335;\n" " mov.u32 %r156, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f337, %f338, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f49, %f338, %f337;\n" " // begin inline asm\n" " shfl.down.b32 %f339, %f439, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f342, %f439, %f339;\n" " // begin inline asm\n" " shfl.down.b32 %f341, %f342, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f344, %f342, %f341;\n" " // begin inline asm\n" " shfl.down.b32 %f343, %f344, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f346, %f344, %f343;\n" " // begin inline asm\n" " shfl.down.b32 %f345, %f346, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f348, %f346, %f345;\n" " // begin inline asm\n" " shfl.down.b32 %f347, %f348, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f50, %f348, %f347;\n" " setp.lt.s32 %p13, %r228, 1;\n" " setp.ne.s32 %p14, %r5, 0;\n" " or.pred %p15, %p14, %p13;\n" " @%p15 bra $L__BB10_25;\n" "\n" " mov.u32 %r245, %ctaid.y;\n" " shl.b32 %r244, %r245, 2;\n" " cvt.u32.u64 %r158, %rd2;\n" " setp.ge.s32 %p16, %r158, %r22;\n" " cvt.s64.s32 %rd62, %r244;\n" " mul.lo.s64 %rd63, %rd62, %rd27;\n" " add.s64 %rd64, %rd63, %rd2;\n" " shl.b64 %rd65, %rd64, 2;\n" " add.s64 %rd28, %rd1, %rd65;\n" " @%p16 bra $L__BB10_23;\n" "\n" " st.global.f32 [%rd28], %f49;\n" "\n" "$L__BB10_23:\n" " @%p1 bra $L__BB10_25;\n" "\n" " st.global.f32 [%rd28+4], %f50;\n" "\n" "$L__BB10_25:\n" " // begin inline asm\n" " shfl.down.b32 %f349, %f434, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f352, %f434, %f349;\n" " // begin inline asm\n" " shfl.down.b32 %f351, %f352, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f354, %f352, %f351;\n" " // begin inline asm\n" " shfl.down.b32 %f353, %f354, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f356, %f354, %f353;\n" " // begin inline asm\n" " shfl.down.b32 %f355, %f356, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f358, %f356, %f355;\n" " // begin inline asm\n" " shfl.down.b32 %f357, %f358, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f51, %f358, %f357;\n" " // begin inline asm\n" " shfl.down.b32 %f359, %f438, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f362, %f438, %f359;\n" " // begin inline asm\n" " shfl.down.b32 %f361, %f362, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f364, %f362, %f361;\n" " // begin inline asm\n" " shfl.down.b32 %f363, %f364, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f366, %f364, %f363;\n" " // begin inline asm\n" " shfl.down.b32 %f365, %f366, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f368, %f366, %f365;\n" " // begin inline asm\n" " shfl.down.b32 %f367, %f368, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f52, %f368, %f367;\n" " setp.lt.s32 %p19, %r3, 2;\n" " or.pred %p20, %p14, %p19;\n" " @%p20 bra $L__BB10_30;\n" "\n" " mov.u32 %r243, %ctaid.y;\n" " shl.b32 %r242, %r243, 2;\n" " cvt.u32.u64 %r179, %rd2;\n" " setp.ge.s32 %p21, %r179, %r22;\n" " add.s32 %r180, %r242, 1;\n" " cvt.s64.s32 %rd66, %r180;\n" " mul.lo.s64 %rd67, %rd66, %rd27;\n" " add.s64 %rd68, %rd67, %rd2;\n" " shl.b64 %rd69, %rd68, 2;\n" " add.s64 %rd29, %rd1, %rd69;\n" " @%p21 bra $L__BB10_28;\n" "\n" " st.global.f32 [%rd29], %f51;\n" "\n" "$L__BB10_28:\n" " @%p1 bra $L__BB10_30;\n" "\n" " st.global.f32 [%rd29+4], %f52;\n" "\n" "$L__BB10_30:\n" " // begin inline asm\n" " shfl.down.b32 %f369, %f433, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f372, %f433, %f369;\n" " // begin inline asm\n" " shfl.down.b32 %f371, %f372, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f374, %f372, %f371;\n" " // begin inline asm\n" " shfl.down.b32 %f373, %f374, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f376, %f374, %f373;\n" " // begin inline asm\n" " shfl.down.b32 %f375, %f376, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f378, %f376, %f375;\n" " // begin inline asm\n" " shfl.down.b32 %f377, %f378, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f53, %f378, %f377;\n" " // begin inline asm\n" " shfl.down.b32 %f379, %f437, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f382, %f437, %f379;\n" " // begin inline asm\n" " shfl.down.b32 %f381, %f382, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f384, %f382, %f381;\n" " // begin inline asm\n" " shfl.down.b32 %f383, %f384, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f386, %f384, %f383;\n" " // begin inline asm\n" " shfl.down.b32 %f385, %f386, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f388, %f386, %f385;\n" " // begin inline asm\n" " shfl.down.b32 %f387, %f388, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f54, %f388, %f387;\n" " setp.lt.s32 %p24, %r3, 3;\n" " or.pred %p25, %p14, %p24;\n" " @%p25 bra $L__BB10_35;\n" "\n" " mov.u32 %r241, %ctaid.y;\n" " shl.b32 %r240, %r241, 2;\n" " cvt.u32.u64 %r201, %rd2;\n" " setp.ge.s32 %p26, %r201, %r22;\n" " add.s32 %r202, %r240, 2;\n" " cvt.s64.s32 %rd70, %r202;\n" " mul.lo.s64 %rd71, %rd70, %rd27;\n" " add.s64 %rd72, %rd71, %rd2;\n" " shl.b64 %rd73, %rd72, 2;\n" " add.s64 %rd30, %rd1, %rd73;\n" " @%p26 bra $L__BB10_33;\n" "\n" " st.global.f32 [%rd30], %f53;\n" "\n" "$L__BB10_33:\n" " @%p1 bra $L__BB10_35;\n" "\n" " st.global.f32 [%rd30+4], %f54;\n" "\n" "$L__BB10_35:\n" " // begin inline asm\n" " shfl.down.b32 %f389, %f432, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f392, %f432, %f389;\n" " // begin inline asm\n" " shfl.down.b32 %f391, %f392, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f394, %f392, %f391;\n" " // begin inline asm\n" " shfl.down.b32 %f393, %f394, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f396, %f394, %f393;\n" " // begin inline asm\n" " shfl.down.b32 %f395, %f396, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f398, %f396, %f395;\n" " // begin inline asm\n" " shfl.down.b32 %f397, %f398, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f55, %f398, %f397;\n" " // begin inline asm\n" " shfl.down.b32 %f399, %f436, %r148, %r157;\n" " // end inline asm\n" " add.f32 %f402, %f436, %f399;\n" " // begin inline asm\n" " shfl.down.b32 %f401, %f402, %r150, %r157;\n" " // end inline asm\n" " add.f32 %f404, %f402, %f401;\n" " // begin inline asm\n" " shfl.down.b32 %f403, %f404, %r152, %r157;\n" " // end inline asm\n" " add.f32 %f406, %f404, %f403;\n" " // begin inline asm\n" " shfl.down.b32 %f405, %f406, %r154, %r157;\n" " // end inline asm\n" " add.f32 %f408, %f406, %f405;\n" " // begin inline asm\n" " shfl.down.b32 %f407, %f408, %r156, %r157;\n" " // end inline asm\n" " add.f32 %f56, %f408, %f407;\n" " setp.lt.s32 %p29, %r3, 4;\n" " or.pred %p30, %p14, %p29;\n" " @%p30 bra $L__BB10_40;\n" "\n" " mov.u32 %r239, %ctaid.y;\n" " shl.b32 %r238, %r239, 2;\n" " cvt.u32.u64 %r223, %rd2;\n" " setp.ge.s32 %p31, %r223, %r22;\n" " add.s32 %r224, %r238, 3;\n" " cvt.s64.s32 %rd74, %r224;\n" " mul.lo.s64 %rd75, %rd74, %rd27;\n" " add.s64 %rd76, %rd75, %rd2;\n" " shl.b64 %rd77, %rd76, 2;\n" " add.s64 %rd31, %rd1, %rd77;\n" " @%p31 bra $L__BB10_38;\n" "\n" " st.global.f32 [%rd31], %f55;\n" "\n" "$L__BB10_38:\n" " @%p1 bra $L__BB10_40;\n" "\n" " st.global.f32 [%rd31+4], %f56;\n" "\n" "$L__BB10_40:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q6k_nx8\n" ".visible .entry fc_matvec_q6k_nx8(\n" " .param .u64 fc_matvec_q6k_nx8_param_0,\n" " .param .u64 fc_matvec_q6k_nx8_param_1,\n" " .param .u64 fc_matvec_q6k_nx8_param_2,\n" " .param .u32 fc_matvec_q6k_nx8_param_3,\n" " .param .u32 fc_matvec_q6k_nx8_param_4,\n" " .param .u32 fc_matvec_q6k_nx8_param_5,\n" " .param .u32 fc_matvec_q6k_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<57>;\n" " .reg .b16 %rs<3>;\n" " .reg .f32 %f<744>;\n" " .reg .b32 %r<342>;\n" " .reg .b64 %rd<142>;\n" "\n" "\n" " ld.param.u64 %rd45, [fc_matvec_q6k_nx8_param_0];\n" " ld.param.u64 %rd44, [fc_matvec_q6k_nx8_param_1];\n" " ld.param.u64 %rd46, [fc_matvec_q6k_nx8_param_2];\n" " ld.param.u32 %r22, [fc_matvec_q6k_nx8_param_3];\n" " ld.param.u32 %r23, [fc_matvec_q6k_nx8_param_4];\n" " ld.param.u32 %r24, [fc_matvec_q6k_nx8_param_5];\n" " ld.param.u32 %r25, [fc_matvec_q6k_nx8_param_6];\n" " cvta.to.global.u64 %rd1, %rd46;\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r1, %r26, 3;\n" " sub.s32 %r2, %r25, %r1;\n" " min.s32 %r3, %r2, 8;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r27, %ctaid.x;\n" " shl.b32 %r28, %r27, 4;\n" " shr.u32 %r29, %r4, 4;\n" " and.b32 %r30, %r29, 268435454;\n" " add.s32 %r31, %r30, %r28;\n" " or.b32 %r6, %r31, 1;\n" " setp.ge.s32 %p1, %r6, %r22;\n" " cvt.s64.s32 %rd2, %r31;\n" " cvt.s64.s32 %rd3, %r24;\n" " mul.wide.s32 %rd47, %r24, %r31;\n" " add.s64 %rd140, %rd45, %rd47;\n" " mov.u64 %rd141, %rd140;\n" " @%p1 bra $L__BB11_2;\n" "\n" " add.s64 %rd141, %rd140, %rd3;\n" "\n" "$L__BB11_2:\n" " setp.lt.s32 %p2, %r23, 1;\n" " mov.f32 %f712, 0f00000000;\n" " mov.f32 %f713, %f712;\n" " mov.f32 %f714, %f712;\n" " mov.f32 %f715, %f712;\n" " mov.f32 %f716, %f712;\n" " mov.f32 %f717, %f712;\n" " mov.f32 %f718, %f712;\n" " mov.f32 %f719, %f712;\n" " mov.f32 %f720, %f712;\n" " mov.f32 %f721, %f712;\n" " mov.f32 %f722, %f712;\n" " mov.f32 %f723, %f712;\n" " mov.f32 %f724, %f712;\n" " mov.f32 %f725, %f712;\n" " mov.f32 %f726, %f712;\n" " mov.f32 %f727, %f712;\n" " @%p2 bra $L__BB11_28;\n" "\n" " cvt.u64.u32 %rd7, %r5;\n" " shr.u32 %r33, %r5, 1;\n" " or.b32 %r34, %r33, 192;\n" " and.b32 %r35, %r34, 200;\n" " cvt.u64.u32 %rd8, %r35;\n" " add.s32 %r36, %r1, 1;\n" " add.s32 %r37, %r1, 2;\n" " add.s32 %r38, %r1, 3;\n" " add.s32 %r39, %r1, 4;\n" " add.s32 %r40, %r1, 5;\n" " add.s32 %r41, %r1, 6;\n" " add.s32 %r42, %r1, 7;\n" " mul.wide.s32 %rd48, %r23, %r1;\n" " shl.b64 %rd49, %rd48, 2;\n" " mul.wide.s32 %rd50, %r4, 4;\n" " add.s64 %rd9, %rd49, %rd50;\n" " mul.wide.s32 %rd51, %r23, %r36;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd10, %rd52, %rd50;\n" " mul.wide.s32 %rd53, %r23, %r37;\n" " shl.b64 %rd54, %rd53, 2;\n" " add.s64 %rd11, %rd54, %rd50;\n" " mul.wide.s32 %rd55, %r23, %r38;\n" " shl.b64 %rd56, %rd55, 2;\n" " add.s64 %rd12, %rd56, %rd50;\n" " mul.wide.s32 %rd57, %r23, %r39;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd13, %rd58, %rd50;\n" " mul.wide.s32 %rd59, %r23, %r40;\n" " shl.b64 %rd60, %rd59, 2;\n" " add.s64 %rd14, %rd60, %rd50;\n" " mul.wide.s32 %rd61, %r23, %r41;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd15, %rd62, %rd50;\n" " shl.b32 %r43, %r4, 2;\n" " mov.u32 %r44, fc_kq_shared;\n" " add.s32 %r45, %r44, %r43;\n" " add.s32 %r7, %r45, 8192;\n" " mul.wide.s32 %rd63, %r23, %r42;\n" " shl.b64 %rd64, %rd63, 2;\n" " add.s64 %rd16, %rd64, %rd50;\n" " mov.u32 %r337, 0;\n" " mov.f32 %f712, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " setp.lt.s32 %p6, %r3, 3;\n" " setp.lt.s32 %p7, %r3, 4;\n" " mov.f32 %f713, %f712;\n" " mov.f32 %f714, %f712;\n" " mov.f32 %f715, %f712;\n" " mov.f32 %f716, %f712;\n" " mov.f32 %f717, %f712;\n" " mov.f32 %f718, %f712;\n" " mov.f32 %f719, %f712;\n" " mov.f32 %f720, %f712;\n" " mov.f32 %f721, %f712;\n" " mov.f32 %f722, %f712;\n" " mov.f32 %f723, %f712;\n" " mov.f32 %f724, %f712;\n" " mov.f32 %f725, %f712;\n" " mov.f32 %f726, %f712;\n" " mov.f32 %f727, %f712;\n" "\n" "$L__BB11_4:\n" " sub.s32 %r46, %r23, %r337;\n" " min.s32 %r9, %r46, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r9;\n" " @%p3 bra $L__BB11_23;\n" "\n" " mov.u32 %r338, %r7;\n" " mov.u64 %rd137, %rd44;\n" " mov.u32 %r339, %r4;\n" "\n" "$L__BB11_6:\n" " .pragma \"nounroll\";\n" " mov.f32 %f688, 0f00000000;\n" " @%p4 bra $L__BB11_8;\n" "\n" " mul.wide.s32 %rd119, %r337, 4;\n" " add.s64 %rd118, %rd9, %rd119;\n" " add.s64 %rd66, %rd137, %rd118;\n" " // begin inline asm\n" " ld.global.nc.f32 %f688, [%rd66];\n" " // end inline asm\n" "\n" "$L__BB11_8:\n" " mov.f32 %f689, 0f00000000;\n" " st.shared.f32 [%r338+-8192], %f688;\n" " @%p5 bra $L__BB11_10;\n" "\n" " mul.wide.s32 %rd131, %r337, 4;\n" " add.s64 %rd130, %rd10, %rd131;\n" " add.s64 %rd67, %rd137, %rd130;\n" " // begin inline asm\n" " ld.global.nc.f32 %f689, [%rd67];\n" " // end inline asm\n" "\n" "$L__BB11_10:\n" " mov.f32 %f690, 0f00000000;\n" " add.s32 %r317, %r338, -8192;\n" " st.shared.f32 [%r317+2048], %f689;\n" " @%p6 bra $L__BB11_12;\n" "\n" " mul.wide.s32 %rd129, %r337, 4;\n" " add.s64 %rd128, %rd11, %rd129;\n" " add.s64 %rd68, %rd137, %rd128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f690, [%rd68];\n" " // end inline asm\n" "\n" "$L__BB11_12:\n" " mov.f32 %f691, 0f00000000;\n" " add.s32 %r318, %r338, -8192;\n" " st.shared.f32 [%r318+4096], %f690;\n" " @%p7 bra $L__BB11_14;\n" "\n" " mul.wide.s32 %rd127, %r337, 4;\n" " add.s64 %rd126, %rd12, %rd127;\n" " add.s64 %rd69, %rd137, %rd126;\n" " // begin inline asm\n" " ld.global.nc.f32 %f691, [%rd69];\n" " // end inline asm\n" "\n" "$L__BB11_14:\n" " mov.f32 %f692, 0f00000000;\n" " add.s32 %r319, %r338, -8192;\n" " st.shared.f32 [%r319+6144], %f691;\n" " setp.lt.s32 %p8, %r3, 5;\n" " @%p8 bra $L__BB11_16;\n" "\n" " mul.wide.s32 %rd125, %r337, 4;\n" " add.s64 %rd124, %rd13, %rd125;\n" " add.s64 %rd70, %rd137, %rd124;\n" " // begin inline asm\n" " ld.global.nc.f32 %f692, [%rd70];\n" " // end inline asm\n" "\n" "$L__BB11_16:\n" " mov.f32 %f693, 0f00000000;\n" " add.s32 %r320, %r338, -8192;\n" " st.shared.f32 [%r320+8192], %f692;\n" " setp.lt.s32 %p9, %r3, 6;\n" " @%p9 bra $L__BB11_18;\n" "\n" " mul.wide.s32 %rd123, %r337, 4;\n" " add.s64 %rd122, %rd14, %rd123;\n" " add.s64 %rd71, %rd137, %rd122;\n" " // begin inline asm\n" " ld.global.nc.f32 %f693, [%rd71];\n" " // end inline asm\n" "\n" "$L__BB11_18:\n" " mov.f32 %f694, 0f00000000;\n" " add.s32 %r321, %r338, -8192;\n" " st.shared.f32 [%r321+10240], %f693;\n" " setp.lt.s32 %p10, %r3, 7;\n" " @%p10 bra $L__BB11_20;\n" "\n" " mul.wide.s32 %rd121, %r337, 4;\n" " add.s64 %rd120, %rd15, %rd121;\n" " add.s64 %rd72, %rd137, %rd120;\n" " // begin inline asm\n" " ld.global.nc.f32 %f694, [%rd72];\n" " // end inline asm\n" "\n" "$L__BB11_20:\n" " mov.f32 %f695, 0f00000000;\n" " add.s32 %r322, %r338, -8192;\n" " st.shared.f32 [%r322+12288], %f694;\n" " setp.lt.s32 %p11, %r3, 8;\n" " @%p11 bra $L__BB11_22;\n" "\n" " mul.wide.s32 %rd133, %r337, 4;\n" " add.s64 %rd132, %rd16, %rd133;\n" " add.s64 %rd73, %rd137, %rd132;\n" " // begin inline asm\n" " ld.global.nc.f32 %f695, [%rd73];\n" " // end inline asm\n" "\n" "$L__BB11_22:\n" " sub.s32 %r329, %r23, %r337;\n" " min.s32 %r328, %r329, 512;\n" " add.s32 %r323, %r338, -8192;\n" " st.shared.f32 [%r323+14336], %f695;\n" " add.s64 %rd137, %rd137, 1024;\n" " add.s32 %r338, %r338, 1024;\n" " add.s32 %r339, %r339, 256;\n" " setp.lt.s32 %p12, %r339, %r328;\n" " @%p12 bra $L__BB11_6;\n" "\n" "$L__BB11_23:\n" " cvt.u32.u64 %r47, %rd2;\n" " setp.ge.s32 %p13, %r47, %r22;\n" " bar.sync 0;\n" " @%p13 bra $L__BB11_27;\n" "\n" " sub.s32 %r331, %r23, %r337;\n" " min.s32 %r330, %r331, 512;\n" " setp.lt.s32 %p14, %r330, 256;\n" " @%p14 bra $L__BB11_27;\n" "\n" " mov.u32 %r340, 0;\n" " mov.u32 %r341, %r44;\n" "\n" "$L__BB11_26:\n" " sub.s32 %r336, %r23, %r337;\n" " min.s32 %r335, %r336, 512;\n" " shr.s32 %r334, %r335, 8;\n" " shl.b64 %rd82, %rd7, 2;\n" " add.s64 %rd74, %rd140, %rd82;\n" " // begin inline asm\n" " ld.global.nc.u32 %r50, [%rd74];\n" " // end inline asm\n" " add.s64 %rd75, %rd141, %rd82;\n" " // begin inline asm\n" " ld.global.nc.u32 %r51, [%rd75];\n" " // end inline asm\n" " shl.b64 %rd83, %rd7, 1;\n" " add.s64 %rd84, %rd140, %rd83;\n" " add.s64 %rd76, %rd84, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs1, [%rd76];\n" " // end inline asm\n" " add.s64 %rd85, %rd141, %rd83;\n" " add.s64 %rd77, %rd85, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs2, [%rd77];\n" " // end inline asm\n" " add.s64 %rd78, %rd140, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f161, [%rd78];\n" " // end inline asm\n" " add.s64 %rd79, %rd141, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f162, [%rd79];\n" " // end inline asm\n" " add.s64 %rd80, %rd140, %rd8;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r52,%r53}, [%rd80];\n" " // end inline asm\n" " add.s64 %rd81, %rd141, %rd8;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r54,%r55}, [%rd81];\n" " // end inline asm\n" " cvt.u32.u16 %r56, %rs1;\n" " cvt.u32.u16 %r57, %rs2;\n" " shr.u32 %r58, %r50, 16;\n" " shr.u32 %r59, %r51, 16;\n" " and.b32 %r60, %r50, 15;\n" " or.b32 %r61, %r60, 1258291200;\n" " mov.b32 %f163, %r61;\n" " mov.f32 %f164, 0fCB000020;\n" " mov.f32 %f165, 0f3F800000;\n" " fma.rn.f32 %f166, %f163, %f165, %f164;\n" " and.b32 %r62, %r56, 3;\n" " or.b32 %r63, %r62, 1258291200;\n" " mov.b32 %f167, %r63;\n" " mov.f32 %f168, 0fCD000000;\n" " mov.f32 %f169, 0f41800000;\n" " fma.rn.f32 %f170, %f167, %f169, %f168;\n" " add.f32 %f171, %f166, %f170;\n" " and.b32 %r64, %r51, 15;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f172, %r65;\n" " fma.rn.f32 %f173, %f172, %f165, %f164;\n" " and.b32 %r66, %r57, 3;\n" " or.b32 %r67, %r66, 1258291200;\n" " mov.b32 %f174, %r67;\n" " fma.rn.f32 %f175, %f174, %f169, %f168;\n" " add.f32 %f176, %f173, %f175;\n" " and.b32 %r68, %r52, 255;\n" " cvt.rn.f32.u32 %f177, %r68;\n" " add.f32 %f178, %f177, 0fC3000000;\n" " mul.rn.f32 %f179, %f178, %f171;\n" " and.b32 %r69, %r54, 255;\n" " cvt.rn.f32.u32 %f180, %r69;\n" " add.f32 %f181, %f180, 0fC3000000;\n" " mul.rn.f32 %f182, %f181, %f176;\n" " shl.b32 %r70, %r5, 2;\n" " add.s32 %r71, %r341, %r70;\n" " ld.shared.f32 %f183, [%r71];\n" " mov.f32 %f184, 0f00000000;\n" " fma.rn.f32 %f185, %f179, %f183, %f184;\n" " fma.rn.f32 %f186, %f182, %f183, %f184;\n" " ld.shared.f32 %f187, [%r71+2048];\n" " fma.rn.f32 %f188, %f179, %f187, %f184;\n" " fma.rn.f32 %f189, %f182, %f187, %f184;\n" " ld.shared.f32 %f190, [%r71+4096];\n" " fma.rn.f32 %f191, %f179, %f190, %f184;\n" " fma.rn.f32 %f192, %f182, %f190, %f184;\n" " ld.shared.f32 %f193, [%r71+6144];\n" " fma.rn.f32 %f194, %f179, %f193, %f184;\n" " fma.rn.f32 %f195, %f182, %f193, %f184;\n" " ld.shared.f32 %f196, [%r71+8192];\n" " fma.rn.f32 %f197, %f179, %f196, %f184;\n" " fma.rn.f32 %f198, %f182, %f196, %f184;\n" " ld.shared.f32 %f199, [%r71+10240];\n" " fma.rn.f32 %f200, %f179, %f199, %f184;\n" " fma.rn.f32 %f201, %f182, %f199, %f184;\n" " ld.shared.f32 %f202, [%r71+12288];\n" " fma.rn.f32 %f203, %f179, %f202, %f184;\n" " fma.rn.f32 %f204, %f182, %f202, %f184;\n" " ld.shared.f32 %f205, [%r71+14336];\n" " fma.rn.f32 %f206, %f179, %f205, %f184;\n" " fma.rn.f32 %f207, %f182, %f205, %f184;\n" " and.b32 %r72, %r50, 3840;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f208, %r73;\n" " mov.f32 %f209, 0fC7002000;\n" " mov.f32 %f210, 0f3B800000;\n" " fma.rn.f32 %f211, %f208, %f210, %f209;\n" " and.b32 %r74, %r56, 12;\n" " or.b32 %r75, %r74, 1258291200;\n" " mov.b32 %f212, %r75;\n" " mov.f32 %f213, 0fCC000000;\n" " mov.f32 %f214, 0f40800000;\n" " fma.rn.f32 %f215, %f212, %f214, %f213;\n" " add.f32 %f216, %f211, %f215;\n" " and.b32 %r76, %r51, 3840;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f217, %r77;\n" " fma.rn.f32 %f218, %f217, %f210, %f209;\n" " and.b32 %r78, %r57, 12;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f219, %r79;\n" " fma.rn.f32 %f220, %f219, %f214, %f213;\n" " add.f32 %f221, %f218, %f220;\n" " bfe.u32 %r80, %r52, 8, 8;\n" " cvt.rn.f32.u32 %f222, %r80;\n" " add.f32 %f223, %f222, 0fC3000000;\n" " mul.rn.f32 %f224, %f223, %f216;\n" " bfe.u32 %r81, %r54, 8, 8;\n" " cvt.rn.f32.u32 %f225, %r81;\n" " add.f32 %f226, %f225, 0fC3000000;\n" " mul.rn.f32 %f227, %f226, %f221;\n" " ld.shared.f32 %f228, [%r71+128];\n" " fma.rn.f32 %f229, %f224, %f228, %f185;\n" " fma.rn.f32 %f230, %f227, %f228, %f186;\n" " ld.shared.f32 %f231, [%r71+2176];\n" " fma.rn.f32 %f232, %f224, %f231, %f188;\n" " fma.rn.f32 %f233, %f227, %f231, %f189;\n" " ld.shared.f32 %f234, [%r71+4224];\n" " fma.rn.f32 %f235, %f224, %f234, %f191;\n" " fma.rn.f32 %f236, %f227, %f234, %f192;\n" " ld.shared.f32 %f237, [%r71+6272];\n" " fma.rn.f32 %f238, %f224, %f237, %f194;\n" " fma.rn.f32 %f239, %f227, %f237, %f195;\n" " ld.shared.f32 %f240, [%r71+8320];\n" " fma.rn.f32 %f241, %f224, %f240, %f197;\n" " fma.rn.f32 %f242, %f227, %f240, %f198;\n" " ld.shared.f32 %f243, [%r71+10368];\n" " fma.rn.f32 %f244, %f224, %f243, %f200;\n" " fma.rn.f32 %f245, %f227, %f243, %f201;\n" " ld.shared.f32 %f246, [%r71+12416];\n" " fma.rn.f32 %f247, %f224, %f246, %f203;\n" " fma.rn.f32 %f248, %f227, %f246, %f204;\n" " ld.shared.f32 %f249, [%r71+14464];\n" " fma.rn.f32 %f250, %f224, %f249, %f206;\n" " fma.rn.f32 %f251, %f227, %f249, %f207;\n" " and.b32 %r82, %r50, 240;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f252, %r83;\n" " mov.f32 %f253, 0fC9000200;\n" " mov.f32 %f254, 0f3D800000;\n" " fma.rn.f32 %f255, %f252, %f254, %f253;\n" " and.b32 %r84, %r56, 48;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f256, %r85;\n" " mov.f32 %f257, 0fCB000000;\n" " fma.rn.f32 %f258, %f256, %f165, %f257;\n" " add.f32 %f259, %f255, %f258;\n" " and.b32 %r86, %r51, 240;\n" " or.b32 %r87, %r86, 1258291200;\n" " mov.b32 %f260, %r87;\n" " fma.rn.f32 %f261, %f260, %f254, %f253;\n" " and.b32 %r88, %r57, 48;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f262, %r89;\n" " fma.rn.f32 %f263, %f262, %f165, %f257;\n" " add.f32 %f264, %f261, %f263;\n" " bfe.u32 %r90, %r52, 16, 8;\n" " cvt.rn.f32.u32 %f265, %r90;\n" " add.f32 %f266, %f265, 0fC3000000;\n" " mul.rn.f32 %f267, %f266, %f259;\n" " bfe.u32 %r91, %r54, 16, 8;\n" " cvt.rn.f32.u32 %f268, %r91;\n" " add.f32 %f269, %f268, 0fC3000000;\n" " mul.rn.f32 %f270, %f269, %f264;\n" " ld.shared.f32 %f271, [%r71+256];\n" " fma.rn.f32 %f272, %f267, %f271, %f229;\n" " fma.rn.f32 %f273, %f270, %f271, %f230;\n" " ld.shared.f32 %f274, [%r71+2304];\n" " fma.rn.f32 %f275, %f267, %f274, %f232;\n" " fma.rn.f32 %f276, %f270, %f274, %f233;\n" " ld.shared.f32 %f277, [%r71+4352];\n" " fma.rn.f32 %f278, %f267, %f277, %f235;\n" " fma.rn.f32 %f279, %f270, %f277, %f236;\n" " ld.shared.f32 %f280, [%r71+6400];\n" " fma.rn.f32 %f281, %f267, %f280, %f238;\n" " fma.rn.f32 %f282, %f270, %f280, %f239;\n" " ld.shared.f32 %f283, [%r71+8448];\n" " fma.rn.f32 %f284, %f267, %f283, %f241;\n" " fma.rn.f32 %f285, %f270, %f283, %f242;\n" " ld.shared.f32 %f286, [%r71+10496];\n" " fma.rn.f32 %f287, %f267, %f286, %f244;\n" " fma.rn.f32 %f288, %f270, %f286, %f245;\n" " ld.shared.f32 %f289, [%r71+12544];\n" " fma.rn.f32 %f290, %f267, %f289, %f247;\n" " fma.rn.f32 %f291, %f270, %f289, %f248;\n" " ld.shared.f32 %f292, [%r71+14592];\n" " fma.rn.f32 %f293, %f267, %f292, %f250;\n" " fma.rn.f32 %f294, %f270, %f292, %f251;\n" " and.b32 %r92, %r50, 61440;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f295, %r93;\n" " mov.f32 %f296, 0fC5020000;\n" " mov.f32 %f297, 0f39800000;\n" " fma.rn.f32 %f298, %f295, %f297, %f296;\n" " and.b32 %r94, %r56, 192;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f299, %r95;\n" " mov.f32 %f300, 0fCA000000;\n" " mov.f32 %f301, 0f3E800000;\n" " fma.rn.f32 %f302, %f299, %f301, %f300;\n" " add.f32 %f303, %f298, %f302;\n" " and.b32 %r96, %r51, 61440;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f304, %r97;\n" " fma.rn.f32 %f305, %f304, %f297, %f296;\n" " and.b32 %r98, %r57, 192;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f306, %r99;\n" " fma.rn.f32 %f307, %f306, %f301, %f300;\n" " add.f32 %f308, %f305, %f307;\n" " shr.u32 %r100, %r52, 24;\n" " cvt.rn.f32.u32 %f309, %r100;\n" " add.f32 %f310, %f309, 0fC3000000;\n" " mul.rn.f32 %f311, %f310, %f303;\n" " shr.u32 %r101, %r54, 24;\n" " cvt.rn.f32.u32 %f312, %r101;\n" " add.f32 %f313, %f312, 0fC3000000;\n" " mul.rn.f32 %f314, %f313, %f308;\n" " ld.shared.f32 %f315, [%r71+384];\n" " fma.rn.f32 %f316, %f311, %f315, %f272;\n" " fma.rn.f32 %f317, %f314, %f315, %f273;\n" " ld.shared.f32 %f318, [%r71+2432];\n" " fma.rn.f32 %f319, %f311, %f318, %f275;\n" " fma.rn.f32 %f320, %f314, %f318, %f276;\n" " ld.shared.f32 %f321, [%r71+4480];\n" " fma.rn.f32 %f322, %f311, %f321, %f278;\n" " fma.rn.f32 %f323, %f314, %f321, %f279;\n" " ld.shared.f32 %f324, [%r71+6528];\n" " fma.rn.f32 %f325, %f311, %f324, %f281;\n" " fma.rn.f32 %f326, %f314, %f324, %f282;\n" " ld.shared.f32 %f327, [%r71+8576];\n" " fma.rn.f32 %f328, %f311, %f327, %f284;\n" " fma.rn.f32 %f329, %f314, %f327, %f285;\n" " ld.shared.f32 %f330, [%r71+10624];\n" " fma.rn.f32 %f331, %f311, %f330, %f287;\n" " fma.rn.f32 %f332, %f314, %f330, %f288;\n" " ld.shared.f32 %f333, [%r71+12672];\n" " fma.rn.f32 %f334, %f311, %f333, %f290;\n" " fma.rn.f32 %f335, %f314, %f333, %f291;\n" " ld.shared.f32 %f336, [%r71+14720];\n" " fma.rn.f32 %f337, %f311, %f336, %f293;\n" " fma.rn.f32 %f338, %f314, %f336, %f294;\n" " bfe.u32 %r102, %r50, 16, 4;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f339, %r103;\n" " fma.rn.f32 %f340, %f339, %f165, %f164;\n" " and.b32 %r104, %r56, 768;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f341, %r105;\n" " mov.f32 %f342, 0fC9000000;\n" " fma.rn.f32 %f343, %f341, %f254, %f342;\n" " add.f32 %f344, %f340, %f343;\n" " bfe.u32 %r106, %r51, 16, 4;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f345, %r107;\n" " fma.rn.f32 %f346, %f345, %f165, %f164;\n" " and.b32 %r108, %r57, 768;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f347, %r109;\n" " fma.rn.f32 %f348, %f347, %f254, %f342;\n" " add.f32 %f349, %f346, %f348;\n" " and.b32 %r110, %r53, 255;\n" " cvt.rn.f32.u32 %f350, %r110;\n" " add.f32 %f351, %f350, 0fC3000000;\n" " mul.rn.f32 %f352, %f351, %f344;\n" " and.b32 %r111, %r55, 255;\n" " cvt.rn.f32.u32 %f353, %r111;\n" " add.f32 %f354, %f353, 0fC3000000;\n" " mul.rn.f32 %f355, %f354, %f349;\n" " ld.shared.f32 %f356, [%r71+512];\n" " fma.rn.f32 %f357, %f352, %f356, %f316;\n" " fma.rn.f32 %f358, %f355, %f356, %f317;\n" " ld.shared.f32 %f359, [%r71+2560];\n" " fma.rn.f32 %f360, %f352, %f359, %f319;\n" " fma.rn.f32 %f361, %f355, %f359, %f320;\n" " ld.shared.f32 %f362, [%r71+4608];\n" " fma.rn.f32 %f363, %f352, %f362, %f322;\n" " fma.rn.f32 %f364, %f355, %f362, %f323;\n" " ld.shared.f32 %f365, [%r71+6656];\n" " fma.rn.f32 %f366, %f352, %f365, %f325;\n" " fma.rn.f32 %f367, %f355, %f365, %f326;\n" " ld.shared.f32 %f368, [%r71+8704];\n" " fma.rn.f32 %f369, %f352, %f368, %f328;\n" " fma.rn.f32 %f370, %f355, %f368, %f329;\n" " ld.shared.f32 %f371, [%r71+10752];\n" " fma.rn.f32 %f372, %f352, %f371, %f331;\n" " fma.rn.f32 %f373, %f355, %f371, %f332;\n" " ld.shared.f32 %f374, [%r71+12800];\n" " fma.rn.f32 %f375, %f352, %f374, %f334;\n" " fma.rn.f32 %f376, %f355, %f374, %f335;\n" " ld.shared.f32 %f377, [%r71+14848];\n" " fma.rn.f32 %f378, %f352, %f377, %f337;\n" " fma.rn.f32 %f379, %f355, %f377, %f338;\n" " and.b32 %r112, %r58, 3840;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f380, %r113;\n" " fma.rn.f32 %f381, %f380, %f210, %f209;\n" " and.b32 %r114, %r56, 3072;\n" " or.b32 %r115, %r114, 1258291200;\n" " mov.b32 %f382, %r115;\n" " mov.f32 %f383, 0fC8000000;\n" " mov.f32 %f384, 0f3C800000;\n" " fma.rn.f32 %f385, %f382, %f384, %f383;\n" " add.f32 %f386, %f381, %f385;\n" " and.b32 %r116, %r59, 3840;\n" " or.b32 %r117, %r116, 1258291200;\n" " mov.b32 %f387, %r117;\n" " fma.rn.f32 %f388, %f387, %f210, %f209;\n" " and.b32 %r118, %r57, 3072;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f389, %r119;\n" " fma.rn.f32 %f390, %f389, %f384, %f383;\n" " add.f32 %f391, %f388, %f390;\n" " bfe.u32 %r120, %r53, 8, 8;\n" " cvt.rn.f32.u32 %f392, %r120;\n" " add.f32 %f393, %f392, 0fC3000000;\n" " mul.rn.f32 %f394, %f393, %f386;\n" " bfe.u32 %r121, %r55, 8, 8;\n" " cvt.rn.f32.u32 %f395, %r121;\n" " add.f32 %f396, %f395, 0fC3000000;\n" " mul.rn.f32 %f397, %f396, %f391;\n" " ld.shared.f32 %f398, [%r71+640];\n" " fma.rn.f32 %f399, %f394, %f398, %f357;\n" " fma.rn.f32 %f400, %f397, %f398, %f358;\n" " ld.shared.f32 %f401, [%r71+2688];\n" " fma.rn.f32 %f402, %f394, %f401, %f360;\n" " fma.rn.f32 %f403, %f397, %f401, %f361;\n" " ld.shared.f32 %f404, [%r71+4736];\n" " fma.rn.f32 %f405, %f394, %f404, %f363;\n" " fma.rn.f32 %f406, %f397, %f404, %f364;\n" " ld.shared.f32 %f407, [%r71+6784];\n" " fma.rn.f32 %f408, %f394, %f407, %f366;\n" " fma.rn.f32 %f409, %f397, %f407, %f367;\n" " ld.shared.f32 %f410, [%r71+8832];\n" " fma.rn.f32 %f411, %f394, %f410, %f369;\n" " fma.rn.f32 %f412, %f397, %f410, %f370;\n" " ld.shared.f32 %f413, [%r71+10880];\n" " fma.rn.f32 %f414, %f394, %f413, %f372;\n" " fma.rn.f32 %f415, %f397, %f413, %f373;\n" " ld.shared.f32 %f416, [%r71+12928];\n" " fma.rn.f32 %f417, %f394, %f416, %f375;\n" " fma.rn.f32 %f418, %f397, %f416, %f376;\n" " ld.shared.f32 %f419, [%r71+14976];\n" " fma.rn.f32 %f420, %f394, %f419, %f378;\n" " fma.rn.f32 %f421, %f397, %f419, %f379;\n" " and.b32 %r122, %r58, 240;\n" " or.b32 %r123, %r122, 1258291200;\n" " mov.b32 %f422, %r123;\n" " fma.rn.f32 %f423, %f422, %f254, %f253;\n" " and.b32 %r124, %r56, 12288;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f424, %r125;\n" " mov.f32 %f425, 0fC7000000;\n" " fma.rn.f32 %f426, %f424, %f210, %f425;\n" " add.f32 %f427, %f423, %f426;\n" " and.b32 %r126, %r59, 240;\n" " or.b32 %r127, %r126, 1258291200;\n" " mov.b32 %f428, %r127;\n" " fma.rn.f32 %f429, %f428, %f254, %f253;\n" " and.b32 %r128, %r57, 12288;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f430, %r129;\n" " fma.rn.f32 %f431, %f430, %f210, %f425;\n" " add.f32 %f432, %f429, %f431;\n" " bfe.u32 %r130, %r53, 16, 8;\n" " cvt.rn.f32.u32 %f433, %r130;\n" " add.f32 %f434, %f433, 0fC3000000;\n" " mul.rn.f32 %f435, %f434, %f427;\n" " bfe.u32 %r131, %r55, 16, 8;\n" " cvt.rn.f32.u32 %f436, %r131;\n" " add.f32 %f437, %f436, 0fC3000000;\n" " mul.rn.f32 %f438, %f437, %f432;\n" " ld.shared.f32 %f439, [%r71+768];\n" " fma.rn.f32 %f440, %f435, %f439, %f399;\n" " fma.rn.f32 %f441, %f438, %f439, %f400;\n" " ld.shared.f32 %f442, [%r71+2816];\n" " fma.rn.f32 %f443, %f435, %f442, %f402;\n" " fma.rn.f32 %f444, %f438, %f442, %f403;\n" " ld.shared.f32 %f445, [%r71+4864];\n" " fma.rn.f32 %f446, %f435, %f445, %f405;\n" " fma.rn.f32 %f447, %f438, %f445, %f406;\n" " ld.shared.f32 %f448, [%r71+6912];\n" " fma.rn.f32 %f449, %f435, %f448, %f408;\n" " fma.rn.f32 %f450, %f438, %f448, %f409;\n" " ld.shared.f32 %f451, [%r71+8960];\n" " fma.rn.f32 %f452, %f435, %f451, %f411;\n" " fma.rn.f32 %f453, %f438, %f451, %f412;\n" " ld.shared.f32 %f454, [%r71+11008];\n" " fma.rn.f32 %f455, %f435, %f454, %f414;\n" " fma.rn.f32 %f456, %f438, %f454, %f415;\n" " ld.shared.f32 %f457, [%r71+13056];\n" " fma.rn.f32 %f458, %f435, %f457, %f417;\n" " fma.rn.f32 %f459, %f438, %f457, %f418;\n" " ld.shared.f32 %f460, [%r71+15104];\n" " fma.rn.f32 %f461, %f435, %f460, %f420;\n" " fma.rn.f32 %f462, %f438, %f460, %f421;\n" " and.b32 %r132, %r58, 61440;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f463, %r133;\n" " fma.rn.f32 %f464, %f463, %f297, %f296;\n" " and.b32 %r134, %r56, 49152;\n" " or.b32 %r135, %r134, 1258291200;\n" " mov.b32 %f465, %r135;\n" " mov.f32 %f466, 0fC6000000;\n" " mov.f32 %f467, 0f3A800000;\n" " fma.rn.f32 %f468, %f465, %f467, %f466;\n" " add.f32 %f469, %f464, %f468;\n" " and.b32 %r136, %r59, 61440;\n" " or.b32 %r137, %r136, 1258291200;\n" " mov.b32 %f470, %r137;\n" " fma.rn.f32 %f471, %f470, %f297, %f296;\n" " and.b32 %r138, %r57, 49152;\n" " or.b32 %r139, %r138, 1258291200;\n" " mov.b32 %f472, %r139;\n" " fma.rn.f32 %f473, %f472, %f467, %f466;\n" " add.f32 %f474, %f471, %f473;\n" " shr.u32 %r140, %r53, 24;\n" " cvt.rn.f32.u32 %f475, %r140;\n" " add.f32 %f476, %f475, 0fC3000000;\n" " mul.rn.f32 %f477, %f476, %f469;\n" " shr.u32 %r141, %r55, 24;\n" " cvt.rn.f32.u32 %f478, %r141;\n" " add.f32 %f479, %f478, 0fC3000000;\n" " mul.rn.f32 %f480, %f479, %f474;\n" " ld.shared.f32 %f481, [%r71+896];\n" " fma.rn.f32 %f482, %f477, %f481, %f440;\n" " fma.rn.f32 %f483, %f480, %f481, %f441;\n" " ld.shared.f32 %f484, [%r71+2944];\n" " fma.rn.f32 %f485, %f477, %f484, %f443;\n" " fma.rn.f32 %f486, %f480, %f484, %f444;\n" " ld.shared.f32 %f487, [%r71+4992];\n" " fma.rn.f32 %f488, %f477, %f487, %f446;\n" " fma.rn.f32 %f489, %f480, %f487, %f447;\n" " ld.shared.f32 %f490, [%r71+7040];\n" " fma.rn.f32 %f491, %f477, %f490, %f449;\n" " fma.rn.f32 %f492, %f480, %f490, %f450;\n" " ld.shared.f32 %f493, [%r71+9088];\n" " fma.rn.f32 %f494, %f477, %f493, %f452;\n" " fma.rn.f32 %f495, %f480, %f493, %f453;\n" " ld.shared.f32 %f496, [%r71+11136];\n" " fma.rn.f32 %f497, %f477, %f496, %f455;\n" " fma.rn.f32 %f498, %f480, %f496, %f456;\n" " ld.shared.f32 %f499, [%r71+13184];\n" " fma.rn.f32 %f500, %f477, %f499, %f458;\n" " fma.rn.f32 %f501, %f480, %f499, %f459;\n" " ld.shared.f32 %f502, [%r71+15232];\n" " fma.rn.f32 %f503, %f477, %f502, %f461;\n" " fma.rn.f32 %f504, %f480, %f502, %f462;\n" " fma.rn.f32 %f719, %f161, %f482, %f719;\n" " fma.rn.f32 %f727, %f162, %f483, %f727;\n" " fma.rn.f32 %f718, %f161, %f485, %f718;\n" " fma.rn.f32 %f726, %f162, %f486, %f726;\n" " fma.rn.f32 %f717, %f161, %f488, %f717;\n" " fma.rn.f32 %f725, %f162, %f489, %f725;\n" " fma.rn.f32 %f716, %f161, %f491, %f716;\n" " fma.rn.f32 %f724, %f162, %f492, %f724;\n" " fma.rn.f32 %f715, %f161, %f494, %f715;\n" " fma.rn.f32 %f723, %f162, %f495, %f723;\n" " fma.rn.f32 %f714, %f161, %f497, %f714;\n" " fma.rn.f32 %f722, %f162, %f498, %f722;\n" " fma.rn.f32 %f713, %f161, %f500, %f713;\n" " fma.rn.f32 %f721, %f162, %f501, %f721;\n" " fma.rn.f32 %f712, %f161, %f503, %f712;\n" " fma.rn.f32 %f720, %f162, %f504, %f720;\n" " add.s64 %rd140, %rd140, 216;\n" " add.s64 %rd141, %rd141, 216;\n" " add.s32 %r341, %r341, 1024;\n" " add.s32 %r340, %r340, 1;\n" " setp.lt.s32 %p15, %r340, %r334;\n" " @%p15 bra $L__BB11_26;\n" "\n" "$L__BB11_27:\n" " add.s32 %r337, %r337, 512;\n" " setp.lt.s32 %p16, %r337, %r23;\n" " @%p16 bra $L__BB11_4;\n" "\n" "$L__BB11_28:\n" " mov.u32 %r327, %ctaid.y;\n" " shl.b32 %r326, %r327, 3;\n" " ld.param.u32 %r325, [fc_matvec_q6k_nx8_param_6];\n" " sub.s32 %r324, %r325, %r326;\n" " cvt.s64.s32 %rd35, %r22;\n" " mov.u32 %r152, 16;\n" " mov.u32 %r161, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f505, %f719, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f508, %f719, %f505;\n" " mov.u32 %r154, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f507, %f508, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f510, %f508, %f507;\n" " mov.u32 %r156, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f509, %f510, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f512, %f510, %f509;\n" " mov.u32 %r158, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f511, %f512, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f514, %f512, %f511;\n" " mov.u32 %r160, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f513, %f514, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f97, %f514, %f513;\n" " // begin inline asm\n" " shfl.down.b32 %f515, %f727, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f518, %f727, %f515;\n" " // begin inline asm\n" " shfl.down.b32 %f517, %f518, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f520, %f518, %f517;\n" " // begin inline asm\n" " shfl.down.b32 %f519, %f520, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f522, %f520, %f519;\n" " // begin inline asm\n" " shfl.down.b32 %f521, %f522, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f524, %f522, %f521;\n" " // begin inline asm\n" " shfl.down.b32 %f523, %f524, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f98, %f524, %f523;\n" " setp.lt.s32 %p17, %r324, 1;\n" " setp.ne.s32 %p18, %r5, 0;\n" " or.pred %p19, %p18, %p17;\n" " @%p19 bra $L__BB11_33;\n" "\n" " cvt.u32.u64 %r162, %rd2;\n" " setp.ge.s32 %p20, %r162, %r22;\n" " cvt.s64.s32 %rd86, %r1;\n" " mul.lo.s64 %rd87, %rd86, %rd35;\n" " add.s64 %rd88, %rd87, %rd2;\n" " shl.b64 %rd89, %rd88, 2;\n" " add.s64 %rd36, %rd1, %rd89;\n" " @%p20 bra $L__BB11_31;\n" "\n" " st.global.f32 [%rd36], %f97;\n" "\n" "$L__BB11_31:\n" " @%p1 bra $L__BB11_33;\n" "\n" " st.global.f32 [%rd36+4], %f98;\n" "\n" "$L__BB11_33:\n" " // begin inline asm\n" " shfl.down.b32 %f525, %f718, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f528, %f718, %f525;\n" " // begin inline asm\n" " shfl.down.b32 %f527, %f528, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f530, %f528, %f527;\n" " // begin inline asm\n" " shfl.down.b32 %f529, %f530, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f532, %f530, %f529;\n" " // begin inline asm\n" " shfl.down.b32 %f531, %f532, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f534, %f532, %f531;\n" " // begin inline asm\n" " shfl.down.b32 %f533, %f534, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f99, %f534, %f533;\n" " // begin inline asm\n" " shfl.down.b32 %f535, %f726, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f538, %f726, %f535;\n" " // begin inline asm\n" " shfl.down.b32 %f537, %f538, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f540, %f538, %f537;\n" " // begin inline asm\n" " shfl.down.b32 %f539, %f540, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f542, %f540, %f539;\n" " // begin inline asm\n" " shfl.down.b32 %f541, %f542, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f544, %f542, %f541;\n" " // begin inline asm\n" " shfl.down.b32 %f543, %f544, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f100, %f544, %f543;\n" " setp.lt.s32 %p23, %r3, 2;\n" " or.pred %p24, %p18, %p23;\n" " @%p24 bra $L__BB11_38;\n" "\n" " cvt.u32.u64 %r183, %rd2;\n" " setp.ge.s32 %p25, %r183, %r22;\n" " add.s32 %r184, %r1, 1;\n" " cvt.s64.s32 %rd90, %r184;\n" " mul.lo.s64 %rd91, %rd90, %rd35;\n" " add.s64 %rd92, %rd91, %rd2;\n" " shl.b64 %rd93, %rd92, 2;\n" " add.s64 %rd37, %rd1, %rd93;\n" " @%p25 bra $L__BB11_36;\n" "\n" " st.global.f32 [%rd37], %f99;\n" "\n" "$L__BB11_36:\n" " @%p1 bra $L__BB11_38;\n" "\n" " st.global.f32 [%rd37+4], %f100;\n" "\n" "$L__BB11_38:\n" " // begin inline asm\n" " shfl.down.b32 %f545, %f717, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f548, %f717, %f545;\n" " // begin inline asm\n" " shfl.down.b32 %f547, %f548, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f550, %f548, %f547;\n" " // begin inline asm\n" " shfl.down.b32 %f549, %f550, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f552, %f550, %f549;\n" " // begin inline asm\n" " shfl.down.b32 %f551, %f552, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f554, %f552, %f551;\n" " // begin inline asm\n" " shfl.down.b32 %f553, %f554, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f101, %f554, %f553;\n" " // begin inline asm\n" " shfl.down.b32 %f555, %f725, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f558, %f725, %f555;\n" " // begin inline asm\n" " shfl.down.b32 %f557, %f558, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f560, %f558, %f557;\n" " // begin inline asm\n" " shfl.down.b32 %f559, %f560, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f562, %f560, %f559;\n" " // begin inline asm\n" " shfl.down.b32 %f561, %f562, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f564, %f562, %f561;\n" " // begin inline asm\n" " shfl.down.b32 %f563, %f564, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f102, %f564, %f563;\n" " setp.lt.s32 %p28, %r3, 3;\n" " or.pred %p29, %p18, %p28;\n" " @%p29 bra $L__BB11_43;\n" "\n" " cvt.u32.u64 %r205, %rd2;\n" " setp.ge.s32 %p30, %r205, %r22;\n" " add.s32 %r206, %r1, 2;\n" " cvt.s64.s32 %rd94, %r206;\n" " mul.lo.s64 %rd95, %rd94, %rd35;\n" " add.s64 %rd96, %rd95, %rd2;\n" " shl.b64 %rd97, %rd96, 2;\n" " add.s64 %rd38, %rd1, %rd97;\n" " @%p30 bra $L__BB11_41;\n" "\n" " st.global.f32 [%rd38], %f101;\n" "\n" "$L__BB11_41:\n" " @%p1 bra $L__BB11_43;\n" "\n" " st.global.f32 [%rd38+4], %f102;\n" "\n" "$L__BB11_43:\n" " // begin inline asm\n" " shfl.down.b32 %f565, %f716, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f568, %f716, %f565;\n" " // begin inline asm\n" " shfl.down.b32 %f567, %f568, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f570, %f568, %f567;\n" " // begin inline asm\n" " shfl.down.b32 %f569, %f570, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f572, %f570, %f569;\n" " // begin inline asm\n" " shfl.down.b32 %f571, %f572, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f574, %f572, %f571;\n" " // begin inline asm\n" " shfl.down.b32 %f573, %f574, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f103, %f574, %f573;\n" " // begin inline asm\n" " shfl.down.b32 %f575, %f724, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f578, %f724, %f575;\n" " // begin inline asm\n" " shfl.down.b32 %f577, %f578, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f580, %f578, %f577;\n" " // begin inline asm\n" " shfl.down.b32 %f579, %f580, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f582, %f580, %f579;\n" " // begin inline asm\n" " shfl.down.b32 %f581, %f582, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f584, %f582, %f581;\n" " // begin inline asm\n" " shfl.down.b32 %f583, %f584, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f104, %f584, %f583;\n" " setp.lt.s32 %p33, %r3, 4;\n" " or.pred %p34, %p18, %p33;\n" " @%p34 bra $L__BB11_48;\n" "\n" " cvt.u32.u64 %r227, %rd2;\n" " setp.ge.s32 %p35, %r227, %r22;\n" " add.s32 %r228, %r1, 3;\n" " cvt.s64.s32 %rd98, %r228;\n" " mul.lo.s64 %rd99, %rd98, %rd35;\n" " add.s64 %rd100, %rd99, %rd2;\n" " shl.b64 %rd101, %rd100, 2;\n" " add.s64 %rd39, %rd1, %rd101;\n" " @%p35 bra $L__BB11_46;\n" "\n" " st.global.f32 [%rd39], %f103;\n" "\n" "$L__BB11_46:\n" " @%p1 bra $L__BB11_48;\n" "\n" " st.global.f32 [%rd39+4], %f104;\n" "\n" "$L__BB11_48:\n" " // begin inline asm\n" " shfl.down.b32 %f585, %f715, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f588, %f715, %f585;\n" " // begin inline asm\n" " shfl.down.b32 %f587, %f588, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f590, %f588, %f587;\n" " // begin inline asm\n" " shfl.down.b32 %f589, %f590, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f592, %f590, %f589;\n" " // begin inline asm\n" " shfl.down.b32 %f591, %f592, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f594, %f592, %f591;\n" " // begin inline asm\n" " shfl.down.b32 %f593, %f594, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f105, %f594, %f593;\n" " // begin inline asm\n" " shfl.down.b32 %f595, %f723, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f598, %f723, %f595;\n" " // begin inline asm\n" " shfl.down.b32 %f597, %f598, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f600, %f598, %f597;\n" " // begin inline asm\n" " shfl.down.b32 %f599, %f600, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f602, %f600, %f599;\n" " // begin inline asm\n" " shfl.down.b32 %f601, %f602, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f604, %f602, %f601;\n" " // begin inline asm\n" " shfl.down.b32 %f603, %f604, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f106, %f604, %f603;\n" " setp.lt.s32 %p38, %r3, 5;\n" " or.pred %p39, %p18, %p38;\n" " @%p39 bra $L__BB11_53;\n" "\n" " cvt.u32.u64 %r249, %rd2;\n" " setp.ge.s32 %p40, %r249, %r22;\n" " add.s32 %r250, %r1, 4;\n" " cvt.s64.s32 %rd102, %r250;\n" " mul.lo.s64 %rd103, %rd102, %rd35;\n" " add.s64 %rd104, %rd103, %rd2;\n" " shl.b64 %rd105, %rd104, 2;\n" " add.s64 %rd40, %rd1, %rd105;\n" " @%p40 bra $L__BB11_51;\n" "\n" " st.global.f32 [%rd40], %f105;\n" "\n" "$L__BB11_51:\n" " @%p1 bra $L__BB11_53;\n" "\n" " st.global.f32 [%rd40+4], %f106;\n" "\n" "$L__BB11_53:\n" " // begin inline asm\n" " shfl.down.b32 %f605, %f714, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f608, %f714, %f605;\n" " // begin inline asm\n" " shfl.down.b32 %f607, %f608, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f610, %f608, %f607;\n" " // begin inline asm\n" " shfl.down.b32 %f609, %f610, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f612, %f610, %f609;\n" " // begin inline asm\n" " shfl.down.b32 %f611, %f612, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f614, %f612, %f611;\n" " // begin inline asm\n" " shfl.down.b32 %f613, %f614, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f107, %f614, %f613;\n" " // begin inline asm\n" " shfl.down.b32 %f615, %f722, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f618, %f722, %f615;\n" " // begin inline asm\n" " shfl.down.b32 %f617, %f618, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f620, %f618, %f617;\n" " // begin inline asm\n" " shfl.down.b32 %f619, %f620, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f622, %f620, %f619;\n" " // begin inline asm\n" " shfl.down.b32 %f621, %f622, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f624, %f622, %f621;\n" " // begin inline asm\n" " shfl.down.b32 %f623, %f624, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f108, %f624, %f623;\n" " setp.lt.s32 %p43, %r3, 6;\n" " or.pred %p44, %p18, %p43;\n" " @%p44 bra $L__BB11_58;\n" "\n" " cvt.u32.u64 %r271, %rd2;\n" " setp.ge.s32 %p45, %r271, %r22;\n" " add.s32 %r272, %r1, 5;\n" " cvt.s64.s32 %rd106, %r272;\n" " mul.lo.s64 %rd107, %rd106, %rd35;\n" " add.s64 %rd108, %rd107, %rd2;\n" " shl.b64 %rd109, %rd108, 2;\n" " add.s64 %rd41, %rd1, %rd109;\n" " @%p45 bra $L__BB11_56;\n" "\n" " st.global.f32 [%rd41], %f107;\n" "\n" "$L__BB11_56:\n" " @%p1 bra $L__BB11_58;\n" "\n" " st.global.f32 [%rd41+4], %f108;\n" "\n" "$L__BB11_58:\n" " // begin inline asm\n" " shfl.down.b32 %f625, %f713, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f628, %f713, %f625;\n" " // begin inline asm\n" " shfl.down.b32 %f627, %f628, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f630, %f628, %f627;\n" " // begin inline asm\n" " shfl.down.b32 %f629, %f630, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f632, %f630, %f629;\n" " // begin inline asm\n" " shfl.down.b32 %f631, %f632, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f634, %f632, %f631;\n" " // begin inline asm\n" " shfl.down.b32 %f633, %f634, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f109, %f634, %f633;\n" " // begin inline asm\n" " shfl.down.b32 %f635, %f721, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f638, %f721, %f635;\n" " // begin inline asm\n" " shfl.down.b32 %f637, %f638, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f640, %f638, %f637;\n" " // begin inline asm\n" " shfl.down.b32 %f639, %f640, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f642, %f640, %f639;\n" " // begin inline asm\n" " shfl.down.b32 %f641, %f642, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f644, %f642, %f641;\n" " // begin inline asm\n" " shfl.down.b32 %f643, %f644, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f110, %f644, %f643;\n" " setp.lt.s32 %p48, %r3, 7;\n" " or.pred %p49, %p18, %p48;\n" " @%p49 bra $L__BB11_63;\n" "\n" " cvt.u32.u64 %r293, %rd2;\n" " setp.ge.s32 %p50, %r293, %r22;\n" " add.s32 %r294, %r1, 6;\n" " cvt.s64.s32 %rd110, %r294;\n" " mul.lo.s64 %rd111, %rd110, %rd35;\n" " add.s64 %rd112, %rd111, %rd2;\n" " shl.b64 %rd113, %rd112, 2;\n" " add.s64 %rd42, %rd1, %rd113;\n" " @%p50 bra $L__BB11_61;\n" "\n" " st.global.f32 [%rd42], %f109;\n" "\n" "$L__BB11_61:\n" " @%p1 bra $L__BB11_63;\n" "\n" " st.global.f32 [%rd42+4], %f110;\n" "\n" "$L__BB11_63:\n" " // begin inline asm\n" " shfl.down.b32 %f645, %f712, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f648, %f712, %f645;\n" " // begin inline asm\n" " shfl.down.b32 %f647, %f648, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f650, %f648, %f647;\n" " // begin inline asm\n" " shfl.down.b32 %f649, %f650, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f652, %f650, %f649;\n" " // begin inline asm\n" " shfl.down.b32 %f651, %f652, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f654, %f652, %f651;\n" " // begin inline asm\n" " shfl.down.b32 %f653, %f654, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f111, %f654, %f653;\n" " // begin inline asm\n" " shfl.down.b32 %f655, %f720, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f658, %f720, %f655;\n" " // begin inline asm\n" " shfl.down.b32 %f657, %f658, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f660, %f658, %f657;\n" " // begin inline asm\n" " shfl.down.b32 %f659, %f660, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f662, %f660, %f659;\n" " // begin inline asm\n" " shfl.down.b32 %f661, %f662, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f664, %f662, %f661;\n" " // begin inline asm\n" " shfl.down.b32 %f663, %f664, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f112, %f664, %f663;\n" " setp.lt.s32 %p53, %r3, 8;\n" " or.pred %p54, %p18, %p53;\n" " @%p54 bra $L__BB11_68;\n" "\n" " cvt.u32.u64 %r315, %rd2;\n" " setp.ge.s32 %p55, %r315, %r22;\n" " add.s32 %r316, %r1, 7;\n" " cvt.s64.s32 %rd114, %r316;\n" " mul.lo.s64 %rd115, %rd114, %rd35;\n" " add.s64 %rd116, %rd115, %rd2;\n" " shl.b64 %rd117, %rd116, 2;\n" " add.s64 %rd43, %rd1, %rd117;\n" " @%p55 bra $L__BB11_66;\n" "\n" " st.global.f32 [%rd43], %f111;\n" "\n" "$L__BB11_66:\n" " @%p1 bra $L__BB11_68;\n" "\n" " st.global.f32 [%rd43+4], %f112;\n" "\n" "$L__BB11_68:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q80\n" ".visible .entry fc_matvec_q80(\n" " .param .u64 fc_matvec_q80_param_0,\n" " .param .u64 fc_matvec_q80_param_1,\n" " .param .u64 fc_matvec_q80_param_2,\n" " .param .u32 fc_matvec_q80_param_3,\n" " .param .u32 fc_matvec_q80_param_4,\n" " .param .u32 fc_matvec_q80_param_5\n" ")\n" "{\n" " .reg .pred %p<19>;\n" " .reg .b16 %rs<15>;\n" " .reg .f32 %f<69>;\n" " .reg .b32 %r<92>;\n" " .reg .b64 %rd<66>;\n" "\n" "\n" " ld.param.u64 %rd21, [fc_matvec_q80_param_0];\n" " ld.param.u64 %rd22, [fc_matvec_q80_param_1];\n" " ld.param.u64 %rd23, [fc_matvec_q80_param_2];\n" " ld.param.u32 %r31, [fc_matvec_q80_param_3];\n" " ld.param.u32 %r32, [fc_matvec_q80_param_4];\n" " ld.param.u32 %r33, [fc_matvec_q80_param_5];\n" " mov.u32 %r1, %tid.x;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r34, %ctaid.x;\n" " shl.b32 %r35, %r34, 3;\n" " shr.u32 %r36, %r1, 5;\n" " add.s32 %r37, %r35, %r36;\n" " cvt.s64.s32 %rd1, %r37;\n" " setp.lt.s32 %p1, %r32, 1;\n" " mov.f32 %f67, 0f00000000;\n" " @%p1 bra $L__BB12_20;\n" "\n" " add.s32 %r40, %r2, 4;\n" " cvt.u64.u32 %rd2, %r40;\n" " mov.u32 %r41, -2;\n" " sub.s32 %r3, %r41, %r1;\n" " not.b32 %r4, %r32;\n" " cvt.s64.s32 %rd3, %r1;\n" " shl.b32 %r42, %r1, 2;\n" " mov.u32 %r43, fc_kq_shared;\n" " add.s32 %r5, %r43, %r42;\n" " add.s32 %r6, %r1, 256;\n" " cvt.s64.s32 %rd4, %r6;\n" " add.s32 %r7, %r1, 512;\n" " cvt.s64.s32 %rd5, %r7;\n" " add.s32 %r8, %r1, 768;\n" " cvt.s64.s32 %rd24, %r33;\n" " mul.lo.s64 %rd25, %rd24, %rd1;\n" " add.s64 %rd65, %rd21, %rd25;\n" " mov.f32 %f67, 0f00000000;\n" " mov.u32 %r84, 0;\n" " mov.u32 %r85, %r84;\n" "\n" "$L__BB12_2:\n" " add.s32 %r44, %r84, %r4;\n" " max.s32 %r45, %r44, -2049;\n" " sub.s32 %r11, %r3, %r45;\n" " sub.s32 %r46, %r32, %r85;\n" " min.s32 %r12, %r46, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r12;\n" " @%p2 bra $L__BB12_10;\n" "\n" " shr.u32 %r47, %r11, 8;\n" " add.s32 %r48, %r47, 1;\n" " cvt.s64.s32 %rd8, %r85;\n" " and.b32 %r13, %r48, 3;\n" " setp.eq.s32 %p3, %r13, 0;\n" " mov.u32 %r86, %r1;\n" " @%p3 bra $L__BB12_7;\n" "\n" " add.s64 %rd27, %rd3, %rd8;\n" " shl.b64 %rd28, %rd27, 2;\n" " add.s64 %rd26, %rd22, %rd28;\n" " // begin inline asm\n" " ld.global.nc.f32 %f14, [%rd26];\n" " // end inline asm\n" " st.shared.f32 [%r5], %f14;\n" " setp.eq.s32 %p4, %r13, 1;\n" " mov.u32 %r86, %r6;\n" " @%p4 bra $L__BB12_7;\n" "\n" " add.s64 %rd30, %rd4, %rd8;\n" " shl.b64 %rd31, %rd30, 2;\n" " add.s64 %rd29, %rd22, %rd31;\n" " // begin inline asm\n" " ld.global.nc.f32 %f15, [%rd29];\n" " // end inline asm\n" " st.shared.f32 [%r5+1024], %f15;\n" " setp.eq.s32 %p5, %r13, 2;\n" " mov.u32 %r86, %r7;\n" " @%p5 bra $L__BB12_7;\n" "\n" " add.s64 %rd33, %rd5, %rd8;\n" " shl.b64 %rd34, %rd33, 2;\n" " add.s64 %rd32, %rd22, %rd34;\n" " // begin inline asm\n" " ld.global.nc.f32 %f16, [%rd32];\n" " // end inline asm\n" " st.shared.f32 [%r5+2048], %f16;\n" " mov.u32 %r86, %r8;\n" "\n" "$L__BB12_7:\n" " setp.lt.u32 %p6, %r11, 768;\n" " @%p6 bra $L__BB12_10;\n" "\n" " cvt.s64.s32 %rd35, %r86;\n" " add.s64 %rd36, %rd35, %rd8;\n" " shl.b64 %rd37, %rd36, 2;\n" " add.s64 %rd61, %rd22, %rd37;\n" " shl.b32 %r49, %r86, 2;\n" " add.s32 %r51, %r43, %r49;\n" " add.s32 %r87, %r51, 2048;\n" "\n" "$L__BB12_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd61];\n" " // end inline asm\n" " st.shared.f32 [%r87+-2048], %f17;\n" " add.s64 %rd39, %rd61, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r87+-1024], %f18;\n" " add.s64 %rd40, %rd61, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd40];\n" " // end inline asm\n" " st.shared.f32 [%r87], %f19;\n" " add.s64 %rd41, %rd61, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r87+1024], %f20;\n" " add.s64 %rd61, %rd61, 4096;\n" " add.s32 %r87, %r87, 4096;\n" " add.s32 %r86, %r86, 1024;\n" " setp.lt.s32 %p7, %r86, %r12;\n" " @%p7 bra $L__BB12_9;\n" "\n" "$L__BB12_10:\n" " cvt.u32.u64 %r52, %rd1;\n" " setp.ge.s32 %p8, %r52, %r31;\n" " bar.sync 0;\n" " @%p8 bra $L__BB12_19;\n" "\n" " setp.lt.s32 %p9, %r12, 32;\n" " @%p9 bra $L__BB12_19;\n" "\n" " shr.s32 %r54, %r12, 5;\n" " max.s32 %r20, %r54, 1;\n" " add.s32 %r55, %r20, -1;\n" " and.b32 %r21, %r20, 3;\n" " setp.lt.u32 %p10, %r55, 3;\n" " mov.u32 %r91, 0;\n" " mov.u64 %rd63, %rd65;\n" " @%p10 bra $L__BB12_15;\n" "\n" " sub.s32 %r90, %r20, %r21;\n" " mov.u32 %r91, 0;\n" " mov.u64 %rd63, %rd65;\n" "\n" "$L__BB12_14:\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd63];\n" " // end inline asm\n" " add.s64 %rd44, %rd63, %rd2;\n" " // begin inline asm\n" " ld.global.nc.s8 %r57, [%rd44];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r57;\n" " cvt.s16.s8 %rs2, %rs1;\n" " cvt.rn.f32.s16 %f26, %rs2;\n" " mul.rn.f32 %f27, %f22, %f26;\n" " shl.b32 %r61, %r91, 5;\n" " or.b32 %r62, %r61, %r2;\n" " shl.b32 %r63, %r62, 2;\n" " add.s32 %r65, %r43, %r63;\n" " ld.shared.f32 %f28, [%r65];\n" " fma.rn.f32 %f29, %f27, %f28, %f67;\n" " add.s64 %rd45, %rd63, 36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd45];\n" " // end inline asm\n" " add.s64 %rd46, %rd44, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r58, [%rd46];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r58;\n" " cvt.s16.s8 %rs4, %rs3;\n" " cvt.rn.f32.s16 %f30, %rs4;\n" " mul.rn.f32 %f31, %f23, %f30;\n" " ld.shared.f32 %f32, [%r65+128];\n" " fma.rn.f32 %f33, %f31, %f32, %f29;\n" " add.s64 %rd47, %rd63, 72;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd47];\n" " // end inline asm\n" " add.s64 %rd48, %rd44, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r59, [%rd48];\n" " // end inline asm\n" " cvt.u16.u32 %rs5, %r59;\n" " cvt.s16.s8 %rs6, %rs5;\n" " cvt.rn.f32.s16 %f34, %rs6;\n" " mul.rn.f32 %f35, %f24, %f34;\n" " ld.shared.f32 %f36, [%r65+256];\n" " fma.rn.f32 %f37, %f35, %f36, %f33;\n" " add.s64 %rd49, %rd63, 108;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd49];\n" " // end inline asm\n" " add.s64 %rd50, %rd44, 108;\n" " // begin inline asm\n" " ld.global.nc.s8 %r60, [%rd50];\n" " // end inline asm\n" " cvt.u16.u32 %rs7, %r60;\n" " cvt.s16.s8 %rs8, %rs7;\n" " cvt.rn.f32.s16 %f38, %rs8;\n" " mul.rn.f32 %f39, %f25, %f38;\n" " ld.shared.f32 %f40, [%r65+384];\n" " fma.rn.f32 %f67, %f39, %f40, %f37;\n" " add.s64 %rd63, %rd63, 144;\n" " add.s32 %r91, %r91, 4;\n" " add.s32 %r90, %r90, -4;\n" " setp.ne.s32 %p11, %r90, 0;\n" " @%p11 bra $L__BB12_14;\n" "\n" "$L__BB12_15:\n" " setp.eq.s32 %p12, %r21, 0;\n" " mov.u64 %rd65, %rd63;\n" " @%p12 bra $L__BB12_19;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f41, [%rd63];\n" " // end inline asm\n" " add.s64 %rd52, %rd63, %rd2;\n" " // begin inline asm\n" " ld.global.nc.s8 %r66, [%rd52];\n" " // end inline asm\n" " cvt.u16.u32 %rs9, %r66;\n" " cvt.s16.s8 %rs10, %rs9;\n" " cvt.rn.f32.s16 %f42, %rs10;\n" " mul.rn.f32 %f43, %f41, %f42;\n" " shl.b32 %r67, %r91, 5;\n" " or.b32 %r68, %r67, %r2;\n" " shl.b32 %r69, %r68, 2;\n" " add.s32 %r28, %r43, %r69;\n" " ld.shared.f32 %f44, [%r28];\n" " fma.rn.f32 %f67, %f43, %f44, %f67;\n" " add.s64 %rd65, %rd63, 36;\n" " setp.eq.s32 %p13, %r21, 1;\n" " @%p13 bra $L__BB12_19;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f45, [%rd65];\n" " // end inline asm\n" " add.s64 %rd54, %rd52, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r71, [%rd54];\n" " // end inline asm\n" " cvt.u16.u32 %rs11, %r71;\n" " cvt.s16.s8 %rs12, %rs11;\n" " cvt.rn.f32.s16 %f46, %rs12;\n" " mul.rn.f32 %f47, %f45, %f46;\n" " ld.shared.f32 %f48, [%r28+128];\n" " fma.rn.f32 %f67, %f47, %f48, %f67;\n" " add.s64 %rd65, %rd63, 72;\n" " setp.eq.s32 %p14, %r21, 2;\n" " @%p14 bra $L__BB12_19;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f49, [%rd65];\n" " // end inline asm\n" " add.s64 %rd56, %rd52, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r72, [%rd56];\n" " // end inline asm\n" " cvt.u16.u32 %rs13, %r72;\n" " cvt.s16.s8 %rs14, %rs13;\n" " cvt.rn.f32.s16 %f50, %rs14;\n" " mul.rn.f32 %f51, %f49, %f50;\n" " ld.shared.f32 %f52, [%r28+256];\n" " fma.rn.f32 %f67, %f51, %f52, %f67;\n" " add.s64 %rd65, %rd63, 108;\n" "\n" "$L__BB12_19:\n" " add.s32 %r85, %r85, 2048;\n" " setp.lt.s32 %p15, %r85, %r32;\n" " add.s32 %r84, %r84, 2048;\n" " @%p15 bra $L__BB12_2;\n" "\n" "$L__BB12_20:\n" " cvt.u32.u64 %r83, %rd1;\n" " setp.ge.s32 %p16, %r83, %r31;\n" " mov.u32 %r73, 16;\n" " mov.u32 %r82, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f53, %f67, %r73, %r82;\n" " // end inline asm\n" " add.f32 %f56, %f67, %f53;\n" " mov.u32 %r75, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f55, %f56, %r75, %r82;\n" " // end inline asm\n" " add.f32 %f58, %f56, %f55;\n" " mov.u32 %r77, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f57, %f58, %r77, %r82;\n" " // end inline asm\n" " add.f32 %f60, %f58, %f57;\n" " mov.u32 %r79, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f59, %f60, %r79, %r82;\n" " // end inline asm\n" " add.f32 %f62, %f60, %f59;\n" " mov.u32 %r81, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f61, %f62, %r81, %r82;\n" " // end inline asm\n" " add.f32 %f11, %f62, %f61;\n" " setp.ne.s32 %p17, %r2, 0;\n" " or.pred %p18, %p17, %p16;\n" " @%p18 bra $L__BB12_22;\n" "\n" " cvta.to.global.u64 %rd57, %rd23;\n" " shl.b64 %rd58, %rd1, 2;\n" " add.s64 %rd59, %rd57, %rd58;\n" " st.global.f32 [%rd59], %f11;\n" "\n" "$L__BB12_22:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q80_nx2\n" ".visible .entry fc_matvec_q80_nx2(\n" " .param .u64 fc_matvec_q80_nx2_param_0,\n" " .param .u64 fc_matvec_q80_nx2_param_1,\n" " .param .u64 fc_matvec_q80_nx2_param_2,\n" " .param .u32 fc_matvec_q80_nx2_param_3,\n" " .param .u32 fc_matvec_q80_nx2_param_4,\n" " .param .u32 fc_matvec_q80_nx2_param_5,\n" " .param .u32 fc_matvec_q80_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<23>;\n" " .reg .b16 %rs<15>;\n" " .reg .f32 %f<106>;\n" " .reg .b32 %r<91>;\n" " .reg .b64 %rd<71>;\n" "\n" "\n" " ld.param.u64 %rd24, [fc_matvec_q80_nx2_param_0];\n" " ld.param.u64 %rd25, [fc_matvec_q80_nx2_param_1];\n" " ld.param.u64 %rd26, [fc_matvec_q80_nx2_param_2];\n" " ld.param.u32 %r25, [fc_matvec_q80_nx2_param_3];\n" " ld.param.u32 %r26, [fc_matvec_q80_nx2_param_4];\n" " ld.param.u32 %r27, [fc_matvec_q80_nx2_param_5];\n" " ld.param.u32 %r28, [fc_matvec_q80_nx2_param_6];\n" " cvta.to.global.u64 %rd1, %rd26;\n" " mov.u32 %r29, %ctaid.y;\n" " shl.b32 %r1, %r29, 1;\n" " sub.s32 %r2, %r28, %r1;\n" " min.s32 %r3, %r2, 2;\n" " mov.u32 %r4, %tid.x;\n" " and.b32 %r5, %r4, 31;\n" " mov.u32 %r30, %ctaid.x;\n" " shl.b32 %r31, %r30, 3;\n" " shr.u32 %r32, %r4, 5;\n" " add.s32 %r33, %r31, %r32;\n" " cvt.s64.s32 %rd2, %r33;\n" " setp.lt.s32 %p2, %r26, 1;\n" " mov.f32 %f102, 0f00000000;\n" " mov.f32 %f103, %f102;\n" " @%p2 bra $L__BB13_19;\n" "\n" " add.s32 %r35, %r5, 4;\n" " cvt.u64.u32 %rd3, %r35;\n" " add.s32 %r36, %r1, 1;\n" " mul.wide.s32 %rd27, %r26, %r1;\n" " cvt.s64.s32 %rd28, %r4;\n" " add.s64 %rd29, %rd27, %rd28;\n" " shl.b64 %rd30, %rd29, 2;\n" " add.s64 %rd4, %rd25, %rd30;\n" " shl.b32 %r37, %r4, 2;\n" " mov.u32 %r38, fc_kq_shared;\n" " add.s32 %r39, %r38, %r37;\n" " add.s32 %r6, %r39, 2048;\n" " mul.wide.s32 %rd31, %r26, %r36;\n" " add.s64 %rd32, %rd31, %rd28;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd5, %rd25, %rd33;\n" " cvt.s64.s32 %rd34, %r27;\n" " mul.lo.s64 %rd35, %rd34, %rd2;\n" " add.s64 %rd70, %rd24, %rd35;\n" " mov.u32 %r85, 0;\n" " mov.f32 %f102, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " mov.f32 %f103, %f102;\n" "\n" "$L__BB13_2:\n" " sub.s32 %r40, %r26, %r85;\n" " min.s32 %r8, %r40, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r4, %r8;\n" " @%p3 bra $L__BB13_9;\n" "\n" " mul.wide.s32 %rd36, %r85, 4;\n" " add.s64 %rd66, %rd4, %rd36;\n" " add.s64 %rd65, %rd5, %rd36;\n" " mov.u32 %r86, %r6;\n" " mov.u32 %r87, %r4;\n" "\n" "$L__BB13_4:\n" " .pragma \"nounroll\";\n" " mov.f32 %f97, 0f00000000;\n" " mov.f32 %f96, %f97;\n" " @%p4 bra $L__BB13_6;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f96, [%rd66];\n" " // end inline asm\n" "\n" "$L__BB13_6:\n" " st.shared.f32 [%r86+-2048], %f96;\n" " @%p5 bra $L__BB13_8;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f97, [%rd65];\n" " // end inline asm\n" "\n" "$L__BB13_8:\n" " add.s32 %r84, %r86, -2048;\n" " st.shared.f32 [%r84+2048], %f97;\n" " add.s64 %rd66, %rd66, 1024;\n" " add.s32 %r86, %r86, 1024;\n" " add.s64 %rd65, %rd65, 1024;\n" " add.s32 %r87, %r87, 256;\n" " setp.lt.s32 %p6, %r87, %r8;\n" " @%p6 bra $L__BB13_4;\n" "\n" "$L__BB13_9:\n" " cvt.u32.u64 %r41, %rd2;\n" " setp.ge.s32 %p7, %r41, %r25;\n" " bar.sync 0;\n" " @%p7 bra $L__BB13_18;\n" "\n" " setp.lt.s32 %p8, %r8, 32;\n" " @%p8 bra $L__BB13_18;\n" "\n" " shr.s32 %r43, %r8, 5;\n" " max.s32 %r15, %r43, 1;\n" " add.s32 %r44, %r15, -1;\n" " and.b32 %r16, %r15, 3;\n" " setp.lt.u32 %p9, %r44, 3;\n" " mov.u32 %r90, 0;\n" " mov.u64 %rd68, %rd70;\n" " @%p9 bra $L__BB13_14;\n" "\n" " sub.s32 %r89, %r15, %r16;\n" " mov.u32 %r90, 0;\n" " mov.u64 %rd68, %rd70;\n" "\n" "$L__BB13_13:\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd68];\n" " // end inline asm\n" " add.s64 %rd41, %rd68, %rd3;\n" " // begin inline asm\n" " ld.global.nc.s8 %r46, [%rd41];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r46;\n" " cvt.s16.s8 %rs2, %rs1;\n" " cvt.rn.f32.s16 %f37, %rs2;\n" " mul.rn.f32 %f38, %f33, %f37;\n" " shl.b32 %r50, %r90, 5;\n" " or.b32 %r51, %r50, %r5;\n" " shl.b32 %r52, %r51, 2;\n" " add.s32 %r54, %r38, %r52;\n" " ld.shared.f32 %f39, [%r54];\n" " fma.rn.f32 %f40, %f38, %f39, %f103;\n" " ld.shared.f32 %f41, [%r54+2048];\n" " fma.rn.f32 %f42, %f38, %f41, %f102;\n" " add.s64 %rd42, %rd68, 36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd42];\n" " // end inline asm\n" " add.s64 %rd43, %rd41, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r47, [%rd43];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r47;\n" " cvt.s16.s8 %rs4, %rs3;\n" " cvt.rn.f32.s16 %f43, %rs4;\n" " mul.rn.f32 %f44, %f34, %f43;\n" " ld.shared.f32 %f45, [%r54+128];\n" " fma.rn.f32 %f46, %f44, %f45, %f40;\n" " ld.shared.f32 %f47, [%r54+2176];\n" " fma.rn.f32 %f48, %f44, %f47, %f42;\n" " add.s64 %rd44, %rd68, 72;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd44];\n" " // end inline asm\n" " add.s64 %rd45, %rd41, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r48, [%rd45];\n" " // end inline asm\n" " cvt.u16.u32 %rs5, %r48;\n" " cvt.s16.s8 %rs6, %rs5;\n" " cvt.rn.f32.s16 %f49, %rs6;\n" " mul.rn.f32 %f50, %f35, %f49;\n" " ld.shared.f32 %f51, [%r54+256];\n" " fma.rn.f32 %f52, %f50, %f51, %f46;\n" " ld.shared.f32 %f53, [%r54+2304];\n" " fma.rn.f32 %f54, %f50, %f53, %f48;\n" " add.s64 %rd46, %rd68, 108;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd46];\n" " // end inline asm\n" " add.s64 %rd47, %rd41, 108;\n" " // begin inline asm\n" " ld.global.nc.s8 %r49, [%rd47];\n" " // end inline asm\n" " cvt.u16.u32 %rs7, %r49;\n" " cvt.s16.s8 %rs8, %rs7;\n" " cvt.rn.f32.s16 %f55, %rs8;\n" " mul.rn.f32 %f56, %f36, %f55;\n" " ld.shared.f32 %f57, [%r54+384];\n" " fma.rn.f32 %f103, %f56, %f57, %f52;\n" " ld.shared.f32 %f58, [%r54+2432];\n" " fma.rn.f32 %f102, %f56, %f58, %f54;\n" " add.s64 %rd68, %rd68, 144;\n" " add.s32 %r90, %r90, 4;\n" " add.s32 %r89, %r89, -4;\n" " setp.ne.s32 %p10, %r89, 0;\n" " @%p10 bra $L__BB13_13;\n" "\n" "$L__BB13_14:\n" " setp.eq.s32 %p11, %r16, 0;\n" " mov.u64 %rd70, %rd68;\n" " @%p11 bra $L__BB13_18;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f59, [%rd68];\n" " // end inline asm\n" " add.s64 %rd49, %rd68, %rd3;\n" " // begin inline asm\n" " ld.global.nc.s8 %r55, [%rd49];\n" " // end inline asm\n" " cvt.u16.u32 %rs9, %r55;\n" " cvt.s16.s8 %rs10, %rs9;\n" " cvt.rn.f32.s16 %f60, %rs10;\n" " mul.rn.f32 %f61, %f59, %f60;\n" " shl.b32 %r56, %r90, 5;\n" " or.b32 %r57, %r56, %r5;\n" " shl.b32 %r58, %r57, 2;\n" " add.s32 %r23, %r38, %r58;\n" " ld.shared.f32 %f62, [%r23];\n" " fma.rn.f32 %f103, %f61, %f62, %f103;\n" " ld.shared.f32 %f63, [%r23+2048];\n" " fma.rn.f32 %f102, %f61, %f63, %f102;\n" " add.s64 %rd70, %rd68, 36;\n" " setp.eq.s32 %p12, %r16, 1;\n" " @%p12 bra $L__BB13_18;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f64, [%rd70];\n" " // end inline asm\n" " add.s64 %rd51, %rd49, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r60, [%rd51];\n" " // end inline asm\n" " cvt.u16.u32 %rs11, %r60;\n" " cvt.s16.s8 %rs12, %rs11;\n" " cvt.rn.f32.s16 %f65, %rs12;\n" " mul.rn.f32 %f66, %f64, %f65;\n" " ld.shared.f32 %f67, [%r23+128];\n" " fma.rn.f32 %f103, %f66, %f67, %f103;\n" " ld.shared.f32 %f68, [%r23+2176];\n" " fma.rn.f32 %f102, %f66, %f68, %f102;\n" " add.s64 %rd70, %rd68, 72;\n" " setp.eq.s32 %p13, %r16, 2;\n" " @%p13 bra $L__BB13_18;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f69, [%rd70];\n" " // end inline asm\n" " add.s64 %rd53, %rd49, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r61, [%rd53];\n" " // end inline asm\n" " cvt.u16.u32 %rs13, %r61;\n" " cvt.s16.s8 %rs14, %rs13;\n" " cvt.rn.f32.s16 %f70, %rs14;\n" " mul.rn.f32 %f71, %f69, %f70;\n" " ld.shared.f32 %f72, [%r23+256];\n" " fma.rn.f32 %f103, %f71, %f72, %f103;\n" " ld.shared.f32 %f73, [%r23+2304];\n" " fma.rn.f32 %f102, %f71, %f73, %f102;\n" " add.s64 %rd70, %rd68, 108;\n" "\n" "$L__BB13_18:\n" " add.s32 %r85, %r85, 512;\n" " setp.lt.s32 %p14, %r85, %r26;\n" " @%p14 bra $L__BB13_2;\n" "\n" "$L__BB13_19:\n" " cvt.u32.u64 %r72, %rd2;\n" " setp.lt.s32 %p15, %r72, %r25;\n" " setp.eq.s32 %p16, %r5, 0;\n" " and.pred %p1, %p16, %p15;\n" " not.pred %p17, %p1;\n" " cvt.s64.s32 %rd23, %r25;\n" " mov.u32 %r62, 16;\n" " mov.u32 %r71, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f74, %f103, %r62, %r71;\n" " // end inline asm\n" " add.f32 %f77, %f103, %f74;\n" " mov.u32 %r64, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f76, %f77, %r64, %r71;\n" " // end inline asm\n" " add.f32 %f79, %f77, %f76;\n" " mov.u32 %r66, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f78, %f79, %r66, %r71;\n" " // end inline asm\n" " add.f32 %f81, %f79, %f78;\n" " mov.u32 %r68, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f80, %f81, %r68, %r71;\n" " // end inline asm\n" " add.f32 %f83, %f81, %f80;\n" " mov.u32 %r70, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f82, %f83, %r70, %r71;\n" " // end inline asm\n" " add.f32 %f23, %f83, %f82;\n" " setp.lt.s32 %p18, %r2, 1;\n" " or.pred %p19, %p17, %p18;\n" " @%p19 bra $L__BB13_21;\n" "\n" " cvt.s64.s32 %rd54, %r1;\n" " mul.lo.s64 %rd55, %rd54, %rd23;\n" " add.s64 %rd56, %rd55, %rd2;\n" " shl.b64 %rd57, %rd56, 2;\n" " add.s64 %rd58, %rd1, %rd57;\n" " st.global.f32 [%rd58], %f23;\n" "\n" "$L__BB13_21:\n" " // begin inline asm\n" " shfl.down.b32 %f84, %f102, %r62, %r71;\n" " // end inline asm\n" " add.f32 %f87, %f102, %f84;\n" " // begin inline asm\n" " shfl.down.b32 %f86, %f87, %r64, %r71;\n" " // end inline asm\n" " add.f32 %f89, %f87, %f86;\n" " // begin inline asm\n" " shfl.down.b32 %f88, %f89, %r66, %r71;\n" " // end inline asm\n" " add.f32 %f91, %f89, %f88;\n" " // begin inline asm\n" " shfl.down.b32 %f90, %f91, %r68, %r71;\n" " // end inline asm\n" " add.f32 %f93, %f91, %f90;\n" " // begin inline asm\n" " shfl.down.b32 %f92, %f93, %r70, %r71;\n" " // end inline asm\n" " add.f32 %f24, %f93, %f92;\n" " setp.lt.s32 %p20, %r3, 2;\n" " or.pred %p22, %p17, %p20;\n" " @%p22 bra $L__BB13_23;\n" "\n" " add.s32 %r83, %r1, 1;\n" " cvt.s64.s32 %rd59, %r83;\n" " mul.lo.s64 %rd60, %rd59, %rd23;\n" " add.s64 %rd61, %rd60, %rd2;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd63, %rd1, %rd62;\n" " st.global.f32 [%rd63], %f24;\n" "\n" "$L__BB13_23:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q80_nx\n" ".visible .entry fc_matvec_q80_nx(\n" " .param .u64 fc_matvec_q80_nx_param_0,\n" " .param .u64 fc_matvec_q80_nx_param_1,\n" " .param .u64 fc_matvec_q80_nx_param_2,\n" " .param .u32 fc_matvec_q80_nx_param_3,\n" " .param .u32 fc_matvec_q80_nx_param_4,\n" " .param .u32 fc_matvec_q80_nx_param_5,\n" " .param .u32 fc_matvec_q80_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<31>;\n" " .reg .b16 %rs<15>;\n" " .reg .f32 %f<190>;\n" " .reg .b32 %r<117>;\n" " .reg .b64 %rd<86>;\n" "\n" "\n" " ld.param.u64 %rd26, [fc_matvec_q80_nx_param_0];\n" " ld.param.u64 %rd27, [fc_matvec_q80_nx_param_1];\n" " ld.param.u64 %rd28, [fc_matvec_q80_nx_param_2];\n" " ld.param.u32 %r25, [fc_matvec_q80_nx_param_3];\n" " ld.param.u32 %r26, [fc_matvec_q80_nx_param_4];\n" " ld.param.u32 %r27, [fc_matvec_q80_nx_param_5];\n" " ld.param.u32 %r28, [fc_matvec_q80_nx_param_6];\n" " mov.u32 %r29, %ctaid.y;\n" " shl.b32 %r1, %r29, 2;\n" " sub.s32 %r2, %r28, %r1;\n" " mov.u32 %r30, %ctaid.x;\n" " shl.b32 %r31, %r30, 3;\n" " mov.u32 %r3, %tid.x;\n" " shr.u32 %r32, %r3, 5;\n" " add.s32 %r33, %r31, %r32;\n" " cvt.s64.s32 %rd1, %r33;\n" " min.s32 %r4, %r2, 4;\n" " cvta.to.global.u64 %rd2, %rd28;\n" " and.b32 %r5, %r3, 31;\n" " setp.lt.s32 %p2, %r26, 1;\n" " mov.f32 %f182, 0f00000000;\n" " mov.f32 %f183, %f182;\n" " mov.f32 %f184, %f182;\n" " mov.f32 %f185, %f182;\n" " @%p2 bra $L__BB14_23;\n" "\n" " add.s32 %r35, %r5, 4;\n" " cvt.u64.u32 %rd3, %r35;\n" " add.s32 %r36, %r1, 1;\n" " add.s32 %r37, %r1, 2;\n" " add.s32 %r38, %r1, 3;\n" " mul.wide.s32 %rd29, %r26, %r1;\n" " shl.b64 %rd30, %rd29, 2;\n" " mul.wide.s32 %rd31, %r3, 4;\n" " add.s64 %rd4, %rd30, %rd31;\n" " mul.wide.s32 %rd32, %r26, %r36;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd5, %rd33, %rd31;\n" " mul.wide.s32 %rd34, %r26, %r37;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd6, %rd35, %rd31;\n" " shl.b32 %r39, %r3, 2;\n" " mov.u32 %r40, fc_kq_shared;\n" " add.s32 %r41, %r40, %r39;\n" " add.s32 %r6, %r41, 4096;\n" " mul.wide.s32 %rd36, %r26, %r38;\n" " shl.b64 %rd37, %rd36, 2;\n" " add.s64 %rd7, %rd37, %rd31;\n" " cvt.s64.s32 %rd38, %r27;\n" " mul.lo.s64 %rd39, %rd38, %rd1;\n" " add.s64 %rd85, %rd26, %rd39;\n" " mov.u32 %r111, 0;\n" " mov.f32 %f182, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r4, 2;\n" " setp.lt.s32 %p6, %r4, 3;\n" " setp.lt.s32 %p7, %r4, 4;\n" " mov.f32 %f183, %f182;\n" " mov.f32 %f184, %f182;\n" " mov.f32 %f185, %f182;\n" "\n" "$L__BB14_2:\n" " sub.s32 %r42, %r26, %r111;\n" " min.s32 %r8, %r42, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r3, %r8;\n" " @%p3 bra $L__BB14_13;\n" "\n" " mul.wide.s32 %rd40, %r111, 4;\n" " add.s64 %rd10, %rd4, %rd40;\n" " add.s64 %rd11, %rd5, %rd40;\n" " add.s64 %rd12, %rd6, %rd40;\n" " add.s64 %rd13, %rd7, %rd40;\n" " mov.u32 %r112, %r6;\n" " mov.u64 %rd81, %rd27;\n" " mov.u32 %r113, %r3;\n" "\n" "$L__BB14_4:\n" " .pragma \"nounroll\";\n" " mov.f32 %f173, 0f00000000;\n" " mov.f32 %f170, %f173;\n" " @%p4 bra $L__BB14_6;\n" "\n" " add.s64 %rd41, %rd81, %rd10;\n" " // begin inline asm\n" " ld.global.nc.f32 %f170, [%rd41];\n" " // end inline asm\n" "\n" "$L__BB14_6:\n" " st.shared.f32 [%r112+-4096], %f170;\n" " mov.f32 %f171, %f173;\n" " @%p5 bra $L__BB14_8;\n" "\n" " add.s64 %rd42, %rd81, %rd11;\n" " // begin inline asm\n" " ld.global.nc.f32 %f171, [%rd42];\n" " // end inline asm\n" "\n" "$L__BB14_8:\n" " add.s32 %r108, %r112, -4096;\n" " st.shared.f32 [%r108+2048], %f171;\n" " mov.f32 %f172, %f173;\n" " @%p6 bra $L__BB14_10;\n" "\n" " add.s64 %rd43, %rd81, %rd12;\n" " // begin inline asm\n" " ld.global.nc.f32 %f172, [%rd43];\n" " // end inline asm\n" "\n" "$L__BB14_10:\n" " add.s32 %r109, %r112, -4096;\n" " st.shared.f32 [%r109+4096], %f172;\n" " @%p7 bra $L__BB14_12;\n" "\n" " add.s64 %rd44, %rd81, %rd13;\n" " // begin inline asm\n" " ld.global.nc.f32 %f173, [%rd44];\n" " // end inline asm\n" "\n" "$L__BB14_12:\n" " add.s32 %r110, %r112, -4096;\n" " st.shared.f32 [%r110+6144], %f173;\n" " add.s64 %rd81, %rd81, 1024;\n" " add.s32 %r112, %r112, 1024;\n" " add.s32 %r113, %r113, 256;\n" " setp.lt.s32 %p8, %r113, %r8;\n" " @%p8 bra $L__BB14_4;\n" "\n" "$L__BB14_13:\n" " cvt.u32.u64 %r43, %rd1;\n" " setp.ge.s32 %p9, %r43, %r25;\n" " bar.sync 0;\n" " @%p9 bra $L__BB14_22;\n" "\n" " setp.lt.s32 %p10, %r8, 32;\n" " @%p10 bra $L__BB14_22;\n" "\n" " shr.s32 %r45, %r8, 5;\n" " max.s32 %r15, %r45, 1;\n" " add.s32 %r46, %r15, -1;\n" " and.b32 %r16, %r15, 3;\n" " setp.lt.u32 %p11, %r46, 3;\n" " mov.u32 %r116, 0;\n" " mov.u64 %rd83, %rd85;\n" " @%p11 bra $L__BB14_18;\n" "\n" " sub.s32 %r115, %r15, %r16;\n" " mov.u32 %r116, 0;\n" " mov.u64 %rd83, %rd85;\n" "\n" "$L__BB14_17:\n" " // begin inline asm\n" " ld.global.nc.f32 %f65, [%rd83];\n" " // end inline asm\n" " add.s64 %rd47, %rd83, %rd3;\n" " // begin inline asm\n" " ld.global.nc.s8 %r48, [%rd47];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r48;\n" " cvt.s16.s8 %rs2, %rs1;\n" " cvt.rn.f32.s16 %f69, %rs2;\n" " mul.rn.f32 %f70, %f65, %f69;\n" " shl.b32 %r52, %r116, 5;\n" " or.b32 %r53, %r52, %r5;\n" " shl.b32 %r54, %r53, 2;\n" " add.s32 %r56, %r40, %r54;\n" " ld.shared.f32 %f71, [%r56];\n" " fma.rn.f32 %f72, %f70, %f71, %f185;\n" " ld.shared.f32 %f73, [%r56+2048];\n" " fma.rn.f32 %f74, %f70, %f73, %f184;\n" " ld.shared.f32 %f75, [%r56+4096];\n" " fma.rn.f32 %f76, %f70, %f75, %f183;\n" " ld.shared.f32 %f77, [%r56+6144];\n" " fma.rn.f32 %f78, %f70, %f77, %f182;\n" " add.s64 %rd48, %rd83, 36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f66, [%rd48];\n" " // end inline asm\n" " add.s64 %rd49, %rd47, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r49, [%rd49];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r49;\n" " cvt.s16.s8 %rs4, %rs3;\n" " cvt.rn.f32.s16 %f79, %rs4;\n" " mul.rn.f32 %f80, %f66, %f79;\n" " ld.shared.f32 %f81, [%r56+128];\n" " fma.rn.f32 %f82, %f80, %f81, %f72;\n" " ld.shared.f32 %f83, [%r56+2176];\n" " fma.rn.f32 %f84, %f80, %f83, %f74;\n" " ld.shared.f32 %f85, [%r56+4224];\n" " fma.rn.f32 %f86, %f80, %f85, %f76;\n" " ld.shared.f32 %f87, [%r56+6272];\n" " fma.rn.f32 %f88, %f80, %f87, %f78;\n" " add.s64 %rd50, %rd83, 72;\n" " // begin inline asm\n" " ld.global.nc.f32 %f67, [%rd50];\n" " // end inline asm\n" " add.s64 %rd51, %rd47, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r50, [%rd51];\n" " // end inline asm\n" " cvt.u16.u32 %rs5, %r50;\n" " cvt.s16.s8 %rs6, %rs5;\n" " cvt.rn.f32.s16 %f89, %rs6;\n" " mul.rn.f32 %f90, %f67, %f89;\n" " ld.shared.f32 %f91, [%r56+256];\n" " fma.rn.f32 %f92, %f90, %f91, %f82;\n" " ld.shared.f32 %f93, [%r56+2304];\n" " fma.rn.f32 %f94, %f90, %f93, %f84;\n" " ld.shared.f32 %f95, [%r56+4352];\n" " fma.rn.f32 %f96, %f90, %f95, %f86;\n" " ld.shared.f32 %f97, [%r56+6400];\n" " fma.rn.f32 %f98, %f90, %f97, %f88;\n" " add.s64 %rd52, %rd83, 108;\n" " // begin inline asm\n" " ld.global.nc.f32 %f68, [%rd52];\n" " // end inline asm\n" " add.s64 %rd53, %rd47, 108;\n" " // begin inline asm\n" " ld.global.nc.s8 %r51, [%rd53];\n" " // end inline asm\n" " cvt.u16.u32 %rs7, %r51;\n" " cvt.s16.s8 %rs8, %rs7;\n" " cvt.rn.f32.s16 %f99, %rs8;\n" " mul.rn.f32 %f100, %f68, %f99;\n" " ld.shared.f32 %f101, [%r56+384];\n" " fma.rn.f32 %f185, %f100, %f101, %f92;\n" " ld.shared.f32 %f102, [%r56+2432];\n" " fma.rn.f32 %f184, %f100, %f102, %f94;\n" " ld.shared.f32 %f103, [%r56+4480];\n" " fma.rn.f32 %f183, %f100, %f103, %f96;\n" " ld.shared.f32 %f104, [%r56+6528];\n" " fma.rn.f32 %f182, %f100, %f104, %f98;\n" " add.s64 %rd83, %rd83, 144;\n" " add.s32 %r116, %r116, 4;\n" " add.s32 %r115, %r115, -4;\n" " setp.ne.s32 %p12, %r115, 0;\n" " @%p12 bra $L__BB14_17;\n" "\n" "$L__BB14_18:\n" " setp.eq.s32 %p13, %r16, 0;\n" " mov.u64 %rd85, %rd83;\n" " @%p13 bra $L__BB14_22;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f105, [%rd83];\n" " // end inline asm\n" " add.s64 %rd55, %rd83, %rd3;\n" " // begin inline asm\n" " ld.global.nc.s8 %r57, [%rd55];\n" " // end inline asm\n" " cvt.u16.u32 %rs9, %r57;\n" " cvt.s16.s8 %rs10, %rs9;\n" " cvt.rn.f32.s16 %f106, %rs10;\n" " mul.rn.f32 %f107, %f105, %f106;\n" " shl.b32 %r58, %r116, 5;\n" " or.b32 %r59, %r58, %r5;\n" " shl.b32 %r60, %r59, 2;\n" " add.s32 %r23, %r40, %r60;\n" " ld.shared.f32 %f108, [%r23];\n" " fma.rn.f32 %f185, %f107, %f108, %f185;\n" " ld.shared.f32 %f109, [%r23+2048];\n" " fma.rn.f32 %f184, %f107, %f109, %f184;\n" " ld.shared.f32 %f110, [%r23+4096];\n" " fma.rn.f32 %f183, %f107, %f110, %f183;\n" " ld.shared.f32 %f111, [%r23+6144];\n" " fma.rn.f32 %f182, %f107, %f111, %f182;\n" " add.s64 %rd85, %rd83, 36;\n" " setp.eq.s32 %p14, %r16, 1;\n" " @%p14 bra $L__BB14_22;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f112, [%rd85];\n" " // end inline asm\n" " add.s64 %rd57, %rd55, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r62, [%rd57];\n" " // end inline asm\n" " cvt.u16.u32 %rs11, %r62;\n" " cvt.s16.s8 %rs12, %rs11;\n" " cvt.rn.f32.s16 %f113, %rs12;\n" " mul.rn.f32 %f114, %f112, %f113;\n" " ld.shared.f32 %f115, [%r23+128];\n" " fma.rn.f32 %f185, %f114, %f115, %f185;\n" " ld.shared.f32 %f116, [%r23+2176];\n" " fma.rn.f32 %f184, %f114, %f116, %f184;\n" " ld.shared.f32 %f117, [%r23+4224];\n" " fma.rn.f32 %f183, %f114, %f117, %f183;\n" " ld.shared.f32 %f118, [%r23+6272];\n" " fma.rn.f32 %f182, %f114, %f118, %f182;\n" " add.s64 %rd85, %rd83, 72;\n" " setp.eq.s32 %p15, %r16, 2;\n" " @%p15 bra $L__BB14_22;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f119, [%rd85];\n" " // end inline asm\n" " add.s64 %rd59, %rd55, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r63, [%rd59];\n" " // end inline asm\n" " cvt.u16.u32 %rs13, %r63;\n" " cvt.s16.s8 %rs14, %rs13;\n" " cvt.rn.f32.s16 %f120, %rs14;\n" " mul.rn.f32 %f121, %f119, %f120;\n" " ld.shared.f32 %f122, [%r23+256];\n" " fma.rn.f32 %f185, %f121, %f122, %f185;\n" " ld.shared.f32 %f123, [%r23+2304];\n" " fma.rn.f32 %f184, %f121, %f123, %f184;\n" " ld.shared.f32 %f124, [%r23+4352];\n" " fma.rn.f32 %f183, %f121, %f124, %f183;\n" " ld.shared.f32 %f125, [%r23+6400];\n" " fma.rn.f32 %f182, %f121, %f125, %f182;\n" " add.s64 %rd85, %rd83, 108;\n" "\n" "$L__BB14_22:\n" " add.s32 %r111, %r111, 512;\n" " setp.lt.s32 %p16, %r111, %r26;\n" " @%p16 bra $L__BB14_2;\n" "\n" "$L__BB14_23:\n" " cvt.u32.u64 %r74, %rd1;\n" " setp.lt.s32 %p17, %r74, %r25;\n" " setp.eq.s32 %p18, %r5, 0;\n" " and.pred %p1, %p18, %p17;\n" " not.pred %p19, %p1;\n" " cvt.s64.s32 %rd25, %r25;\n" " mov.u32 %r64, 16;\n" " mov.u32 %r73, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f126, %f185, %r64, %r73;\n" " // end inline asm\n" " add.f32 %f129, %f185, %f126;\n" " mov.u32 %r66, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f128, %f129, %r66, %r73;\n" " // end inline asm\n" " add.f32 %f131, %f129, %f128;\n" " mov.u32 %r68, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f130, %f131, %r68, %r73;\n" " // end inline asm\n" " add.f32 %f133, %f131, %f130;\n" " mov.u32 %r70, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f132, %f133, %r70, %r73;\n" " // end inline asm\n" " add.f32 %f135, %f133, %f132;\n" " mov.u32 %r72, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f134, %f135, %r72, %r73;\n" " // end inline asm\n" " add.f32 %f45, %f135, %f134;\n" " setp.lt.s32 %p20, %r2, 1;\n" " or.pred %p21, %p19, %p20;\n" " @%p21 bra $L__BB14_25;\n" "\n" " cvt.s64.s32 %rd60, %r1;\n" " mul.lo.s64 %rd61, %rd60, %rd25;\n" " add.s64 %rd62, %rd61, %rd1;\n" " shl.b64 %rd63, %rd62, 2;\n" " add.s64 %rd64, %rd2, %rd63;\n" " st.global.f32 [%rd64], %f45;\n" "\n" "$L__BB14_25:\n" " // begin inline asm\n" " shfl.down.b32 %f136, %f184, %r64, %r73;\n" " // end inline asm\n" " add.f32 %f139, %f184, %f136;\n" " // begin inline asm\n" " shfl.down.b32 %f138, %f139, %r66, %r73;\n" " // end inline asm\n" " add.f32 %f141, %f139, %f138;\n" " // begin inline asm\n" " shfl.down.b32 %f140, %f141, %r68, %r73;\n" " // end inline asm\n" " add.f32 %f143, %f141, %f140;\n" " // begin inline asm\n" " shfl.down.b32 %f142, %f143, %r70, %r73;\n" " // end inline asm\n" " add.f32 %f145, %f143, %f142;\n" " // begin inline asm\n" " shfl.down.b32 %f144, %f145, %r72, %r73;\n" " // end inline asm\n" " add.f32 %f46, %f145, %f144;\n" " setp.lt.s32 %p22, %r4, 2;\n" " or.pred %p24, %p19, %p22;\n" " @%p24 bra $L__BB14_27;\n" "\n" " add.s32 %r85, %r1, 1;\n" " cvt.s64.s32 %rd65, %r85;\n" " mul.lo.s64 %rd66, %rd65, %rd25;\n" " add.s64 %rd67, %rd66, %rd1;\n" " shl.b64 %rd68, %rd67, 2;\n" " add.s64 %rd69, %rd2, %rd68;\n" " st.global.f32 [%rd69], %f46;\n" "\n" "$L__BB14_27:\n" " // begin inline asm\n" " shfl.down.b32 %f146, %f183, %r64, %r73;\n" " // end inline asm\n" " add.f32 %f149, %f183, %f146;\n" " // begin inline asm\n" " shfl.down.b32 %f148, %f149, %r66, %r73;\n" " // end inline asm\n" " add.f32 %f151, %f149, %f148;\n" " // begin inline asm\n" " shfl.down.b32 %f150, %f151, %r68, %r73;\n" " // end inline asm\n" " add.f32 %f153, %f151, %f150;\n" " // begin inline asm\n" " shfl.down.b32 %f152, %f153, %r70, %r73;\n" " // end inline asm\n" " add.f32 %f155, %f153, %f152;\n" " // begin inline asm\n" " shfl.down.b32 %f154, %f155, %r72, %r73;\n" " // end inline asm\n" " add.f32 %f47, %f155, %f154;\n" " setp.lt.s32 %p25, %r4, 3;\n" " or.pred %p27, %p19, %p25;\n" " @%p27 bra $L__BB14_29;\n" "\n" " add.s32 %r96, %r1, 2;\n" " cvt.s64.s32 %rd70, %r96;\n" " mul.lo.s64 %rd71, %rd70, %rd25;\n" " add.s64 %rd72, %rd71, %rd1;\n" " shl.b64 %rd73, %rd72, 2;\n" " add.s64 %rd74, %rd2, %rd73;\n" " st.global.f32 [%rd74], %f47;\n" "\n" "$L__BB14_29:\n" " // begin inline asm\n" " shfl.down.b32 %f156, %f182, %r64, %r73;\n" " // end inline asm\n" " add.f32 %f159, %f182, %f156;\n" " // begin inline asm\n" " shfl.down.b32 %f158, %f159, %r66, %r73;\n" " // end inline asm\n" " add.f32 %f161, %f159, %f158;\n" " // begin inline asm\n" " shfl.down.b32 %f160, %f161, %r68, %r73;\n" " // end inline asm\n" " add.f32 %f163, %f161, %f160;\n" " // begin inline asm\n" " shfl.down.b32 %f162, %f163, %r70, %r73;\n" " // end inline asm\n" " add.f32 %f165, %f163, %f162;\n" " // begin inline asm\n" " shfl.down.b32 %f164, %f165, %r72, %r73;\n" " // end inline asm\n" " add.f32 %f48, %f165, %f164;\n" " setp.lt.s32 %p28, %r4, 4;\n" " or.pred %p30, %p19, %p28;\n" " @%p30 bra $L__BB14_31;\n" "\n" " add.s32 %r107, %r1, 3;\n" " cvt.s64.s32 %rd75, %r107;\n" " mul.lo.s64 %rd76, %rd75, %rd25;\n" " add.s64 %rd77, %rd76, %rd1;\n" " shl.b64 %rd78, %rd77, 2;\n" " add.s64 %rd79, %rd2, %rd78;\n" " st.global.f32 [%rd79], %f48;\n" "\n" "$L__BB14_31:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q80_nx8\n" ".visible .entry fc_matvec_q80_nx8(\n" " .param .u64 fc_matvec_q80_nx8_param_0,\n" " .param .u64 fc_matvec_q80_nx8_param_1,\n" " .param .u64 fc_matvec_q80_nx8_param_2,\n" " .param .u32 fc_matvec_q80_nx8_param_3,\n" " .param .u32 fc_matvec_q80_nx8_param_4,\n" " .param .u32 fc_matvec_q80_nx8_param_5,\n" " .param .u32 fc_matvec_q80_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<47>;\n" " .reg .b16 %rs<15>;\n" " .reg .f32 %f<365>;\n" " .reg .b32 %r<183>;\n" " .reg .b64 %rd<144>;\n" "\n" "\n" " ld.param.u64 %rd34, [fc_matvec_q80_nx8_param_0];\n" " ld.param.u64 %rd35, [fc_matvec_q80_nx8_param_1];\n" " ld.param.u64 %rd36, [fc_matvec_q80_nx8_param_2];\n" " ld.param.u32 %r25, [fc_matvec_q80_nx8_param_3];\n" " ld.param.u32 %r26, [fc_matvec_q80_nx8_param_4];\n" " ld.param.u32 %r27, [fc_matvec_q80_nx8_param_5];\n" " ld.param.u32 %r28, [fc_matvec_q80_nx8_param_6];\n" " mov.u32 %r29, %ctaid.y;\n" " shl.b32 %r1, %r29, 3;\n" " sub.s32 %r2, %r28, %r1;\n" " mov.u32 %r30, %ctaid.x;\n" " shl.b32 %r31, %r30, 3;\n" " mov.u32 %r3, %tid.x;\n" " shr.u32 %r32, %r3, 5;\n" " add.s32 %r33, %r31, %r32;\n" " cvt.s64.s32 %rd1, %r33;\n" " min.s32 %r4, %r2, 8;\n" " cvta.to.global.u64 %rd2, %rd36;\n" " and.b32 %r5, %r3, 31;\n" " setp.lt.s32 %p2, %r26, 1;\n" " mov.f32 %f349, 0f00000000;\n" " mov.f32 %f350, %f349;\n" " mov.f32 %f351, %f349;\n" " mov.f32 %f352, %f349;\n" " mov.f32 %f353, %f349;\n" " mov.f32 %f354, %f349;\n" " mov.f32 %f355, %f349;\n" " mov.f32 %f356, %f349;\n" " @%p2 bra $L__BB15_31;\n" "\n" " add.s32 %r35, %r5, 4;\n" " cvt.u64.u32 %rd3, %r35;\n" " add.s32 %r36, %r1, 1;\n" " add.s32 %r37, %r1, 2;\n" " add.s32 %r38, %r1, 3;\n" " add.s32 %r39, %r1, 4;\n" " add.s32 %r40, %r1, 5;\n" " add.s32 %r41, %r1, 6;\n" " add.s32 %r42, %r1, 7;\n" " mul.wide.s32 %rd37, %r26, %r1;\n" " shl.b64 %rd38, %rd37, 2;\n" " mul.wide.s32 %rd39, %r3, 4;\n" " add.s64 %rd4, %rd38, %rd39;\n" " mul.wide.s32 %rd40, %r26, %r36;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd5, %rd41, %rd39;\n" " mul.wide.s32 %rd42, %r26, %r37;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd6, %rd43, %rd39;\n" " mul.wide.s32 %rd44, %r26, %r38;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd7, %rd45, %rd39;\n" " mul.wide.s32 %rd46, %r26, %r39;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd8, %rd47, %rd39;\n" " mul.wide.s32 %rd48, %r26, %r40;\n" " shl.b64 %rd49, %rd48, 2;\n" " add.s64 %rd9, %rd49, %rd39;\n" " mul.wide.s32 %rd50, %r26, %r41;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd10, %rd51, %rd39;\n" " shl.b32 %r43, %r3, 2;\n" " mov.u32 %r44, fc_kq_shared;\n" " add.s32 %r45, %r44, %r43;\n" " add.s32 %r6, %r45, 8192;\n" " mul.wide.s32 %rd52, %r26, %r42;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd11, %rd53, %rd39;\n" " cvt.s64.s32 %rd54, %r27;\n" " mul.lo.s64 %rd55, %rd54, %rd1;\n" " add.s64 %rd143, %rd34, %rd55;\n" " mov.u32 %r177, 0;\n" " mov.f32 %f349, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r4, 2;\n" " setp.lt.s32 %p6, %r4, 3;\n" " setp.lt.s32 %p7, %r4, 4;\n" " mov.f32 %f350, %f349;\n" " mov.f32 %f351, %f349;\n" " mov.f32 %f352, %f349;\n" " mov.f32 %f353, %f349;\n" " mov.f32 %f354, %f349;\n" " mov.f32 %f355, %f349;\n" " mov.f32 %f356, %f349;\n" "\n" "$L__BB15_2:\n" " sub.s32 %r46, %r26, %r177;\n" " min.s32 %r8, %r46, 512;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r3, %r8;\n" " @%p3 bra $L__BB15_21;\n" "\n" " mov.u32 %r178, %r6;\n" " mov.u64 %rd139, %rd35;\n" " mov.u32 %r179, %r3;\n" "\n" "$L__BB15_4:\n" " .pragma \"nounroll\";\n" " mov.f32 %f325, 0f00000000;\n" " @%p4 bra $L__BB15_6;\n" "\n" " mul.wide.s32 %rd121, %r177, 4;\n" " add.s64 %rd120, %rd4, %rd121;\n" " add.s64 %rd57, %rd139, %rd120;\n" " // begin inline asm\n" " ld.global.nc.f32 %f325, [%rd57];\n" " // end inline asm\n" "\n" "$L__BB15_6:\n" " mov.f32 %f326, 0f00000000;\n" " st.shared.f32 [%r178+-8192], %f325;\n" " @%p5 bra $L__BB15_8;\n" "\n" " mul.wide.s32 %rd131, %r177, 4;\n" " add.s64 %rd130, %rd5, %rd131;\n" " add.s64 %rd58, %rd139, %rd130;\n" " // begin inline asm\n" " ld.global.nc.f32 %f326, [%rd58];\n" " // end inline asm\n" "\n" "$L__BB15_8:\n" " mov.f32 %f327, 0f00000000;\n" " add.s32 %r156, %r178, -8192;\n" " st.shared.f32 [%r156+2048], %f326;\n" " @%p6 bra $L__BB15_10;\n" "\n" " mul.wide.s32 %rd129, %r177, 4;\n" " add.s64 %rd128, %rd6, %rd129;\n" " add.s64 %rd59, %rd139, %rd128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f327, [%rd59];\n" " // end inline asm\n" "\n" "$L__BB15_10:\n" " mov.f32 %f328, 0f00000000;\n" " add.s32 %r157, %r178, -8192;\n" " st.shared.f32 [%r157+4096], %f327;\n" " @%p7 bra $L__BB15_12;\n" "\n" " mul.wide.s32 %rd127, %r177, 4;\n" " add.s64 %rd126, %rd7, %rd127;\n" " add.s64 %rd60, %rd139, %rd126;\n" " // begin inline asm\n" " ld.global.nc.f32 %f328, [%rd60];\n" " // end inline asm\n" "\n" "$L__BB15_12:\n" " mov.f32 %f329, 0f00000000;\n" " add.s32 %r158, %r178, -8192;\n" " st.shared.f32 [%r158+6144], %f328;\n" " setp.lt.s32 %p8, %r4, 5;\n" " @%p8 bra $L__BB15_14;\n" "\n" " mul.wide.s32 %rd125, %r177, 4;\n" " add.s64 %rd124, %rd8, %rd125;\n" " add.s64 %rd61, %rd139, %rd124;\n" " // begin inline asm\n" " ld.global.nc.f32 %f329, [%rd61];\n" " // end inline asm\n" "\n" "$L__BB15_14:\n" " mov.f32 %f330, 0f00000000;\n" " add.s32 %r159, %r178, -8192;\n" " st.shared.f32 [%r159+8192], %f329;\n" " setp.lt.s32 %p9, %r4, 6;\n" " @%p9 bra $L__BB15_16;\n" "\n" " mul.wide.s32 %rd123, %r177, 4;\n" " add.s64 %rd122, %rd9, %rd123;\n" " add.s64 %rd62, %rd139, %rd122;\n" " // begin inline asm\n" " ld.global.nc.f32 %f330, [%rd62];\n" " // end inline asm\n" "\n" "$L__BB15_16:\n" " mov.f32 %f331, 0f00000000;\n" " add.s32 %r160, %r178, -8192;\n" " st.shared.f32 [%r160+10240], %f330;\n" " setp.lt.s32 %p10, %r4, 7;\n" " @%p10 bra $L__BB15_18;\n" "\n" " mul.wide.s32 %rd135, %r177, 4;\n" " add.s64 %rd134, %rd10, %rd135;\n" " add.s64 %rd63, %rd139, %rd134;\n" " // begin inline asm\n" " ld.global.nc.f32 %f331, [%rd63];\n" " // end inline asm\n" "\n" "$L__BB15_18:\n" " mov.f32 %f332, 0f00000000;\n" " add.s32 %r161, %r178, -8192;\n" " st.shared.f32 [%r161+12288], %f331;\n" " setp.lt.s32 %p11, %r4, 8;\n" " @%p11 bra $L__BB15_20;\n" "\n" " mul.wide.s32 %rd133, %r177, 4;\n" " add.s64 %rd132, %rd11, %rd133;\n" " add.s64 %rd64, %rd139, %rd132;\n" " // begin inline asm\n" " ld.global.nc.f32 %f332, [%rd64];\n" " // end inline asm\n" "\n" "$L__BB15_20:\n" " sub.s32 %r168, %r26, %r177;\n" " min.s32 %r167, %r168, 512;\n" " add.s32 %r162, %r178, -8192;\n" " st.shared.f32 [%r162+14336], %f332;\n" " add.s64 %rd139, %rd139, 1024;\n" " add.s32 %r178, %r178, 1024;\n" " add.s32 %r179, %r179, 256;\n" " setp.lt.s32 %p12, %r179, %r167;\n" " @%p12 bra $L__BB15_4;\n" "\n" "$L__BB15_21:\n" " cvt.u32.u64 %r47, %rd1;\n" " setp.ge.s32 %p13, %r47, %r25;\n" " bar.sync 0;\n" " @%p13 bra $L__BB15_30;\n" "\n" " sub.s32 %r170, %r26, %r177;\n" " min.s32 %r169, %r170, 512;\n" " setp.lt.s32 %p14, %r169, 32;\n" " @%p14 bra $L__BB15_30;\n" "\n" " sub.s32 %r172, %r26, %r177;\n" " min.s32 %r171, %r172, 512;\n" " shr.s32 %r49, %r171, 5;\n" " max.s32 %r15, %r49, 1;\n" " add.s32 %r50, %r15, -1;\n" " setp.lt.u32 %p15, %r50, 3;\n" " mov.u32 %r182, 0;\n" " mov.u64 %rd141, %rd143;\n" " @%p15 bra $L__BB15_26;\n" "\n" " and.b32 %r173, %r15, 3;\n" " sub.s32 %r181, %r15, %r173;\n" " mov.u32 %r182, 0;\n" " mov.u64 %rd141, %rd143;\n" "\n" "$L__BB15_25:\n" " // begin inline asm\n" " ld.global.nc.f32 %f129, [%rd141];\n" " // end inline asm\n" " add.s64 %rd67, %rd141, %rd3;\n" " // begin inline asm\n" " ld.global.nc.s8 %r52, [%rd67];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r52;\n" " cvt.s16.s8 %rs2, %rs1;\n" " cvt.rn.f32.s16 %f133, %rs2;\n" " mul.rn.f32 %f134, %f129, %f133;\n" " shl.b32 %r56, %r182, 5;\n" " or.b32 %r57, %r56, %r5;\n" " shl.b32 %r58, %r57, 2;\n" " add.s32 %r60, %r44, %r58;\n" " ld.shared.f32 %f135, [%r60];\n" " fma.rn.f32 %f136, %f134, %f135, %f356;\n" " ld.shared.f32 %f137, [%r60+2048];\n" " fma.rn.f32 %f138, %f134, %f137, %f355;\n" " ld.shared.f32 %f139, [%r60+4096];\n" " fma.rn.f32 %f140, %f134, %f139, %f354;\n" " ld.shared.f32 %f141, [%r60+6144];\n" " fma.rn.f32 %f142, %f134, %f141, %f353;\n" " ld.shared.f32 %f143, [%r60+8192];\n" " fma.rn.f32 %f144, %f134, %f143, %f352;\n" " ld.shared.f32 %f145, [%r60+10240];\n" " fma.rn.f32 %f146, %f134, %f145, %f351;\n" " ld.shared.f32 %f147, [%r60+12288];\n" " fma.rn.f32 %f148, %f134, %f147, %f350;\n" " ld.shared.f32 %f149, [%r60+14336];\n" " fma.rn.f32 %f150, %f134, %f149, %f349;\n" " add.s64 %rd68, %rd141, 36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f130, [%rd68];\n" " // end inline asm\n" " add.s64 %rd69, %rd67, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r53, [%rd69];\n" " // end inline asm\n" " cvt.u16.u32 %rs3, %r53;\n" " cvt.s16.s8 %rs4, %rs3;\n" " cvt.rn.f32.s16 %f151, %rs4;\n" " mul.rn.f32 %f152, %f130, %f151;\n" " ld.shared.f32 %f153, [%r60+128];\n" " fma.rn.f32 %f154, %f152, %f153, %f136;\n" " ld.shared.f32 %f155, [%r60+2176];\n" " fma.rn.f32 %f156, %f152, %f155, %f138;\n" " ld.shared.f32 %f157, [%r60+4224];\n" " fma.rn.f32 %f158, %f152, %f157, %f140;\n" " ld.shared.f32 %f159, [%r60+6272];\n" " fma.rn.f32 %f160, %f152, %f159, %f142;\n" " ld.shared.f32 %f161, [%r60+8320];\n" " fma.rn.f32 %f162, %f152, %f161, %f144;\n" " ld.shared.f32 %f163, [%r60+10368];\n" " fma.rn.f32 %f164, %f152, %f163, %f146;\n" " ld.shared.f32 %f165, [%r60+12416];\n" " fma.rn.f32 %f166, %f152, %f165, %f148;\n" " ld.shared.f32 %f167, [%r60+14464];\n" " fma.rn.f32 %f168, %f152, %f167, %f150;\n" " add.s64 %rd70, %rd141, 72;\n" " // begin inline asm\n" " ld.global.nc.f32 %f131, [%rd70];\n" " // end inline asm\n" " add.s64 %rd71, %rd67, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r54, [%rd71];\n" " // end inline asm\n" " cvt.u16.u32 %rs5, %r54;\n" " cvt.s16.s8 %rs6, %rs5;\n" " cvt.rn.f32.s16 %f169, %rs6;\n" " mul.rn.f32 %f170, %f131, %f169;\n" " ld.shared.f32 %f171, [%r60+256];\n" " fma.rn.f32 %f172, %f170, %f171, %f154;\n" " ld.shared.f32 %f173, [%r60+2304];\n" " fma.rn.f32 %f174, %f170, %f173, %f156;\n" " ld.shared.f32 %f175, [%r60+4352];\n" " fma.rn.f32 %f176, %f170, %f175, %f158;\n" " ld.shared.f32 %f177, [%r60+6400];\n" " fma.rn.f32 %f178, %f170, %f177, %f160;\n" " ld.shared.f32 %f179, [%r60+8448];\n" " fma.rn.f32 %f180, %f170, %f179, %f162;\n" " ld.shared.f32 %f181, [%r60+10496];\n" " fma.rn.f32 %f182, %f170, %f181, %f164;\n" " ld.shared.f32 %f183, [%r60+12544];\n" " fma.rn.f32 %f184, %f170, %f183, %f166;\n" " ld.shared.f32 %f185, [%r60+14592];\n" " fma.rn.f32 %f186, %f170, %f185, %f168;\n" " add.s64 %rd72, %rd141, 108;\n" " // begin inline asm\n" " ld.global.nc.f32 %f132, [%rd72];\n" " // end inline asm\n" " add.s64 %rd73, %rd67, 108;\n" " // begin inline asm\n" " ld.global.nc.s8 %r55, [%rd73];\n" " // end inline asm\n" " cvt.u16.u32 %rs7, %r55;\n" " cvt.s16.s8 %rs8, %rs7;\n" " cvt.rn.f32.s16 %f187, %rs8;\n" " mul.rn.f32 %f188, %f132, %f187;\n" " ld.shared.f32 %f189, [%r60+384];\n" " fma.rn.f32 %f356, %f188, %f189, %f172;\n" " ld.shared.f32 %f190, [%r60+2432];\n" " fma.rn.f32 %f355, %f188, %f190, %f174;\n" " ld.shared.f32 %f191, [%r60+4480];\n" " fma.rn.f32 %f354, %f188, %f191, %f176;\n" " ld.shared.f32 %f192, [%r60+6528];\n" " fma.rn.f32 %f353, %f188, %f192, %f178;\n" " ld.shared.f32 %f193, [%r60+8576];\n" " fma.rn.f32 %f352, %f188, %f193, %f180;\n" " ld.shared.f32 %f194, [%r60+10624];\n" " fma.rn.f32 %f351, %f188, %f194, %f182;\n" " ld.shared.f32 %f195, [%r60+12672];\n" " fma.rn.f32 %f350, %f188, %f195, %f184;\n" " ld.shared.f32 %f196, [%r60+14720];\n" " fma.rn.f32 %f349, %f188, %f196, %f186;\n" " add.s64 %rd141, %rd141, 144;\n" " add.s32 %r182, %r182, 4;\n" " add.s32 %r181, %r181, -4;\n" " setp.ne.s32 %p16, %r181, 0;\n" " @%p16 bra $L__BB15_25;\n" "\n" "$L__BB15_26:\n" " and.b32 %r174, %r15, 3;\n" " setp.eq.s32 %p17, %r174, 0;\n" " mov.u64 %rd143, %rd141;\n" " @%p17 bra $L__BB15_30;\n" "\n" " and.b32 %r175, %r15, 3;\n" " // begin inline asm\n" " ld.global.nc.f32 %f197, [%rd141];\n" " // end inline asm\n" " add.s64 %rd75, %rd141, %rd3;\n" " // begin inline asm\n" " ld.global.nc.s8 %r61, [%rd75];\n" " // end inline asm\n" " cvt.u16.u32 %rs9, %r61;\n" " cvt.s16.s8 %rs10, %rs9;\n" " cvt.rn.f32.s16 %f198, %rs10;\n" " mul.rn.f32 %f199, %f197, %f198;\n" " shl.b32 %r62, %r182, 5;\n" " or.b32 %r63, %r62, %r5;\n" " shl.b32 %r64, %r63, 2;\n" " add.s32 %r23, %r44, %r64;\n" " ld.shared.f32 %f200, [%r23];\n" " fma.rn.f32 %f356, %f199, %f200, %f356;\n" " ld.shared.f32 %f201, [%r23+2048];\n" " fma.rn.f32 %f355, %f199, %f201, %f355;\n" " ld.shared.f32 %f202, [%r23+4096];\n" " fma.rn.f32 %f354, %f199, %f202, %f354;\n" " ld.shared.f32 %f203, [%r23+6144];\n" " fma.rn.f32 %f353, %f199, %f203, %f353;\n" " ld.shared.f32 %f204, [%r23+8192];\n" " fma.rn.f32 %f352, %f199, %f204, %f352;\n" " ld.shared.f32 %f205, [%r23+10240];\n" " fma.rn.f32 %f351, %f199, %f205, %f351;\n" " ld.shared.f32 %f206, [%r23+12288];\n" " fma.rn.f32 %f350, %f199, %f206, %f350;\n" " ld.shared.f32 %f207, [%r23+14336];\n" " fma.rn.f32 %f349, %f199, %f207, %f349;\n" " add.s64 %rd143, %rd141, 36;\n" " setp.eq.s32 %p18, %r175, 1;\n" " @%p18 bra $L__BB15_30;\n" "\n" " and.b32 %r176, %r15, 3;\n" " add.s64 %rd136, %rd141, %rd3;\n" " // begin inline asm\n" " ld.global.nc.f32 %f208, [%rd143];\n" " // end inline asm\n" " add.s64 %rd77, %rd136, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r66, [%rd77];\n" " // end inline asm\n" " cvt.u16.u32 %rs11, %r66;\n" " cvt.s16.s8 %rs12, %rs11;\n" " cvt.rn.f32.s16 %f209, %rs12;\n" " mul.rn.f32 %f210, %f208, %f209;\n" " ld.shared.f32 %f211, [%r23+128];\n" " fma.rn.f32 %f356, %f210, %f211, %f356;\n" " ld.shared.f32 %f212, [%r23+2176];\n" " fma.rn.f32 %f355, %f210, %f212, %f355;\n" " ld.shared.f32 %f213, [%r23+4224];\n" " fma.rn.f32 %f354, %f210, %f213, %f354;\n" " ld.shared.f32 %f214, [%r23+6272];\n" " fma.rn.f32 %f353, %f210, %f214, %f353;\n" " ld.shared.f32 %f215, [%r23+8320];\n" " fma.rn.f32 %f352, %f210, %f215, %f352;\n" " ld.shared.f32 %f216, [%r23+10368];\n" " fma.rn.f32 %f351, %f210, %f216, %f351;\n" " ld.shared.f32 %f217, [%r23+12416];\n" " fma.rn.f32 %f350, %f210, %f217, %f350;\n" " ld.shared.f32 %f218, [%r23+14464];\n" " fma.rn.f32 %f349, %f210, %f218, %f349;\n" " add.s64 %rd143, %rd141, 72;\n" " setp.eq.s32 %p19, %r176, 2;\n" " @%p19 bra $L__BB15_30;\n" "\n" " add.s64 %rd137, %rd141, %rd3;\n" " // begin inline asm\n" " ld.global.nc.f32 %f219, [%rd143];\n" " // end inline asm\n" " add.s64 %rd79, %rd137, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r67, [%rd79];\n" " // end inline asm\n" " cvt.u16.u32 %rs13, %r67;\n" " cvt.s16.s8 %rs14, %rs13;\n" " cvt.rn.f32.s16 %f220, %rs14;\n" " mul.rn.f32 %f221, %f219, %f220;\n" " ld.shared.f32 %f222, [%r23+256];\n" " fma.rn.f32 %f356, %f221, %f222, %f356;\n" " ld.shared.f32 %f223, [%r23+2304];\n" " fma.rn.f32 %f355, %f221, %f223, %f355;\n" " ld.shared.f32 %f224, [%r23+4352];\n" " fma.rn.f32 %f354, %f221, %f224, %f354;\n" " ld.shared.f32 %f225, [%r23+6400];\n" " fma.rn.f32 %f353, %f221, %f225, %f353;\n" " ld.shared.f32 %f226, [%r23+8448];\n" " fma.rn.f32 %f352, %f221, %f226, %f352;\n" " ld.shared.f32 %f227, [%r23+10496];\n" " fma.rn.f32 %f351, %f221, %f227, %f351;\n" " ld.shared.f32 %f228, [%r23+12544];\n" " fma.rn.f32 %f350, %f221, %f228, %f350;\n" " ld.shared.f32 %f229, [%r23+14592];\n" " fma.rn.f32 %f349, %f221, %f229, %f349;\n" " add.s64 %rd143, %rd141, 108;\n" "\n" "$L__BB15_30:\n" " add.s32 %r177, %r177, 512;\n" " setp.lt.s32 %p20, %r177, %r26;\n" " @%p20 bra $L__BB15_2;\n" "\n" "$L__BB15_31:\n" " mov.u32 %r166, %ctaid.y;\n" " shl.b32 %r165, %r166, 3;\n" " ld.param.u32 %r164, [fc_matvec_q80_nx8_param_6];\n" " sub.s32 %r163, %r164, %r165;\n" " cvt.u32.u64 %r78, %rd1;\n" " setp.lt.s32 %p21, %r78, %r25;\n" " setp.eq.s32 %p22, %r5, 0;\n" " and.pred %p1, %p22, %p21;\n" " not.pred %p23, %p1;\n" " cvt.s64.s32 %rd33, %r25;\n" " mov.u32 %r68, 16;\n" " mov.u32 %r77, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f230, %f356, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f233, %f356, %f230;\n" " mov.u32 %r70, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f232, %f233, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f235, %f233, %f232;\n" " mov.u32 %r72, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f234, %f235, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f237, %f235, %f234;\n" " mov.u32 %r74, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f236, %f237, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f239, %f237, %f236;\n" " mov.u32 %r76, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f238, %f239, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f89, %f239, %f238;\n" " setp.lt.s32 %p24, %r163, 1;\n" " or.pred %p25, %p23, %p24;\n" " @%p25 bra $L__BB15_33;\n" "\n" " cvt.s64.s32 %rd80, %r1;\n" " mul.lo.s64 %rd81, %rd80, %rd33;\n" " add.s64 %rd82, %rd81, %rd1;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd84, %rd2, %rd83;\n" " st.global.f32 [%rd84], %f89;\n" "\n" "$L__BB15_33:\n" " // begin inline asm\n" " shfl.down.b32 %f240, %f355, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f243, %f355, %f240;\n" " // begin inline asm\n" " shfl.down.b32 %f242, %f243, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f245, %f243, %f242;\n" " // begin inline asm\n" " shfl.down.b32 %f244, %f245, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f247, %f245, %f244;\n" " // begin inline asm\n" " shfl.down.b32 %f246, %f247, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f249, %f247, %f246;\n" " // begin inline asm\n" " shfl.down.b32 %f248, %f249, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f90, %f249, %f248;\n" " setp.lt.s32 %p26, %r4, 2;\n" " or.pred %p28, %p23, %p26;\n" " @%p28 bra $L__BB15_35;\n" "\n" " add.s32 %r89, %r1, 1;\n" " cvt.s64.s32 %rd85, %r89;\n" " mul.lo.s64 %rd86, %rd85, %rd33;\n" " add.s64 %rd87, %rd86, %rd1;\n" " shl.b64 %rd88, %rd87, 2;\n" " add.s64 %rd89, %rd2, %rd88;\n" " st.global.f32 [%rd89], %f90;\n" "\n" "$L__BB15_35:\n" " // begin inline asm\n" " shfl.down.b32 %f250, %f354, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f253, %f354, %f250;\n" " // begin inline asm\n" " shfl.down.b32 %f252, %f253, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f255, %f253, %f252;\n" " // begin inline asm\n" " shfl.down.b32 %f254, %f255, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f257, %f255, %f254;\n" " // begin inline asm\n" " shfl.down.b32 %f256, %f257, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f259, %f257, %f256;\n" " // begin inline asm\n" " shfl.down.b32 %f258, %f259, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f91, %f259, %f258;\n" " setp.lt.s32 %p29, %r4, 3;\n" " or.pred %p31, %p23, %p29;\n" " @%p31 bra $L__BB15_37;\n" "\n" " add.s32 %r100, %r1, 2;\n" " cvt.s64.s32 %rd90, %r100;\n" " mul.lo.s64 %rd91, %rd90, %rd33;\n" " add.s64 %rd92, %rd91, %rd1;\n" " shl.b64 %rd93, %rd92, 2;\n" " add.s64 %rd94, %rd2, %rd93;\n" " st.global.f32 [%rd94], %f91;\n" "\n" "$L__BB15_37:\n" " // begin inline asm\n" " shfl.down.b32 %f260, %f353, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f263, %f353, %f260;\n" " // begin inline asm\n" " shfl.down.b32 %f262, %f263, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f265, %f263, %f262;\n" " // begin inline asm\n" " shfl.down.b32 %f264, %f265, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f267, %f265, %f264;\n" " // begin inline asm\n" " shfl.down.b32 %f266, %f267, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f269, %f267, %f266;\n" " // begin inline asm\n" " shfl.down.b32 %f268, %f269, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f92, %f269, %f268;\n" " setp.lt.s32 %p32, %r4, 4;\n" " or.pred %p34, %p23, %p32;\n" " @%p34 bra $L__BB15_39;\n" "\n" " add.s32 %r111, %r1, 3;\n" " cvt.s64.s32 %rd95, %r111;\n" " mul.lo.s64 %rd96, %rd95, %rd33;\n" " add.s64 %rd97, %rd96, %rd1;\n" " shl.b64 %rd98, %rd97, 2;\n" " add.s64 %rd99, %rd2, %rd98;\n" " st.global.f32 [%rd99], %f92;\n" "\n" "$L__BB15_39:\n" " // begin inline asm\n" " shfl.down.b32 %f270, %f352, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f273, %f352, %f270;\n" " // begin inline asm\n" " shfl.down.b32 %f272, %f273, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f275, %f273, %f272;\n" " // begin inline asm\n" " shfl.down.b32 %f274, %f275, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f277, %f275, %f274;\n" " // begin inline asm\n" " shfl.down.b32 %f276, %f277, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f279, %f277, %f276;\n" " // begin inline asm\n" " shfl.down.b32 %f278, %f279, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f93, %f279, %f278;\n" " setp.lt.s32 %p35, %r4, 5;\n" " or.pred %p37, %p23, %p35;\n" " @%p37 bra $L__BB15_41;\n" "\n" " add.s32 %r122, %r1, 4;\n" " cvt.s64.s32 %rd100, %r122;\n" " mul.lo.s64 %rd101, %rd100, %rd33;\n" " add.s64 %rd102, %rd101, %rd1;\n" " shl.b64 %rd103, %rd102, 2;\n" " add.s64 %rd104, %rd2, %rd103;\n" " st.global.f32 [%rd104], %f93;\n" "\n" "$L__BB15_41:\n" " // begin inline asm\n" " shfl.down.b32 %f280, %f351, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f283, %f351, %f280;\n" " // begin inline asm\n" " shfl.down.b32 %f282, %f283, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f285, %f283, %f282;\n" " // begin inline asm\n" " shfl.down.b32 %f284, %f285, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f287, %f285, %f284;\n" " // begin inline asm\n" " shfl.down.b32 %f286, %f287, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f289, %f287, %f286;\n" " // begin inline asm\n" " shfl.down.b32 %f288, %f289, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f94, %f289, %f288;\n" " setp.lt.s32 %p38, %r4, 6;\n" " or.pred %p40, %p23, %p38;\n" " @%p40 bra $L__BB15_43;\n" "\n" " add.s32 %r133, %r1, 5;\n" " cvt.s64.s32 %rd105, %r133;\n" " mul.lo.s64 %rd106, %rd105, %rd33;\n" " add.s64 %rd107, %rd106, %rd1;\n" " shl.b64 %rd108, %rd107, 2;\n" " add.s64 %rd109, %rd2, %rd108;\n" " st.global.f32 [%rd109], %f94;\n" "\n" "$L__BB15_43:\n" " // begin inline asm\n" " shfl.down.b32 %f290, %f350, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f293, %f350, %f290;\n" " // begin inline asm\n" " shfl.down.b32 %f292, %f293, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f295, %f293, %f292;\n" " // begin inline asm\n" " shfl.down.b32 %f294, %f295, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f297, %f295, %f294;\n" " // begin inline asm\n" " shfl.down.b32 %f296, %f297, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f299, %f297, %f296;\n" " // begin inline asm\n" " shfl.down.b32 %f298, %f299, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f95, %f299, %f298;\n" " setp.lt.s32 %p41, %r4, 7;\n" " or.pred %p43, %p23, %p41;\n" " @%p43 bra $L__BB15_45;\n" "\n" " add.s32 %r144, %r1, 6;\n" " cvt.s64.s32 %rd110, %r144;\n" " mul.lo.s64 %rd111, %rd110, %rd33;\n" " add.s64 %rd112, %rd111, %rd1;\n" " shl.b64 %rd113, %rd112, 2;\n" " add.s64 %rd114, %rd2, %rd113;\n" " st.global.f32 [%rd114], %f95;\n" "\n" "$L__BB15_45:\n" " // begin inline asm\n" " shfl.down.b32 %f300, %f349, %r68, %r77;\n" " // end inline asm\n" " add.f32 %f303, %f349, %f300;\n" " // begin inline asm\n" " shfl.down.b32 %f302, %f303, %r70, %r77;\n" " // end inline asm\n" " add.f32 %f305, %f303, %f302;\n" " // begin inline asm\n" " shfl.down.b32 %f304, %f305, %r72, %r77;\n" " // end inline asm\n" " add.f32 %f307, %f305, %f304;\n" " // begin inline asm\n" " shfl.down.b32 %f306, %f307, %r74, %r77;\n" " // end inline asm\n" " add.f32 %f309, %f307, %f306;\n" " // begin inline asm\n" " shfl.down.b32 %f308, %f309, %r76, %r77;\n" " // end inline asm\n" " add.f32 %f96, %f309, %f308;\n" " setp.lt.s32 %p44, %r4, 8;\n" " or.pred %p46, %p23, %p44;\n" " @%p46 bra $L__BB15_47;\n" "\n" " add.s32 %r155, %r1, 7;\n" " cvt.s64.s32 %rd115, %r155;\n" " mul.lo.s64 %rd116, %rd115, %rd33;\n" " add.s64 %rd117, %rd116, %rd1;\n" " shl.b64 %rd118, %rd117, 2;\n" " add.s64 %rd119, %rd2, %rd118;\n" " st.global.f32 [%rd119], %f96;\n" "\n" "$L__BB15_47:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f32\n" ".visible .entry fc_matvec_f32(\n" " .param .u64 fc_matvec_f32_param_0,\n" " .param .u64 fc_matvec_f32_param_1,\n" " .param .u64 fc_matvec_f32_param_2,\n" " .param .u32 fc_matvec_f32_param_3,\n" " .param .u32 fc_matvec_f32_param_4,\n" " .param .u32 fc_matvec_f32_param_5\n" ")\n" "{\n" " .reg .pred %p<11>;\n" " .reg .f32 %f<34>;\n" " .reg .b32 %r<37>;\n" " .reg .b64 %rd<33>;\n" "\n" "\n" " ld.param.u64 %rd14, [fc_matvec_f32_param_0];\n" " ld.param.u64 %rd16, [fc_matvec_f32_param_1];\n" " ld.param.u64 %rd15, [fc_matvec_f32_param_2];\n" " ld.param.u32 %r19, [fc_matvec_f32_param_3];\n" " ld.param.u32 %r17, [fc_matvec_f32_param_4];\n" " ld.param.u32 %r18, [fc_matvec_f32_param_5];\n" " cvta.to.global.u64 %rd1, %rd16;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r19;\n" " @%p1 bra $L__BB16_15;\n" "\n" " cvta.to.global.u64 %rd2, %rd14;\n" " mov.u32 %r2, %ntid.x;\n" " cvt.s64.s32 %rd3, %r1;\n" " mul.wide.s32 %rd4, %r18, %r1;\n" " mov.u32 %r3, %tid.x;\n" " setp.ge.s32 %p2, %r3, %r17;\n" " mov.f32 %f33, 0f00000000;\n" " @%p2 bra $L__BB16_8;\n" "\n" " not.b32 %r20, %r3;\n" " add.s32 %r21, %r20, %r17;\n" " div.u32 %r4, %r21, %r2;\n" " add.s32 %r22, %r4, 1;\n" " and.b32 %r33, %r22, 3;\n" " setp.eq.s32 %p3, %r33, 0;\n" " mov.f32 %f33, 0f00000000;\n" " mov.u32 %r34, %r3;\n" " @%p3 bra $L__BB16_5;\n" "\n" " mul.wide.s32 %rd17, %r3, 4;\n" " add.s64 %rd32, %rd1, %rd17;\n" " mul.wide.s32 %rd6, %r2, 4;\n" " add.s64 %rd18, %rd17, %rd4;\n" " add.s64 %rd31, %rd2, %rd18;\n" " mov.f32 %f33, 0f00000000;\n" " mov.u32 %r34, %r3;\n" "\n" "$L__BB16_4:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f12, [%rd32];\n" " ld.global.f32 %f13, [%rd31];\n" " fma.rn.f32 %f33, %f13, %f12, %f33;\n" " add.s32 %r34, %r34, %r2;\n" " add.s64 %rd32, %rd32, %rd6;\n" " add.s64 %rd31, %rd31, %rd6;\n" " add.s32 %r33, %r33, -1;\n" " setp.ne.s32 %p4, %r33, 0;\n" " @%p4 bra $L__BB16_4;\n" "\n" "$L__BB16_5:\n" " setp.lt.u32 %p5, %r4, 3;\n" " add.s64 %rd12, %rd2, %rd4;\n" " @%p5 bra $L__BB16_8;\n" "\n" " mul.wide.s32 %rd13, %r2, 4;\n" "\n" "$L__BB16_7:\n" " mul.wide.s32 %rd19, %r34, 4;\n" " add.s64 %rd20, %rd12, %rd19;\n" " add.s64 %rd21, %rd1, %rd19;\n" " ld.global.f32 %f14, [%rd21];\n" " ld.global.f32 %f15, [%rd20];\n" " fma.rn.f32 %f16, %f15, %f14, %f33;\n" " add.s64 %rd22, %rd20, %rd13;\n" " add.s64 %rd23, %rd21, %rd13;\n" " ld.global.f32 %f17, [%rd23];\n" " ld.global.f32 %f18, [%rd22];\n" " fma.rn.f32 %f19, %f18, %f17, %f16;\n" " add.s32 %r23, %r34, %r2;\n" " add.s32 %r24, %r23, %r2;\n" " add.s64 %rd24, %rd22, %rd13;\n" " add.s64 %rd25, %rd23, %rd13;\n" " ld.global.f32 %f20, [%rd25];\n" " ld.global.f32 %f21, [%rd24];\n" " fma.rn.f32 %f22, %f21, %f20, %f19;\n" " add.s32 %r25, %r24, %r2;\n" " add.s64 %rd26, %rd24, %rd13;\n" " add.s64 %rd27, %rd25, %rd13;\n" " ld.global.f32 %f23, [%rd27];\n" " ld.global.f32 %f24, [%rd26];\n" " fma.rn.f32 %f33, %f24, %f23, %f22;\n" " add.s32 %r34, %r25, %r2;\n" " setp.lt.s32 %p6, %r34, %r17;\n" " @%p6 bra $L__BB16_7;\n" "\n" "$L__BB16_8:\n" " shl.b32 %r26, %r3, 2;\n" " mov.u32 %r27, fc_kq_shared;\n" " add.s32 %r13, %r27, %r26;\n" " st.shared.f32 [%r13], %f33;\n" " bar.sync 0;\n" " setp.lt.s32 %p7, %r2, 2;\n" " @%p7 bra $L__BB16_13;\n" "\n" " shr.u32 %r28, %r2, 31;\n" " add.s32 %r29, %r2, %r28;\n" " shr.s32 %r36, %r29, 1;\n" "\n" "$L__BB16_10:\n" " setp.ge.s32 %p8, %r3, %r36;\n" " @%p8 bra $L__BB16_12;\n" "\n" " shl.b32 %r30, %r36, 2;\n" " add.s32 %r31, %r13, %r30;\n" " ld.shared.f32 %f25, [%r13];\n" " ld.shared.f32 %f26, [%r31];\n" " add.f32 %f27, %f26, %f25;\n" " st.shared.f32 [%r13], %f27;\n" "\n" "$L__BB16_12:\n" " bar.sync 0;\n" " shr.s32 %r16, %r36, 1;\n" " setp.gt.s32 %p9, %r36, 1;\n" " mov.u32 %r36, %r16;\n" " @%p9 bra $L__BB16_10;\n" "\n" "$L__BB16_13:\n" " setp.ne.s32 %p10, %r3, 0;\n" " @%p10 bra $L__BB16_15;\n" "\n" " ld.shared.f32 %f28, [fc_kq_shared];\n" " cvta.to.global.u64 %rd28, %rd15;\n" " shl.b64 %rd29, %rd3, 2;\n" " add.s64 %rd30, %rd28, %rd29;\n" " st.global.f32 [%rd30], %f28;\n" "\n" "$L__BB16_15:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f32_nx2\n" ".visible .entry fc_matvec_f32_nx2(\n" " .param .u64 fc_matvec_f32_nx2_param_0,\n" " .param .u64 fc_matvec_f32_nx2_param_1,\n" " .param .u64 fc_matvec_f32_nx2_param_2,\n" " .param .u32 fc_matvec_f32_nx2_param_3,\n" " .param .u32 fc_matvec_f32_nx2_param_4,\n" " .param .u32 fc_matvec_f32_nx2_param_5,\n" " .param .u32 fc_matvec_f32_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<24>;\n" " .reg .f32 %f<75>;\n" " .reg .b32 %r<60>;\n" " .reg .b64 %rd<82>;\n" "\n" "\n" " ld.param.u64 %rd32, [fc_matvec_f32_nx2_param_0];\n" " ld.param.u64 %rd34, [fc_matvec_f32_nx2_param_1];\n" " ld.param.u64 %rd33, [fc_matvec_f32_nx2_param_2];\n" " ld.param.u32 %r26, [fc_matvec_f32_nx2_param_3];\n" " ld.param.u32 %r27, [fc_matvec_f32_nx2_param_4];\n" " ld.param.u32 %r28, [fc_matvec_f32_nx2_param_5];\n" " ld.param.u32 %r29, [fc_matvec_f32_nx2_param_6];\n" " cvta.to.global.u64 %rd1, %rd34;\n" " mov.u32 %r30, %ctaid.y;\n" " shl.b32 %r1, %r30, 1;\n" " sub.s32 %r2, %r29, %r1;\n" " min.s32 %r3, %r2, 2;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r26;\n" " @%p1 bra $L__BB17_36;\n" "\n" " cvta.to.global.u64 %rd2, %rd32;\n" " mov.u32 %r5, %ntid.x;\n" " cvt.s64.s32 %rd3, %r4;\n" " mul.wide.s32 %rd4, %r28, %r4;\n" " mov.u32 %r6, %tid.x;\n" " setp.ge.s32 %p2, %r6, %r27;\n" " mov.f32 %f66, 0f00000000;\n" " mov.f32 %f65, %f66;\n" " @%p2 bra $L__BB17_28;\n" "\n" " cvt.s64.s32 %rd5, %r27;\n" " not.b32 %r31, %r6;\n" " add.s32 %r32, %r31, %r27;\n" " div.u32 %r7, %r32, %r5;\n" " add.s32 %r33, %r7, 1;\n" " and.b32 %r54, %r33, 3;\n" " setp.eq.s32 %p3, %r54, 0;\n" " mov.f32 %f66, 0f00000000;\n" " mov.f32 %f65, %f66;\n" " mov.u32 %r55, %r6;\n" " @%p3 bra $L__BB17_9;\n" "\n" " cvt.s64.s32 %rd35, %r1;\n" " add.s32 %r34, %r1, 1;\n" " cvt.s64.s32 %rd36, %r34;\n" " mul.lo.s64 %rd37, %rd5, %rd36;\n" " cvt.s64.s32 %rd38, %r6;\n" " add.s64 %rd39, %rd37, %rd38;\n" " shl.b64 %rd40, %rd39, 2;\n" " add.s64 %rd80, %rd1, %rd40;\n" " mul.wide.s32 %rd7, %r5, 4;\n" " mul.lo.s64 %rd41, %rd5, %rd35;\n" " add.s64 %rd42, %rd41, %rd38;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd79, %rd1, %rd43;\n" " mul.wide.s32 %rd44, %r6, 4;\n" " add.s64 %rd45, %rd44, %rd4;\n" " add.s64 %rd78, %rd2, %rd45;\n" " mov.f32 %f66, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " mov.f32 %f65, %f66;\n" " mov.u32 %r55, %r6;\n" "\n" "$L__BB17_4:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f3, [%rd78];\n" " @%p4 bra $L__BB17_6;\n" "\n" " ld.global.f32 %f40, [%rd79];\n" " fma.rn.f32 %f65, %f3, %f40, %f65;\n" "\n" "$L__BB17_6:\n" " @%p5 bra $L__BB17_8;\n" "\n" " ld.global.f32 %f41, [%rd80];\n" " fma.rn.f32 %f66, %f3, %f41, %f66;\n" "\n" "$L__BB17_8:\n" " add.s32 %r55, %r55, %r5;\n" " add.s64 %rd80, %rd80, %rd7;\n" " add.s64 %rd79, %rd79, %rd7;\n" " add.s64 %rd78, %rd78, %rd7;\n" " add.s32 %r54, %r54, -1;\n" " setp.ne.s32 %p6, %r54, 0;\n" " @%p6 bra $L__BB17_4;\n" "\n" "$L__BB17_9:\n" " setp.lt.u32 %p7, %r7, 3;\n" " add.s64 %rd16, %rd2, %rd4;\n" " @%p7 bra $L__BB17_28;\n" "\n" " cvt.s64.s32 %rd46, %r1;\n" " mul.lo.s64 %rd17, %rd46, %rd5;\n" " add.s32 %r35, %r1, 1;\n" " cvt.s64.s32 %rd47, %r35;\n" " mul.lo.s64 %rd18, %rd47, %rd5;\n" " mul.wide.s32 %rd19, %r5, 4;\n" " setp.lt.s32 %p8, %r2, 1;\n" " setp.lt.s32 %p9, %r3, 2;\n" "\n" "$L__BB17_11:\n" " cvt.s64.s32 %rd20, %r55;\n" " mul.wide.s32 %rd48, %r55, 4;\n" " add.s64 %rd21, %rd16, %rd48;\n" " ld.global.f32 %f12, [%rd21];\n" " @%p8 bra $L__BB17_13;\n" "\n" " add.s64 %rd49, %rd17, %rd20;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd51, %rd1, %rd50;\n" " ld.global.f32 %f42, [%rd51];\n" " fma.rn.f32 %f65, %f12, %f42, %f65;\n" "\n" "$L__BB17_13:\n" " @%p9 bra $L__BB17_15;\n" "\n" " add.s64 %rd52, %rd18, %rd20;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd54, %rd1, %rd53;\n" " ld.global.f32 %f43, [%rd54];\n" " fma.rn.f32 %f66, %f12, %f43, %f66;\n" "\n" "$L__BB17_15:\n" " cvt.u32.u64 %r36, %rd20;\n" " add.s32 %r37, %r36, %r5;\n" " cvt.s64.s32 %rd22, %r37;\n" " add.s64 %rd23, %rd21, %rd19;\n" " ld.global.f32 %f17, [%rd23];\n" " @%p8 bra $L__BB17_17;\n" "\n" " add.s64 %rd55, %rd17, %rd22;\n" " shl.b64 %rd56, %rd55, 2;\n" " add.s64 %rd57, %rd1, %rd56;\n" " ld.global.f32 %f44, [%rd57];\n" " fma.rn.f32 %f65, %f17, %f44, %f65;\n" "\n" "$L__BB17_17:\n" " @%p9 bra $L__BB17_19;\n" "\n" " add.s64 %rd58, %rd18, %rd22;\n" " shl.b64 %rd59, %rd58, 2;\n" " add.s64 %rd60, %rd1, %rd59;\n" " ld.global.f32 %f45, [%rd60];\n" " fma.rn.f32 %f66, %f17, %f45, %f66;\n" "\n" "$L__BB17_19:\n" " cvt.u32.u64 %r38, %rd22;\n" " add.s32 %r39, %r38, %r5;\n" " cvt.s64.s32 %rd24, %r39;\n" " add.s64 %rd25, %rd23, %rd19;\n" " ld.global.f32 %f22, [%rd25];\n" " @%p8 bra $L__BB17_21;\n" "\n" " add.s64 %rd61, %rd17, %rd24;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd63, %rd1, %rd62;\n" " ld.global.f32 %f46, [%rd63];\n" " fma.rn.f32 %f65, %f22, %f46, %f65;\n" "\n" "$L__BB17_21:\n" " @%p9 bra $L__BB17_23;\n" "\n" " add.s64 %rd64, %rd18, %rd24;\n" " shl.b64 %rd65, %rd64, 2;\n" " add.s64 %rd66, %rd1, %rd65;\n" " ld.global.f32 %f47, [%rd66];\n" " fma.rn.f32 %f66, %f22, %f47, %f66;\n" "\n" "$L__BB17_23:\n" " cvt.u32.u64 %r40, %rd24;\n" " add.s32 %r41, %r40, %r5;\n" " cvt.s64.s32 %rd26, %r41;\n" " add.s64 %rd67, %rd25, %rd19;\n" " ld.global.f32 %f27, [%rd67];\n" " @%p8 bra $L__BB17_25;\n" "\n" " add.s64 %rd68, %rd17, %rd26;\n" " shl.b64 %rd69, %rd68, 2;\n" " add.s64 %rd70, %rd1, %rd69;\n" " ld.global.f32 %f48, [%rd70];\n" " fma.rn.f32 %f65, %f27, %f48, %f65;\n" "\n" "$L__BB17_25:\n" " @%p9 bra $L__BB17_27;\n" "\n" " add.s64 %rd71, %rd18, %rd26;\n" " shl.b64 %rd72, %rd71, 2;\n" " add.s64 %rd73, %rd1, %rd72;\n" " ld.global.f32 %f49, [%rd73];\n" " fma.rn.f32 %f66, %f27, %f49, %f66;\n" "\n" "$L__BB17_27:\n" " cvt.u32.u64 %r42, %rd26;\n" " add.s32 %r55, %r42, %r5;\n" " setp.lt.s32 %p16, %r55, %r27;\n" " @%p16 bra $L__BB17_11;\n" "\n" "$L__BB17_28:\n" " shl.b32 %r43, %r6, 2;\n" " mov.u32 %r58, fc_kq_shared;\n" " add.s32 %r16, %r58, %r43;\n" " st.shared.f32 [%r16], %f65;\n" " shl.b32 %r45, %r5, 2;\n" " add.s32 %r17, %r16, %r45;\n" " st.shared.f32 [%r17], %f66;\n" " bar.sync 0;\n" " setp.lt.s32 %p17, %r5, 2;\n" " @%p17 bra $L__BB17_33;\n" "\n" " shr.u32 %r46, %r5, 31;\n" " add.s32 %r47, %r5, %r46;\n" " shr.s32 %r57, %r47, 1;\n" "\n" "$L__BB17_30:\n" " setp.ge.s32 %p18, %r6, %r57;\n" " @%p18 bra $L__BB17_32;\n" "\n" " shl.b32 %r48, %r57, 2;\n" " add.s32 %r49, %r16, %r48;\n" " ld.shared.f32 %f50, [%r16];\n" " ld.shared.f32 %f51, [%r49];\n" " add.f32 %f52, %f51, %f50;\n" " st.shared.f32 [%r16], %f52;\n" " ld.shared.f32 %f53, [%r17];\n" " add.s32 %r50, %r17, %r48;\n" " ld.shared.f32 %f54, [%r50];\n" " add.f32 %f55, %f54, %f53;\n" " st.shared.f32 [%r17], %f55;\n" "\n" "$L__BB17_32:\n" " bar.sync 0;\n" " shr.s32 %r20, %r57, 1;\n" " setp.gt.s32 %p19, %r57, 1;\n" " mov.u32 %r57, %r20;\n" " @%p19 bra $L__BB17_30;\n" "\n" "$L__BB17_33:\n" " setp.ne.s32 %p20, %r6, 0;\n" " setp.lt.s32 %p21, %r2, 1;\n" " or.pred %p22, %p20, %p21;\n" " @%p22 bra $L__BB17_36;\n" "\n" " cvt.s64.s32 %rd27, %r26;\n" " cvt.s64.s32 %rd81, %r1;\n" " cvta.to.global.u64 %rd29, %rd33;\n" " mov.u32 %r59, 0;\n" "\n" "$L__BB17_35:\n" " ld.shared.f32 %f56, [%r58];\n" " mul.lo.s64 %rd74, %rd81, %rd27;\n" " add.s64 %rd75, %rd74, %rd3;\n" " shl.b64 %rd76, %rd75, 2;\n" " add.s64 %rd77, %rd29, %rd76;\n" " st.global.f32 [%rd77], %f56;\n" " add.s32 %r58, %r58, %r45;\n" " add.s64 %rd81, %rd81, 1;\n" " add.s32 %r59, %r59, 1;\n" " setp.lt.s32 %p23, %r59, %r3;\n" " @%p23 bra $L__BB17_35;\n" "\n" "$L__BB17_36:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f32_nx\n" ".visible .entry fc_matvec_f32_nx(\n" " .param .u64 fc_matvec_f32_nx_param_0,\n" " .param .u64 fc_matvec_f32_nx_param_1,\n" " .param .u64 fc_matvec_f32_nx_param_2,\n" " .param .u32 fc_matvec_f32_nx_param_3,\n" " .param .u32 fc_matvec_f32_nx_param_4,\n" " .param .u32 fc_matvec_f32_nx_param_5,\n" " .param .u32 fc_matvec_f32_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<34>;\n" " .reg .f32 %f<143>;\n" " .reg .b32 %r<69>;\n" " .reg .b64 %rd<126>;\n" "\n" "\n" " ld.param.u64 %rd40, [fc_matvec_f32_nx_param_0];\n" " ld.param.u64 %rd42, [fc_matvec_f32_nx_param_1];\n" " ld.param.u64 %rd41, [fc_matvec_f32_nx_param_2];\n" " ld.param.u32 %r29, [fc_matvec_f32_nx_param_3];\n" " ld.param.u32 %r30, [fc_matvec_f32_nx_param_4];\n" " ld.param.u32 %r31, [fc_matvec_f32_nx_param_5];\n" " ld.param.u32 %r32, [fc_matvec_f32_nx_param_6];\n" " cvta.to.global.u64 %rd1, %rd42;\n" " mov.u32 %r33, %ctaid.y;\n" " shl.b32 %r1, %r33, 2;\n" " sub.s32 %r2, %r32, %r1;\n" " min.s32 %r3, %r2, 4;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r29;\n" " @%p1 bra $L__BB18_56;\n" "\n" " mov.u32 %r5, %tid.x;\n" " cvt.s64.s32 %rd2, %r4;\n" " mul.wide.s32 %rd3, %r31, %r4;\n" " cvta.to.global.u64 %rd4, %rd40;\n" " mov.u32 %r6, %ntid.x;\n" " setp.ge.s32 %p2, %r5, %r30;\n" " mov.f32 %f126, 0f00000000;\n" " mov.f32 %f125, %f126;\n" " mov.f32 %f124, %f126;\n" " mov.f32 %f123, %f126;\n" " @%p2 bra $L__BB18_48;\n" "\n" " cvt.s64.s32 %rd5, %r30;\n" " not.b32 %r34, %r5;\n" " add.s32 %r35, %r34, %r30;\n" " div.u32 %r7, %r35, %r6;\n" " add.s32 %r36, %r7, 1;\n" " and.b32 %r63, %r36, 3;\n" " setp.eq.s32 %p3, %r63, 0;\n" " mov.f32 %f126, 0f00000000;\n" " mov.f32 %f125, %f126;\n" " mov.f32 %f124, %f126;\n" " mov.f32 %f123, %f126;\n" " mov.u32 %r64, %r5;\n" " @%p3 bra $L__BB18_13;\n" "\n" " cvt.s64.s32 %rd43, %r1;\n" " add.s32 %r37, %r1, 1;\n" " cvt.s64.s32 %rd44, %r37;\n" " add.s32 %r38, %r1, 2;\n" " cvt.s64.s32 %rd45, %r38;\n" " add.s32 %r39, %r1, 3;\n" " cvt.s64.s32 %rd46, %r39;\n" " mul.lo.s64 %rd47, %rd5, %rd46;\n" " cvt.s64.s32 %rd48, %r5;\n" " add.s64 %rd49, %rd47, %rd48;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd124, %rd1, %rd50;\n" " mul.wide.s32 %rd7, %r6, 4;\n" " mul.lo.s64 %rd51, %rd5, %rd45;\n" " add.s64 %rd52, %rd51, %rd48;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd123, %rd1, %rd53;\n" " mul.lo.s64 %rd54, %rd5, %rd44;\n" " add.s64 %rd55, %rd54, %rd48;\n" " shl.b64 %rd56, %rd55, 2;\n" " add.s64 %rd122, %rd1, %rd56;\n" " mul.lo.s64 %rd57, %rd5, %rd43;\n" " add.s64 %rd58, %rd57, %rd48;\n" " shl.b64 %rd59, %rd58, 2;\n" " add.s64 %rd121, %rd1, %rd59;\n" " mul.wide.s32 %rd60, %r5, 4;\n" " add.s64 %rd61, %rd60, %rd3;\n" " add.s64 %rd120, %rd4, %rd61;\n" " mov.f32 %f126, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " setp.lt.s32 %p6, %r3, 3;\n" " setp.lt.s32 %p7, %r3, 4;\n" " mov.f32 %f125, %f126;\n" " mov.f32 %f124, %f126;\n" " mov.f32 %f123, %f126;\n" " mov.u32 %r64, %r5;\n" "\n" "$L__BB18_4:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f5, [%rd120];\n" " @%p4 bra $L__BB18_6;\n" "\n" " ld.global.f32 %f74, [%rd121];\n" " fma.rn.f32 %f123, %f5, %f74, %f123;\n" "\n" "$L__BB18_6:\n" " @%p5 bra $L__BB18_8;\n" "\n" " ld.global.f32 %f75, [%rd122];\n" " fma.rn.f32 %f124, %f5, %f75, %f124;\n" "\n" "$L__BB18_8:\n" " @%p6 bra $L__BB18_10;\n" "\n" " ld.global.f32 %f76, [%rd123];\n" " fma.rn.f32 %f125, %f5, %f76, %f125;\n" "\n" "$L__BB18_10:\n" " @%p7 bra $L__BB18_12;\n" "\n" " ld.global.f32 %f77, [%rd124];\n" " fma.rn.f32 %f126, %f5, %f77, %f126;\n" "\n" "$L__BB18_12:\n" " add.s32 %r64, %r64, %r6;\n" " add.s64 %rd124, %rd124, %rd7;\n" " add.s64 %rd123, %rd123, %rd7;\n" " add.s64 %rd122, %rd122, %rd7;\n" " add.s64 %rd121, %rd121, %rd7;\n" " add.s64 %rd120, %rd120, %rd7;\n" " add.s32 %r63, %r63, -1;\n" " setp.ne.s32 %p8, %r63, 0;\n" " @%p8 bra $L__BB18_4;\n" "\n" "$L__BB18_13:\n" " setp.lt.u32 %p9, %r7, 3;\n" " add.s64 %rd22, %rd4, %rd3;\n" " @%p9 bra $L__BB18_48;\n" "\n" " cvt.s64.s32 %rd62, %r1;\n" " mul.lo.s64 %rd23, %rd62, %rd5;\n" " add.s32 %r40, %r1, 1;\n" " cvt.s64.s32 %rd63, %r40;\n" " mul.lo.s64 %rd24, %rd63, %rd5;\n" " add.s32 %r41, %r1, 2;\n" " cvt.s64.s32 %rd64, %r41;\n" " mul.lo.s64 %rd25, %rd64, %rd5;\n" " add.s32 %r42, %r1, 3;\n" " cvt.s64.s32 %rd65, %r42;\n" " mul.lo.s64 %rd26, %rd65, %rd5;\n" " mul.wide.s32 %rd27, %r6, 4;\n" " setp.lt.s32 %p10, %r2, 1;\n" " setp.lt.s32 %p11, %r3, 2;\n" " setp.lt.s32 %p12, %r3, 3;\n" " setp.lt.s32 %p13, %r3, 4;\n" "\n" "$L__BB18_15:\n" " cvt.s64.s32 %rd28, %r64;\n" " mul.wide.s32 %rd66, %r64, 4;\n" " add.s64 %rd29, %rd22, %rd66;\n" " ld.global.f32 %f22, [%rd29];\n" " @%p10 bra $L__BB18_17;\n" "\n" " add.s64 %rd67, %rd23, %rd28;\n" " shl.b64 %rd68, %rd67, 2;\n" " add.s64 %rd69, %rd1, %rd68;\n" " ld.global.f32 %f78, [%rd69];\n" " fma.rn.f32 %f123, %f22, %f78, %f123;\n" "\n" "$L__BB18_17:\n" " @%p11 bra $L__BB18_19;\n" "\n" " add.s64 %rd70, %rd24, %rd28;\n" " shl.b64 %rd71, %rd70, 2;\n" " add.s64 %rd72, %rd1, %rd71;\n" " ld.global.f32 %f79, [%rd72];\n" " fma.rn.f32 %f124, %f22, %f79, %f124;\n" "\n" "$L__BB18_19:\n" " @%p12 bra $L__BB18_21;\n" "\n" " add.s64 %rd73, %rd25, %rd28;\n" " shl.b64 %rd74, %rd73, 2;\n" " add.s64 %rd75, %rd1, %rd74;\n" " ld.global.f32 %f80, [%rd75];\n" " fma.rn.f32 %f125, %f22, %f80, %f125;\n" "\n" "$L__BB18_21:\n" " @%p13 bra $L__BB18_23;\n" "\n" " add.s64 %rd76, %rd26, %rd28;\n" " shl.b64 %rd77, %rd76, 2;\n" " add.s64 %rd78, %rd1, %rd77;\n" " ld.global.f32 %f81, [%rd78];\n" " fma.rn.f32 %f126, %f22, %f81, %f126;\n" "\n" "$L__BB18_23:\n" " cvt.u32.u64 %r43, %rd28;\n" " add.s32 %r44, %r43, %r6;\n" " cvt.s64.s32 %rd30, %r44;\n" " add.s64 %rd31, %rd29, %rd27;\n" " ld.global.f32 %f31, [%rd31];\n" " @%p10 bra $L__BB18_25;\n" "\n" " add.s64 %rd79, %rd23, %rd30;\n" " shl.b64 %rd80, %rd79, 2;\n" " add.s64 %rd81, %rd1, %rd80;\n" " ld.global.f32 %f82, [%rd81];\n" " fma.rn.f32 %f123, %f31, %f82, %f123;\n" "\n" "$L__BB18_25:\n" " @%p11 bra $L__BB18_27;\n" "\n" " add.s64 %rd82, %rd24, %rd30;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd84, %rd1, %rd83;\n" " ld.global.f32 %f83, [%rd84];\n" " fma.rn.f32 %f124, %f31, %f83, %f124;\n" "\n" "$L__BB18_27:\n" " @%p12 bra $L__BB18_29;\n" "\n" " add.s64 %rd85, %rd25, %rd30;\n" " shl.b64 %rd86, %rd85, 2;\n" " add.s64 %rd87, %rd1, %rd86;\n" " ld.global.f32 %f84, [%rd87];\n" " fma.rn.f32 %f125, %f31, %f84, %f125;\n" "\n" "$L__BB18_29:\n" " @%p13 bra $L__BB18_31;\n" "\n" " add.s64 %rd88, %rd26, %rd30;\n" " shl.b64 %rd89, %rd88, 2;\n" " add.s64 %rd90, %rd1, %rd89;\n" " ld.global.f32 %f85, [%rd90];\n" " fma.rn.f32 %f126, %f31, %f85, %f126;\n" "\n" "$L__BB18_31:\n" " cvt.u32.u64 %r45, %rd30;\n" " add.s32 %r46, %r45, %r6;\n" " cvt.s64.s32 %rd32, %r46;\n" " add.s64 %rd33, %rd31, %rd27;\n" " ld.global.f32 %f40, [%rd33];\n" " @%p10 bra $L__BB18_33;\n" "\n" " add.s64 %rd91, %rd23, %rd32;\n" " shl.b64 %rd92, %rd91, 2;\n" " add.s64 %rd93, %rd1, %rd92;\n" " ld.global.f32 %f86, [%rd93];\n" " fma.rn.f32 %f123, %f40, %f86, %f123;\n" "\n" "$L__BB18_33:\n" " @%p11 bra $L__BB18_35;\n" "\n" " add.s64 %rd94, %rd24, %rd32;\n" " shl.b64 %rd95, %rd94, 2;\n" " add.s64 %rd96, %rd1, %rd95;\n" " ld.global.f32 %f87, [%rd96];\n" " fma.rn.f32 %f124, %f40, %f87, %f124;\n" "\n" "$L__BB18_35:\n" " @%p12 bra $L__BB18_37;\n" "\n" " add.s64 %rd97, %rd25, %rd32;\n" " shl.b64 %rd98, %rd97, 2;\n" " add.s64 %rd99, %rd1, %rd98;\n" " ld.global.f32 %f88, [%rd99];\n" " fma.rn.f32 %f125, %f40, %f88, %f125;\n" "\n" "$L__BB18_37:\n" " @%p13 bra $L__BB18_39;\n" "\n" " add.s64 %rd100, %rd26, %rd32;\n" " shl.b64 %rd101, %rd100, 2;\n" " add.s64 %rd102, %rd1, %rd101;\n" " ld.global.f32 %f89, [%rd102];\n" " fma.rn.f32 %f126, %f40, %f89, %f126;\n" "\n" "$L__BB18_39:\n" " cvt.u32.u64 %r47, %rd32;\n" " add.s32 %r48, %r47, %r6;\n" " cvt.s64.s32 %rd34, %r48;\n" " add.s64 %rd103, %rd33, %rd27;\n" " ld.global.f32 %f49, [%rd103];\n" " @%p10 bra $L__BB18_41;\n" "\n" " add.s64 %rd104, %rd23, %rd34;\n" " shl.b64 %rd105, %rd104, 2;\n" " add.s64 %rd106, %rd1, %rd105;\n" " ld.global.f32 %f90, [%rd106];\n" " fma.rn.f32 %f123, %f49, %f90, %f123;\n" "\n" "$L__BB18_41:\n" " @%p11 bra $L__BB18_43;\n" "\n" " add.s64 %rd107, %rd24, %rd34;\n" " shl.b64 %rd108, %rd107, 2;\n" " add.s64 %rd109, %rd1, %rd108;\n" " ld.global.f32 %f91, [%rd109];\n" " fma.rn.f32 %f124, %f49, %f91, %f124;\n" "\n" "$L__BB18_43:\n" " @%p12 bra $L__BB18_45;\n" "\n" " add.s64 %rd110, %rd25, %rd34;\n" " shl.b64 %rd111, %rd110, 2;\n" " add.s64 %rd112, %rd1, %rd111;\n" " ld.global.f32 %f92, [%rd112];\n" " fma.rn.f32 %f125, %f49, %f92, %f125;\n" "\n" "$L__BB18_45:\n" " @%p13 bra $L__BB18_47;\n" "\n" " add.s64 %rd113, %rd26, %rd34;\n" " shl.b64 %rd114, %rd113, 2;\n" " add.s64 %rd115, %rd1, %rd114;\n" " ld.global.f32 %f93, [%rd115];\n" " fma.rn.f32 %f126, %f49, %f93, %f126;\n" "\n" "$L__BB18_47:\n" " cvt.u32.u64 %r49, %rd34;\n" " add.s32 %r64, %r49, %r6;\n" " setp.lt.s32 %p26, %r64, %r30;\n" " @%p26 bra $L__BB18_15;\n" "\n" "$L__BB18_48:\n" " shl.b32 %r50, %r5, 2;\n" " mov.u32 %r67, fc_kq_shared;\n" " add.s32 %r16, %r67, %r50;\n" " st.shared.f32 [%r16], %f123;\n" " shl.b32 %r52, %r6, 2;\n" " add.s32 %r17, %r16, %r52;\n" " st.shared.f32 [%r17], %f124;\n" " add.s32 %r18, %r17, %r52;\n" " st.shared.f32 [%r18], %f125;\n" " add.s32 %r19, %r18, %r52;\n" " st.shared.f32 [%r19], %f126;\n" " bar.sync 0;\n" " setp.lt.s32 %p27, %r6, 2;\n" " @%p27 bra $L__BB18_53;\n" "\n" " shr.u32 %r53, %r6, 31;\n" " add.s32 %r54, %r6, %r53;\n" " shr.s32 %r66, %r54, 1;\n" "\n" "$L__BB18_50:\n" " setp.ge.s32 %p28, %r5, %r66;\n" " @%p28 bra $L__BB18_52;\n" "\n" " shl.b32 %r55, %r66, 2;\n" " add.s32 %r56, %r16, %r55;\n" " ld.shared.f32 %f94, [%r16];\n" " ld.shared.f32 %f95, [%r56];\n" " add.f32 %f96, %f95, %f94;\n" " st.shared.f32 [%r16], %f96;\n" " add.s32 %r57, %r56, %r52;\n" " ld.shared.f32 %f97, [%r17];\n" " ld.shared.f32 %f98, [%r57];\n" " add.f32 %f99, %f98, %f97;\n" " st.shared.f32 [%r17], %f99;\n" " add.s32 %r58, %r57, %r52;\n" " ld.shared.f32 %f100, [%r18];\n" " ld.shared.f32 %f101, [%r58];\n" " add.f32 %f102, %f101, %f100;\n" " st.shared.f32 [%r18], %f102;\n" " add.s32 %r59, %r58, %r52;\n" " ld.shared.f32 %f103, [%r19];\n" " ld.shared.f32 %f104, [%r59];\n" " add.f32 %f105, %f104, %f103;\n" " st.shared.f32 [%r19], %f105;\n" "\n" "$L__BB18_52:\n" " bar.sync 0;\n" " shr.s32 %r23, %r66, 1;\n" " setp.gt.s32 %p29, %r66, 1;\n" " mov.u32 %r66, %r23;\n" " @%p29 bra $L__BB18_50;\n" "\n" "$L__BB18_53:\n" " setp.ne.s32 %p30, %r5, 0;\n" " setp.lt.s32 %p31, %r2, 1;\n" " or.pred %p32, %p30, %p31;\n" " @%p32 bra $L__BB18_56;\n" "\n" " cvt.s64.s32 %rd35, %r29;\n" " cvt.s64.s32 %rd125, %r1;\n" " cvta.to.global.u64 %rd37, %rd41;\n" " mov.u32 %r68, 0;\n" "\n" "$L__BB18_55:\n" " ld.shared.f32 %f106, [%r67];\n" " mul.lo.s64 %rd116, %rd125, %rd35;\n" " add.s64 %rd117, %rd116, %rd2;\n" " shl.b64 %rd118, %rd117, 2;\n" " add.s64 %rd119, %rd37, %rd118;\n" " st.global.f32 [%rd119], %f106;\n" " add.s32 %r67, %r67, %r52;\n" " add.s64 %rd125, %rd125, 1;\n" " add.s32 %r68, %r68, 1;\n" " setp.lt.s32 %p33, %r68, %r3;\n" " @%p33 bra $L__BB18_55;\n" "\n" "$L__BB18_56:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f32_nx8\n" ".visible .entry fc_matvec_f32_nx8(\n" " .param .u64 fc_matvec_f32_nx8_param_0,\n" " .param .u64 fc_matvec_f32_nx8_param_1,\n" " .param .u64 fc_matvec_f32_nx8_param_2,\n" " .param .u32 fc_matvec_f32_nx8_param_3,\n" " .param .u32 fc_matvec_f32_nx8_param_4,\n" " .param .u32 fc_matvec_f32_nx8_param_5,\n" " .param .u32 fc_matvec_f32_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<39>;\n" " .reg .f32 %f<205>;\n" " .reg .b32 %r<75>;\n" " .reg .b64 %rd<134>;\n" "\n" "\n" " ld.param.u64 %rd22, [fc_matvec_f32_nx8_param_0];\n" " ld.param.u64 %rd24, [fc_matvec_f32_nx8_param_1];\n" " ld.param.u64 %rd23, [fc_matvec_f32_nx8_param_2];\n" " ld.param.u32 %r29, [fc_matvec_f32_nx8_param_3];\n" " ld.param.u32 %r30, [fc_matvec_f32_nx8_param_4];\n" " ld.param.u32 %r31, [fc_matvec_f32_nx8_param_5];\n" " ld.param.u32 %r32, [fc_matvec_f32_nx8_param_6];\n" " cvta.to.global.u64 %rd1, %rd24;\n" " mov.u32 %r33, %ctaid.y;\n" " shl.b32 %r1, %r33, 3;\n" " sub.s32 %r2, %r32, %r1;\n" " min.s32 %r3, %r2, 8;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r29;\n" " @%p1 bra $L__BB19_63;\n" "\n" " mov.u32 %r5, %tid.x;\n" " cvt.s64.s32 %rd2, %r4;\n" " mov.u32 %r6, %ntid.x;\n" " setp.ge.s32 %p2, %r5, %r30;\n" " mov.f32 %f188, 0f00000000;\n" " mov.f32 %f187, %f188;\n" " mov.f32 %f186, %f188;\n" " mov.f32 %f185, %f188;\n" " mov.f32 %f184, %f188;\n" " mov.f32 %f183, %f188;\n" " mov.f32 %f182, %f188;\n" " mov.f32 %f181, %f188;\n" " @%p2 bra $L__BB19_55;\n" "\n" " cvta.to.global.u64 %rd25, %rd22;\n" " cvt.s64.s32 %rd26, %r31;\n" " mul.lo.s64 %rd27, %rd26, %rd2;\n" " add.s64 %rd3, %rd25, %rd27;\n" " cvt.s64.s32 %rd4, %r30;\n" " not.b32 %r34, %r5;\n" " add.s32 %r7, %r34, %r30;\n" " div.u32 %r35, %r7, %r6;\n" " and.b32 %r36, %r35, 1;\n" " setp.eq.b32 %p3, %r36, 1;\n" " mov.pred %p4, 0;\n" " xor.pred %p5, %p3, %p4;\n" " mov.f32 %f188, 0f00000000;\n" " mov.f32 %f187, %f188;\n" " mov.f32 %f186, %f188;\n" " mov.f32 %f185, %f188;\n" " mov.f32 %f184, %f188;\n" " mov.f32 %f183, %f188;\n" " mov.f32 %f182, %f188;\n" " mov.f32 %f181, %f188;\n" " mov.u32 %r71, %r5;\n" " @%p5 bra $L__BB19_20;\n" "\n" " cvt.s64.s32 %rd5, %r5;\n" " mul.wide.s32 %rd28, %r5, 4;\n" " add.s64 %rd29, %rd3, %rd28;\n" " ld.global.f32 %f1, [%rd29];\n" " setp.lt.s32 %p6, %r2, 1;\n" " mov.f32 %f188, 0f00000000;\n" " mov.f32 %f181, %f188;\n" " @%p6 bra $L__BB19_5;\n" "\n" " cvt.s64.s32 %rd30, %r1;\n" " mul.lo.s64 %rd31, %rd30, %rd4;\n" " add.s64 %rd32, %rd31, %rd5;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd34, %rd1, %rd33;\n" " ld.global.f32 %f93, [%rd34];\n" " mov.f32 %f94, 0f00000000;\n" " fma.rn.f32 %f181, %f1, %f93, %f94;\n" "\n" "$L__BB19_5:\n" " setp.lt.s32 %p7, %r3, 2;\n" " mov.f32 %f182, %f188;\n" " @%p7 bra $L__BB19_7;\n" "\n" " add.s32 %r37, %r1, 1;\n" " cvt.s64.s32 %rd35, %r37;\n" " mul.lo.s64 %rd36, %rd35, %rd4;\n" " add.s64 %rd37, %rd36, %rd5;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd39, %rd1, %rd38;\n" " ld.global.f32 %f96, [%rd39];\n" " mov.f32 %f97, 0f00000000;\n" " fma.rn.f32 %f182, %f1, %f96, %f97;\n" "\n" "$L__BB19_7:\n" " setp.lt.s32 %p8, %r3, 3;\n" " mov.f32 %f183, %f188;\n" " @%p8 bra $L__BB19_9;\n" "\n" " add.s32 %r38, %r1, 2;\n" " cvt.s64.s32 %rd40, %r38;\n" " mul.lo.s64 %rd41, %rd40, %rd4;\n" " add.s64 %rd42, %rd41, %rd5;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd44, %rd1, %rd43;\n" " ld.global.f32 %f99, [%rd44];\n" " mov.f32 %f100, 0f00000000;\n" " fma.rn.f32 %f183, %f1, %f99, %f100;\n" "\n" "$L__BB19_9:\n" " setp.lt.s32 %p9, %r3, 4;\n" " mov.f32 %f184, %f188;\n" " @%p9 bra $L__BB19_11;\n" "\n" " add.s32 %r39, %r1, 3;\n" " cvt.s64.s32 %rd45, %r39;\n" " mul.lo.s64 %rd46, %rd45, %rd4;\n" " add.s64 %rd47, %rd46, %rd5;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd49, %rd1, %rd48;\n" " ld.global.f32 %f102, [%rd49];\n" " mov.f32 %f103, 0f00000000;\n" " fma.rn.f32 %f184, %f1, %f102, %f103;\n" "\n" "$L__BB19_11:\n" " setp.lt.s32 %p10, %r3, 5;\n" " mov.f32 %f185, %f188;\n" " @%p10 bra $L__BB19_13;\n" "\n" " add.s32 %r40, %r1, 4;\n" " cvt.s64.s32 %rd50, %r40;\n" " mul.lo.s64 %rd51, %rd50, %rd4;\n" " add.s64 %rd52, %rd51, %rd5;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd54, %rd1, %rd53;\n" " ld.global.f32 %f105, [%rd54];\n" " mov.f32 %f106, 0f00000000;\n" " fma.rn.f32 %f185, %f1, %f105, %f106;\n" "\n" "$L__BB19_13:\n" " setp.lt.s32 %p11, %r3, 6;\n" " mov.f32 %f186, %f188;\n" " @%p11 bra $L__BB19_15;\n" "\n" " add.s32 %r41, %r1, 5;\n" " cvt.s64.s32 %rd55, %r41;\n" " mul.lo.s64 %rd56, %rd55, %rd4;\n" " add.s64 %rd57, %rd56, %rd5;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd59, %rd1, %rd58;\n" " ld.global.f32 %f108, [%rd59];\n" " mov.f32 %f109, 0f00000000;\n" " fma.rn.f32 %f186, %f1, %f108, %f109;\n" "\n" "$L__BB19_15:\n" " setp.lt.s32 %p12, %r3, 7;\n" " mov.f32 %f187, %f188;\n" " @%p12 bra $L__BB19_17;\n" "\n" " add.s32 %r42, %r1, 6;\n" " cvt.s64.s32 %rd60, %r42;\n" " mul.lo.s64 %rd61, %rd60, %rd4;\n" " add.s64 %rd62, %rd61, %rd5;\n" " shl.b64 %rd63, %rd62, 2;\n" " add.s64 %rd64, %rd1, %rd63;\n" " ld.global.f32 %f111, [%rd64];\n" " mov.f32 %f112, 0f00000000;\n" " fma.rn.f32 %f187, %f1, %f111, %f112;\n" "\n" "$L__BB19_17:\n" " setp.lt.s32 %p13, %r3, 8;\n" " @%p13 bra $L__BB19_19;\n" "\n" " add.s32 %r43, %r1, 7;\n" " cvt.s64.s32 %rd65, %r43;\n" " mul.lo.s64 %rd66, %rd65, %rd4;\n" " add.s64 %rd67, %rd66, %rd5;\n" " shl.b64 %rd68, %rd67, 2;\n" " add.s64 %rd69, %rd1, %rd68;\n" " ld.global.f32 %f114, [%rd69];\n" " mov.f32 %f115, 0f00000000;\n" " fma.rn.f32 %f188, %f1, %f114, %f115;\n" "\n" "$L__BB19_19:\n" " add.s32 %r71, %r5, %r6;\n" "\n" "$L__BB19_20:\n" " setp.gt.u32 %p14, %r6, %r7;\n" " @%p14 bra $L__BB19_55;\n" "\n" " cvt.s64.s32 %rd70, %r1;\n" " mul.lo.s64 %rd6, %rd70, %rd4;\n" " add.s32 %r44, %r1, 1;\n" " cvt.s64.s32 %rd71, %r44;\n" " mul.lo.s64 %rd7, %rd71, %rd4;\n" " add.s32 %r45, %r1, 2;\n" " cvt.s64.s32 %rd72, %r45;\n" " mul.lo.s64 %rd8, %rd72, %rd4;\n" " add.s32 %r46, %r1, 3;\n" " cvt.s64.s32 %rd73, %r46;\n" " mul.lo.s64 %rd9, %rd73, %rd4;\n" " add.s32 %r47, %r1, 4;\n" " cvt.s64.s32 %rd74, %r47;\n" " mul.lo.s64 %rd10, %rd74, %rd4;\n" " add.s32 %r48, %r1, 5;\n" " cvt.s64.s32 %rd75, %r48;\n" " mul.lo.s64 %rd11, %rd75, %rd4;\n" " add.s32 %r49, %r1, 6;\n" " cvt.s64.s32 %rd76, %r49;\n" " mul.lo.s64 %rd12, %rd76, %rd4;\n" " add.s32 %r50, %r1, 7;\n" " cvt.s64.s32 %rd77, %r50;\n" " mul.lo.s64 %rd13, %rd77, %rd4;\n" " setp.lt.s32 %p15, %r2, 1;\n" " setp.lt.s32 %p16, %r3, 2;\n" " setp.lt.s32 %p17, %r3, 3;\n" " setp.lt.s32 %p18, %r3, 4;\n" "\n" "$L__BB19_22:\n" " cvt.s64.s32 %rd14, %r71;\n" " mul.wide.s32 %rd78, %r71, 4;\n" " add.s64 %rd15, %rd3, %rd78;\n" " ld.global.f32 %f34, [%rd15];\n" " @%p15 bra $L__BB19_24;\n" "\n" " add.s64 %rd79, %rd6, %rd14;\n" " shl.b64 %rd80, %rd79, 2;\n" " add.s64 %rd81, %rd1, %rd80;\n" " ld.global.f32 %f116, [%rd81];\n" " fma.rn.f32 %f181, %f34, %f116, %f181;\n" "\n" "$L__BB19_24:\n" " @%p16 bra $L__BB19_26;\n" "\n" " add.s64 %rd82, %rd7, %rd14;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd84, %rd1, %rd83;\n" " ld.global.f32 %f117, [%rd84];\n" " fma.rn.f32 %f182, %f34, %f117, %f182;\n" "\n" "$L__BB19_26:\n" " @%p17 bra $L__BB19_28;\n" "\n" " add.s64 %rd85, %rd8, %rd14;\n" " shl.b64 %rd86, %rd85, 2;\n" " add.s64 %rd87, %rd1, %rd86;\n" " ld.global.f32 %f118, [%rd87];\n" " fma.rn.f32 %f183, %f34, %f118, %f183;\n" "\n" "$L__BB19_28:\n" " @%p18 bra $L__BB19_30;\n" "\n" " add.s64 %rd88, %rd9, %rd14;\n" " shl.b64 %rd89, %rd88, 2;\n" " add.s64 %rd90, %rd1, %rd89;\n" " ld.global.f32 %f119, [%rd90];\n" " fma.rn.f32 %f184, %f34, %f119, %f184;\n" "\n" "$L__BB19_30:\n" " setp.lt.s32 %p19, %r3, 5;\n" " @%p19 bra $L__BB19_32;\n" "\n" " add.s64 %rd91, %rd10, %rd14;\n" " shl.b64 %rd92, %rd91, 2;\n" " add.s64 %rd93, %rd1, %rd92;\n" " ld.global.f32 %f120, [%rd93];\n" " fma.rn.f32 %f185, %f34, %f120, %f185;\n" "\n" "$L__BB19_32:\n" " setp.lt.s32 %p20, %r3, 6;\n" " @%p20 bra $L__BB19_34;\n" "\n" " add.s64 %rd94, %rd11, %rd14;\n" " shl.b64 %rd95, %rd94, 2;\n" " add.s64 %rd96, %rd1, %rd95;\n" " ld.global.f32 %f121, [%rd96];\n" " fma.rn.f32 %f186, %f34, %f121, %f186;\n" "\n" "$L__BB19_34:\n" " setp.lt.s32 %p21, %r3, 7;\n" " @%p21 bra $L__BB19_36;\n" "\n" " add.s64 %rd97, %rd12, %rd14;\n" " shl.b64 %rd98, %rd97, 2;\n" " add.s64 %rd99, %rd1, %rd98;\n" " ld.global.f32 %f122, [%rd99];\n" " fma.rn.f32 %f187, %f34, %f122, %f187;\n" "\n" "$L__BB19_36:\n" " setp.lt.s32 %p22, %r3, 8;\n" " @%p22 bra $L__BB19_38;\n" "\n" " add.s64 %rd100, %rd13, %rd14;\n" " shl.b64 %rd101, %rd100, 2;\n" " add.s64 %rd102, %rd1, %rd101;\n" " ld.global.f32 %f123, [%rd102];\n" " fma.rn.f32 %f188, %f34, %f123, %f188;\n" "\n" "$L__BB19_38:\n" " cvt.u32.u64 %r51, %rd14;\n" " add.s32 %r52, %r51, %r6;\n" " cvt.s64.s32 %rd16, %r52;\n" " mul.wide.s32 %rd103, %r6, 4;\n" " add.s64 %rd104, %rd15, %rd103;\n" " ld.global.f32 %f51, [%rd104];\n" " @%p15 bra $L__BB19_40;\n" "\n" " add.s64 %rd105, %rd6, %rd16;\n" " shl.b64 %rd106, %rd105, 2;\n" " add.s64 %rd107, %rd1, %rd106;\n" " ld.global.f32 %f124, [%rd107];\n" " fma.rn.f32 %f181, %f51, %f124, %f181;\n" "\n" "$L__BB19_40:\n" " @%p16 bra $L__BB19_42;\n" "\n" " add.s64 %rd108, %rd7, %rd16;\n" " shl.b64 %rd109, %rd108, 2;\n" " add.s64 %rd110, %rd1, %rd109;\n" " ld.global.f32 %f125, [%rd110];\n" " fma.rn.f32 %f182, %f51, %f125, %f182;\n" "\n" "$L__BB19_42:\n" " @%p17 bra $L__BB19_44;\n" "\n" " add.s64 %rd111, %rd8, %rd16;\n" " shl.b64 %rd112, %rd111, 2;\n" " add.s64 %rd113, %rd1, %rd112;\n" " ld.global.f32 %f126, [%rd113];\n" " fma.rn.f32 %f183, %f51, %f126, %f183;\n" "\n" "$L__BB19_44:\n" " @%p18 bra $L__BB19_46;\n" "\n" " add.s64 %rd114, %rd9, %rd16;\n" " shl.b64 %rd115, %rd114, 2;\n" " add.s64 %rd116, %rd1, %rd115;\n" " ld.global.f32 %f127, [%rd116];\n" " fma.rn.f32 %f184, %f51, %f127, %f184;\n" "\n" "$L__BB19_46:\n" " @%p19 bra $L__BB19_48;\n" "\n" " add.s64 %rd117, %rd10, %rd16;\n" " shl.b64 %rd118, %rd117, 2;\n" " add.s64 %rd119, %rd1, %rd118;\n" " ld.global.f32 %f128, [%rd119];\n" " fma.rn.f32 %f185, %f51, %f128, %f185;\n" "\n" "$L__BB19_48:\n" " @%p20 bra $L__BB19_50;\n" "\n" " add.s64 %rd120, %rd11, %rd16;\n" " shl.b64 %rd121, %rd120, 2;\n" " add.s64 %rd122, %rd1, %rd121;\n" " ld.global.f32 %f129, [%rd122];\n" " fma.rn.f32 %f186, %f51, %f129, %f186;\n" "\n" "$L__BB19_50:\n" " @%p21 bra $L__BB19_52;\n" "\n" " add.s64 %rd123, %rd12, %rd16;\n" " shl.b64 %rd124, %rd123, 2;\n" " add.s64 %rd125, %rd1, %rd124;\n" " ld.global.f32 %f130, [%rd125];\n" " fma.rn.f32 %f187, %f51, %f130, %f187;\n" "\n" "$L__BB19_52:\n" " @%p22 bra $L__BB19_54;\n" "\n" " add.s64 %rd126, %rd13, %rd16;\n" " shl.b64 %rd127, %rd126, 2;\n" " add.s64 %rd128, %rd1, %rd127;\n" " ld.global.f32 %f131, [%rd128];\n" " fma.rn.f32 %f188, %f51, %f131, %f188;\n" "\n" "$L__BB19_54:\n" " cvt.u32.u64 %r53, %rd16;\n" " add.s32 %r71, %r53, %r6;\n" " setp.lt.s32 %p31, %r71, %r30;\n" " @%p31 bra $L__BB19_22;\n" "\n" "$L__BB19_55:\n" " shl.b32 %r54, %r5, 2;\n" " mov.u32 %r73, fc_kq_shared;\n" " add.s32 %r12, %r73, %r54;\n" " st.shared.f32 [%r12], %f181;\n" " shl.b32 %r56, %r6, 2;\n" " add.s32 %r13, %r12, %r56;\n" " st.shared.f32 [%r13], %f182;\n" " add.s32 %r14, %r13, %r56;\n" " st.shared.f32 [%r14], %f183;\n" " add.s32 %r15, %r14, %r56;\n" " st.shared.f32 [%r15], %f184;\n" " add.s32 %r16, %r15, %r56;\n" " st.shared.f32 [%r16], %f185;\n" " add.s32 %r17, %r16, %r56;\n" " st.shared.f32 [%r17], %f186;\n" " add.s32 %r18, %r17, %r56;\n" " st.shared.f32 [%r18], %f187;\n" " add.s32 %r19, %r18, %r56;\n" " st.shared.f32 [%r19], %f188;\n" " bar.sync 0;\n" " setp.lt.s32 %p32, %r6, 2;\n" " @%p32 bra $L__BB19_60;\n" "\n" " shr.u32 %r57, %r6, 31;\n" " add.s32 %r58, %r6, %r57;\n" " shr.s32 %r72, %r58, 1;\n" "\n" "$L__BB19_57:\n" " setp.ge.s32 %p33, %r5, %r72;\n" " @%p33 bra $L__BB19_59;\n" "\n" " shl.b32 %r59, %r72, 2;\n" " add.s32 %r60, %r12, %r59;\n" " ld.shared.f32 %f132, [%r12];\n" " ld.shared.f32 %f133, [%r60];\n" " add.f32 %f134, %f133, %f132;\n" " st.shared.f32 [%r12], %f134;\n" " add.s32 %r61, %r60, %r56;\n" " ld.shared.f32 %f135, [%r13];\n" " ld.shared.f32 %f136, [%r61];\n" " add.f32 %f137, %f136, %f135;\n" " st.shared.f32 [%r13], %f137;\n" " add.s32 %r62, %r61, %r56;\n" " ld.shared.f32 %f138, [%r14];\n" " ld.shared.f32 %f139, [%r62];\n" " add.f32 %f140, %f139, %f138;\n" " st.shared.f32 [%r14], %f140;\n" " add.s32 %r63, %r62, %r56;\n" " ld.shared.f32 %f141, [%r15];\n" " ld.shared.f32 %f142, [%r63];\n" " add.f32 %f143, %f142, %f141;\n" " st.shared.f32 [%r15], %f143;\n" " add.s32 %r64, %r63, %r56;\n" " ld.shared.f32 %f144, [%r16];\n" " ld.shared.f32 %f145, [%r64];\n" " add.f32 %f146, %f145, %f144;\n" " st.shared.f32 [%r16], %f146;\n" " add.s32 %r65, %r64, %r56;\n" " ld.shared.f32 %f147, [%r17];\n" " ld.shared.f32 %f148, [%r65];\n" " add.f32 %f149, %f148, %f147;\n" " st.shared.f32 [%r17], %f149;\n" " add.s32 %r66, %r65, %r56;\n" " ld.shared.f32 %f150, [%r18];\n" " ld.shared.f32 %f151, [%r66];\n" " add.f32 %f152, %f151, %f150;\n" " st.shared.f32 [%r18], %f152;\n" " add.s32 %r67, %r66, %r56;\n" " ld.shared.f32 %f153, [%r19];\n" " ld.shared.f32 %f154, [%r67];\n" " add.f32 %f155, %f154, %f153;\n" " st.shared.f32 [%r19], %f155;\n" "\n" "$L__BB19_59:\n" " bar.sync 0;\n" " shr.s32 %r23, %r72, 1;\n" " setp.gt.s32 %p34, %r72, 1;\n" " mov.u32 %r72, %r23;\n" " @%p34 bra $L__BB19_57;\n" "\n" "$L__BB19_60:\n" " setp.ne.s32 %p35, %r5, 0;\n" " setp.lt.s32 %p36, %r2, 1;\n" " or.pred %p37, %p35, %p36;\n" " @%p37 bra $L__BB19_63;\n" "\n" " cvt.s64.s32 %rd17, %r29;\n" " cvt.s64.s32 %rd133, %r1;\n" " cvta.to.global.u64 %rd19, %rd23;\n" " mov.u32 %r74, 0;\n" "\n" "$L__BB19_62:\n" " ld.shared.f32 %f156, [%r73];\n" " mul.lo.s64 %rd129, %rd133, %rd17;\n" " add.s64 %rd130, %rd129, %rd2;\n" " shl.b64 %rd131, %rd130, 2;\n" " add.s64 %rd132, %rd19, %rd131;\n" " st.global.f32 [%rd132], %f156;\n" " add.s32 %r73, %r73, %r56;\n" " add.s64 %rd133, %rd133, 1;\n" " add.s32 %r74, %r74, 1;\n" " setp.lt.s32 %p38, %r74, %r3;\n" " @%p38 bra $L__BB19_62;\n" "\n" "$L__BB19_63:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f16\n" ".visible .entry fc_matvec_f16(\n" " .param .u64 fc_matvec_f16_param_0,\n" " .param .u64 fc_matvec_f16_param_1,\n" " .param .u64 fc_matvec_f16_param_2,\n" " .param .u32 fc_matvec_f16_param_3,\n" " .param .u32 fc_matvec_f16_param_4,\n" " .param .u32 fc_matvec_f16_param_5\n" ")\n" "{\n" " .reg .pred %p<8>;\n" " .reg .b16 %rs<2>;\n" " .reg .f32 %f<14>;\n" " .reg .b32 %r<21>;\n" " .reg .b64 %rd<17>;\n" "\n" "\n" " ld.param.u64 %rd4, [fc_matvec_f16_param_0];\n" " ld.param.u64 %rd5, [fc_matvec_f16_param_1];\n" " ld.param.u64 %rd6, [fc_matvec_f16_param_2];\n" " ld.param.u32 %r12, [fc_matvec_f16_param_3];\n" " ld.param.u32 %r10, [fc_matvec_f16_param_4];\n" " ld.param.u32 %r11, [fc_matvec_f16_param_5];\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r12;\n" " @%p1 bra $L__BB20_11;\n" "\n" " mov.u32 %r2, %ntid.x;\n" " cvt.s64.s32 %rd1, %r1;\n" " mov.u32 %r3, %tid.x;\n" " setp.ge.s32 %p2, %r3, %r10;\n" " mov.f32 %f13, 0f00000000;\n" " @%p2 bra $L__BB20_4;\n" "\n" " cvta.to.global.u64 %rd2, %rd5;\n" " cvt.s64.s32 %rd7, %r11;\n" " mul.lo.s64 %rd8, %rd7, %rd1;\n" " cvta.to.global.u64 %rd9, %rd4;\n" " add.s64 %rd3, %rd9, %rd8;\n" " mov.f32 %f13, 0f00000000;\n" " mov.u32 %r19, %r3;\n" "\n" "$L__BB20_3:\n" " mul.wide.s32 %rd10, %r19, 2;\n" " add.s64 %rd11, %rd3, %rd10;\n" " ld.global.u16 %rs1, [%rd11];\n" " // begin inline asm\n" " cvt.f32.f16 %f6, %rs1;\n" " // end inline asm\n" " mul.wide.s32 %rd12, %r19, 4;\n" " add.s64 %rd13, %rd2, %rd12;\n" " ld.global.f32 %f7, [%rd13];\n" " fma.rn.f32 %f13, %f6, %f7, %f13;\n" " add.s32 %r19, %r19, %r2;\n" " setp.lt.s32 %p3, %r19, %r10;\n" " @%p3 bra $L__BB20_3;\n" "\n" "$L__BB20_4:\n" " shl.b32 %r13, %r3, 2;\n" " mov.u32 %r14, fc_kq_shared;\n" " add.s32 %r6, %r14, %r13;\n" " st.shared.f32 [%r6], %f13;\n" " bar.sync 0;\n" " setp.lt.s32 %p4, %r2, 2;\n" " @%p4 bra $L__BB20_9;\n" "\n" " shr.u32 %r15, %r2, 31;\n" " add.s32 %r16, %r2, %r15;\n" " shr.s32 %r20, %r16, 1;\n" "\n" "$L__BB20_6:\n" " setp.ge.s32 %p5, %r3, %r20;\n" " @%p5 bra $L__BB20_8;\n" "\n" " shl.b32 %r17, %r20, 2;\n" " add.s32 %r18, %r6, %r17;\n" " ld.shared.f32 %f8, [%r6];\n" " ld.shared.f32 %f9, [%r18];\n" " add.f32 %f10, %f9, %f8;\n" " st.shared.f32 [%r6], %f10;\n" "\n" "$L__BB20_8:\n" " bar.sync 0;\n" " shr.s32 %r9, %r20, 1;\n" " setp.gt.s32 %p6, %r20, 1;\n" " mov.u32 %r20, %r9;\n" " @%p6 bra $L__BB20_6;\n" "\n" "$L__BB20_9:\n" " setp.ne.s32 %p7, %r3, 0;\n" " @%p7 bra $L__BB20_11;\n" "\n" " ld.shared.f32 %f11, [fc_kq_shared];\n" " cvta.to.global.u64 %rd14, %rd6;\n" " shl.b64 %rd15, %rd1, 2;\n" " add.s64 %rd16, %rd14, %rd15;\n" " st.global.f32 [%rd16], %f11;\n" "\n" "$L__BB20_11:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f16_nx2\n" ".visible .entry fc_matvec_f16_nx2(\n" " .param .u64 fc_matvec_f16_nx2_param_0,\n" " .param .u64 fc_matvec_f16_nx2_param_1,\n" " .param .u64 fc_matvec_f16_nx2_param_2,\n" " .param .u32 fc_matvec_f16_nx2_param_3,\n" " .param .u32 fc_matvec_f16_nx2_param_4,\n" " .param .u32 fc_matvec_f16_nx2_param_5,\n" " .param .u32 fc_matvec_f16_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<13>;\n" " .reg .b16 %rs<2>;\n" " .reg .f32 %f<30>;\n" " .reg .b32 %r<40>;\n" " .reg .b64 %rd<31>;\n" "\n" "\n" " ld.param.u64 %rd12, [fc_matvec_f16_nx2_param_0];\n" " ld.param.u64 %rd14, [fc_matvec_f16_nx2_param_1];\n" " ld.param.u64 %rd13, [fc_matvec_f16_nx2_param_2];\n" " ld.param.u32 %r19, [fc_matvec_f16_nx2_param_3];\n" " ld.param.u32 %r20, [fc_matvec_f16_nx2_param_4];\n" " ld.param.u32 %r21, [fc_matvec_f16_nx2_param_5];\n" " ld.param.u32 %r22, [fc_matvec_f16_nx2_param_6];\n" " cvta.to.global.u64 %rd1, %rd14;\n" " mov.u32 %r23, %ctaid.y;\n" " shl.b32 %r1, %r23, 1;\n" " sub.s32 %r2, %r22, %r1;\n" " min.s32 %r3, %r2, 2;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r19;\n" " @%p1 bra $L__BB21_16;\n" "\n" " mov.u32 %r5, %ntid.x;\n" " cvt.s64.s32 %rd2, %r4;\n" " mov.u32 %r6, %tid.x;\n" " setp.ge.s32 %p2, %r6, %r20;\n" " mov.f32 %f27, 0f00000000;\n" " mov.f32 %f26, %f27;\n" " @%p2 bra $L__BB21_8;\n" "\n" " mul.wide.s32 %rd3, %r1, %r20;\n" " add.s32 %r24, %r1, 1;\n" " mul.wide.s32 %rd4, %r24, %r20;\n" " cvt.s64.s32 %rd15, %r21;\n" " mul.lo.s64 %rd16, %rd15, %rd2;\n" " cvta.to.global.u64 %rd17, %rd12;\n" " add.s64 %rd5, %rd17, %rd16;\n" " mov.f32 %f27, 0f00000000;\n" " setp.lt.s32 %p3, %r2, 1;\n" " setp.lt.s32 %p4, %r3, 2;\n" " mov.f32 %f26, %f27;\n" " mov.u32 %r36, %r6;\n" "\n" "$L__BB21_3:\n" " cvt.s64.s32 %rd6, %r36;\n" " mul.wide.s32 %rd18, %r36, 2;\n" " add.s64 %rd19, %rd5, %rd18;\n" " ld.global.u16 %rs1, [%rd19];\n" " // begin inline asm\n" " cvt.f32.f16 %f14, %rs1;\n" " // end inline asm\n" " @%p3 bra $L__BB21_5;\n" "\n" " add.s64 %rd20, %rd3, %rd6;\n" " shl.b64 %rd21, %rd20, 2;\n" " add.s64 %rd22, %rd1, %rd21;\n" " ld.global.f32 %f15, [%rd22];\n" " fma.rn.f32 %f26, %f14, %f15, %f26;\n" "\n" "$L__BB21_5:\n" " @%p4 bra $L__BB21_7;\n" "\n" " add.s64 %rd23, %rd4, %rd6;\n" " shl.b64 %rd24, %rd23, 2;\n" " add.s64 %rd25, %rd1, %rd24;\n" " ld.global.f32 %f16, [%rd25];\n" " fma.rn.f32 %f27, %f14, %f16, %f27;\n" "\n" "$L__BB21_7:\n" " cvt.u32.u64 %r25, %rd6;\n" " add.s32 %r36, %r25, %r5;\n" " setp.lt.s32 %p5, %r36, %r20;\n" " @%p5 bra $L__BB21_3;\n" "\n" "$L__BB21_8:\n" " shl.b32 %r26, %r6, 2;\n" " mov.u32 %r38, fc_kq_shared;\n" " add.s32 %r9, %r38, %r26;\n" " st.shared.f32 [%r9], %f26;\n" " shl.b32 %r28, %r5, 2;\n" " add.s32 %r10, %r9, %r28;\n" " st.shared.f32 [%r10], %f27;\n" " bar.sync 0;\n" " setp.lt.s32 %p6, %r5, 2;\n" " @%p6 bra $L__BB21_13;\n" "\n" " shr.u32 %r29, %r5, 31;\n" " add.s32 %r30, %r5, %r29;\n" " shr.s32 %r37, %r30, 1;\n" "\n" "$L__BB21_10:\n" " setp.ge.s32 %p7, %r6, %r37;\n" " @%p7 bra $L__BB21_12;\n" "\n" " shl.b32 %r31, %r37, 2;\n" " add.s32 %r32, %r9, %r31;\n" " ld.shared.f32 %f17, [%r9];\n" " ld.shared.f32 %f18, [%r32];\n" " add.f32 %f19, %f18, %f17;\n" " st.shared.f32 [%r9], %f19;\n" " ld.shared.f32 %f20, [%r10];\n" " add.s32 %r33, %r10, %r31;\n" " ld.shared.f32 %f21, [%r33];\n" " add.f32 %f22, %f21, %f20;\n" " st.shared.f32 [%r10], %f22;\n" "\n" "$L__BB21_12:\n" " bar.sync 0;\n" " shr.s32 %r13, %r37, 1;\n" " setp.gt.s32 %p8, %r37, 1;\n" " mov.u32 %r37, %r13;\n" " @%p8 bra $L__BB21_10;\n" "\n" "$L__BB21_13:\n" " setp.ne.s32 %p9, %r6, 0;\n" " setp.lt.s32 %p10, %r2, 1;\n" " or.pred %p11, %p9, %p10;\n" " @%p11 bra $L__BB21_16;\n" "\n" " cvt.s64.s32 %rd7, %r19;\n" " cvt.s64.s32 %rd30, %r1;\n" " cvta.to.global.u64 %rd9, %rd13;\n" " mov.u32 %r39, 0;\n" "\n" "$L__BB21_15:\n" " ld.shared.f32 %f23, [%r38];\n" " mul.lo.s64 %rd26, %rd30, %rd7;\n" " add.s64 %rd27, %rd26, %rd2;\n" " shl.b64 %rd28, %rd27, 2;\n" " add.s64 %rd29, %rd9, %rd28;\n" " st.global.f32 [%rd29], %f23;\n" " add.s32 %r38, %r38, %r28;\n" " add.s64 %rd30, %rd30, 1;\n" " add.s32 %r39, %r39, 1;\n" " setp.lt.s32 %p12, %r39, %r3;\n" " @%p12 bra $L__BB21_15;\n" "\n" "$L__BB21_16:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f16_nx\n" ".visible .entry fc_matvec_f16_nx(\n" " .param .u64 fc_matvec_f16_nx_param_0,\n" " .param .u64 fc_matvec_f16_nx_param_1,\n" " .param .u64 fc_matvec_f16_nx_param_2,\n" " .param .u32 fc_matvec_f16_nx_param_3,\n" " .param .u32 fc_matvec_f16_nx_param_4,\n" " .param .u32 fc_matvec_f16_nx_param_5,\n" " .param .u32 fc_matvec_f16_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .b16 %rs<2>;\n" " .reg .f32 %f<56>;\n" " .reg .b32 %r<47>;\n" " .reg .b64 %rd<39>;\n" "\n" "\n" " ld.param.u64 %rd14, [fc_matvec_f16_nx_param_0];\n" " ld.param.u64 %rd16, [fc_matvec_f16_nx_param_1];\n" " ld.param.u64 %rd15, [fc_matvec_f16_nx_param_2];\n" " ld.param.u32 %r22, [fc_matvec_f16_nx_param_3];\n" " ld.param.u32 %r23, [fc_matvec_f16_nx_param_4];\n" " ld.param.u32 %r24, [fc_matvec_f16_nx_param_5];\n" " ld.param.u32 %r25, [fc_matvec_f16_nx_param_6];\n" " cvta.to.global.u64 %rd1, %rd16;\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r1, %r26, 2;\n" " sub.s32 %r2, %r25, %r1;\n" " min.s32 %r3, %r2, 4;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r22;\n" " @%p1 bra $L__BB22_20;\n" "\n" " mov.u32 %r5, %tid.x;\n" " cvt.s64.s32 %rd2, %r4;\n" " mov.u32 %r6, %ntid.x;\n" " setp.ge.s32 %p2, %r5, %r23;\n" " mov.f32 %f51, 0f00000000;\n" " mov.f32 %f50, %f51;\n" " mov.f32 %f49, %f51;\n" " mov.f32 %f48, %f51;\n" " @%p2 bra $L__BB22_12;\n" "\n" " mul.wide.s32 %rd3, %r1, %r23;\n" " add.s32 %r27, %r1, 1;\n" " mul.wide.s32 %rd4, %r27, %r23;\n" " add.s32 %r28, %r1, 2;\n" " mul.wide.s32 %rd5, %r28, %r23;\n" " add.s32 %r29, %r1, 3;\n" " mul.wide.s32 %rd6, %r29, %r23;\n" " cvta.to.global.u64 %rd17, %rd14;\n" " cvt.s64.s32 %rd18, %r24;\n" " mul.lo.s64 %rd19, %rd18, %rd2;\n" " add.s64 %rd7, %rd17, %rd19;\n" " mov.f32 %f51, 0f00000000;\n" " setp.lt.s32 %p3, %r2, 1;\n" " setp.lt.s32 %p4, %r3, 2;\n" " setp.lt.s32 %p5, %r3, 3;\n" " setp.lt.s32 %p6, %r3, 4;\n" " mov.f32 %f50, %f51;\n" " mov.f32 %f49, %f51;\n" " mov.f32 %f48, %f51;\n" " mov.u32 %r43, %r5;\n" "\n" "$L__BB22_3:\n" " cvt.s64.s32 %rd8, %r43;\n" " mul.wide.s32 %rd20, %r43, 2;\n" " add.s64 %rd21, %rd7, %rd20;\n" " ld.global.u16 %rs1, [%rd21];\n" " // begin inline asm\n" " cvt.f32.f16 %f26, %rs1;\n" " // end inline asm\n" " @%p3 bra $L__BB22_5;\n" "\n" " add.s64 %rd22, %rd3, %rd8;\n" " shl.b64 %rd23, %rd22, 2;\n" " add.s64 %rd24, %rd1, %rd23;\n" " ld.global.f32 %f27, [%rd24];\n" " fma.rn.f32 %f48, %f26, %f27, %f48;\n" "\n" "$L__BB22_5:\n" " @%p4 bra $L__BB22_7;\n" "\n" " add.s64 %rd25, %rd4, %rd8;\n" " shl.b64 %rd26, %rd25, 2;\n" " add.s64 %rd27, %rd1, %rd26;\n" " ld.global.f32 %f28, [%rd27];\n" " fma.rn.f32 %f49, %f26, %f28, %f49;\n" "\n" "$L__BB22_7:\n" " @%p5 bra $L__BB22_9;\n" "\n" " add.s64 %rd28, %rd5, %rd8;\n" " shl.b64 %rd29, %rd28, 2;\n" " add.s64 %rd30, %rd1, %rd29;\n" " ld.global.f32 %f29, [%rd30];\n" " fma.rn.f32 %f50, %f26, %f29, %f50;\n" "\n" "$L__BB22_9:\n" " @%p6 bra $L__BB22_11;\n" "\n" " add.s64 %rd31, %rd6, %rd8;\n" " shl.b64 %rd32, %rd31, 2;\n" " add.s64 %rd33, %rd1, %rd32;\n" " ld.global.f32 %f30, [%rd33];\n" " fma.rn.f32 %f51, %f26, %f30, %f51;\n" "\n" "$L__BB22_11:\n" " cvt.u32.u64 %r30, %rd8;\n" " add.s32 %r43, %r30, %r6;\n" " setp.lt.s32 %p7, %r43, %r23;\n" " @%p7 bra $L__BB22_3;\n" "\n" "$L__BB22_12:\n" " shl.b32 %r31, %r5, 2;\n" " mov.u32 %r45, fc_kq_shared;\n" " add.s32 %r9, %r45, %r31;\n" " st.shared.f32 [%r9], %f48;\n" " shl.b32 %r33, %r6, 2;\n" " add.s32 %r10, %r9, %r33;\n" " st.shared.f32 [%r10], %f49;\n" " add.s32 %r11, %r10, %r33;\n" " st.shared.f32 [%r11], %f50;\n" " add.s32 %r12, %r11, %r33;\n" " st.shared.f32 [%r12], %f51;\n" " bar.sync 0;\n" " setp.lt.s32 %p8, %r6, 2;\n" " @%p8 bra $L__BB22_17;\n" "\n" " shr.u32 %r34, %r6, 31;\n" " add.s32 %r35, %r6, %r34;\n" " shr.s32 %r44, %r35, 1;\n" "\n" "$L__BB22_14:\n" " setp.ge.s32 %p9, %r5, %r44;\n" " @%p9 bra $L__BB22_16;\n" "\n" " shl.b32 %r36, %r44, 2;\n" " add.s32 %r37, %r9, %r36;\n" " ld.shared.f32 %f31, [%r9];\n" " ld.shared.f32 %f32, [%r37];\n" " add.f32 %f33, %f32, %f31;\n" " st.shared.f32 [%r9], %f33;\n" " add.s32 %r38, %r37, %r33;\n" " ld.shared.f32 %f34, [%r10];\n" " ld.shared.f32 %f35, [%r38];\n" " add.f32 %f36, %f35, %f34;\n" " st.shared.f32 [%r10], %f36;\n" " add.s32 %r39, %r38, %r33;\n" " ld.shared.f32 %f37, [%r11];\n" " ld.shared.f32 %f38, [%r39];\n" " add.f32 %f39, %f38, %f37;\n" " st.shared.f32 [%r11], %f39;\n" " add.s32 %r40, %r39, %r33;\n" " ld.shared.f32 %f40, [%r12];\n" " ld.shared.f32 %f41, [%r40];\n" " add.f32 %f42, %f41, %f40;\n" " st.shared.f32 [%r12], %f42;\n" "\n" "$L__BB22_16:\n" " bar.sync 0;\n" " shr.s32 %r16, %r44, 1;\n" " setp.gt.s32 %p10, %r44, 1;\n" " mov.u32 %r44, %r16;\n" " @%p10 bra $L__BB22_14;\n" "\n" "$L__BB22_17:\n" " setp.ne.s32 %p11, %r5, 0;\n" " setp.lt.s32 %p12, %r2, 1;\n" " or.pred %p13, %p11, %p12;\n" " @%p13 bra $L__BB22_20;\n" "\n" " cvt.s64.s32 %rd9, %r22;\n" " cvt.s64.s32 %rd38, %r1;\n" " cvta.to.global.u64 %rd11, %rd15;\n" " mov.u32 %r46, 0;\n" "\n" "$L__BB22_19:\n" " ld.shared.f32 %f43, [%r45];\n" " mul.lo.s64 %rd34, %rd38, %rd9;\n" " add.s64 %rd35, %rd34, %rd2;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd37, %rd11, %rd36;\n" " st.global.f32 [%rd37], %f43;\n" " add.s32 %r45, %r45, %r33;\n" " add.s64 %rd38, %rd38, 1;\n" " add.s32 %r46, %r46, 1;\n" " setp.lt.s32 %p14, %r46, %r3;\n" " @%p14 bra $L__BB22_19;\n" "\n" "$L__BB22_20:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_f16_nx8\n" ".visible .entry fc_matvec_f16_nx8(\n" " .param .u64 fc_matvec_f16_nx8_param_0,\n" " .param .u64 fc_matvec_f16_nx8_param_1,\n" " .param .u64 fc_matvec_f16_nx8_param_2,\n" " .param .u32 fc_matvec_f16_nx8_param_3,\n" " .param .u32 fc_matvec_f16_nx8_param_4,\n" " .param .u32 fc_matvec_f16_nx8_param_5,\n" " .param .u32 fc_matvec_f16_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<19>;\n" " .reg .b16 %rs<2>;\n" " .reg .f32 %f<108>;\n" " .reg .b32 %r<59>;\n" " .reg .b64 %rd<55>;\n" "\n" "\n" " ld.param.u64 %rd18, [fc_matvec_f16_nx8_param_0];\n" " ld.param.u64 %rd20, [fc_matvec_f16_nx8_param_1];\n" " ld.param.u64 %rd19, [fc_matvec_f16_nx8_param_2];\n" " ld.param.u32 %r26, [fc_matvec_f16_nx8_param_3];\n" " ld.param.u32 %r27, [fc_matvec_f16_nx8_param_4];\n" " ld.param.u32 %r28, [fc_matvec_f16_nx8_param_5];\n" " ld.param.u32 %r29, [fc_matvec_f16_nx8_param_6];\n" " cvta.to.global.u64 %rd1, %rd20;\n" " mov.u32 %r30, %ctaid.y;\n" " shl.b32 %r1, %r30, 3;\n" " sub.s32 %r2, %r29, %r1;\n" " min.s32 %r3, %r2, 8;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r26;\n" " @%p1 bra $L__BB23_28;\n" "\n" " mov.u32 %r5, %tid.x;\n" " cvt.s64.s32 %rd2, %r4;\n" " mov.u32 %r6, %ntid.x;\n" " setp.ge.s32 %p2, %r5, %r27;\n" " mov.f32 %f99, 0f00000000;\n" " mov.f32 %f98, %f99;\n" " mov.f32 %f97, %f99;\n" " mov.f32 %f96, %f99;\n" " mov.f32 %f95, %f99;\n" " mov.f32 %f94, %f99;\n" " mov.f32 %f93, %f99;\n" " mov.f32 %f92, %f99;\n" " @%p2 bra $L__BB23_20;\n" "\n" " mul.wide.s32 %rd3, %r1, %r27;\n" " add.s32 %r31, %r1, 1;\n" " mul.wide.s32 %rd4, %r31, %r27;\n" " add.s32 %r32, %r1, 2;\n" " mul.wide.s32 %rd5, %r32, %r27;\n" " add.s32 %r33, %r1, 3;\n" " mul.wide.s32 %rd6, %r33, %r27;\n" " add.s32 %r34, %r1, 4;\n" " mul.wide.s32 %rd7, %r34, %r27;\n" " add.s32 %r35, %r1, 5;\n" " mul.wide.s32 %rd8, %r35, %r27;\n" " add.s32 %r36, %r1, 6;\n" " mul.wide.s32 %rd9, %r36, %r27;\n" " add.s32 %r37, %r1, 7;\n" " mul.wide.s32 %rd10, %r37, %r27;\n" " cvta.to.global.u64 %rd21, %rd18;\n" " cvt.s64.s32 %rd22, %r28;\n" " mul.lo.s64 %rd23, %rd22, %rd2;\n" " add.s64 %rd11, %rd21, %rd23;\n" " mov.f32 %f99, 0f00000000;\n" " setp.lt.s32 %p3, %r2, 1;\n" " setp.lt.s32 %p4, %r3, 2;\n" " setp.lt.s32 %p5, %r3, 3;\n" " setp.lt.s32 %p6, %r3, 4;\n" " mov.f32 %f98, %f99;\n" " mov.f32 %f97, %f99;\n" " mov.f32 %f96, %f99;\n" " mov.f32 %f95, %f99;\n" " mov.f32 %f94, %f99;\n" " mov.f32 %f93, %f99;\n" " mov.f32 %f92, %f99;\n" " mov.u32 %r55, %r5;\n" "\n" "$L__BB23_3:\n" " cvt.s64.s32 %rd12, %r55;\n" " mul.wide.s32 %rd24, %r55, 2;\n" " add.s64 %rd25, %rd11, %rd24;\n" " ld.global.u16 %rs1, [%rd25];\n" " // begin inline asm\n" " cvt.f32.f16 %f50, %rs1;\n" " // end inline asm\n" " @%p3 bra $L__BB23_5;\n" "\n" " add.s64 %rd26, %rd3, %rd12;\n" " shl.b64 %rd27, %rd26, 2;\n" " add.s64 %rd28, %rd1, %rd27;\n" " ld.global.f32 %f51, [%rd28];\n" " fma.rn.f32 %f92, %f50, %f51, %f92;\n" "\n" "$L__BB23_5:\n" " @%p4 bra $L__BB23_7;\n" "\n" " add.s64 %rd29, %rd4, %rd12;\n" " shl.b64 %rd30, %rd29, 2;\n" " add.s64 %rd31, %rd1, %rd30;\n" " ld.global.f32 %f52, [%rd31];\n" " fma.rn.f32 %f93, %f50, %f52, %f93;\n" "\n" "$L__BB23_7:\n" " @%p5 bra $L__BB23_9;\n" "\n" " add.s64 %rd32, %rd5, %rd12;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd34, %rd1, %rd33;\n" " ld.global.f32 %f53, [%rd34];\n" " fma.rn.f32 %f94, %f50, %f53, %f94;\n" "\n" "$L__BB23_9:\n" " @%p6 bra $L__BB23_11;\n" "\n" " add.s64 %rd35, %rd6, %rd12;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd37, %rd1, %rd36;\n" " ld.global.f32 %f54, [%rd37];\n" " fma.rn.f32 %f95, %f50, %f54, %f95;\n" "\n" "$L__BB23_11:\n" " setp.lt.s32 %p7, %r3, 5;\n" " @%p7 bra $L__BB23_13;\n" "\n" " add.s64 %rd38, %rd7, %rd12;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd40, %rd1, %rd39;\n" " ld.global.f32 %f55, [%rd40];\n" " fma.rn.f32 %f96, %f50, %f55, %f96;\n" "\n" "$L__BB23_13:\n" " setp.lt.s32 %p8, %r3, 6;\n" " @%p8 bra $L__BB23_15;\n" "\n" " add.s64 %rd41, %rd8, %rd12;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd43, %rd1, %rd42;\n" " ld.global.f32 %f56, [%rd43];\n" " fma.rn.f32 %f97, %f50, %f56, %f97;\n" "\n" "$L__BB23_15:\n" " setp.lt.s32 %p9, %r3, 7;\n" " @%p9 bra $L__BB23_17;\n" "\n" " add.s64 %rd44, %rd9, %rd12;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd46, %rd1, %rd45;\n" " ld.global.f32 %f57, [%rd46];\n" " fma.rn.f32 %f98, %f50, %f57, %f98;\n" "\n" "$L__BB23_17:\n" " setp.lt.s32 %p10, %r3, 8;\n" " @%p10 bra $L__BB23_19;\n" "\n" " add.s64 %rd47, %rd10, %rd12;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd49, %rd1, %rd48;\n" " ld.global.f32 %f58, [%rd49];\n" " fma.rn.f32 %f99, %f50, %f58, %f99;\n" "\n" "$L__BB23_19:\n" " cvt.u32.u64 %r38, %rd12;\n" " add.s32 %r55, %r38, %r6;\n" " setp.lt.s32 %p11, %r55, %r27;\n" " @%p11 bra $L__BB23_3;\n" "\n" "$L__BB23_20:\n" " shl.b32 %r39, %r5, 2;\n" " mov.u32 %r57, fc_kq_shared;\n" " add.s32 %r9, %r57, %r39;\n" " st.shared.f32 [%r9], %f92;\n" " shl.b32 %r41, %r6, 2;\n" " add.s32 %r10, %r9, %r41;\n" " st.shared.f32 [%r10], %f93;\n" " add.s32 %r11, %r10, %r41;\n" " st.shared.f32 [%r11], %f94;\n" " add.s32 %r12, %r11, %r41;\n" " st.shared.f32 [%r12], %f95;\n" " add.s32 %r13, %r12, %r41;\n" " st.shared.f32 [%r13], %f96;\n" " add.s32 %r14, %r13, %r41;\n" " st.shared.f32 [%r14], %f97;\n" " add.s32 %r15, %r14, %r41;\n" " st.shared.f32 [%r15], %f98;\n" " add.s32 %r16, %r15, %r41;\n" " st.shared.f32 [%r16], %f99;\n" " bar.sync 0;\n" " setp.lt.s32 %p12, %r6, 2;\n" " @%p12 bra $L__BB23_25;\n" "\n" " shr.u32 %r42, %r6, 31;\n" " add.s32 %r43, %r6, %r42;\n" " shr.s32 %r56, %r43, 1;\n" "\n" "$L__BB23_22:\n" " setp.ge.s32 %p13, %r5, %r56;\n" " @%p13 bra $L__BB23_24;\n" "\n" " shl.b32 %r44, %r56, 2;\n" " add.s32 %r45, %r9, %r44;\n" " ld.shared.f32 %f59, [%r9];\n" " ld.shared.f32 %f60, [%r45];\n" " add.f32 %f61, %f60, %f59;\n" " st.shared.f32 [%r9], %f61;\n" " add.s32 %r46, %r45, %r41;\n" " ld.shared.f32 %f62, [%r10];\n" " ld.shared.f32 %f63, [%r46];\n" " add.f32 %f64, %f63, %f62;\n" " st.shared.f32 [%r10], %f64;\n" " add.s32 %r47, %r46, %r41;\n" " ld.shared.f32 %f65, [%r11];\n" " ld.shared.f32 %f66, [%r47];\n" " add.f32 %f67, %f66, %f65;\n" " st.shared.f32 [%r11], %f67;\n" " add.s32 %r48, %r47, %r41;\n" " ld.shared.f32 %f68, [%r12];\n" " ld.shared.f32 %f69, [%r48];\n" " add.f32 %f70, %f69, %f68;\n" " st.shared.f32 [%r12], %f70;\n" " add.s32 %r49, %r48, %r41;\n" " ld.shared.f32 %f71, [%r13];\n" " ld.shared.f32 %f72, [%r49];\n" " add.f32 %f73, %f72, %f71;\n" " st.shared.f32 [%r13], %f73;\n" " add.s32 %r50, %r49, %r41;\n" " ld.shared.f32 %f74, [%r14];\n" " ld.shared.f32 %f75, [%r50];\n" " add.f32 %f76, %f75, %f74;\n" " st.shared.f32 [%r14], %f76;\n" " add.s32 %r51, %r50, %r41;\n" " ld.shared.f32 %f77, [%r15];\n" " ld.shared.f32 %f78, [%r51];\n" " add.f32 %f79, %f78, %f77;\n" " st.shared.f32 [%r15], %f79;\n" " add.s32 %r52, %r51, %r41;\n" " ld.shared.f32 %f80, [%r16];\n" " ld.shared.f32 %f81, [%r52];\n" " add.f32 %f82, %f81, %f80;\n" " st.shared.f32 [%r16], %f82;\n" "\n" "$L__BB23_24:\n" " bar.sync 0;\n" " shr.s32 %r20, %r56, 1;\n" " setp.gt.s32 %p14, %r56, 1;\n" " mov.u32 %r56, %r20;\n" " @%p14 bra $L__BB23_22;\n" "\n" "$L__BB23_25:\n" " setp.ne.s32 %p15, %r5, 0;\n" " setp.lt.s32 %p16, %r2, 1;\n" " or.pred %p17, %p15, %p16;\n" " @%p17 bra $L__BB23_28;\n" "\n" " cvt.s64.s32 %rd13, %r26;\n" " cvt.s64.s32 %rd54, %r1;\n" " cvta.to.global.u64 %rd15, %rd19;\n" " mov.u32 %r58, 0;\n" "\n" "$L__BB23_27:\n" " ld.shared.f32 %f83, [%r57];\n" " mul.lo.s64 %rd50, %rd54, %rd13;\n" " add.s64 %rd51, %rd50, %rd2;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd53, %rd15, %rd52;\n" " st.global.f32 [%rd53], %f83;\n" " add.s32 %r57, %r57, %r41;\n" " add.s64 %rd54, %rd54, 1;\n" " add.s32 %r58, %r58, 1;\n" " setp.lt.s32 %p18, %r58, %r3;\n" " @%p18 bra $L__BB23_27;\n" "\n" "$L__BB23_28:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_bf16\n" ".visible .entry fc_matvec_bf16(\n" " .param .u64 fc_matvec_bf16_param_0,\n" " .param .u64 fc_matvec_bf16_param_1,\n" " .param .u64 fc_matvec_bf16_param_2,\n" " .param .u32 fc_matvec_bf16_param_3,\n" " .param .u32 fc_matvec_bf16_param_4,\n" " .param .u32 fc_matvec_bf16_param_5\n" ")\n" "{\n" " .reg .pred %p<11>;\n" " .reg .f32 %f<34>;\n" " .reg .b32 %r<47>;\n" " .reg .b64 %rd<37>;\n" "\n" "\n" " ld.param.u64 %rd16, [fc_matvec_bf16_param_0];\n" " ld.param.u64 %rd18, [fc_matvec_bf16_param_1];\n" " ld.param.u64 %rd17, [fc_matvec_bf16_param_2];\n" " ld.param.u32 %r19, [fc_matvec_bf16_param_3];\n" " ld.param.u32 %r17, [fc_matvec_bf16_param_4];\n" " ld.param.u32 %r18, [fc_matvec_bf16_param_5];\n" " cvta.to.global.u64 %rd1, %rd18;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r19;\n" " @%p1 bra $L__BB24_15;\n" "\n" " cvta.to.global.u64 %rd2, %rd16;\n" " mov.u32 %r2, %ntid.x;\n" " cvt.s64.s32 %rd3, %r1;\n" " mul.wide.s32 %rd4, %r18, %r1;\n" " mov.u32 %r3, %tid.x;\n" " setp.ge.s32 %p2, %r3, %r17;\n" " mov.f32 %f33, 0f00000000;\n" " @%p2 bra $L__BB24_8;\n" "\n" " not.b32 %r20, %r3;\n" " add.s32 %r21, %r20, %r17;\n" " div.u32 %r4, %r21, %r2;\n" " add.s32 %r22, %r4, 1;\n" " and.b32 %r43, %r22, 3;\n" " setp.eq.s32 %p3, %r43, 0;\n" " mov.f32 %f33, 0f00000000;\n" " mov.u32 %r44, %r3;\n" " @%p3 bra $L__BB24_5;\n" "\n" " mul.wide.s32 %rd19, %r3, 4;\n" " add.s64 %rd36, %rd1, %rd19;\n" " mul.wide.s32 %rd6, %r2, 4;\n" " mul.wide.s32 %rd20, %r3, 2;\n" " add.s64 %rd21, %rd20, %rd4;\n" " add.s64 %rd35, %rd2, %rd21;\n" " mul.wide.s32 %rd8, %r2, 2;\n" " mov.f32 %f33, 0f00000000;\n" " mov.u32 %r44, %r3;\n" "\n" "$L__BB24_4:\n" " .pragma \"nounroll\";\n" " ld.global.u16 %r23, [%rd35];\n" " shl.b32 %r24, %r23, 16;\n" " mov.b32 %f12, %r24;\n" " ld.global.f32 %f13, [%rd36];\n" " fma.rn.f32 %f33, %f12, %f13, %f33;\n" " add.s32 %r44, %r44, %r2;\n" " add.s64 %rd36, %rd36, %rd6;\n" " add.s64 %rd35, %rd35, %rd8;\n" " add.s32 %r43, %r43, -1;\n" " setp.ne.s32 %p4, %r43, 0;\n" " @%p4 bra $L__BB24_4;\n" "\n" "$L__BB24_5:\n" " setp.lt.u32 %p5, %r4, 3;\n" " add.s64 %rd13, %rd2, %rd4;\n" " @%p5 bra $L__BB24_8;\n" "\n" " mul.wide.s32 %rd14, %r2, 2;\n" " mul.wide.s32 %rd15, %r2, 4;\n" "\n" "$L__BB24_7:\n" " mul.wide.s32 %rd22, %r44, 2;\n" " add.s64 %rd23, %rd13, %rd22;\n" " ld.global.u16 %r25, [%rd23];\n" " shl.b32 %r26, %r25, 16;\n" " mov.b32 %f14, %r26;\n" " mul.wide.s32 %rd24, %r44, 4;\n" " add.s64 %rd25, %rd1, %rd24;\n" " ld.global.f32 %f15, [%rd25];\n" " fma.rn.f32 %f16, %f14, %f15, %f33;\n" " add.s64 %rd26, %rd23, %rd14;\n" " ld.global.u16 %r27, [%rd26];\n" " shl.b32 %r28, %r27, 16;\n" " mov.b32 %f17, %r28;\n" " add.s64 %rd27, %rd25, %rd15;\n" " ld.global.f32 %f18, [%rd27];\n" " fma.rn.f32 %f19, %f17, %f18, %f16;\n" " add.s32 %r29, %r44, %r2;\n" " add.s32 %r30, %r29, %r2;\n" " add.s64 %rd28, %rd26, %rd14;\n" " ld.global.u16 %r31, [%rd28];\n" " shl.b32 %r32, %r31, 16;\n" " mov.b32 %f20, %r32;\n" " add.s64 %rd29, %rd27, %rd15;\n" " ld.global.f32 %f21, [%rd29];\n" " fma.rn.f32 %f22, %f20, %f21, %f19;\n" " add.s32 %r33, %r30, %r2;\n" " add.s64 %rd30, %rd28, %rd14;\n" " ld.global.u16 %r34, [%rd30];\n" " shl.b32 %r35, %r34, 16;\n" " mov.b32 %f23, %r35;\n" " add.s64 %rd31, %rd29, %rd15;\n" " ld.global.f32 %f24, [%rd31];\n" " fma.rn.f32 %f33, %f23, %f24, %f22;\n" " add.s32 %r44, %r33, %r2;\n" " setp.lt.s32 %p6, %r44, %r17;\n" " @%p6 bra $L__BB24_7;\n" "\n" "$L__BB24_8:\n" " shl.b32 %r36, %r3, 2;\n" " mov.u32 %r37, fc_kq_shared;\n" " add.s32 %r13, %r37, %r36;\n" " st.shared.f32 [%r13], %f33;\n" " bar.sync 0;\n" " setp.lt.s32 %p7, %r2, 2;\n" " @%p7 bra $L__BB24_13;\n" "\n" " shr.u32 %r38, %r2, 31;\n" " add.s32 %r39, %r2, %r38;\n" " shr.s32 %r46, %r39, 1;\n" "\n" "$L__BB24_10:\n" " setp.ge.s32 %p8, %r3, %r46;\n" " @%p8 bra $L__BB24_12;\n" "\n" " shl.b32 %r40, %r46, 2;\n" " add.s32 %r41, %r13, %r40;\n" " ld.shared.f32 %f25, [%r13];\n" " ld.shared.f32 %f26, [%r41];\n" " add.f32 %f27, %f26, %f25;\n" " st.shared.f32 [%r13], %f27;\n" "\n" "$L__BB24_12:\n" " bar.sync 0;\n" " shr.s32 %r16, %r46, 1;\n" " setp.gt.s32 %p9, %r46, 1;\n" " mov.u32 %r46, %r16;\n" " @%p9 bra $L__BB24_10;\n" "\n" "$L__BB24_13:\n" " setp.ne.s32 %p10, %r3, 0;\n" " @%p10 bra $L__BB24_15;\n" "\n" " ld.shared.f32 %f28, [fc_kq_shared];\n" " cvta.to.global.u64 %rd32, %rd17;\n" " shl.b64 %rd33, %rd3, 2;\n" " add.s64 %rd34, %rd32, %rd33;\n" " st.global.f32 [%rd34], %f28;\n" "\n" "$L__BB24_15:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_bf16_nx2\n" ".visible .entry fc_matvec_bf16_nx2(\n" " .param .u64 fc_matvec_bf16_nx2_param_0,\n" " .param .u64 fc_matvec_bf16_nx2_param_1,\n" " .param .u64 fc_matvec_bf16_nx2_param_2,\n" " .param .u32 fc_matvec_bf16_nx2_param_3,\n" " .param .u32 fc_matvec_bf16_nx2_param_4,\n" " .param .u32 fc_matvec_bf16_nx2_param_5,\n" " .param .u32 fc_matvec_bf16_nx2_param_6\n" ")\n" "{\n" " .reg .pred %p<24>;\n" " .reg .f32 %f<75>;\n" " .reg .b32 %r<70>;\n" " .reg .b64 %rd<83>;\n" "\n" "\n" " ld.param.u64 %rd33, [fc_matvec_bf16_nx2_param_0];\n" " ld.param.u64 %rd35, [fc_matvec_bf16_nx2_param_1];\n" " ld.param.u64 %rd34, [fc_matvec_bf16_nx2_param_2];\n" " ld.param.u32 %r26, [fc_matvec_bf16_nx2_param_3];\n" " ld.param.u32 %r27, [fc_matvec_bf16_nx2_param_4];\n" " ld.param.u32 %r28, [fc_matvec_bf16_nx2_param_5];\n" " ld.param.u32 %r29, [fc_matvec_bf16_nx2_param_6];\n" " cvta.to.global.u64 %rd1, %rd35;\n" " mov.u32 %r30, %ctaid.y;\n" " shl.b32 %r1, %r30, 1;\n" " sub.s32 %r2, %r29, %r1;\n" " min.s32 %r3, %r2, 2;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r26;\n" " @%p1 bra $L__BB25_36;\n" "\n" " cvta.to.global.u64 %rd2, %rd33;\n" " mov.u32 %r5, %ntid.x;\n" " cvt.s64.s32 %rd3, %r4;\n" " mul.wide.s32 %rd4, %r28, %r4;\n" " mov.u32 %r6, %tid.x;\n" " setp.ge.s32 %p2, %r6, %r27;\n" " mov.f32 %f66, 0f00000000;\n" " mov.f32 %f65, %f66;\n" " @%p2 bra $L__BB25_28;\n" "\n" " cvt.s64.s32 %rd5, %r27;\n" " not.b32 %r31, %r6;\n" " add.s32 %r32, %r31, %r27;\n" " div.u32 %r7, %r32, %r5;\n" " add.s32 %r33, %r7, 1;\n" " and.b32 %r64, %r33, 3;\n" " setp.eq.s32 %p3, %r64, 0;\n" " mov.f32 %f66, 0f00000000;\n" " mov.f32 %f65, %f66;\n" " mov.u32 %r65, %r6;\n" " @%p3 bra $L__BB25_9;\n" "\n" " cvt.s64.s32 %rd36, %r1;\n" " add.s32 %r34, %r1, 1;\n" " cvt.s64.s32 %rd37, %r34;\n" " mul.lo.s64 %rd38, %rd5, %rd37;\n" " cvt.s64.s32 %rd39, %r6;\n" " add.s64 %rd40, %rd38, %rd39;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd81, %rd1, %rd41;\n" " mul.wide.s32 %rd7, %r5, 4;\n" " mul.lo.s64 %rd42, %rd5, %rd36;\n" " add.s64 %rd43, %rd42, %rd39;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd80, %rd1, %rd44;\n" " mul.wide.s32 %rd45, %r6, 2;\n" " add.s64 %rd46, %rd45, %rd4;\n" " add.s64 %rd79, %rd2, %rd46;\n" " mul.wide.s32 %rd10, %r5, 2;\n" " mov.f32 %f66, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " mov.f32 %f65, %f66;\n" " mov.u32 %r65, %r6;\n" "\n" "$L__BB25_4:\n" " .pragma \"nounroll\";\n" " ld.global.u16 %r35, [%rd79];\n" " shl.b32 %r36, %r35, 16;\n" " mov.b32 %f3, %r36;\n" " @%p4 bra $L__BB25_6;\n" "\n" " ld.global.f32 %f40, [%rd80];\n" " fma.rn.f32 %f65, %f3, %f40, %f65;\n" "\n" "$L__BB25_6:\n" " @%p5 bra $L__BB25_8;\n" "\n" " ld.global.f32 %f41, [%rd81];\n" " fma.rn.f32 %f66, %f3, %f41, %f66;\n" "\n" "$L__BB25_8:\n" " add.s32 %r65, %r65, %r5;\n" " add.s64 %rd81, %rd81, %rd7;\n" " add.s64 %rd80, %rd80, %rd7;\n" " add.s64 %rd79, %rd79, %rd10;\n" " add.s32 %r64, %r64, -1;\n" " setp.ne.s32 %p6, %r64, 0;\n" " @%p6 bra $L__BB25_4;\n" "\n" "$L__BB25_9:\n" " setp.lt.u32 %p7, %r7, 3;\n" " add.s64 %rd17, %rd2, %rd4;\n" " @%p7 bra $L__BB25_28;\n" "\n" " cvt.s64.s32 %rd47, %r1;\n" " mul.lo.s64 %rd18, %rd47, %rd5;\n" " add.s32 %r37, %r1, 1;\n" " cvt.s64.s32 %rd48, %r37;\n" " mul.lo.s64 %rd19, %rd48, %rd5;\n" " mul.wide.s32 %rd20, %r5, 2;\n" " setp.lt.s32 %p8, %r2, 1;\n" " setp.lt.s32 %p9, %r3, 2;\n" "\n" "$L__BB25_11:\n" " cvt.s64.s32 %rd21, %r65;\n" " mul.wide.s32 %rd49, %r65, 2;\n" " add.s64 %rd22, %rd17, %rd49;\n" " ld.global.u16 %r38, [%rd22];\n" " shl.b32 %r39, %r38, 16;\n" " mov.b32 %f12, %r39;\n" " @%p8 bra $L__BB25_13;\n" "\n" " add.s64 %rd50, %rd18, %rd21;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd52, %rd1, %rd51;\n" " ld.global.f32 %f42, [%rd52];\n" " fma.rn.f32 %f65, %f12, %f42, %f65;\n" "\n" "$L__BB25_13:\n" " @%p9 bra $L__BB25_15;\n" "\n" " add.s64 %rd53, %rd19, %rd21;\n" " shl.b64 %rd54, %rd53, 2;\n" " add.s64 %rd55, %rd1, %rd54;\n" " ld.global.f32 %f43, [%rd55];\n" " fma.rn.f32 %f66, %f12, %f43, %f66;\n" "\n" "$L__BB25_15:\n" " cvt.u32.u64 %r40, %rd21;\n" " add.s32 %r41, %r40, %r5;\n" " cvt.s64.s32 %rd23, %r41;\n" " add.s64 %rd24, %rd22, %rd20;\n" " ld.global.u16 %r42, [%rd24];\n" " shl.b32 %r43, %r42, 16;\n" " mov.b32 %f17, %r43;\n" " @%p8 bra $L__BB25_17;\n" "\n" " add.s64 %rd56, %rd18, %rd23;\n" " shl.b64 %rd57, %rd56, 2;\n" " add.s64 %rd58, %rd1, %rd57;\n" " ld.global.f32 %f44, [%rd58];\n" " fma.rn.f32 %f65, %f17, %f44, %f65;\n" "\n" "$L__BB25_17:\n" " @%p9 bra $L__BB25_19;\n" "\n" " add.s64 %rd59, %rd19, %rd23;\n" " shl.b64 %rd60, %rd59, 2;\n" " add.s64 %rd61, %rd1, %rd60;\n" " ld.global.f32 %f45, [%rd61];\n" " fma.rn.f32 %f66, %f17, %f45, %f66;\n" "\n" "$L__BB25_19:\n" " cvt.u32.u64 %r44, %rd23;\n" " add.s32 %r45, %r44, %r5;\n" " cvt.s64.s32 %rd25, %r45;\n" " add.s64 %rd26, %rd24, %rd20;\n" " ld.global.u16 %r46, [%rd26];\n" " shl.b32 %r47, %r46, 16;\n" " mov.b32 %f22, %r47;\n" " @%p8 bra $L__BB25_21;\n" "\n" " add.s64 %rd62, %rd18, %rd25;\n" " shl.b64 %rd63, %rd62, 2;\n" " add.s64 %rd64, %rd1, %rd63;\n" " ld.global.f32 %f46, [%rd64];\n" " fma.rn.f32 %f65, %f22, %f46, %f65;\n" "\n" "$L__BB25_21:\n" " @%p9 bra $L__BB25_23;\n" "\n" " add.s64 %rd65, %rd19, %rd25;\n" " shl.b64 %rd66, %rd65, 2;\n" " add.s64 %rd67, %rd1, %rd66;\n" " ld.global.f32 %f47, [%rd67];\n" " fma.rn.f32 %f66, %f22, %f47, %f66;\n" "\n" "$L__BB25_23:\n" " cvt.u32.u64 %r48, %rd25;\n" " add.s32 %r49, %r48, %r5;\n" " cvt.s64.s32 %rd27, %r49;\n" " add.s64 %rd68, %rd26, %rd20;\n" " ld.global.u16 %r50, [%rd68];\n" " shl.b32 %r51, %r50, 16;\n" " mov.b32 %f27, %r51;\n" " @%p8 bra $L__BB25_25;\n" "\n" " add.s64 %rd69, %rd18, %rd27;\n" " shl.b64 %rd70, %rd69, 2;\n" " add.s64 %rd71, %rd1, %rd70;\n" " ld.global.f32 %f48, [%rd71];\n" " fma.rn.f32 %f65, %f27, %f48, %f65;\n" "\n" "$L__BB25_25:\n" " @%p9 bra $L__BB25_27;\n" "\n" " add.s64 %rd72, %rd19, %rd27;\n" " shl.b64 %rd73, %rd72, 2;\n" " add.s64 %rd74, %rd1, %rd73;\n" " ld.global.f32 %f49, [%rd74];\n" " fma.rn.f32 %f66, %f27, %f49, %f66;\n" "\n" "$L__BB25_27:\n" " cvt.u32.u64 %r52, %rd27;\n" " add.s32 %r65, %r52, %r5;\n" " setp.lt.s32 %p16, %r65, %r27;\n" " @%p16 bra $L__BB25_11;\n" "\n" "$L__BB25_28:\n" " shl.b32 %r53, %r6, 2;\n" " mov.u32 %r68, fc_kq_shared;\n" " add.s32 %r16, %r68, %r53;\n" " st.shared.f32 [%r16], %f65;\n" " shl.b32 %r55, %r5, 2;\n" " add.s32 %r17, %r16, %r55;\n" " st.shared.f32 [%r17], %f66;\n" " bar.sync 0;\n" " setp.lt.s32 %p17, %r5, 2;\n" " @%p17 bra $L__BB25_33;\n" "\n" " shr.u32 %r56, %r5, 31;\n" " add.s32 %r57, %r5, %r56;\n" " shr.s32 %r67, %r57, 1;\n" "\n" "$L__BB25_30:\n" " setp.ge.s32 %p18, %r6, %r67;\n" " @%p18 bra $L__BB25_32;\n" "\n" " shl.b32 %r58, %r67, 2;\n" " add.s32 %r59, %r16, %r58;\n" " ld.shared.f32 %f50, [%r16];\n" " ld.shared.f32 %f51, [%r59];\n" " add.f32 %f52, %f51, %f50;\n" " st.shared.f32 [%r16], %f52;\n" " ld.shared.f32 %f53, [%r17];\n" " add.s32 %r60, %r17, %r58;\n" " ld.shared.f32 %f54, [%r60];\n" " add.f32 %f55, %f54, %f53;\n" " st.shared.f32 [%r17], %f55;\n" "\n" "$L__BB25_32:\n" " bar.sync 0;\n" " shr.s32 %r20, %r67, 1;\n" " setp.gt.s32 %p19, %r67, 1;\n" " mov.u32 %r67, %r20;\n" " @%p19 bra $L__BB25_30;\n" "\n" "$L__BB25_33:\n" " setp.ne.s32 %p20, %r6, 0;\n" " setp.lt.s32 %p21, %r2, 1;\n" " or.pred %p22, %p20, %p21;\n" " @%p22 bra $L__BB25_36;\n" "\n" " cvt.s64.s32 %rd28, %r26;\n" " cvt.s64.s32 %rd82, %r1;\n" " cvta.to.global.u64 %rd30, %rd34;\n" " mov.u32 %r69, 0;\n" "\n" "$L__BB25_35:\n" " ld.shared.f32 %f56, [%r68];\n" " mul.lo.s64 %rd75, %rd82, %rd28;\n" " add.s64 %rd76, %rd75, %rd3;\n" " shl.b64 %rd77, %rd76, 2;\n" " add.s64 %rd78, %rd30, %rd77;\n" " st.global.f32 [%rd78], %f56;\n" " add.s32 %r68, %r68, %r55;\n" " add.s64 %rd82, %rd82, 1;\n" " add.s32 %r69, %r69, 1;\n" " setp.lt.s32 %p23, %r69, %r3;\n" " @%p23 bra $L__BB25_35;\n" "\n" "$L__BB25_36:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_bf16_nx\n" ".visible .entry fc_matvec_bf16_nx(\n" " .param .u64 fc_matvec_bf16_nx_param_0,\n" " .param .u64 fc_matvec_bf16_nx_param_1,\n" " .param .u64 fc_matvec_bf16_nx_param_2,\n" " .param .u32 fc_matvec_bf16_nx_param_3,\n" " .param .u32 fc_matvec_bf16_nx_param_4,\n" " .param .u32 fc_matvec_bf16_nx_param_5,\n" " .param .u32 fc_matvec_bf16_nx_param_6\n" ")\n" "{\n" " .reg .pred %p<34>;\n" " .reg .f32 %f<143>;\n" " .reg .b32 %r<79>;\n" " .reg .b64 %rd<127>;\n" "\n" "\n" " ld.param.u64 %rd41, [fc_matvec_bf16_nx_param_0];\n" " ld.param.u64 %rd43, [fc_matvec_bf16_nx_param_1];\n" " ld.param.u64 %rd42, [fc_matvec_bf16_nx_param_2];\n" " ld.param.u32 %r29, [fc_matvec_bf16_nx_param_3];\n" " ld.param.u32 %r30, [fc_matvec_bf16_nx_param_4];\n" " ld.param.u32 %r31, [fc_matvec_bf16_nx_param_5];\n" " ld.param.u32 %r32, [fc_matvec_bf16_nx_param_6];\n" " cvta.to.global.u64 %rd1, %rd43;\n" " mov.u32 %r33, %ctaid.y;\n" " shl.b32 %r1, %r33, 2;\n" " sub.s32 %r2, %r32, %r1;\n" " min.s32 %r3, %r2, 4;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r29;\n" " @%p1 bra $L__BB26_56;\n" "\n" " mov.u32 %r5, %tid.x;\n" " cvt.s64.s32 %rd2, %r4;\n" " mul.wide.s32 %rd3, %r31, %r4;\n" " cvta.to.global.u64 %rd4, %rd41;\n" " mov.u32 %r6, %ntid.x;\n" " setp.ge.s32 %p2, %r5, %r30;\n" " mov.f32 %f126, 0f00000000;\n" " mov.f32 %f125, %f126;\n" " mov.f32 %f124, %f126;\n" " mov.f32 %f123, %f126;\n" " @%p2 bra $L__BB26_48;\n" "\n" " cvt.s64.s32 %rd5, %r30;\n" " not.b32 %r34, %r5;\n" " add.s32 %r35, %r34, %r30;\n" " div.u32 %r7, %r35, %r6;\n" " add.s32 %r36, %r7, 1;\n" " and.b32 %r73, %r36, 3;\n" " setp.eq.s32 %p3, %r73, 0;\n" " mov.f32 %f126, 0f00000000;\n" " mov.f32 %f125, %f126;\n" " mov.f32 %f124, %f126;\n" " mov.f32 %f123, %f126;\n" " mov.u32 %r74, %r5;\n" " @%p3 bra $L__BB26_13;\n" "\n" " cvt.s64.s32 %rd44, %r1;\n" " add.s32 %r37, %r1, 1;\n" " cvt.s64.s32 %rd45, %r37;\n" " add.s32 %r38, %r1, 2;\n" " cvt.s64.s32 %rd46, %r38;\n" " add.s32 %r39, %r1, 3;\n" " cvt.s64.s32 %rd47, %r39;\n" " mul.lo.s64 %rd48, %rd5, %rd47;\n" " cvt.s64.s32 %rd49, %r5;\n" " add.s64 %rd50, %rd48, %rd49;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd125, %rd1, %rd51;\n" " mul.wide.s32 %rd7, %r6, 4;\n" " mul.lo.s64 %rd52, %rd5, %rd46;\n" " add.s64 %rd53, %rd52, %rd49;\n" " shl.b64 %rd54, %rd53, 2;\n" " add.s64 %rd124, %rd1, %rd54;\n" " mul.lo.s64 %rd55, %rd5, %rd45;\n" " add.s64 %rd56, %rd55, %rd49;\n" " shl.b64 %rd57, %rd56, 2;\n" " add.s64 %rd123, %rd1, %rd57;\n" " mul.lo.s64 %rd58, %rd5, %rd44;\n" " add.s64 %rd59, %rd58, %rd49;\n" " shl.b64 %rd60, %rd59, 2;\n" " add.s64 %rd122, %rd1, %rd60;\n" " mul.wide.s32 %rd61, %r5, 2;\n" " add.s64 %rd62, %rd61, %rd3;\n" " add.s64 %rd121, %rd4, %rd62;\n" " mul.wide.s32 %rd12, %r6, 2;\n" " mov.f32 %f126, 0f00000000;\n" " setp.lt.s32 %p4, %r2, 1;\n" " setp.lt.s32 %p5, %r3, 2;\n" " setp.lt.s32 %p6, %r3, 3;\n" " setp.lt.s32 %p7, %r3, 4;\n" " mov.f32 %f125, %f126;\n" " mov.f32 %f124, %f126;\n" " mov.f32 %f123, %f126;\n" " mov.u32 %r74, %r5;\n" "\n" "$L__BB26_4:\n" " .pragma \"nounroll\";\n" " ld.global.u16 %r40, [%rd121];\n" " shl.b32 %r41, %r40, 16;\n" " mov.b32 %f5, %r41;\n" " @%p4 bra $L__BB26_6;\n" "\n" " ld.global.f32 %f74, [%rd122];\n" " fma.rn.f32 %f123, %f5, %f74, %f123;\n" "\n" "$L__BB26_6:\n" " @%p5 bra $L__BB26_8;\n" "\n" " ld.global.f32 %f75, [%rd123];\n" " fma.rn.f32 %f124, %f5, %f75, %f124;\n" "\n" "$L__BB26_8:\n" " @%p6 bra $L__BB26_10;\n" "\n" " ld.global.f32 %f76, [%rd124];\n" " fma.rn.f32 %f125, %f5, %f76, %f125;\n" "\n" "$L__BB26_10:\n" " @%p7 bra $L__BB26_12;\n" "\n" " ld.global.f32 %f77, [%rd125];\n" " fma.rn.f32 %f126, %f5, %f77, %f126;\n" "\n" "$L__BB26_12:\n" " add.s32 %r74, %r74, %r6;\n" " add.s64 %rd125, %rd125, %rd7;\n" " add.s64 %rd124, %rd124, %rd7;\n" " add.s64 %rd123, %rd123, %rd7;\n" " add.s64 %rd122, %rd122, %rd7;\n" " add.s64 %rd121, %rd121, %rd12;\n" " add.s32 %r73, %r73, -1;\n" " setp.ne.s32 %p8, %r73, 0;\n" " @%p8 bra $L__BB26_4;\n" "\n" "$L__BB26_13:\n" " setp.lt.u32 %p9, %r7, 3;\n" " add.s64 %rd23, %rd4, %rd3;\n" " @%p9 bra $L__BB26_48;\n" "\n" " cvt.s64.s32 %rd63, %r1;\n" " mul.lo.s64 %rd24, %rd63, %rd5;\n" " add.s32 %r42, %r1, 1;\n" " cvt.s64.s32 %rd64, %r42;\n" " mul.lo.s64 %rd25, %rd64, %rd5;\n" " add.s32 %r43, %r1, 2;\n" " cvt.s64.s32 %rd65, %r43;\n" " mul.lo.s64 %rd26, %rd65, %rd5;\n" " add.s32 %r44, %r1, 3;\n" " cvt.s64.s32 %rd66, %r44;\n" " mul.lo.s64 %rd27, %rd66, %rd5;\n" " mul.wide.s32 %rd28, %r6, 2;\n" " setp.lt.s32 %p10, %r2, 1;\n" " setp.lt.s32 %p11, %r3, 2;\n" " setp.lt.s32 %p12, %r3, 3;\n" " setp.lt.s32 %p13, %r3, 4;\n" "\n" "$L__BB26_15:\n" " cvt.s64.s32 %rd29, %r74;\n" " mul.wide.s32 %rd67, %r74, 2;\n" " add.s64 %rd30, %rd23, %rd67;\n" " ld.global.u16 %r45, [%rd30];\n" " shl.b32 %r46, %r45, 16;\n" " mov.b32 %f22, %r46;\n" " @%p10 bra $L__BB26_17;\n" "\n" " add.s64 %rd68, %rd24, %rd29;\n" " shl.b64 %rd69, %rd68, 2;\n" " add.s64 %rd70, %rd1, %rd69;\n" " ld.global.f32 %f78, [%rd70];\n" " fma.rn.f32 %f123, %f22, %f78, %f123;\n" "\n" "$L__BB26_17:\n" " @%p11 bra $L__BB26_19;\n" "\n" " add.s64 %rd71, %rd25, %rd29;\n" " shl.b64 %rd72, %rd71, 2;\n" " add.s64 %rd73, %rd1, %rd72;\n" " ld.global.f32 %f79, [%rd73];\n" " fma.rn.f32 %f124, %f22, %f79, %f124;\n" "\n" "$L__BB26_19:\n" " @%p12 bra $L__BB26_21;\n" "\n" " add.s64 %rd74, %rd26, %rd29;\n" " shl.b64 %rd75, %rd74, 2;\n" " add.s64 %rd76, %rd1, %rd75;\n" " ld.global.f32 %f80, [%rd76];\n" " fma.rn.f32 %f125, %f22, %f80, %f125;\n" "\n" "$L__BB26_21:\n" " @%p13 bra $L__BB26_23;\n" "\n" " add.s64 %rd77, %rd27, %rd29;\n" " shl.b64 %rd78, %rd77, 2;\n" " add.s64 %rd79, %rd1, %rd78;\n" " ld.global.f32 %f81, [%rd79];\n" " fma.rn.f32 %f126, %f22, %f81, %f126;\n" "\n" "$L__BB26_23:\n" " cvt.u32.u64 %r47, %rd29;\n" " add.s32 %r48, %r47, %r6;\n" " cvt.s64.s32 %rd31, %r48;\n" " add.s64 %rd32, %rd30, %rd28;\n" " ld.global.u16 %r49, [%rd32];\n" " shl.b32 %r50, %r49, 16;\n" " mov.b32 %f31, %r50;\n" " @%p10 bra $L__BB26_25;\n" "\n" " add.s64 %rd80, %rd24, %rd31;\n" " shl.b64 %rd81, %rd80, 2;\n" " add.s64 %rd82, %rd1, %rd81;\n" " ld.global.f32 %f82, [%rd82];\n" " fma.rn.f32 %f123, %f31, %f82, %f123;\n" "\n" "$L__BB26_25:\n" " @%p11 bra $L__BB26_27;\n" "\n" " add.s64 %rd83, %rd25, %rd31;\n" " shl.b64 %rd84, %rd83, 2;\n" " add.s64 %rd85, %rd1, %rd84;\n" " ld.global.f32 %f83, [%rd85];\n" " fma.rn.f32 %f124, %f31, %f83, %f124;\n" "\n" "$L__BB26_27:\n" " @%p12 bra $L__BB26_29;\n" "\n" " add.s64 %rd86, %rd26, %rd31;\n" " shl.b64 %rd87, %rd86, 2;\n" " add.s64 %rd88, %rd1, %rd87;\n" " ld.global.f32 %f84, [%rd88];\n" " fma.rn.f32 %f125, %f31, %f84, %f125;\n" "\n" "$L__BB26_29:\n" " @%p13 bra $L__BB26_31;\n" "\n" " add.s64 %rd89, %rd27, %rd31;\n" " shl.b64 %rd90, %rd89, 2;\n" " add.s64 %rd91, %rd1, %rd90;\n" " ld.global.f32 %f85, [%rd91];\n" " fma.rn.f32 %f126, %f31, %f85, %f126;\n" "\n" "$L__BB26_31:\n" " cvt.u32.u64 %r51, %rd31;\n" " add.s32 %r52, %r51, %r6;\n" " cvt.s64.s32 %rd33, %r52;\n" " add.s64 %rd34, %rd32, %rd28;\n" " ld.global.u16 %r53, [%rd34];\n" " shl.b32 %r54, %r53, 16;\n" " mov.b32 %f40, %r54;\n" " @%p10 bra $L__BB26_33;\n" "\n" " add.s64 %rd92, %rd24, %rd33;\n" " shl.b64 %rd93, %rd92, 2;\n" " add.s64 %rd94, %rd1, %rd93;\n" " ld.global.f32 %f86, [%rd94];\n" " fma.rn.f32 %f123, %f40, %f86, %f123;\n" "\n" "$L__BB26_33:\n" " @%p11 bra $L__BB26_35;\n" "\n" " add.s64 %rd95, %rd25, %rd33;\n" " shl.b64 %rd96, %rd95, 2;\n" " add.s64 %rd97, %rd1, %rd96;\n" " ld.global.f32 %f87, [%rd97];\n" " fma.rn.f32 %f124, %f40, %f87, %f124;\n" "\n" "$L__BB26_35:\n" " @%p12 bra $L__BB26_37;\n" "\n" " add.s64 %rd98, %rd26, %rd33;\n" " shl.b64 %rd99, %rd98, 2;\n" " add.s64 %rd100, %rd1, %rd99;\n" " ld.global.f32 %f88, [%rd100];\n" " fma.rn.f32 %f125, %f40, %f88, %f125;\n" "\n" "$L__BB26_37:\n" " @%p13 bra $L__BB26_39;\n" "\n" " add.s64 %rd101, %rd27, %rd33;\n" " shl.b64 %rd102, %rd101, 2;\n" " add.s64 %rd103, %rd1, %rd102;\n" " ld.global.f32 %f89, [%rd103];\n" " fma.rn.f32 %f126, %f40, %f89, %f126;\n" "\n" "$L__BB26_39:\n" " cvt.u32.u64 %r55, %rd33;\n" " add.s32 %r56, %r55, %r6;\n" " cvt.s64.s32 %rd35, %r56;\n" " add.s64 %rd104, %rd34, %rd28;\n" " ld.global.u16 %r57, [%rd104];\n" " shl.b32 %r58, %r57, 16;\n" " mov.b32 %f49, %r58;\n" " @%p10 bra $L__BB26_41;\n" "\n" " add.s64 %rd105, %rd24, %rd35;\n" " shl.b64 %rd106, %rd105, 2;\n" " add.s64 %rd107, %rd1, %rd106;\n" " ld.global.f32 %f90, [%rd107];\n" " fma.rn.f32 %f123, %f49, %f90, %f123;\n" "\n" "$L__BB26_41:\n" " @%p11 bra $L__BB26_43;\n" "\n" " add.s64 %rd108, %rd25, %rd35;\n" " shl.b64 %rd109, %rd108, 2;\n" " add.s64 %rd110, %rd1, %rd109;\n" " ld.global.f32 %f91, [%rd110];\n" " fma.rn.f32 %f124, %f49, %f91, %f124;\n" "\n" "$L__BB26_43:\n" " @%p12 bra $L__BB26_45;\n" "\n" " add.s64 %rd111, %rd26, %rd35;\n" " shl.b64 %rd112, %rd111, 2;\n" " add.s64 %rd113, %rd1, %rd112;\n" " ld.global.f32 %f92, [%rd113];\n" " fma.rn.f32 %f125, %f49, %f92, %f125;\n" "\n" "$L__BB26_45:\n" " @%p13 bra $L__BB26_47;\n" "\n" " add.s64 %rd114, %rd27, %rd35;\n" " shl.b64 %rd115, %rd114, 2;\n" " add.s64 %rd116, %rd1, %rd115;\n" " ld.global.f32 %f93, [%rd116];\n" " fma.rn.f32 %f126, %f49, %f93, %f126;\n" "\n" "$L__BB26_47:\n" " cvt.u32.u64 %r59, %rd35;\n" " add.s32 %r74, %r59, %r6;\n" " setp.lt.s32 %p26, %r74, %r30;\n" " @%p26 bra $L__BB26_15;\n" "\n" "$L__BB26_48:\n" " shl.b32 %r60, %r5, 2;\n" " mov.u32 %r77, fc_kq_shared;\n" " add.s32 %r16, %r77, %r60;\n" " st.shared.f32 [%r16], %f123;\n" " shl.b32 %r62, %r6, 2;\n" " add.s32 %r17, %r16, %r62;\n" " st.shared.f32 [%r17], %f124;\n" " add.s32 %r18, %r17, %r62;\n" " st.shared.f32 [%r18], %f125;\n" " add.s32 %r19, %r18, %r62;\n" " st.shared.f32 [%r19], %f126;\n" " bar.sync 0;\n" " setp.lt.s32 %p27, %r6, 2;\n" " @%p27 bra $L__BB26_53;\n" "\n" " shr.u32 %r63, %r6, 31;\n" " add.s32 %r64, %r6, %r63;\n" " shr.s32 %r76, %r64, 1;\n" "\n" "$L__BB26_50:\n" " setp.ge.s32 %p28, %r5, %r76;\n" " @%p28 bra $L__BB26_52;\n" "\n" " shl.b32 %r65, %r76, 2;\n" " add.s32 %r66, %r16, %r65;\n" " ld.shared.f32 %f94, [%r16];\n" " ld.shared.f32 %f95, [%r66];\n" " add.f32 %f96, %f95, %f94;\n" " st.shared.f32 [%r16], %f96;\n" " add.s32 %r67, %r66, %r62;\n" " ld.shared.f32 %f97, [%r17];\n" " ld.shared.f32 %f98, [%r67];\n" " add.f32 %f99, %f98, %f97;\n" " st.shared.f32 [%r17], %f99;\n" " add.s32 %r68, %r67, %r62;\n" " ld.shared.f32 %f100, [%r18];\n" " ld.shared.f32 %f101, [%r68];\n" " add.f32 %f102, %f101, %f100;\n" " st.shared.f32 [%r18], %f102;\n" " add.s32 %r69, %r68, %r62;\n" " ld.shared.f32 %f103, [%r19];\n" " ld.shared.f32 %f104, [%r69];\n" " add.f32 %f105, %f104, %f103;\n" " st.shared.f32 [%r19], %f105;\n" "\n" "$L__BB26_52:\n" " bar.sync 0;\n" " shr.s32 %r23, %r76, 1;\n" " setp.gt.s32 %p29, %r76, 1;\n" " mov.u32 %r76, %r23;\n" " @%p29 bra $L__BB26_50;\n" "\n" "$L__BB26_53:\n" " setp.ne.s32 %p30, %r5, 0;\n" " setp.lt.s32 %p31, %r2, 1;\n" " or.pred %p32, %p30, %p31;\n" " @%p32 bra $L__BB26_56;\n" "\n" " cvt.s64.s32 %rd36, %r29;\n" " cvt.s64.s32 %rd126, %r1;\n" " cvta.to.global.u64 %rd38, %rd42;\n" " mov.u32 %r78, 0;\n" "\n" "$L__BB26_55:\n" " ld.shared.f32 %f106, [%r77];\n" " mul.lo.s64 %rd117, %rd126, %rd36;\n" " add.s64 %rd118, %rd117, %rd2;\n" " shl.b64 %rd119, %rd118, 2;\n" " add.s64 %rd120, %rd38, %rd119;\n" " st.global.f32 [%rd120], %f106;\n" " add.s32 %r77, %r77, %r62;\n" " add.s64 %rd126, %rd126, 1;\n" " add.s32 %r78, %r78, 1;\n" " setp.lt.s32 %p33, %r78, %r3;\n" " @%p33 bra $L__BB26_55;\n" "\n" "$L__BB26_56:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_bf16_nx8\n" ".visible .entry fc_matvec_bf16_nx8(\n" " .param .u64 fc_matvec_bf16_nx8_param_0,\n" " .param .u64 fc_matvec_bf16_nx8_param_1,\n" " .param .u64 fc_matvec_bf16_nx8_param_2,\n" " .param .u32 fc_matvec_bf16_nx8_param_3,\n" " .param .u32 fc_matvec_bf16_nx8_param_4,\n" " .param .u32 fc_matvec_bf16_nx8_param_5,\n" " .param .u32 fc_matvec_bf16_nx8_param_6\n" ")\n" "{\n" " .reg .pred %p<39>;\n" " .reg .f32 %f<205>;\n" " .reg .b32 %r<81>;\n" " .reg .b64 %rd<134>;\n" "\n" "\n" " ld.param.u64 %rd22, [fc_matvec_bf16_nx8_param_0];\n" " ld.param.u64 %rd24, [fc_matvec_bf16_nx8_param_1];\n" " ld.param.u64 %rd23, [fc_matvec_bf16_nx8_param_2];\n" " ld.param.u32 %r29, [fc_matvec_bf16_nx8_param_3];\n" " ld.param.u32 %r30, [fc_matvec_bf16_nx8_param_4];\n" " ld.param.u32 %r31, [fc_matvec_bf16_nx8_param_5];\n" " ld.param.u32 %r32, [fc_matvec_bf16_nx8_param_6];\n" " cvta.to.global.u64 %rd1, %rd24;\n" " mov.u32 %r33, %ctaid.y;\n" " shl.b32 %r1, %r33, 3;\n" " sub.s32 %r2, %r32, %r1;\n" " min.s32 %r3, %r2, 8;\n" " mov.u32 %r4, %ctaid.x;\n" " setp.ge.s32 %p1, %r4, %r29;\n" " @%p1 bra $L__BB27_63;\n" "\n" " mov.u32 %r5, %tid.x;\n" " cvt.s64.s32 %rd2, %r4;\n" " mov.u32 %r6, %ntid.x;\n" " setp.ge.s32 %p2, %r5, %r30;\n" " mov.f32 %f188, 0f00000000;\n" " mov.f32 %f187, %f188;\n" " mov.f32 %f186, %f188;\n" " mov.f32 %f185, %f188;\n" " mov.f32 %f184, %f188;\n" " mov.f32 %f183, %f188;\n" " mov.f32 %f182, %f188;\n" " mov.f32 %f181, %f188;\n" " @%p2 bra $L__BB27_55;\n" "\n" " cvta.to.global.u64 %rd25, %rd22;\n" " cvt.s64.s32 %rd26, %r31;\n" " mul.lo.s64 %rd27, %rd26, %rd2;\n" " add.s64 %rd3, %rd25, %rd27;\n" " cvt.s64.s32 %rd4, %r30;\n" " not.b32 %r34, %r5;\n" " add.s32 %r7, %r34, %r30;\n" " div.u32 %r35, %r7, %r6;\n" " and.b32 %r36, %r35, 1;\n" " setp.eq.b32 %p3, %r36, 1;\n" " mov.pred %p4, 0;\n" " xor.pred %p5, %p3, %p4;\n" " mov.f32 %f188, 0f00000000;\n" " mov.f32 %f187, %f188;\n" " mov.f32 %f186, %f188;\n" " mov.f32 %f185, %f188;\n" " mov.f32 %f184, %f188;\n" " mov.f32 %f183, %f188;\n" " mov.f32 %f182, %f188;\n" " mov.f32 %f181, %f188;\n" " mov.u32 %r77, %r5;\n" " @%p5 bra $L__BB27_20;\n" "\n" " cvt.s64.s32 %rd5, %r5;\n" " mul.wide.s32 %rd28, %r5, 2;\n" " add.s64 %rd29, %rd3, %rd28;\n" " ld.global.u16 %r37, [%rd29];\n" " shl.b32 %r38, %r37, 16;\n" " mov.b32 %f1, %r38;\n" " setp.lt.s32 %p6, %r2, 1;\n" " mov.f32 %f188, 0f00000000;\n" " mov.f32 %f181, %f188;\n" " @%p6 bra $L__BB27_5;\n" "\n" " cvt.s64.s32 %rd30, %r1;\n" " mul.lo.s64 %rd31, %rd30, %rd4;\n" " add.s64 %rd32, %rd31, %rd5;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd34, %rd1, %rd33;\n" " ld.global.f32 %f93, [%rd34];\n" " mov.f32 %f94, 0f00000000;\n" " fma.rn.f32 %f181, %f1, %f93, %f94;\n" "\n" "$L__BB27_5:\n" " setp.lt.s32 %p7, %r3, 2;\n" " mov.f32 %f182, %f188;\n" " @%p7 bra $L__BB27_7;\n" "\n" " add.s32 %r39, %r1, 1;\n" " cvt.s64.s32 %rd35, %r39;\n" " mul.lo.s64 %rd36, %rd35, %rd4;\n" " add.s64 %rd37, %rd36, %rd5;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd39, %rd1, %rd38;\n" " ld.global.f32 %f96, [%rd39];\n" " mov.f32 %f97, 0f00000000;\n" " fma.rn.f32 %f182, %f1, %f96, %f97;\n" "\n" "$L__BB27_7:\n" " setp.lt.s32 %p8, %r3, 3;\n" " mov.f32 %f183, %f188;\n" " @%p8 bra $L__BB27_9;\n" "\n" " add.s32 %r40, %r1, 2;\n" " cvt.s64.s32 %rd40, %r40;\n" " mul.lo.s64 %rd41, %rd40, %rd4;\n" " add.s64 %rd42, %rd41, %rd5;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd44, %rd1, %rd43;\n" " ld.global.f32 %f99, [%rd44];\n" " mov.f32 %f100, 0f00000000;\n" " fma.rn.f32 %f183, %f1, %f99, %f100;\n" "\n" "$L__BB27_9:\n" " setp.lt.s32 %p9, %r3, 4;\n" " mov.f32 %f184, %f188;\n" " @%p9 bra $L__BB27_11;\n" "\n" " add.s32 %r41, %r1, 3;\n" " cvt.s64.s32 %rd45, %r41;\n" " mul.lo.s64 %rd46, %rd45, %rd4;\n" " add.s64 %rd47, %rd46, %rd5;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd49, %rd1, %rd48;\n" " ld.global.f32 %f102, [%rd49];\n" " mov.f32 %f103, 0f00000000;\n" " fma.rn.f32 %f184, %f1, %f102, %f103;\n" "\n" "$L__BB27_11:\n" " setp.lt.s32 %p10, %r3, 5;\n" " mov.f32 %f185, %f188;\n" " @%p10 bra $L__BB27_13;\n" "\n" " add.s32 %r42, %r1, 4;\n" " cvt.s64.s32 %rd50, %r42;\n" " mul.lo.s64 %rd51, %rd50, %rd4;\n" " add.s64 %rd52, %rd51, %rd5;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd54, %rd1, %rd53;\n" " ld.global.f32 %f105, [%rd54];\n" " mov.f32 %f106, 0f00000000;\n" " fma.rn.f32 %f185, %f1, %f105, %f106;\n" "\n" "$L__BB27_13:\n" " setp.lt.s32 %p11, %r3, 6;\n" " mov.f32 %f186, %f188;\n" " @%p11 bra $L__BB27_15;\n" "\n" " add.s32 %r43, %r1, 5;\n" " cvt.s64.s32 %rd55, %r43;\n" " mul.lo.s64 %rd56, %rd55, %rd4;\n" " add.s64 %rd57, %rd56, %rd5;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd59, %rd1, %rd58;\n" " ld.global.f32 %f108, [%rd59];\n" " mov.f32 %f109, 0f00000000;\n" " fma.rn.f32 %f186, %f1, %f108, %f109;\n" "\n" "$L__BB27_15:\n" " setp.lt.s32 %p12, %r3, 7;\n" " mov.f32 %f187, %f188;\n" " @%p12 bra $L__BB27_17;\n" "\n" " add.s32 %r44, %r1, 6;\n" " cvt.s64.s32 %rd60, %r44;\n" " mul.lo.s64 %rd61, %rd60, %rd4;\n" " add.s64 %rd62, %rd61, %rd5;\n" " shl.b64 %rd63, %rd62, 2;\n" " add.s64 %rd64, %rd1, %rd63;\n" " ld.global.f32 %f111, [%rd64];\n" " mov.f32 %f112, 0f00000000;\n" " fma.rn.f32 %f187, %f1, %f111, %f112;\n" "\n" "$L__BB27_17:\n" " setp.lt.s32 %p13, %r3, 8;\n" " @%p13 bra $L__BB27_19;\n" "\n" " add.s32 %r45, %r1, 7;\n" " cvt.s64.s32 %rd65, %r45;\n" " mul.lo.s64 %rd66, %rd65, %rd4;\n" " add.s64 %rd67, %rd66, %rd5;\n" " shl.b64 %rd68, %rd67, 2;\n" " add.s64 %rd69, %rd1, %rd68;\n" " ld.global.f32 %f114, [%rd69];\n" " mov.f32 %f115, 0f00000000;\n" " fma.rn.f32 %f188, %f1, %f114, %f115;\n" "\n" "$L__BB27_19:\n" " add.s32 %r77, %r5, %r6;\n" "\n" "$L__BB27_20:\n" " setp.gt.u32 %p14, %r6, %r7;\n" " @%p14 bra $L__BB27_55;\n" "\n" " cvt.s64.s32 %rd70, %r1;\n" " mul.lo.s64 %rd6, %rd70, %rd4;\n" " add.s32 %r46, %r1, 1;\n" " cvt.s64.s32 %rd71, %r46;\n" " mul.lo.s64 %rd7, %rd71, %rd4;\n" " add.s32 %r47, %r1, 2;\n" " cvt.s64.s32 %rd72, %r47;\n" " mul.lo.s64 %rd8, %rd72, %rd4;\n" " add.s32 %r48, %r1, 3;\n" " cvt.s64.s32 %rd73, %r48;\n" " mul.lo.s64 %rd9, %rd73, %rd4;\n" " add.s32 %r49, %r1, 4;\n" " cvt.s64.s32 %rd74, %r49;\n" " mul.lo.s64 %rd10, %rd74, %rd4;\n" " add.s32 %r50, %r1, 5;\n" " cvt.s64.s32 %rd75, %r50;\n" " mul.lo.s64 %rd11, %rd75, %rd4;\n" " add.s32 %r51, %r1, 6;\n" " cvt.s64.s32 %rd76, %r51;\n" " mul.lo.s64 %rd12, %rd76, %rd4;\n" " add.s32 %r52, %r1, 7;\n" " cvt.s64.s32 %rd77, %r52;\n" " mul.lo.s64 %rd13, %rd77, %rd4;\n" " setp.lt.s32 %p15, %r2, 1;\n" " setp.lt.s32 %p16, %r3, 2;\n" " setp.lt.s32 %p17, %r3, 3;\n" " setp.lt.s32 %p18, %r3, 4;\n" "\n" "$L__BB27_22:\n" " cvt.s64.s32 %rd14, %r77;\n" " mul.wide.s32 %rd78, %r77, 2;\n" " add.s64 %rd15, %rd3, %rd78;\n" " ld.global.u16 %r53, [%rd15];\n" " shl.b32 %r54, %r53, 16;\n" " mov.b32 %f34, %r54;\n" " @%p15 bra $L__BB27_24;\n" "\n" " add.s64 %rd79, %rd6, %rd14;\n" " shl.b64 %rd80, %rd79, 2;\n" " add.s64 %rd81, %rd1, %rd80;\n" " ld.global.f32 %f116, [%rd81];\n" " fma.rn.f32 %f181, %f34, %f116, %f181;\n" "\n" "$L__BB27_24:\n" " @%p16 bra $L__BB27_26;\n" "\n" " add.s64 %rd82, %rd7, %rd14;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd84, %rd1, %rd83;\n" " ld.global.f32 %f117, [%rd84];\n" " fma.rn.f32 %f182, %f34, %f117, %f182;\n" "\n" "$L__BB27_26:\n" " @%p17 bra $L__BB27_28;\n" "\n" " add.s64 %rd85, %rd8, %rd14;\n" " shl.b64 %rd86, %rd85, 2;\n" " add.s64 %rd87, %rd1, %rd86;\n" " ld.global.f32 %f118, [%rd87];\n" " fma.rn.f32 %f183, %f34, %f118, %f183;\n" "\n" "$L__BB27_28:\n" " @%p18 bra $L__BB27_30;\n" "\n" " add.s64 %rd88, %rd9, %rd14;\n" " shl.b64 %rd89, %rd88, 2;\n" " add.s64 %rd90, %rd1, %rd89;\n" " ld.global.f32 %f119, [%rd90];\n" " fma.rn.f32 %f184, %f34, %f119, %f184;\n" "\n" "$L__BB27_30:\n" " setp.lt.s32 %p19, %r3, 5;\n" " @%p19 bra $L__BB27_32;\n" "\n" " add.s64 %rd91, %rd10, %rd14;\n" " shl.b64 %rd92, %rd91, 2;\n" " add.s64 %rd93, %rd1, %rd92;\n" " ld.global.f32 %f120, [%rd93];\n" " fma.rn.f32 %f185, %f34, %f120, %f185;\n" "\n" "$L__BB27_32:\n" " setp.lt.s32 %p20, %r3, 6;\n" " @%p20 bra $L__BB27_34;\n" "\n" " add.s64 %rd94, %rd11, %rd14;\n" " shl.b64 %rd95, %rd94, 2;\n" " add.s64 %rd96, %rd1, %rd95;\n" " ld.global.f32 %f121, [%rd96];\n" " fma.rn.f32 %f186, %f34, %f121, %f186;\n" "\n" "$L__BB27_34:\n" " setp.lt.s32 %p21, %r3, 7;\n" " @%p21 bra $L__BB27_36;\n" "\n" " add.s64 %rd97, %rd12, %rd14;\n" " shl.b64 %rd98, %rd97, 2;\n" " add.s64 %rd99, %rd1, %rd98;\n" " ld.global.f32 %f122, [%rd99];\n" " fma.rn.f32 %f187, %f34, %f122, %f187;\n" "\n" "$L__BB27_36:\n" " setp.lt.s32 %p22, %r3, 8;\n" " @%p22 bra $L__BB27_38;\n" "\n" " add.s64 %rd100, %rd13, %rd14;\n" " shl.b64 %rd101, %rd100, 2;\n" " add.s64 %rd102, %rd1, %rd101;\n" " ld.global.f32 %f123, [%rd102];\n" " fma.rn.f32 %f188, %f34, %f123, %f188;\n" "\n" "$L__BB27_38:\n" " cvt.u32.u64 %r55, %rd14;\n" " add.s32 %r56, %r55, %r6;\n" " cvt.s64.s32 %rd16, %r56;\n" " mul.wide.s32 %rd103, %r6, 2;\n" " add.s64 %rd104, %rd15, %rd103;\n" " ld.global.u16 %r57, [%rd104];\n" " shl.b32 %r58, %r57, 16;\n" " mov.b32 %f51, %r58;\n" " @%p15 bra $L__BB27_40;\n" "\n" " add.s64 %rd105, %rd6, %rd16;\n" " shl.b64 %rd106, %rd105, 2;\n" " add.s64 %rd107, %rd1, %rd106;\n" " ld.global.f32 %f124, [%rd107];\n" " fma.rn.f32 %f181, %f51, %f124, %f181;\n" "\n" "$L__BB27_40:\n" " @%p16 bra $L__BB27_42;\n" "\n" " add.s64 %rd108, %rd7, %rd16;\n" " shl.b64 %rd109, %rd108, 2;\n" " add.s64 %rd110, %rd1, %rd109;\n" " ld.global.f32 %f125, [%rd110];\n" " fma.rn.f32 %f182, %f51, %f125, %f182;\n" "\n" "$L__BB27_42:\n" " @%p17 bra $L__BB27_44;\n" "\n" " add.s64 %rd111, %rd8, %rd16;\n" " shl.b64 %rd112, %rd111, 2;\n" " add.s64 %rd113, %rd1, %rd112;\n" " ld.global.f32 %f126, [%rd113];\n" " fma.rn.f32 %f183, %f51, %f126, %f183;\n" "\n" "$L__BB27_44:\n" " @%p18 bra $L__BB27_46;\n" "\n" " add.s64 %rd114, %rd9, %rd16;\n" " shl.b64 %rd115, %rd114, 2;\n" " add.s64 %rd116, %rd1, %rd115;\n" " ld.global.f32 %f127, [%rd116];\n" " fma.rn.f32 %f184, %f51, %f127, %f184;\n" "\n" "$L__BB27_46:\n" " @%p19 bra $L__BB27_48;\n" "\n" " add.s64 %rd117, %rd10, %rd16;\n" " shl.b64 %rd118, %rd117, 2;\n" " add.s64 %rd119, %rd1, %rd118;\n" " ld.global.f32 %f128, [%rd119];\n" " fma.rn.f32 %f185, %f51, %f128, %f185;\n" "\n" "$L__BB27_48:\n" " @%p20 bra $L__BB27_50;\n" "\n" " add.s64 %rd120, %rd11, %rd16;\n" " shl.b64 %rd121, %rd120, 2;\n" " add.s64 %rd122, %rd1, %rd121;\n" " ld.global.f32 %f129, [%rd122];\n" " fma.rn.f32 %f186, %f51, %f129, %f186;\n" "\n" "$L__BB27_50:\n" " @%p21 bra $L__BB27_52;\n" "\n" " add.s64 %rd123, %rd12, %rd16;\n" " shl.b64 %rd124, %rd123, 2;\n" " add.s64 %rd125, %rd1, %rd124;\n" " ld.global.f32 %f130, [%rd125];\n" " fma.rn.f32 %f187, %f51, %f130, %f187;\n" "\n" "$L__BB27_52:\n" " @%p22 bra $L__BB27_54;\n" "\n" " add.s64 %rd126, %rd13, %rd16;\n" " shl.b64 %rd127, %rd126, 2;\n" " add.s64 %rd128, %rd1, %rd127;\n" " ld.global.f32 %f131, [%rd128];\n" " fma.rn.f32 %f188, %f51, %f131, %f188;\n" "\n" "$L__BB27_54:\n" " cvt.u32.u64 %r59, %rd16;\n" " add.s32 %r77, %r59, %r6;\n" " setp.lt.s32 %p31, %r77, %r30;\n" " @%p31 bra $L__BB27_22;\n" "\n" "$L__BB27_55:\n" " shl.b32 %r60, %r5, 2;\n" " mov.u32 %r79, fc_kq_shared;\n" " add.s32 %r12, %r79, %r60;\n" " st.shared.f32 [%r12], %f181;\n" " shl.b32 %r62, %r6, 2;\n" " add.s32 %r13, %r12, %r62;\n" " st.shared.f32 [%r13], %f182;\n" " add.s32 %r14, %r13, %r62;\n" " st.shared.f32 [%r14], %f183;\n" " add.s32 %r15, %r14, %r62;\n" " st.shared.f32 [%r15], %f184;\n" " add.s32 %r16, %r15, %r62;\n" " st.shared.f32 [%r16], %f185;\n" " add.s32 %r17, %r16, %r62;\n" " st.shared.f32 [%r17], %f186;\n" " add.s32 %r18, %r17, %r62;\n" " st.shared.f32 [%r18], %f187;\n" " add.s32 %r19, %r18, %r62;\n" " st.shared.f32 [%r19], %f188;\n" " bar.sync 0;\n" " setp.lt.s32 %p32, %r6, 2;\n" " @%p32 bra $L__BB27_60;\n" "\n" " shr.u32 %r63, %r6, 31;\n" " add.s32 %r64, %r6, %r63;\n" " shr.s32 %r78, %r64, 1;\n" "\n" "$L__BB27_57:\n" " setp.ge.s32 %p33, %r5, %r78;\n" " @%p33 bra $L__BB27_59;\n" "\n" " shl.b32 %r65, %r78, 2;\n" " add.s32 %r66, %r12, %r65;\n" " ld.shared.f32 %f132, [%r12];\n" " ld.shared.f32 %f133, [%r66];\n" " add.f32 %f134, %f133, %f132;\n" " st.shared.f32 [%r12], %f134;\n" " add.s32 %r67, %r66, %r62;\n" " ld.shared.f32 %f135, [%r13];\n" " ld.shared.f32 %f136, [%r67];\n" " add.f32 %f137, %f136, %f135;\n" " st.shared.f32 [%r13], %f137;\n" " add.s32 %r68, %r67, %r62;\n" " ld.shared.f32 %f138, [%r14];\n" " ld.shared.f32 %f139, [%r68];\n" " add.f32 %f140, %f139, %f138;\n" " st.shared.f32 [%r14], %f140;\n" " add.s32 %r69, %r68, %r62;\n" " ld.shared.f32 %f141, [%r15];\n" " ld.shared.f32 %f142, [%r69];\n" " add.f32 %f143, %f142, %f141;\n" " st.shared.f32 [%r15], %f143;\n" " add.s32 %r70, %r69, %r62;\n" " ld.shared.f32 %f144, [%r16];\n" " ld.shared.f32 %f145, [%r70];\n" " add.f32 %f146, %f145, %f144;\n" " st.shared.f32 [%r16], %f146;\n" " add.s32 %r71, %r70, %r62;\n" " ld.shared.f32 %f147, [%r17];\n" " ld.shared.f32 %f148, [%r71];\n" " add.f32 %f149, %f148, %f147;\n" " st.shared.f32 [%r17], %f149;\n" " add.s32 %r72, %r71, %r62;\n" " ld.shared.f32 %f150, [%r18];\n" " ld.shared.f32 %f151, [%r72];\n" " add.f32 %f152, %f151, %f150;\n" " st.shared.f32 [%r18], %f152;\n" " add.s32 %r73, %r72, %r62;\n" " ld.shared.f32 %f153, [%r19];\n" " ld.shared.f32 %f154, [%r73];\n" " add.f32 %f155, %f154, %f153;\n" " st.shared.f32 [%r19], %f155;\n" "\n" "$L__BB27_59:\n" " bar.sync 0;\n" " shr.s32 %r23, %r78, 1;\n" " setp.gt.s32 %p34, %r78, 1;\n" " mov.u32 %r78, %r23;\n" " @%p34 bra $L__BB27_57;\n" "\n" "$L__BB27_60:\n" " setp.ne.s32 %p35, %r5, 0;\n" " setp.lt.s32 %p36, %r2, 1;\n" " or.pred %p37, %p35, %p36;\n" " @%p37 bra $L__BB27_63;\n" "\n" " cvt.s64.s32 %rd17, %r29;\n" " cvt.s64.s32 %rd133, %r1;\n" " cvta.to.global.u64 %rd19, %rd23;\n" " mov.u32 %r80, 0;\n" "\n" "$L__BB27_62:\n" " ld.shared.f32 %f156, [%r79];\n" " mul.lo.s64 %rd129, %rd133, %rd17;\n" " add.s64 %rd130, %rd129, %rd2;\n" " shl.b64 %rd131, %rd130, 2;\n" " add.s64 %rd132, %rd19, %rd131;\n" " st.global.f32 [%rd132], %f156;\n" " add.s32 %r79, %r79, %r62;\n" " add.s64 %rd133, %rd133, 1;\n" " add.s32 %r80, %r80, 1;\n" " setp.lt.s32 %p38, %r80, %r3;\n" " @%p38 bra $L__BB27_62;\n" "\n" "$L__BB27_63:\n" " ret;\n" "\n" "}\n" " // .globl fc_matvec_q2_0\n" ".visible .entry fc_matvec_q2_0(\n" " .param .u64 fc_matvec_q2_0_param_0,\n" " .param .u64 fc_matvec_q2_0_param_1,\n" " .param .u64 fc_matvec_q2_0_param_2,\n" " .param .u32 fc_matvec_q2_0_param_3,\n" " .param .u32 fc_matvec_q2_0_param_4,\n" " .param .u32 fc_matvec_q2_0_param_5\n" ")\n" "{\n" " .reg .pred %p<22>;\n" " .reg .f32 %f<112>;\n" " .reg .b32 %r<135>;\n" " .reg .b64 %rd<56>;\n" "\n" "\n" " ld.param.u64 %rd21, [fc_matvec_q2_0_param_0];\n" " ld.param.u64 %rd22, [fc_matvec_q2_0_param_1];\n" " ld.param.u64 %rd23, [fc_matvec_q2_0_param_2];\n" " ld.param.u32 %r30, [fc_matvec_q2_0_param_3];\n" " ld.param.u32 %r31, [fc_matvec_q2_0_param_4];\n" " ld.param.u32 %r32, [fc_matvec_q2_0_param_5];\n" " mov.u32 %r1, %tid.x;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r33, %ctaid.x;\n" " shl.b32 %r34, %r33, 4;\n" " shr.u32 %r35, %r1, 4;\n" " and.b32 %r36, %r35, 268435454;\n" " add.s32 %r37, %r36, %r34;\n" " or.b32 %r3, %r37, 1;\n" " cvt.s64.s32 %rd1, %r37;\n" " setp.lt.s32 %p1, %r31, 1;\n" " mov.f32 %f108, 0f00000000;\n" " mov.f32 %f109, %f108;\n" " @%p1 bra $L__BB28_21;\n" "\n" " cvt.s64.s32 %rd24, %r32;\n" " mul.lo.s64 %rd25, %rd24, %rd1;\n" " add.s64 %rd54, %rd21, %rd25;\n" " add.s32 %r40, %r2, 4;\n" " cvt.u64.u32 %rd3, %r40;\n" " mov.u32 %r41, -2;\n" " sub.s32 %r4, %r41, %r1;\n" " not.b32 %r5, %r31;\n" " cvt.s64.s32 %rd4, %r1;\n" " shl.b32 %r42, %r1, 2;\n" " mov.u32 %r43, fc_kq_shared;\n" " add.s32 %r6, %r43, %r42;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd5, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd6, %r8;\n" " add.s32 %r9, %r1, 768;\n" " add.s64 %rd26, %rd54, %rd24;\n" " setp.lt.s32 %p2, %r3, %r30;\n" " selp.b64 %rd55, %rd26, %rd54, %p2;\n" " mov.f32 %f109, 0f00000000;\n" " mov.u32 %r129, 0;\n" " mov.u32 %r130, %r129;\n" " mov.f32 %f108, %f109;\n" "\n" "$L__BB28_2:\n" " add.s32 %r44, %r129, %r5;\n" " max.s32 %r45, %r44, -2049;\n" " sub.s32 %r12, %r4, %r45;\n" " sub.s32 %r46, %r31, %r130;\n" " min.s32 %r13, %r46, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p3, %r1, %r13;\n" " @%p3 bra $L__BB28_10;\n" "\n" " shr.u32 %r47, %r12, 8;\n" " add.s32 %r48, %r47, 1;\n" " cvt.s64.s32 %rd10, %r130;\n" " and.b32 %r14, %r48, 3;\n" " setp.eq.s32 %p4, %r14, 0;\n" " mov.u32 %r131, %r1;\n" " @%p4 bra $L__BB28_7;\n" "\n" " add.s64 %rd28, %rd4, %rd10;\n" " shl.b64 %rd29, %rd28, 2;\n" " add.s64 %rd27, %rd22, %rd29;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd27];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f28;\n" " setp.eq.s32 %p5, %r14, 1;\n" " mov.u32 %r131, %r7;\n" " @%p5 bra $L__BB28_7;\n" "\n" " add.s64 %rd31, %rd5, %rd10;\n" " shl.b64 %rd32, %rd31, 2;\n" " add.s64 %rd30, %rd22, %rd32;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd30];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f29;\n" " setp.eq.s32 %p6, %r14, 2;\n" " mov.u32 %r131, %r8;\n" " @%p6 bra $L__BB28_7;\n" "\n" " add.s64 %rd34, %rd6, %rd10;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd33, %rd22, %rd35;\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd33];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f30;\n" " mov.u32 %r131, %r9;\n" "\n" "$L__BB28_7:\n" " setp.lt.u32 %p7, %r12, 768;\n" " @%p7 bra $L__BB28_10;\n" "\n" " cvt.s64.s32 %rd36, %r131;\n" " add.s64 %rd37, %rd36, %rd10;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd51, %rd22, %rd38;\n" " shl.b32 %r49, %r131, 2;\n" " add.s32 %r51, %r43, %r49;\n" " add.s32 %r132, %r51, 2048;\n" "\n" "$L__BB28_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd51];\n" " // end inline asm\n" " st.shared.f32 [%r132+-2048], %f31;\n" " add.s64 %rd40, %rd51, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd40];\n" " // end inline asm\n" " st.shared.f32 [%r132+-1024], %f32;\n" " add.s64 %rd41, %rd51, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd41];\n" " // end inline asm\n" " st.shared.f32 [%r132], %f33;\n" " add.s64 %rd42, %rd51, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r132+1024], %f34;\n" " add.s64 %rd51, %rd51, 4096;\n" " add.s32 %r132, %r132, 4096;\n" " add.s32 %r131, %r131, 1024;\n" " setp.lt.s32 %p8, %r131, %r13;\n" " @%p8 bra $L__BB28_9;\n" "\n" "$L__BB28_10:\n" " cvt.u32.u64 %r52, %rd1;\n" " setp.ge.s32 %p9, %r52, %r30;\n" " bar.sync 0;\n" " @%p9 bra $L__BB28_20;\n" "\n" " setp.lt.s32 %p10, %r13, 128;\n" " @%p10 bra $L__BB28_20;\n" "\n" " shr.s32 %r21, %r13, 7;\n" " mov.u32 %r134, 0;\n" "\n" "$L__BB28_13:\n" " shl.b32 %r60, %r134, 5;\n" " or.b32 %r61, %r60, %r2;\n" " shl.b32 %r62, %r61, 4;\n" " add.s32 %r64, %r43, %r62;\n" " mov.u32 %r58, 4;\n" " ld.shared.v4.f32 {%f42, %f43, %f44, %f45}, [%r64];\n" " mov.u32 %r54, 1;\n" " mov.u32 %r56, 2;\n" " abs.f32 %f46, %f42;\n" " mov.f32 %f106, 0f00000000;\n" " max.f32 %f47, %f106, %f46;\n" " abs.f32 %f48, %f43;\n" " max.f32 %f49, %f47, %f48;\n" " abs.f32 %f50, %f44;\n" " max.f32 %f51, %f49, %f50;\n" " abs.f32 %f52, %f45;\n" " max.f32 %f36, %f51, %f52;\n" " mov.u32 %r59, 31;\n" " // begin inline asm\n" " shfl.bfly.b32 %f35, %f36, %r54, %r59;\n" " // end inline asm\n" " max.f32 %f38, %f36, %f35;\n" " // begin inline asm\n" " shfl.bfly.b32 %f37, %f38, %r56, %r59;\n" " // end inline asm\n" " max.f32 %f40, %f38, %f37;\n" " // begin inline asm\n" " shfl.bfly.b32 %f39, %f40, %r58, %r59;\n" " // end inline asm\n" " max.f32 %f53, %f40, %f39;\n" " mov.f32 %f54, 0f3C010204;\n" " mul.rn.f32 %f9, %f53, %f54;\n" " setp.eq.f32 %p11, %f9, 0f00000000;\n" " @%p11 bra $L__BB28_15;\n" "\n" " mov.f32 %f55, 0f3F800000;\n" " div.approx.f32 %f106, %f55, %f9;\n" "\n" "$L__BB28_15:\n" " mul.rn.f32 %f57, %f42, %f106;\n" " mov.b32 %r65, %f57;\n" " and.b32 %r66, %r65, -2147483648;\n" " or.b32 %r67, %r66, 1056964608;\n" " mov.b32 %f58, %r67;\n" " add.rz.f32 %f59, %f57, %f58;\n" " cvt.rzi.f32.f32 %f60, %f59;\n" " cvt.rzi.s32.f32 %r23, %f60;\n" " mul.rn.f32 %f61, %f43, %f106;\n" " mov.b32 %r68, %f61;\n" " and.b32 %r69, %r68, -2147483648;\n" " or.b32 %r70, %r69, 1056964608;\n" " mov.b32 %f62, %r70;\n" " add.rz.f32 %f63, %f61, %f62;\n" " cvt.rzi.f32.f32 %f64, %f63;\n" " cvt.rzi.s32.f32 %r24, %f64;\n" " mul.rn.f32 %f65, %f44, %f106;\n" " mov.b32 %r71, %f65;\n" " and.b32 %r72, %r71, -2147483648;\n" " or.b32 %r73, %r72, 1056964608;\n" " mov.b32 %f66, %r73;\n" " add.rz.f32 %f67, %f65, %f66;\n" " cvt.rzi.f32.f32 %f68, %f67;\n" " cvt.rzi.s32.f32 %r25, %f68;\n" " mul.rn.f32 %f12, %f45, %f106;\n" " mov.b32 %r26, %f9;\n" " setp.gt.u32 %p12, %r26, 1199566847;\n" " mov.f32 %f107, 0f7F800000;\n" " @%p12 bra $L__BB28_19;\n" "\n" " setp.lt.u32 %p13, %r26, 947912704;\n" " @%p13 bra $L__BB28_18;\n" " bra.uni $L__BB28_17;\n" "\n" "$L__BB28_18:\n" " mov.f32 %f69, 0f3F000000;\n" " add.rn.f32 %f70, %f9, %f69;\n" " sub.rn.f32 %f107, %f70, %f69;\n" " bra.uni $L__BB28_19;\n" "\n" "$L__BB28_17:\n" " bfe.u32 %r74, %r26, 13, 1;\n" " add.s32 %r75, %r26, %r74;\n" " add.s32 %r76, %r75, 4095;\n" " and.b32 %r77, %r76, -8192;\n" " mov.b32 %f107, %r77;\n" "\n" "$L__BB28_19:\n" " mov.b32 %r80, %f12;\n" " and.b32 %r81, %r80, -2147483648;\n" " or.b32 %r82, %r81, 1056964608;\n" " mov.b32 %f73, %r82;\n" " add.rz.f32 %f74, %f12, %f73;\n" " cvt.rzi.f32.f32 %f75, %f74;\n" " cvt.rzi.s32.f32 %r83, %f75;\n" " add.s64 %rd43, %rd54, %rd3;\n" " // begin inline asm\n" " ld.global.nc.u8 %r78, [%rd43];\n" " // end inline asm\n" " and.b32 %r84, %r78, 3;\n" " add.s32 %r85, %r84, -1;\n" " mul.lo.s32 %r86, %r23, %r85;\n" " bfe.u32 %r87, %r78, 2, 2;\n" " add.s32 %r88, %r87, -1;\n" " mad.lo.s32 %r89, %r24, %r88, %r86;\n" " bfe.u32 %r90, %r78, 4, 2;\n" " add.s32 %r91, %r90, -1;\n" " mad.lo.s32 %r92, %r25, %r91, %r89;\n" " bfe.u32 %r93, %r78, 6, 2;\n" " add.s32 %r94, %r93, -1;\n" " add.s64 %rd44, %rd55, %rd3;\n" " // begin inline asm\n" " ld.global.nc.u8 %r79, [%rd44];\n" " // end inline asm\n" " and.b32 %r95, %r79, 3;\n" " add.s32 %r96, %r95, -1;\n" " mul.lo.s32 %r97, %r23, %r96;\n" " bfe.u32 %r98, %r79, 2, 2;\n" " add.s32 %r99, %r98, -1;\n" " mad.lo.s32 %r100, %r24, %r99, %r97;\n" " bfe.u32 %r101, %r79, 4, 2;\n" " add.s32 %r102, %r101, -1;\n" " mad.lo.s32 %r103, %r25, %r102, %r100;\n" " bfe.u32 %r104, %r79, 6, 2;\n" " add.s32 %r105, %r104, -1;\n" " mad.lo.s32 %r106, %r83, %r105, %r103;\n" " mad.lo.s32 %r107, %r83, %r94, %r92;\n" " // begin inline asm\n" " ld.global.nc.f32 %f71, [%rd54];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.f32 %f72, [%rd55];\n" " // end inline asm\n" " cvt.rn.f32.s32 %f76, %r107;\n" " mul.rn.f32 %f77, %f107, %f76;\n" " mul.rn.f32 %f78, %f71, %f77;\n" " add.rn.f32 %f108, %f108, %f78;\n" " cvt.rn.f32.s32 %f79, %r106;\n" " mul.rn.f32 %f80, %f107, %f79;\n" " mul.rn.f32 %f81, %f72, %f80;\n" " add.rn.f32 %f109, %f109, %f81;\n" " add.s64 %rd54, %rd54, 36;\n" " add.s64 %rd55, %rd55, 36;\n" " add.s32 %r134, %r134, 1;\n" " setp.lt.s32 %p14, %r134, %r21;\n" " @%p14 bra $L__BB28_13;\n" "\n" "$L__BB28_20:\n" " add.s32 %r130, %r130, 2048;\n" " setp.lt.s32 %p15, %r130, %r31;\n" " add.s32 %r129, %r129, 2048;\n" " @%p15 bra $L__BB28_2;\n" "\n" "$L__BB28_21:\n" " cvt.u32.u64 %r118, %rd1;\n" " setp.ge.s32 %p16, %r118, %r30;\n" " mov.u32 %r108, 16;\n" " mov.u32 %r117, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f82, %f108, %r108, %r117;\n" " // end inline asm\n" " add.f32 %f85, %f108, %f82;\n" " mov.u32 %r110, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f84, %f85, %r110, %r117;\n" " // end inline asm\n" " add.f32 %f87, %f85, %f84;\n" " mov.u32 %r112, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f86, %f87, %r112, %r117;\n" " // end inline asm\n" " add.f32 %f89, %f87, %f86;\n" " mov.u32 %r114, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f88, %f89, %r114, %r117;\n" " // end inline asm\n" " add.f32 %f91, %f89, %f88;\n" " mov.u32 %r116, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f90, %f91, %r116, %r117;\n" " // end inline asm\n" " add.f32 %f22, %f91, %f90;\n" " setp.ne.s32 %p17, %r2, 0;\n" " cvta.to.global.u64 %rd47, %rd23;\n" " shl.b64 %rd48, %rd1, 2;\n" " add.s64 %rd20, %rd47, %rd48;\n" " or.pred %p18, %p17, %p16;\n" " @%p18 bra $L__BB28_23;\n" "\n" " st.global.f32 [%rd20], %f22;\n" "\n" "$L__BB28_23:\n" " // begin inline asm\n" " shfl.down.b32 %f92, %f109, %r108, %r117;\n" " // end inline asm\n" " add.f32 %f95, %f109, %f92;\n" " // begin inline asm\n" " shfl.down.b32 %f94, %f95, %r110, %r117;\n" " // end inline asm\n" " add.f32 %f97, %f95, %f94;\n" " // begin inline asm\n" " shfl.down.b32 %f96, %f97, %r112, %r117;\n" " // end inline asm\n" " add.f32 %f99, %f97, %f96;\n" " // begin inline asm\n" " shfl.down.b32 %f98, %f99, %r114, %r117;\n" " // end inline asm\n" " add.f32 %f101, %f99, %f98;\n" " // begin inline asm\n" " shfl.down.b32 %f100, %f101, %r116, %r117;\n" " // end inline asm\n" " add.f32 %f23, %f101, %f100;\n" " setp.ge.s32 %p20, %r3, %r30;\n" " or.pred %p21, %p17, %p20;\n" " @%p21 bra $L__BB28_25;\n" "\n" " st.global.f32 [%rd20+4], %f23;\n" "\n" "$L__BB28_25:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_ROPE[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_rope_neox_partial\n" ".global .align 4 .b8 __cudart_i2opi_f[24] = {65, 144, 67, 60, 153, 149, 98, 219, 192, 221, 52, 245, 209, 87, 39, 252, 41, 21, 68, 78, 110, 131, 249, 162};\n" "\n" ".visible .entry fc_rope_neox_partial(\n" " .param .u64 fc_rope_neox_partial_param_0,\n" " .param .u32 fc_rope_neox_partial_param_1,\n" " .param .u32 fc_rope_neox_partial_param_2,\n" " .param .u32 fc_rope_neox_partial_param_3,\n" " .param .u32 fc_rope_neox_partial_param_4,\n" " .param .f32 fc_rope_neox_partial_param_5\n" ")\n" "{\n" " .local .align 4 .b8 __local_depot0[28];\n" " .reg .b64 %SP;\n" " .reg .b64 %SPL;\n" " .reg .pred %p<53>;\n" " .reg .f32 %f<183>;\n" " .reg .b32 %r<143>;\n" " .reg .f64 %fd<5>;\n" " .reg .b64 %rd<56>;\n" "\n" "\n" " mov.u64 %SPL, __local_depot0;\n" " ld.param.u64 %rd17, [fc_rope_neox_partial_param_0];\n" " ld.param.u32 %r44, [fc_rope_neox_partial_param_1];\n" " ld.param.u32 %r41, [fc_rope_neox_partial_param_2];\n" " ld.param.u32 %r42, [fc_rope_neox_partial_param_3];\n" " ld.param.f32 %f39, [fc_rope_neox_partial_param_5];\n" " add.u64 %rd1, %SPL, 0;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p2, %r1, %r44;\n" " @%p2 bra $L__BB0_41;\n" "\n" " setp.gt.s32 %p3, %r42, 0;\n" " setp.lt.s32 %p4, %r42, %r41;\n" " and.pred %p5, %p3, %p4;\n" " selp.b32 %r2, %r42, %r41, %p5;\n" " shr.u32 %r45, %r2, 31;\n" " add.s32 %r46, %r2, %r45;\n" " shr.s32 %r3, %r46, 1;\n" " mov.u32 %r4, %tid.x;\n" " setp.ge.s32 %p6, %r4, %r3;\n" " @%p6 bra $L__BB0_41;\n" "\n" " cvta.to.global.u64 %rd2, %rd17;\n" " shl.b32 %r47, %r4, 1;\n" " cvt.rn.f32.s32 %f41, %r47;\n" " cvt.rn.f32.s32 %f42, %r2;\n" " div.rn.f32 %f1, %f41, %f42;\n" " mul.f32 %f43, %f1, 0f3F000000;\n" " cvt.rzi.f32.f32 %f44, %f43;\n" " add.f32 %f45, %f44, %f44;\n" " sub.f32 %f46, %f1, %f45;\n" " abs.f32 %f2, %f46;\n" " abs.f32 %f3, %f39;\n" " setp.lt.f32 %p7, %f3, 0f00800000;\n" " mul.f32 %f47, %f3, 0f4B800000;\n" " selp.f32 %f48, %f47, %f3, %p7;\n" " selp.f32 %f49, 0fC3170000, 0fC2FE0000, %p7;\n" " mov.b32 %r48, %f48;\n" " and.b32 %r49, %r48, 8388607;\n" " or.b32 %r50, %r49, 1065353216;\n" " mov.b32 %f50, %r50;\n" " shr.u32 %r51, %r48, 23;\n" " cvt.rn.f32.u32 %f51, %r51;\n" " add.f32 %f52, %f49, %f51;\n" " setp.gt.f32 %p8, %f50, 0f3FB504F3;\n" " mul.f32 %f53, %f50, 0f3F000000;\n" " add.f32 %f54, %f52, 0f3F800000;\n" " selp.f32 %f55, %f54, %f52, %p8;\n" " selp.f32 %f56, %f53, %f50, %p8;\n" " add.f32 %f57, %f56, 0fBF800000;\n" " add.f32 %f58, %f56, 0f3F800000;\n" " rcp.approx.ftz.f32 %f59, %f58;\n" " add.f32 %f60, %f57, %f57;\n" " mul.f32 %f61, %f60, %f59;\n" " mul.f32 %f62, %f61, %f61;\n" " mov.f32 %f63, 0f3C4CAF63;\n" " mov.f32 %f64, 0f3B18F0FE;\n" " fma.rn.f32 %f65, %f64, %f62, %f63;\n" " mov.f32 %f66, 0f3DAAAABD;\n" " fma.rn.f32 %f67, %f65, %f62, %f66;\n" " mul.rn.f32 %f68, %f67, %f62;\n" " mul.rn.f32 %f69, %f68, %f61;\n" " sub.f32 %f70, %f57, %f61;\n" " add.f32 %f71, %f70, %f70;\n" " neg.f32 %f72, %f61;\n" " fma.rn.f32 %f73, %f72, %f57, %f71;\n" " mul.rn.f32 %f74, %f59, %f73;\n" " add.f32 %f75, %f69, %f61;\n" " sub.f32 %f76, %f61, %f75;\n" " add.f32 %f77, %f69, %f76;\n" " add.f32 %f78, %f74, %f77;\n" " add.f32 %f79, %f75, %f78;\n" " sub.f32 %f80, %f75, %f79;\n" " add.f32 %f81, %f78, %f80;\n" " mov.f32 %f82, 0f3F317200;\n" " mul.rn.f32 %f83, %f55, %f82;\n" " mov.f32 %f84, 0f35BFBE8E;\n" " mul.rn.f32 %f85, %f55, %f84;\n" " add.f32 %f86, %f83, %f79;\n" " sub.f32 %f87, %f83, %f86;\n" " add.f32 %f88, %f79, %f87;\n" " add.f32 %f89, %f81, %f88;\n" " add.f32 %f90, %f85, %f89;\n" " add.f32 %f91, %f86, %f90;\n" " sub.f32 %f92, %f86, %f91;\n" " add.f32 %f93, %f90, %f92;\n" " abs.f32 %f4, %f1;\n" " setp.gt.f32 %p9, %f4, 0f77F684DF;\n" " mul.f32 %f94, %f1, 0f39000000;\n" " selp.f32 %f95, %f94, %f1, %p9;\n" " mul.rn.f32 %f96, %f95, %f91;\n" " neg.f32 %f97, %f96;\n" " fma.rn.f32 %f98, %f95, %f91, %f97;\n" " fma.rn.f32 %f99, %f95, %f93, %f98;\n" " mov.f32 %f100, 0f00000000;\n" " fma.rn.f32 %f101, %f100, %f91, %f99;\n" " add.rn.f32 %f102, %f96, %f101;\n" " neg.f32 %f103, %f102;\n" " add.rn.f32 %f104, %f96, %f103;\n" " add.rn.f32 %f105, %f104, %f101;\n" " mov.b32 %r52, %f102;\n" " setp.eq.s32 %p10, %r52, 1118925336;\n" " add.s32 %r53, %r52, -1;\n" " mov.b32 %f106, %r53;\n" " add.f32 %f107, %f105, 0f37000000;\n" " selp.f32 %f5, %f107, %f105, %p10;\n" " selp.f32 %f108, %f106, %f102, %p10;\n" " mov.f32 %f109, 0f3FB8AA3B;\n" " mul.rn.f32 %f110, %f108, %f109;\n" " cvt.rzi.f32.f32 %f111, %f110;\n" " abs.f32 %f112, %f111;\n" " setp.gt.f32 %p11, %f112, 0f42FC0000;\n" " mov.b32 %r54, %f111;\n" " and.b32 %r55, %r54, -2147483648;\n" " or.b32 %r56, %r55, 1123811328;\n" " mov.b32 %f113, %r56;\n" " selp.f32 %f114, %f113, %f111, %p11;\n" " mov.f32 %f115, 0fBF317218;\n" " fma.rn.f32 %f116, %f114, %f115, %f108;\n" " mov.f32 %f117, 0f3102E308;\n" " fma.rn.f32 %f118, %f114, %f117, %f116;\n" " mul.f32 %f119, %f118, 0f3FB8AA3B;\n" " add.f32 %f120, %f114, 0f4B40007F;\n" " mov.b32 %r57, %f120;\n" " shl.b32 %r58, %r57, 23;\n" " mov.b32 %f121, %r58;\n" " ex2.approx.ftz.f32 %f122, %f119;\n" " mul.f32 %f6, %f122, %f121;\n" " setp.eq.f32 %p12, %f6, 0f7F800000;\n" " mov.f32 %f174, 0f7F800000;\n" " @%p12 bra $L__BB0_4;\n" "\n" " fma.rn.f32 %f174, %f6, %f5, %f6;\n" "\n" "$L__BB0_4:\n" " setp.lt.f32 %p13, %f39, 0f00000000;\n" " setp.eq.f32 %p14, %f2, 0f3F800000;\n" " and.pred %p1, %p13, %p14;\n" " setp.eq.f32 %p15, %f39, 0f00000000;\n" " @%p15 bra $L__BB0_8;\n" " bra.uni $L__BB0_5;\n" "\n" "$L__BB0_8:\n" " add.f32 %f126, %f39, %f39;\n" " mov.b32 %r61, %f126;\n" " selp.b32 %r62, %r61, 0, %p14;\n" " or.b32 %r63, %r62, 2139095040;\n" " setp.lt.f32 %p19, %f1, 0f00000000;\n" " selp.b32 %r64, %r63, %r62, %p19;\n" " mov.b32 %f176, %r64;\n" " bra.uni $L__BB0_9;\n" "\n" "$L__BB0_5:\n" " mov.b32 %r59, %f174;\n" " xor.b32 %r60, %r59, -2147483648;\n" " mov.b32 %f123, %r60;\n" " selp.f32 %f176, %f123, %f174, %p1;\n" " setp.geu.f32 %p16, %f39, 0f00000000;\n" " @%p16 bra $L__BB0_9;\n" "\n" " cvt.rzi.f32.f32 %f124, %f1;\n" " setp.eq.f32 %p17, %f124, %f1;\n" " @%p17 bra $L__BB0_9;\n" "\n" " mov.f32 %f176, 0f7FFFFFFF;\n" "\n" "$L__BB0_9:\n" " add.f32 %f127, %f3, %f4;\n" " mov.b32 %r65, %f127;\n" " setp.lt.s32 %p20, %r65, 2139095040;\n" " @%p20 bra $L__BB0_16;\n" "\n" " setp.gtu.f32 %p21, %f3, 0f7F800000;\n" " setp.gtu.f32 %p22, %f4, 0f7F800000;\n" " or.pred %p23, %p21, %p22;\n" " @%p23 bra $L__BB0_15;\n" " bra.uni $L__BB0_11;\n" "\n" "$L__BB0_15:\n" " add.f32 %f176, %f1, %f39;\n" " bra.uni $L__BB0_16;\n" "\n" "$L__BB0_11:\n" " setp.eq.f32 %p24, %f4, 0f7F800000;\n" " @%p24 bra $L__BB0_14;\n" " bra.uni $L__BB0_12;\n" "\n" "$L__BB0_14:\n" " setp.gt.f32 %p27, %f3, 0f3F800000;\n" " selp.b32 %r69, 2139095040, 0, %p27;\n" " xor.b32 %r70, %r69, 2139095040;\n" " setp.lt.f32 %p28, %f1, 0f00000000;\n" " selp.b32 %r71, %r70, %r69, %p28;\n" " mov.b32 %f128, %r71;\n" " setp.eq.f32 %p29, %f39, 0fBF800000;\n" " selp.f32 %f176, 0f3F800000, %f128, %p29;\n" " bra.uni $L__BB0_16;\n" "\n" "$L__BB0_12:\n" " setp.neu.f32 %p25, %f3, 0f7F800000;\n" " @%p25 bra $L__BB0_16;\n" "\n" " setp.ge.f32 %p26, %f1, 0f00000000;\n" " selp.b32 %r66, 2139095040, 0, %p26;\n" " or.b32 %r67, %r66, -2147483648;\n" " selp.b32 %r68, %r67, %r66, %p1;\n" " mov.b32 %f176, %r68;\n" "\n" "$L__BB0_16:\n" " ld.param.u32 %r131, [fc_rope_neox_partial_param_4];\n" " setp.eq.f32 %p30, %f1, 0f00000000;\n" " setp.eq.f32 %p31, %f39, 0f3F800000;\n" " or.pred %p32, %p31, %p30;\n" " rcp.rn.f32 %f129, %f176;\n" " selp.f32 %f130, 0f3F800000, %f129, %p32;\n" " cvt.rn.f32.s32 %f131, %r131;\n" " mul.f32 %f16, %f130, %f131;\n" " mul.f32 %f132, %f16, 0f3F22F983;\n" " cvt.rni.s32.f32 %r142, %f132;\n" " cvt.rn.f32.s32 %f133, %r142;\n" " mov.f32 %f134, 0fBFC90FDA;\n" " fma.rn.f32 %f135, %f133, %f134, %f16;\n" " mov.f32 %f136, 0fB3A22168;\n" " fma.rn.f32 %f137, %f133, %f136, %f135;\n" " mov.f32 %f138, 0fA7C234C5;\n" " fma.rn.f32 %f180, %f133, %f138, %f137;\n" " abs.f32 %f18, %f16;\n" " setp.leu.f32 %p33, %f18, 0f47CE4780;\n" " add.s64 %rd4, %rd1, 24;\n" " mov.u32 %r138, %r142;\n" " mov.f32 %f177, %f180;\n" " @%p33 bra $L__BB0_24;\n" "\n" " setp.eq.f32 %p34, %f18, 0f7F800000;\n" " @%p34 bra $L__BB0_23;\n" " bra.uni $L__BB0_18;\n" "\n" "$L__BB0_23:\n" " mul.rn.f32 %f177, %f16, %f100;\n" " mov.u32 %r138, %r142;\n" " bra.uni $L__BB0_24;\n" "\n" "$L__BB0_18:\n" " mov.b32 %r6, %f16;\n" " bfe.u32 %r73, %r6, 23, 8;\n" " add.s32 %r7, %r73, -128;\n" " shl.b32 %r74, %r6, 8;\n" " or.b32 %r8, %r74, -2147483648;\n" " shr.u32 %r9, %r7, 5;\n" " mov.u64 %rd52, 0;\n" " mov.u32 %r135, 0;\n" " mov.u64 %rd50, __cudart_i2opi_f;\n" " mov.u64 %rd51, %rd1;\n" "\n" "$L__BB0_19:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r75, [%rd50];\n" " mad.wide.u32 %rd21, %r75, %r8, %rd52;\n" " shr.u64 %rd52, %rd21, 32;\n" " st.local.u32 [%rd51], %rd21;\n" " add.s64 %rd51, %rd51, 4;\n" " add.s64 %rd50, %rd50, 4;\n" " add.s32 %r135, %r135, 1;\n" " setp.ne.s32 %p35, %r135, 6;\n" " @%p35 bra $L__BB0_19;\n" "\n" " st.local.u32 [%rd4], %rd52;\n" " mov.u32 %r76, 4;\n" " sub.s32 %r12, %r76, %r9;\n" " mov.u32 %r77, 6;\n" " sub.s32 %r78, %r77, %r9;\n" " mul.wide.s32 %rd22, %r78, 4;\n" " add.s64 %rd23, %rd1, %rd22;\n" " ld.local.u32 %r136, [%rd23];\n" " ld.local.u32 %r137, [%rd23+-4];\n" " and.b32 %r15, %r7, 31;\n" " setp.eq.s32 %p36, %r15, 0;\n" " @%p36 bra $L__BB0_22;\n" "\n" " mov.u32 %r79, 32;\n" " sub.s32 %r80, %r79, %r15;\n" " shr.u32 %r81, %r137, %r80;\n" " shl.b32 %r82, %r136, %r15;\n" " add.s32 %r136, %r81, %r82;\n" " mul.wide.s32 %rd24, %r12, 4;\n" " add.s64 %rd25, %rd1, %rd24;\n" " ld.local.u32 %r83, [%rd25];\n" " shr.u32 %r84, %r83, %r80;\n" " shl.b32 %r85, %r137, %r15;\n" " add.s32 %r137, %r84, %r85;\n" "\n" "$L__BB0_22:\n" " and.b32 %r86, %r6, -2147483648;\n" " shr.u32 %r87, %r137, 30;\n" " shl.b32 %r88, %r136, 2;\n" " or.b32 %r89, %r87, %r88;\n" " shr.u32 %r90, %r89, 31;\n" " shr.u32 %r91, %r136, 30;\n" " add.s32 %r92, %r90, %r91;\n" " neg.s32 %r93, %r92;\n" " setp.eq.s32 %p37, %r86, 0;\n" " selp.b32 %r138, %r92, %r93, %p37;\n" " setp.ne.s32 %p38, %r90, 0;\n" " xor.b32 %r94, %r86, -2147483648;\n" " selp.b32 %r95, %r94, %r86, %p38;\n" " selp.b32 %r96, -1, 0, %p38;\n" " xor.b32 %r97, %r89, %r96;\n" " shl.b32 %r98, %r137, 2;\n" " xor.b32 %r99, %r98, %r96;\n" " cvt.u64.u32 %rd26, %r97;\n" " shl.b64 %rd27, %rd26, 32;\n" " cvt.u64.u32 %rd28, %r99;\n" " or.b64 %rd29, %rd27, %rd28;\n" " cvt.rn.f64.s64 %fd1, %rd29;\n" " mul.f64 %fd2, %fd1, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f139, %fd2;\n" " setp.eq.s32 %p39, %r95, 0;\n" " neg.f32 %f140, %f139;\n" " selp.f32 %f177, %f139, %f140, %p39;\n" "\n" "$L__BB0_24:\n" " add.s32 %r22, %r138, 1;\n" " and.b32 %r23, %r22, 1;\n" " setp.eq.s32 %p40, %r23, 0;\n" " selp.f32 %f22, %f177, 0f3F800000, %p40;\n" " mul.rn.f32 %f23, %f177, %f177;\n" " mov.f32 %f178, 0fB94D4153;\n" " @%p40 bra $L__BB0_26;\n" "\n" " mov.f32 %f143, 0fBAB607ED;\n" " mov.f32 %f144, 0f37CBAC00;\n" " fma.rn.f32 %f178, %f144, %f23, %f143;\n" "\n" "$L__BB0_26:\n" " selp.f32 %f145, 0f3C0885E4, 0f3D2AAABB, %p40;\n" " fma.rn.f32 %f146, %f178, %f23, %f145;\n" " selp.f32 %f147, 0fBE2AAAA8, 0fBEFFFFFF, %p40;\n" " fma.rn.f32 %f148, %f146, %f23, %f147;\n" " fma.rn.f32 %f150, %f23, %f22, %f100;\n" " fma.rn.f32 %f179, %f148, %f150, %f22;\n" " and.b32 %r100, %r22, 2;\n" " setp.eq.s32 %p42, %r100, 0;\n" " @%p42 bra $L__BB0_28;\n" "\n" " mov.f32 %f152, 0fBF800000;\n" " fma.rn.f32 %f179, %f179, %f152, %f100;\n" "\n" "$L__BB0_28:\n" " @%p33 bra $L__BB0_36;\n" "\n" " setp.eq.f32 %p44, %f18, 0f7F800000;\n" " @%p44 bra $L__BB0_35;\n" " bra.uni $L__BB0_30;\n" "\n" "$L__BB0_35:\n" " mul.rn.f32 %f180, %f16, %f100;\n" " bra.uni $L__BB0_36;\n" "\n" "$L__BB0_30:\n" " mov.b32 %r24, %f16;\n" " bfe.u32 %r102, %r24, 23, 8;\n" " add.s32 %r25, %r102, -128;\n" " shl.b32 %r103, %r24, 8;\n" " or.b32 %r26, %r103, -2147483648;\n" " shr.u32 %r27, %r25, 5;\n" " mov.u64 %rd55, 0;\n" " mov.u32 %r139, 0;\n" " mov.u64 %rd53, __cudart_i2opi_f;\n" " mov.u64 %rd54, %rd1;\n" "\n" "$L__BB0_31:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r104, [%rd53];\n" " mad.wide.u32 %rd32, %r104, %r26, %rd55;\n" " shr.u64 %rd55, %rd32, 32;\n" " st.local.u32 [%rd54], %rd32;\n" " add.s64 %rd54, %rd54, 4;\n" " add.s64 %rd53, %rd53, 4;\n" " add.s32 %r139, %r139, 1;\n" " setp.ne.s32 %p45, %r139, 6;\n" " @%p45 bra $L__BB0_31;\n" "\n" " st.local.u32 [%rd4], %rd55;\n" " mov.u32 %r105, 4;\n" " sub.s32 %r30, %r105, %r27;\n" " mov.u32 %r106, 6;\n" " sub.s32 %r107, %r106, %r27;\n" " mul.wide.s32 %rd33, %r107, 4;\n" " add.s64 %rd34, %rd1, %rd33;\n" " ld.local.u32 %r140, [%rd34];\n" " ld.local.u32 %r141, [%rd34+-4];\n" " and.b32 %r33, %r25, 31;\n" " setp.eq.s32 %p46, %r33, 0;\n" " @%p46 bra $L__BB0_34;\n" "\n" " mov.u32 %r108, 32;\n" " sub.s32 %r109, %r108, %r33;\n" " shr.u32 %r110, %r141, %r109;\n" " shl.b32 %r111, %r140, %r33;\n" " add.s32 %r140, %r110, %r111;\n" " mul.wide.s32 %rd35, %r30, 4;\n" " add.s64 %rd36, %rd1, %rd35;\n" " ld.local.u32 %r112, [%rd36];\n" " shr.u32 %r113, %r112, %r109;\n" " shl.b32 %r114, %r141, %r33;\n" " add.s32 %r141, %r113, %r114;\n" "\n" "$L__BB0_34:\n" " and.b32 %r115, %r24, -2147483648;\n" " shr.u32 %r116, %r141, 30;\n" " shl.b32 %r117, %r140, 2;\n" " or.b32 %r118, %r116, %r117;\n" " shr.u32 %r119, %r118, 31;\n" " shr.u32 %r120, %r140, 30;\n" " add.s32 %r121, %r119, %r120;\n" " neg.s32 %r122, %r121;\n" " setp.eq.s32 %p47, %r115, 0;\n" " selp.b32 %r142, %r121, %r122, %p47;\n" " setp.ne.s32 %p48, %r119, 0;\n" " xor.b32 %r123, %r115, -2147483648;\n" " selp.b32 %r124, %r123, %r115, %p48;\n" " selp.b32 %r125, -1, 0, %p48;\n" " xor.b32 %r126, %r118, %r125;\n" " shl.b32 %r127, %r141, 2;\n" " xor.b32 %r128, %r127, %r125;\n" " cvt.u64.u32 %rd37, %r126;\n" " shl.b64 %rd38, %rd37, 32;\n" " cvt.u64.u32 %rd39, %r128;\n" " or.b64 %rd40, %rd38, %rd39;\n" " cvt.rn.f64.s64 %fd3, %rd40;\n" " mul.f64 %fd4, %fd3, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f153, %fd4;\n" " setp.eq.s32 %p49, %r124, 0;\n" " neg.f32 %f154, %f153;\n" " selp.f32 %f180, %f153, %f154, %p49;\n" "\n" "$L__BB0_36:\n" " and.b32 %r40, %r142, 1;\n" " setp.eq.s32 %p50, %r40, 0;\n" " selp.f32 %f32, %f180, 0f3F800000, %p50;\n" " mul.rn.f32 %f33, %f180, %f180;\n" " mov.f32 %f181, 0fB94D4153;\n" " @%p50 bra $L__BB0_38;\n" "\n" " mov.f32 %f157, 0fBAB607ED;\n" " mov.f32 %f158, 0f37CBAC00;\n" " fma.rn.f32 %f181, %f158, %f33, %f157;\n" "\n" "$L__BB0_38:\n" " selp.f32 %f159, 0f3C0885E4, 0f3D2AAABB, %p50;\n" " fma.rn.f32 %f160, %f181, %f33, %f159;\n" " selp.f32 %f161, 0fBE2AAAA8, 0fBEFFFFFF, %p50;\n" " fma.rn.f32 %f162, %f160, %f33, %f161;\n" " fma.rn.f32 %f164, %f33, %f32, %f100;\n" " fma.rn.f32 %f182, %f162, %f164, %f32;\n" " and.b32 %r129, %r142, 2;\n" " setp.eq.s32 %p52, %r129, 0;\n" " @%p52 bra $L__BB0_40;\n" "\n" " mov.f32 %f166, 0fBF800000;\n" " fma.rn.f32 %f182, %f182, %f166, %f100;\n" "\n" "$L__BB0_40:\n" " mov.u32 %r134, %ctaid.x;\n" " ld.param.u32 %r133, [fc_rope_neox_partial_param_2];\n" " mul.wide.s32 %rd49, %r133, %r134;\n" " mov.u32 %r132, %tid.x;\n" " cvt.s64.s32 %rd41, %r132;\n" " add.s64 %rd42, %rd49, %rd41;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd44, %rd2, %rd43;\n" " add.s32 %r130, %r3, %r132;\n" " cvt.s64.s32 %rd45, %r130;\n" " add.s64 %rd46, %rd49, %rd45;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd48, %rd2, %rd47;\n" " ld.global.f32 %f167, [%rd44];\n" " mul.f32 %f168, %f179, %f167;\n" " ld.global.f32 %f169, [%rd48];\n" " mul.f32 %f170, %f182, %f169;\n" " sub.f32 %f171, %f168, %f170;\n" " st.global.f32 [%rd44], %f171;\n" " mul.f32 %f172, %f179, %f169;\n" " fma.rn.f32 %f173, %f182, %f167, %f172;\n" " st.global.f32 [%rd48], %f173;\n" "\n" "$L__BB0_41:\n" " ret;\n" "\n" "}\n" " // .globl fc_rope_neox_partial_nx\n" ".visible .entry fc_rope_neox_partial_nx(\n" " .param .u64 fc_rope_neox_partial_nx_param_0,\n" " .param .u32 fc_rope_neox_partial_nx_param_1,\n" " .param .u32 fc_rope_neox_partial_nx_param_2,\n" " .param .u32 fc_rope_neox_partial_nx_param_3,\n" " .param .u32 fc_rope_neox_partial_nx_param_4,\n" " .param .f32 fc_rope_neox_partial_nx_param_5\n" ")\n" "{\n" " .local .align 4 .b8 __local_depot1[28];\n" " .reg .b64 %SP;\n" " .reg .b64 %SPL;\n" " .reg .pred %p<53>;\n" " .reg .f32 %f<183>;\n" " .reg .b32 %r<146>;\n" " .reg .f64 %fd<5>;\n" " .reg .b64 %rd<66>;\n" "\n" "\n" " mov.u64 %SPL, __local_depot1;\n" " ld.param.u32 %r42, [fc_rope_neox_partial_nx_param_1];\n" " ld.param.u32 %r43, [fc_rope_neox_partial_nx_param_2];\n" " ld.param.u32 %r44, [fc_rope_neox_partial_nx_param_3];\n" " ld.param.u32 %r45, [fc_rope_neox_partial_nx_param_4];\n" " ld.param.f32 %f39, [fc_rope_neox_partial_nx_param_5];\n" " add.u64 %rd1, %SPL, 0;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p2, %r1, %r42;\n" " @%p2 bra $L__BB1_41;\n" "\n" " setp.gt.s32 %p3, %r44, 0;\n" " setp.lt.s32 %p4, %r44, %r43;\n" " and.pred %p5, %p3, %p4;\n" " selp.b32 %r2, %r44, %r43, %p5;\n" " shr.u32 %r46, %r2, 31;\n" " add.s32 %r47, %r2, %r46;\n" " shr.s32 %r3, %r47, 1;\n" " mov.u32 %r4, %tid.x;\n" " setp.ge.s32 %p6, %r4, %r3;\n" " @%p6 bra $L__BB1_41;\n" "\n" " mov.u32 %r48, %ctaid.y;\n" " add.s32 %r5, %r48, %r45;\n" " shl.b32 %r49, %r4, 1;\n" " cvt.rn.f32.s32 %f41, %r49;\n" " cvt.rn.f32.s32 %f42, %r2;\n" " div.rn.f32 %f1, %f41, %f42;\n" " mul.f32 %f43, %f1, 0f3F000000;\n" " cvt.rzi.f32.f32 %f44, %f43;\n" " add.f32 %f45, %f44, %f44;\n" " sub.f32 %f46, %f1, %f45;\n" " abs.f32 %f2, %f46;\n" " abs.f32 %f3, %f39;\n" " setp.lt.f32 %p7, %f3, 0f00800000;\n" " mul.f32 %f47, %f3, 0f4B800000;\n" " selp.f32 %f48, %f47, %f3, %p7;\n" " selp.f32 %f49, 0fC3170000, 0fC2FE0000, %p7;\n" " mov.b32 %r50, %f48;\n" " and.b32 %r51, %r50, 8388607;\n" " or.b32 %r52, %r51, 1065353216;\n" " mov.b32 %f50, %r52;\n" " shr.u32 %r53, %r50, 23;\n" " cvt.rn.f32.u32 %f51, %r53;\n" " add.f32 %f52, %f49, %f51;\n" " setp.gt.f32 %p8, %f50, 0f3FB504F3;\n" " mul.f32 %f53, %f50, 0f3F000000;\n" " add.f32 %f54, %f52, 0f3F800000;\n" " selp.f32 %f55, %f54, %f52, %p8;\n" " selp.f32 %f56, %f53, %f50, %p8;\n" " add.f32 %f57, %f56, 0fBF800000;\n" " add.f32 %f58, %f56, 0f3F800000;\n" " rcp.approx.ftz.f32 %f59, %f58;\n" " add.f32 %f60, %f57, %f57;\n" " mul.f32 %f61, %f60, %f59;\n" " mul.f32 %f62, %f61, %f61;\n" " mov.f32 %f63, 0f3C4CAF63;\n" " mov.f32 %f64, 0f3B18F0FE;\n" " fma.rn.f32 %f65, %f64, %f62, %f63;\n" " mov.f32 %f66, 0f3DAAAABD;\n" " fma.rn.f32 %f67, %f65, %f62, %f66;\n" " mul.rn.f32 %f68, %f67, %f62;\n" " mul.rn.f32 %f69, %f68, %f61;\n" " sub.f32 %f70, %f57, %f61;\n" " add.f32 %f71, %f70, %f70;\n" " neg.f32 %f72, %f61;\n" " fma.rn.f32 %f73, %f72, %f57, %f71;\n" " mul.rn.f32 %f74, %f59, %f73;\n" " add.f32 %f75, %f69, %f61;\n" " sub.f32 %f76, %f61, %f75;\n" " add.f32 %f77, %f69, %f76;\n" " add.f32 %f78, %f74, %f77;\n" " add.f32 %f79, %f75, %f78;\n" " sub.f32 %f80, %f75, %f79;\n" " add.f32 %f81, %f78, %f80;\n" " mov.f32 %f82, 0f3F317200;\n" " mul.rn.f32 %f83, %f55, %f82;\n" " mov.f32 %f84, 0f35BFBE8E;\n" " mul.rn.f32 %f85, %f55, %f84;\n" " add.f32 %f86, %f83, %f79;\n" " sub.f32 %f87, %f83, %f86;\n" " add.f32 %f88, %f79, %f87;\n" " add.f32 %f89, %f81, %f88;\n" " add.f32 %f90, %f85, %f89;\n" " add.f32 %f91, %f86, %f90;\n" " sub.f32 %f92, %f86, %f91;\n" " add.f32 %f93, %f90, %f92;\n" " abs.f32 %f4, %f1;\n" " setp.gt.f32 %p9, %f4, 0f77F684DF;\n" " mul.f32 %f94, %f1, 0f39000000;\n" " selp.f32 %f95, %f94, %f1, %p9;\n" " mul.rn.f32 %f96, %f95, %f91;\n" " neg.f32 %f97, %f96;\n" " fma.rn.f32 %f98, %f95, %f91, %f97;\n" " fma.rn.f32 %f99, %f95, %f93, %f98;\n" " mov.f32 %f100, 0f00000000;\n" " fma.rn.f32 %f101, %f100, %f91, %f99;\n" " add.rn.f32 %f102, %f96, %f101;\n" " neg.f32 %f103, %f102;\n" " add.rn.f32 %f104, %f96, %f103;\n" " add.rn.f32 %f105, %f104, %f101;\n" " mov.b32 %r54, %f102;\n" " setp.eq.s32 %p10, %r54, 1118925336;\n" " add.s32 %r55, %r54, -1;\n" " mov.b32 %f106, %r55;\n" " add.f32 %f107, %f105, 0f37000000;\n" " selp.f32 %f5, %f107, %f105, %p10;\n" " selp.f32 %f108, %f106, %f102, %p10;\n" " mov.f32 %f109, 0f3FB8AA3B;\n" " mul.rn.f32 %f110, %f108, %f109;\n" " cvt.rzi.f32.f32 %f111, %f110;\n" " abs.f32 %f112, %f111;\n" " setp.gt.f32 %p11, %f112, 0f42FC0000;\n" " mov.b32 %r56, %f111;\n" " and.b32 %r57, %r56, -2147483648;\n" " or.b32 %r58, %r57, 1123811328;\n" " mov.b32 %f113, %r58;\n" " selp.f32 %f114, %f113, %f111, %p11;\n" " mov.f32 %f115, 0fBF317218;\n" " fma.rn.f32 %f116, %f114, %f115, %f108;\n" " mov.f32 %f117, 0f3102E308;\n" " fma.rn.f32 %f118, %f114, %f117, %f116;\n" " mul.f32 %f119, %f118, 0f3FB8AA3B;\n" " add.f32 %f120, %f114, 0f4B40007F;\n" " mov.b32 %r59, %f120;\n" " shl.b32 %r60, %r59, 23;\n" " mov.b32 %f121, %r60;\n" " ex2.approx.ftz.f32 %f122, %f119;\n" " mul.f32 %f6, %f122, %f121;\n" " setp.eq.f32 %p12, %f6, 0f7F800000;\n" " mov.f32 %f174, 0f7F800000;\n" " @%p12 bra $L__BB1_4;\n" "\n" " fma.rn.f32 %f174, %f6, %f5, %f6;\n" "\n" "$L__BB1_4:\n" " setp.lt.f32 %p13, %f39, 0f00000000;\n" " setp.eq.f32 %p14, %f2, 0f3F800000;\n" " and.pred %p1, %p13, %p14;\n" " setp.eq.f32 %p15, %f39, 0f00000000;\n" " @%p15 bra $L__BB1_8;\n" " bra.uni $L__BB1_5;\n" "\n" "$L__BB1_8:\n" " add.f32 %f126, %f39, %f39;\n" " mov.b32 %r63, %f126;\n" " selp.b32 %r64, %r63, 0, %p14;\n" " or.b32 %r65, %r64, 2139095040;\n" " setp.lt.f32 %p19, %f1, 0f00000000;\n" " selp.b32 %r66, %r65, %r64, %p19;\n" " mov.b32 %f176, %r66;\n" " bra.uni $L__BB1_9;\n" "\n" "$L__BB1_5:\n" " mov.b32 %r61, %f174;\n" " xor.b32 %r62, %r61, -2147483648;\n" " mov.b32 %f123, %r62;\n" " selp.f32 %f176, %f123, %f174, %p1;\n" " setp.geu.f32 %p16, %f39, 0f00000000;\n" " @%p16 bra $L__BB1_9;\n" "\n" " cvt.rzi.f32.f32 %f124, %f1;\n" " setp.eq.f32 %p17, %f124, %f1;\n" " @%p17 bra $L__BB1_9;\n" "\n" " mov.f32 %f176, 0f7FFFFFFF;\n" "\n" "$L__BB1_9:\n" " add.f32 %f127, %f3, %f4;\n" " mov.b32 %r67, %f127;\n" " setp.lt.s32 %p20, %r67, 2139095040;\n" " @%p20 bra $L__BB1_16;\n" "\n" " setp.gtu.f32 %p21, %f3, 0f7F800000;\n" " setp.gtu.f32 %p22, %f4, 0f7F800000;\n" " or.pred %p23, %p21, %p22;\n" " @%p23 bra $L__BB1_15;\n" " bra.uni $L__BB1_11;\n" "\n" "$L__BB1_15:\n" " add.f32 %f176, %f1, %f39;\n" " bra.uni $L__BB1_16;\n" "\n" "$L__BB1_11:\n" " setp.eq.f32 %p24, %f4, 0f7F800000;\n" " @%p24 bra $L__BB1_14;\n" " bra.uni $L__BB1_12;\n" "\n" "$L__BB1_14:\n" " setp.gt.f32 %p27, %f3, 0f3F800000;\n" " selp.b32 %r71, 2139095040, 0, %p27;\n" " xor.b32 %r72, %r71, 2139095040;\n" " setp.lt.f32 %p28, %f1, 0f00000000;\n" " selp.b32 %r73, %r72, %r71, %p28;\n" " mov.b32 %f128, %r73;\n" " setp.eq.f32 %p29, %f39, 0fBF800000;\n" " selp.f32 %f176, 0f3F800000, %f128, %p29;\n" " bra.uni $L__BB1_16;\n" "\n" "$L__BB1_12:\n" " setp.neu.f32 %p25, %f3, 0f7F800000;\n" " @%p25 bra $L__BB1_16;\n" "\n" " setp.ge.f32 %p26, %f1, 0f00000000;\n" " selp.b32 %r68, 2139095040, 0, %p26;\n" " or.b32 %r69, %r68, -2147483648;\n" " selp.b32 %r70, %r69, %r68, %p1;\n" " mov.b32 %f176, %r70;\n" "\n" "$L__BB1_16:\n" " setp.eq.f32 %p30, %f1, 0f00000000;\n" " setp.eq.f32 %p31, %f39, 0f3F800000;\n" " or.pred %p32, %p31, %p30;\n" " rcp.rn.f32 %f129, %f176;\n" " selp.f32 %f130, 0f3F800000, %f129, %p32;\n" " cvt.rn.f32.s32 %f131, %r5;\n" " mul.f32 %f16, %f130, %f131;\n" " mul.f32 %f132, %f16, 0f3F22F983;\n" " cvt.rni.s32.f32 %r145, %f132;\n" " cvt.rn.f32.s32 %f133, %r145;\n" " mov.f32 %f134, 0fBFC90FDA;\n" " fma.rn.f32 %f135, %f133, %f134, %f16;\n" " mov.f32 %f136, 0fB3A22168;\n" " fma.rn.f32 %f137, %f133, %f136, %f135;\n" " mov.f32 %f138, 0fA7C234C5;\n" " fma.rn.f32 %f180, %f133, %f138, %f137;\n" " abs.f32 %f18, %f16;\n" " setp.leu.f32 %p33, %f18, 0f47CE4780;\n" " add.s64 %rd4, %rd1, 24;\n" " mov.u32 %r141, %r145;\n" " mov.f32 %f177, %f180;\n" " @%p33 bra $L__BB1_24;\n" "\n" " setp.eq.f32 %p34, %f18, 0f7F800000;\n" " @%p34 bra $L__BB1_23;\n" " bra.uni $L__BB1_18;\n" "\n" "$L__BB1_23:\n" " mul.rn.f32 %f177, %f16, %f100;\n" " mov.u32 %r141, %r145;\n" " bra.uni $L__BB1_24;\n" "\n" "$L__BB1_18:\n" " mov.b32 %r7, %f16;\n" " bfe.u32 %r75, %r7, 23, 8;\n" " add.s32 %r8, %r75, -128;\n" " shl.b32 %r76, %r7, 8;\n" " or.b32 %r9, %r76, -2147483648;\n" " shr.u32 %r10, %r8, 5;\n" " mov.u64 %rd62, 0;\n" " mov.u32 %r138, 0;\n" " mov.u64 %rd60, __cudart_i2opi_f;\n" " mov.u64 %rd61, %rd1;\n" "\n" "$L__BB1_19:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r77, [%rd60];\n" " mad.wide.u32 %rd25, %r77, %r9, %rd62;\n" " shr.u64 %rd62, %rd25, 32;\n" " st.local.u32 [%rd61], %rd25;\n" " add.s64 %rd61, %rd61, 4;\n" " add.s64 %rd60, %rd60, 4;\n" " add.s32 %r138, %r138, 1;\n" " setp.ne.s32 %p35, %r138, 6;\n" " @%p35 bra $L__BB1_19;\n" "\n" " st.local.u32 [%rd4], %rd62;\n" " mov.u32 %r78, 4;\n" " sub.s32 %r13, %r78, %r10;\n" " mov.u32 %r79, 6;\n" " sub.s32 %r80, %r79, %r10;\n" " mul.wide.s32 %rd26, %r80, 4;\n" " add.s64 %rd27, %rd1, %rd26;\n" " ld.local.u32 %r139, [%rd27];\n" " ld.local.u32 %r140, [%rd27+-4];\n" " and.b32 %r16, %r8, 31;\n" " setp.eq.s32 %p36, %r16, 0;\n" " @%p36 bra $L__BB1_22;\n" "\n" " mov.u32 %r81, 32;\n" " sub.s32 %r82, %r81, %r16;\n" " shr.u32 %r83, %r140, %r82;\n" " shl.b32 %r84, %r139, %r16;\n" " add.s32 %r139, %r83, %r84;\n" " mul.wide.s32 %rd28, %r13, 4;\n" " add.s64 %rd29, %rd1, %rd28;\n" " ld.local.u32 %r85, [%rd29];\n" " shr.u32 %r86, %r85, %r82;\n" " shl.b32 %r87, %r140, %r16;\n" " add.s32 %r140, %r86, %r87;\n" "\n" "$L__BB1_22:\n" " and.b32 %r88, %r7, -2147483648;\n" " shr.u32 %r89, %r140, 30;\n" " shl.b32 %r90, %r139, 2;\n" " or.b32 %r91, %r89, %r90;\n" " shr.u32 %r92, %r91, 31;\n" " shr.u32 %r93, %r139, 30;\n" " add.s32 %r94, %r92, %r93;\n" " neg.s32 %r95, %r94;\n" " setp.eq.s32 %p37, %r88, 0;\n" " selp.b32 %r141, %r94, %r95, %p37;\n" " setp.ne.s32 %p38, %r92, 0;\n" " xor.b32 %r96, %r88, -2147483648;\n" " selp.b32 %r97, %r96, %r88, %p38;\n" " selp.b32 %r98, -1, 0, %p38;\n" " xor.b32 %r99, %r91, %r98;\n" " shl.b32 %r100, %r140, 2;\n" " xor.b32 %r101, %r100, %r98;\n" " cvt.u64.u32 %rd30, %r99;\n" " shl.b64 %rd31, %rd30, 32;\n" " cvt.u64.u32 %rd32, %r101;\n" " or.b64 %rd33, %rd31, %rd32;\n" " cvt.rn.f64.s64 %fd1, %rd33;\n" " mul.f64 %fd2, %fd1, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f139, %fd2;\n" " setp.eq.s32 %p39, %r97, 0;\n" " neg.f32 %f140, %f139;\n" " selp.f32 %f177, %f139, %f140, %p39;\n" "\n" "$L__BB1_24:\n" " add.s32 %r23, %r141, 1;\n" " and.b32 %r24, %r23, 1;\n" " setp.eq.s32 %p40, %r24, 0;\n" " selp.f32 %f22, %f177, 0f3F800000, %p40;\n" " mul.rn.f32 %f23, %f177, %f177;\n" " mov.f32 %f178, 0fB94D4153;\n" " @%p40 bra $L__BB1_26;\n" "\n" " mov.f32 %f143, 0fBAB607ED;\n" " mov.f32 %f144, 0f37CBAC00;\n" " fma.rn.f32 %f178, %f144, %f23, %f143;\n" "\n" "$L__BB1_26:\n" " selp.f32 %f145, 0f3C0885E4, 0f3D2AAABB, %p40;\n" " fma.rn.f32 %f146, %f178, %f23, %f145;\n" " selp.f32 %f147, 0fBE2AAAA8, 0fBEFFFFFF, %p40;\n" " fma.rn.f32 %f148, %f146, %f23, %f147;\n" " fma.rn.f32 %f150, %f23, %f22, %f100;\n" " fma.rn.f32 %f179, %f148, %f150, %f22;\n" " and.b32 %r102, %r23, 2;\n" " setp.eq.s32 %p42, %r102, 0;\n" " @%p42 bra $L__BB1_28;\n" "\n" " mov.f32 %f152, 0fBF800000;\n" " fma.rn.f32 %f179, %f179, %f152, %f100;\n" "\n" "$L__BB1_28:\n" " @%p33 bra $L__BB1_36;\n" "\n" " setp.eq.f32 %p44, %f18, 0f7F800000;\n" " @%p44 bra $L__BB1_35;\n" " bra.uni $L__BB1_30;\n" "\n" "$L__BB1_35:\n" " mul.rn.f32 %f180, %f16, %f100;\n" " bra.uni $L__BB1_36;\n" "\n" "$L__BB1_30:\n" " mov.b32 %r25, %f16;\n" " bfe.u32 %r104, %r25, 23, 8;\n" " add.s32 %r26, %r104, -128;\n" " shl.b32 %r105, %r25, 8;\n" " or.b32 %r27, %r105, -2147483648;\n" " shr.u32 %r28, %r26, 5;\n" " mov.u64 %rd65, 0;\n" " mov.u32 %r142, 0;\n" " mov.u64 %rd63, __cudart_i2opi_f;\n" " mov.u64 %rd64, %rd1;\n" "\n" "$L__BB1_31:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r106, [%rd63];\n" " mad.wide.u32 %rd36, %r106, %r27, %rd65;\n" " shr.u64 %rd65, %rd36, 32;\n" " st.local.u32 [%rd64], %rd36;\n" " add.s64 %rd64, %rd64, 4;\n" " add.s64 %rd63, %rd63, 4;\n" " add.s32 %r142, %r142, 1;\n" " setp.ne.s32 %p45, %r142, 6;\n" " @%p45 bra $L__BB1_31;\n" "\n" " st.local.u32 [%rd4], %rd65;\n" " mov.u32 %r107, 4;\n" " sub.s32 %r31, %r107, %r28;\n" " mov.u32 %r108, 6;\n" " sub.s32 %r109, %r108, %r28;\n" " mul.wide.s32 %rd37, %r109, 4;\n" " add.s64 %rd38, %rd1, %rd37;\n" " ld.local.u32 %r143, [%rd38];\n" " ld.local.u32 %r144, [%rd38+-4];\n" " and.b32 %r34, %r26, 31;\n" " setp.eq.s32 %p46, %r34, 0;\n" " @%p46 bra $L__BB1_34;\n" "\n" " mov.u32 %r110, 32;\n" " sub.s32 %r111, %r110, %r34;\n" " shr.u32 %r112, %r144, %r111;\n" " shl.b32 %r113, %r143, %r34;\n" " add.s32 %r143, %r112, %r113;\n" " mul.wide.s32 %rd39, %r31, 4;\n" " add.s64 %rd40, %rd1, %rd39;\n" " ld.local.u32 %r114, [%rd40];\n" " shr.u32 %r115, %r114, %r111;\n" " shl.b32 %r116, %r144, %r34;\n" " add.s32 %r144, %r115, %r116;\n" "\n" "$L__BB1_34:\n" " and.b32 %r117, %r25, -2147483648;\n" " shr.u32 %r118, %r144, 30;\n" " shl.b32 %r119, %r143, 2;\n" " or.b32 %r120, %r118, %r119;\n" " shr.u32 %r121, %r120, 31;\n" " shr.u32 %r122, %r143, 30;\n" " add.s32 %r123, %r121, %r122;\n" " neg.s32 %r124, %r123;\n" " setp.eq.s32 %p47, %r117, 0;\n" " selp.b32 %r145, %r123, %r124, %p47;\n" " setp.ne.s32 %p48, %r121, 0;\n" " xor.b32 %r125, %r117, -2147483648;\n" " selp.b32 %r126, %r125, %r117, %p48;\n" " selp.b32 %r127, -1, 0, %p48;\n" " xor.b32 %r128, %r120, %r127;\n" " shl.b32 %r129, %r144, 2;\n" " xor.b32 %r130, %r129, %r127;\n" " cvt.u64.u32 %rd41, %r128;\n" " shl.b64 %rd42, %rd41, 32;\n" " cvt.u64.u32 %rd43, %r130;\n" " or.b64 %rd44, %rd42, %rd43;\n" " cvt.rn.f64.s64 %fd3, %rd44;\n" " mul.f64 %fd4, %fd3, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f153, %fd4;\n" " setp.eq.s32 %p49, %r126, 0;\n" " neg.f32 %f154, %f153;\n" " selp.f32 %f180, %f153, %f154, %p49;\n" "\n" "$L__BB1_36:\n" " and.b32 %r41, %r145, 1;\n" " setp.eq.s32 %p50, %r41, 0;\n" " selp.f32 %f32, %f180, 0f3F800000, %p50;\n" " mul.rn.f32 %f33, %f180, %f180;\n" " mov.f32 %f181, 0fB94D4153;\n" " @%p50 bra $L__BB1_38;\n" "\n" " mov.f32 %f157, 0fBAB607ED;\n" " mov.f32 %f158, 0f37CBAC00;\n" " fma.rn.f32 %f181, %f158, %f33, %f157;\n" "\n" "$L__BB1_38:\n" " selp.f32 %f159, 0f3C0885E4, 0f3D2AAABB, %p50;\n" " fma.rn.f32 %f160, %f181, %f33, %f159;\n" " selp.f32 %f161, 0fBE2AAAA8, 0fBEFFFFFF, %p50;\n" " fma.rn.f32 %f162, %f160, %f33, %f161;\n" " fma.rn.f32 %f164, %f33, %f32, %f100;\n" " fma.rn.f32 %f182, %f162, %f164, %f32;\n" " and.b32 %r131, %r145, 2;\n" " setp.eq.s32 %p52, %r131, 0;\n" " @%p52 bra $L__BB1_40;\n" "\n" " mov.f32 %f166, 0fBF800000;\n" " fma.rn.f32 %f182, %f182, %f166, %f100;\n" "\n" "$L__BB1_40:\n" " ld.param.u64 %rd59, [fc_rope_neox_partial_nx_param_0];\n" " cvta.to.global.u64 %rd58, %rd59;\n" " mov.u32 %r137, %ctaid.y;\n" " ld.param.u32 %r136, [fc_rope_neox_partial_nx_param_1];\n" " mov.u32 %r135, %ctaid.x;\n" " cvt.s64.s32 %rd57, %r135;\n" " mul.wide.s32 %rd56, %r136, %r137;\n" " ld.param.u32 %r134, [fc_rope_neox_partial_nx_param_2];\n" " cvt.s64.s32 %rd55, %r134;\n" " add.s64 %rd54, %rd56, %rd57;\n" " mul.lo.s64 %rd53, %rd54, %rd55;\n" " mov.u32 %r133, %tid.x;\n" " cvt.s64.s32 %rd45, %r133;\n" " add.s64 %rd46, %rd53, %rd45;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd48, %rd58, %rd47;\n" " add.s32 %r132, %r3, %r133;\n" " cvt.s64.s32 %rd49, %r132;\n" " add.s64 %rd50, %rd53, %rd49;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd52, %rd58, %rd51;\n" " ld.global.f32 %f167, [%rd48];\n" " mul.f32 %f168, %f179, %f167;\n" " ld.global.f32 %f169, [%rd52];\n" " mul.f32 %f170, %f182, %f169;\n" " sub.f32 %f171, %f168, %f170;\n" " st.global.f32 [%rd48], %f171;\n" " mul.f32 %f172, %f179, %f169;\n" " fma.rn.f32 %f173, %f182, %f167, %f172;\n" " st.global.f32 [%rd52], %f173;\n" "\n" "$L__BB1_41:\n" " ret;\n" "\n" "}\n" " // .globl fc_rope_neox_partial_pos\n" ".visible .entry fc_rope_neox_partial_pos(\n" " .param .u64 fc_rope_neox_partial_pos_param_0,\n" " .param .u32 fc_rope_neox_partial_pos_param_1,\n" " .param .u32 fc_rope_neox_partial_pos_param_2,\n" " .param .u32 fc_rope_neox_partial_pos_param_3,\n" " .param .u64 fc_rope_neox_partial_pos_param_4,\n" " .param .f32 fc_rope_neox_partial_pos_param_5\n" ")\n" "{\n" " .local .align 4 .b8 __local_depot2[28];\n" " .reg .b64 %SP;\n" " .reg .b64 %SPL;\n" " .reg .pred %p<53>;\n" " .reg .f32 %f<183>;\n" " .reg .b32 %r<145>;\n" " .reg .f64 %fd<5>;\n" " .reg .b64 %rd<70>;\n" "\n" "\n" " mov.u64 %SPL, __local_depot2;\n" " ld.param.u32 %r42, [fc_rope_neox_partial_pos_param_1];\n" " ld.param.u32 %r43, [fc_rope_neox_partial_pos_param_2];\n" " ld.param.u32 %r44, [fc_rope_neox_partial_pos_param_3];\n" " ld.param.u64 %rd18, [fc_rope_neox_partial_pos_param_4];\n" " ld.param.f32 %f39, [fc_rope_neox_partial_pos_param_5];\n" " add.u64 %rd1, %SPL, 0;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p2, %r1, %r42;\n" " @%p2 bra $L__BB2_41;\n" "\n" " setp.gt.s32 %p3, %r44, 0;\n" " setp.lt.s32 %p4, %r44, %r43;\n" " and.pred %p5, %p3, %p4;\n" " selp.b32 %r2, %r44, %r43, %p5;\n" " shr.u32 %r45, %r2, 31;\n" " add.s32 %r46, %r2, %r45;\n" " shr.s32 %r3, %r46, 1;\n" " mov.u32 %r4, %tid.x;\n" " setp.ge.s32 %p6, %r4, %r3;\n" " @%p6 bra $L__BB2_41;\n" "\n" " mov.u32 %r47, %ctaid.y;\n" " cvta.to.global.u64 %rd24, %rd18;\n" " mul.wide.s32 %rd25, %r47, 4;\n" " add.s64 %rd26, %rd24, %rd25;\n" " ld.global.u32 %r5, [%rd26];\n" " shl.b32 %r48, %r4, 1;\n" " cvt.rn.f32.s32 %f41, %r48;\n" " cvt.rn.f32.s32 %f42, %r2;\n" " div.rn.f32 %f1, %f41, %f42;\n" " mul.f32 %f43, %f1, 0f3F000000;\n" " cvt.rzi.f32.f32 %f44, %f43;\n" " add.f32 %f45, %f44, %f44;\n" " sub.f32 %f46, %f1, %f45;\n" " abs.f32 %f2, %f46;\n" " abs.f32 %f3, %f39;\n" " setp.lt.f32 %p7, %f3, 0f00800000;\n" " mul.f32 %f47, %f3, 0f4B800000;\n" " selp.f32 %f48, %f47, %f3, %p7;\n" " selp.f32 %f49, 0fC3170000, 0fC2FE0000, %p7;\n" " mov.b32 %r49, %f48;\n" " and.b32 %r50, %r49, 8388607;\n" " or.b32 %r51, %r50, 1065353216;\n" " mov.b32 %f50, %r51;\n" " shr.u32 %r52, %r49, 23;\n" " cvt.rn.f32.u32 %f51, %r52;\n" " add.f32 %f52, %f49, %f51;\n" " setp.gt.f32 %p8, %f50, 0f3FB504F3;\n" " mul.f32 %f53, %f50, 0f3F000000;\n" " add.f32 %f54, %f52, 0f3F800000;\n" " selp.f32 %f55, %f54, %f52, %p8;\n" " selp.f32 %f56, %f53, %f50, %p8;\n" " add.f32 %f57, %f56, 0fBF800000;\n" " add.f32 %f58, %f56, 0f3F800000;\n" " rcp.approx.ftz.f32 %f59, %f58;\n" " add.f32 %f60, %f57, %f57;\n" " mul.f32 %f61, %f60, %f59;\n" " mul.f32 %f62, %f61, %f61;\n" " mov.f32 %f63, 0f3C4CAF63;\n" " mov.f32 %f64, 0f3B18F0FE;\n" " fma.rn.f32 %f65, %f64, %f62, %f63;\n" " mov.f32 %f66, 0f3DAAAABD;\n" " fma.rn.f32 %f67, %f65, %f62, %f66;\n" " mul.rn.f32 %f68, %f67, %f62;\n" " mul.rn.f32 %f69, %f68, %f61;\n" " sub.f32 %f70, %f57, %f61;\n" " add.f32 %f71, %f70, %f70;\n" " neg.f32 %f72, %f61;\n" " fma.rn.f32 %f73, %f72, %f57, %f71;\n" " mul.rn.f32 %f74, %f59, %f73;\n" " add.f32 %f75, %f69, %f61;\n" " sub.f32 %f76, %f61, %f75;\n" " add.f32 %f77, %f69, %f76;\n" " add.f32 %f78, %f74, %f77;\n" " add.f32 %f79, %f75, %f78;\n" " sub.f32 %f80, %f75, %f79;\n" " add.f32 %f81, %f78, %f80;\n" " mov.f32 %f82, 0f3F317200;\n" " mul.rn.f32 %f83, %f55, %f82;\n" " mov.f32 %f84, 0f35BFBE8E;\n" " mul.rn.f32 %f85, %f55, %f84;\n" " add.f32 %f86, %f83, %f79;\n" " sub.f32 %f87, %f83, %f86;\n" " add.f32 %f88, %f79, %f87;\n" " add.f32 %f89, %f81, %f88;\n" " add.f32 %f90, %f85, %f89;\n" " add.f32 %f91, %f86, %f90;\n" " sub.f32 %f92, %f86, %f91;\n" " add.f32 %f93, %f90, %f92;\n" " abs.f32 %f4, %f1;\n" " setp.gt.f32 %p9, %f4, 0f77F684DF;\n" " mul.f32 %f94, %f1, 0f39000000;\n" " selp.f32 %f95, %f94, %f1, %p9;\n" " mul.rn.f32 %f96, %f95, %f91;\n" " neg.f32 %f97, %f96;\n" " fma.rn.f32 %f98, %f95, %f91, %f97;\n" " fma.rn.f32 %f99, %f95, %f93, %f98;\n" " mov.f32 %f100, 0f00000000;\n" " fma.rn.f32 %f101, %f100, %f91, %f99;\n" " add.rn.f32 %f102, %f96, %f101;\n" " neg.f32 %f103, %f102;\n" " add.rn.f32 %f104, %f96, %f103;\n" " add.rn.f32 %f105, %f104, %f101;\n" " mov.b32 %r53, %f102;\n" " setp.eq.s32 %p10, %r53, 1118925336;\n" " add.s32 %r54, %r53, -1;\n" " mov.b32 %f106, %r54;\n" " add.f32 %f107, %f105, 0f37000000;\n" " selp.f32 %f5, %f107, %f105, %p10;\n" " selp.f32 %f108, %f106, %f102, %p10;\n" " mov.f32 %f109, 0f3FB8AA3B;\n" " mul.rn.f32 %f110, %f108, %f109;\n" " cvt.rzi.f32.f32 %f111, %f110;\n" " abs.f32 %f112, %f111;\n" " setp.gt.f32 %p11, %f112, 0f42FC0000;\n" " mov.b32 %r55, %f111;\n" " and.b32 %r56, %r55, -2147483648;\n" " or.b32 %r57, %r56, 1123811328;\n" " mov.b32 %f113, %r57;\n" " selp.f32 %f114, %f113, %f111, %p11;\n" " mov.f32 %f115, 0fBF317218;\n" " fma.rn.f32 %f116, %f114, %f115, %f108;\n" " mov.f32 %f117, 0f3102E308;\n" " fma.rn.f32 %f118, %f114, %f117, %f116;\n" " mul.f32 %f119, %f118, 0f3FB8AA3B;\n" " add.f32 %f120, %f114, 0f4B40007F;\n" " mov.b32 %r58, %f120;\n" " shl.b32 %r59, %r58, 23;\n" " mov.b32 %f121, %r59;\n" " ex2.approx.ftz.f32 %f122, %f119;\n" " mul.f32 %f6, %f122, %f121;\n" " setp.eq.f32 %p12, %f6, 0f7F800000;\n" " mov.f32 %f174, 0f7F800000;\n" " @%p12 bra $L__BB2_4;\n" "\n" " fma.rn.f32 %f174, %f6, %f5, %f6;\n" "\n" "$L__BB2_4:\n" " setp.lt.f32 %p13, %f39, 0f00000000;\n" " setp.eq.f32 %p14, %f2, 0f3F800000;\n" " and.pred %p1, %p13, %p14;\n" " setp.eq.f32 %p15, %f39, 0f00000000;\n" " @%p15 bra $L__BB2_8;\n" " bra.uni $L__BB2_5;\n" "\n" "$L__BB2_8:\n" " add.f32 %f126, %f39, %f39;\n" " mov.b32 %r62, %f126;\n" " selp.b32 %r63, %r62, 0, %p14;\n" " or.b32 %r64, %r63, 2139095040;\n" " setp.lt.f32 %p19, %f1, 0f00000000;\n" " selp.b32 %r65, %r64, %r63, %p19;\n" " mov.b32 %f176, %r65;\n" " bra.uni $L__BB2_9;\n" "\n" "$L__BB2_5:\n" " mov.b32 %r60, %f174;\n" " xor.b32 %r61, %r60, -2147483648;\n" " mov.b32 %f123, %r61;\n" " selp.f32 %f176, %f123, %f174, %p1;\n" " setp.geu.f32 %p16, %f39, 0f00000000;\n" " @%p16 bra $L__BB2_9;\n" "\n" " cvt.rzi.f32.f32 %f124, %f1;\n" " setp.eq.f32 %p17, %f124, %f1;\n" " @%p17 bra $L__BB2_9;\n" "\n" " mov.f32 %f176, 0f7FFFFFFF;\n" "\n" "$L__BB2_9:\n" " add.f32 %f127, %f3, %f4;\n" " mov.b32 %r66, %f127;\n" " setp.lt.s32 %p20, %r66, 2139095040;\n" " @%p20 bra $L__BB2_16;\n" "\n" " setp.gtu.f32 %p21, %f3, 0f7F800000;\n" " setp.gtu.f32 %p22, %f4, 0f7F800000;\n" " or.pred %p23, %p21, %p22;\n" " @%p23 bra $L__BB2_15;\n" " bra.uni $L__BB2_11;\n" "\n" "$L__BB2_15:\n" " add.f32 %f176, %f1, %f39;\n" " bra.uni $L__BB2_16;\n" "\n" "$L__BB2_11:\n" " setp.eq.f32 %p24, %f4, 0f7F800000;\n" " @%p24 bra $L__BB2_14;\n" " bra.uni $L__BB2_12;\n" "\n" "$L__BB2_14:\n" " setp.gt.f32 %p27, %f3, 0f3F800000;\n" " selp.b32 %r70, 2139095040, 0, %p27;\n" " xor.b32 %r71, %r70, 2139095040;\n" " setp.lt.f32 %p28, %f1, 0f00000000;\n" " selp.b32 %r72, %r71, %r70, %p28;\n" " mov.b32 %f128, %r72;\n" " setp.eq.f32 %p29, %f39, 0fBF800000;\n" " selp.f32 %f176, 0f3F800000, %f128, %p29;\n" " bra.uni $L__BB2_16;\n" "\n" "$L__BB2_12:\n" " setp.neu.f32 %p25, %f3, 0f7F800000;\n" " @%p25 bra $L__BB2_16;\n" "\n" " setp.ge.f32 %p26, %f1, 0f00000000;\n" " selp.b32 %r67, 2139095040, 0, %p26;\n" " or.b32 %r68, %r67, -2147483648;\n" " selp.b32 %r69, %r68, %r67, %p1;\n" " mov.b32 %f176, %r69;\n" "\n" "$L__BB2_16:\n" " setp.eq.f32 %p30, %f1, 0f00000000;\n" " setp.eq.f32 %p31, %f39, 0f3F800000;\n" " or.pred %p32, %p31, %p30;\n" " rcp.rn.f32 %f129, %f176;\n" " selp.f32 %f130, 0f3F800000, %f129, %p32;\n" " cvt.rn.f32.s32 %f131, %r5;\n" " mul.f32 %f16, %f130, %f131;\n" " mul.f32 %f132, %f16, 0f3F22F983;\n" " cvt.rni.s32.f32 %r144, %f132;\n" " cvt.rn.f32.s32 %f133, %r144;\n" " mov.f32 %f134, 0fBFC90FDA;\n" " fma.rn.f32 %f135, %f133, %f134, %f16;\n" " mov.f32 %f136, 0fB3A22168;\n" " fma.rn.f32 %f137, %f133, %f136, %f135;\n" " mov.f32 %f138, 0fA7C234C5;\n" " fma.rn.f32 %f180, %f133, %f138, %f137;\n" " abs.f32 %f18, %f16;\n" " setp.leu.f32 %p33, %f18, 0f47CE4780;\n" " add.s64 %rd4, %rd1, 24;\n" " mov.u32 %r140, %r144;\n" " mov.f32 %f177, %f180;\n" " @%p33 bra $L__BB2_24;\n" "\n" " setp.eq.f32 %p34, %f18, 0f7F800000;\n" " @%p34 bra $L__BB2_23;\n" " bra.uni $L__BB2_18;\n" "\n" "$L__BB2_23:\n" " mul.rn.f32 %f177, %f16, %f100;\n" " mov.u32 %r140, %r144;\n" " bra.uni $L__BB2_24;\n" "\n" "$L__BB2_18:\n" " mov.b32 %r7, %f16;\n" " bfe.u32 %r74, %r7, 23, 8;\n" " add.s32 %r8, %r74, -128;\n" " shl.b32 %r75, %r7, 8;\n" " or.b32 %r9, %r75, -2147483648;\n" " shr.u32 %r10, %r8, 5;\n" " mov.u64 %rd66, 0;\n" " mov.u32 %r137, 0;\n" " mov.u64 %rd64, __cudart_i2opi_f;\n" " mov.u64 %rd65, %rd1;\n" "\n" "$L__BB2_19:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r76, [%rd64];\n" " mad.wide.u32 %rd29, %r76, %r9, %rd66;\n" " shr.u64 %rd66, %rd29, 32;\n" " st.local.u32 [%rd65], %rd29;\n" " add.s64 %rd65, %rd65, 4;\n" " add.s64 %rd64, %rd64, 4;\n" " add.s32 %r137, %r137, 1;\n" " setp.ne.s32 %p35, %r137, 6;\n" " @%p35 bra $L__BB2_19;\n" "\n" " st.local.u32 [%rd4], %rd66;\n" " mov.u32 %r77, 4;\n" " sub.s32 %r13, %r77, %r10;\n" " mov.u32 %r78, 6;\n" " sub.s32 %r79, %r78, %r10;\n" " mul.wide.s32 %rd30, %r79, 4;\n" " add.s64 %rd31, %rd1, %rd30;\n" " ld.local.u32 %r138, [%rd31];\n" " ld.local.u32 %r139, [%rd31+-4];\n" " and.b32 %r16, %r8, 31;\n" " setp.eq.s32 %p36, %r16, 0;\n" " @%p36 bra $L__BB2_22;\n" "\n" " mov.u32 %r80, 32;\n" " sub.s32 %r81, %r80, %r16;\n" " shr.u32 %r82, %r139, %r81;\n" " shl.b32 %r83, %r138, %r16;\n" " add.s32 %r138, %r82, %r83;\n" " mul.wide.s32 %rd32, %r13, 4;\n" " add.s64 %rd33, %rd1, %rd32;\n" " ld.local.u32 %r84, [%rd33];\n" " shr.u32 %r85, %r84, %r81;\n" " shl.b32 %r86, %r139, %r16;\n" " add.s32 %r139, %r85, %r86;\n" "\n" "$L__BB2_22:\n" " and.b32 %r87, %r7, -2147483648;\n" " shr.u32 %r88, %r139, 30;\n" " shl.b32 %r89, %r138, 2;\n" " or.b32 %r90, %r88, %r89;\n" " shr.u32 %r91, %r90, 31;\n" " shr.u32 %r92, %r138, 30;\n" " add.s32 %r93, %r91, %r92;\n" " neg.s32 %r94, %r93;\n" " setp.eq.s32 %p37, %r87, 0;\n" " selp.b32 %r140, %r93, %r94, %p37;\n" " setp.ne.s32 %p38, %r91, 0;\n" " xor.b32 %r95, %r87, -2147483648;\n" " selp.b32 %r96, %r95, %r87, %p38;\n" " selp.b32 %r97, -1, 0, %p38;\n" " xor.b32 %r98, %r90, %r97;\n" " shl.b32 %r99, %r139, 2;\n" " xor.b32 %r100, %r99, %r97;\n" " cvt.u64.u32 %rd34, %r98;\n" " shl.b64 %rd35, %rd34, 32;\n" " cvt.u64.u32 %rd36, %r100;\n" " or.b64 %rd37, %rd35, %rd36;\n" " cvt.rn.f64.s64 %fd1, %rd37;\n" " mul.f64 %fd2, %fd1, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f139, %fd2;\n" " setp.eq.s32 %p39, %r96, 0;\n" " neg.f32 %f140, %f139;\n" " selp.f32 %f177, %f139, %f140, %p39;\n" "\n" "$L__BB2_24:\n" " add.s32 %r23, %r140, 1;\n" " and.b32 %r24, %r23, 1;\n" " setp.eq.s32 %p40, %r24, 0;\n" " selp.f32 %f22, %f177, 0f3F800000, %p40;\n" " mul.rn.f32 %f23, %f177, %f177;\n" " mov.f32 %f178, 0fB94D4153;\n" " @%p40 bra $L__BB2_26;\n" "\n" " mov.f32 %f143, 0fBAB607ED;\n" " mov.f32 %f144, 0f37CBAC00;\n" " fma.rn.f32 %f178, %f144, %f23, %f143;\n" "\n" "$L__BB2_26:\n" " selp.f32 %f145, 0f3C0885E4, 0f3D2AAABB, %p40;\n" " fma.rn.f32 %f146, %f178, %f23, %f145;\n" " selp.f32 %f147, 0fBE2AAAA8, 0fBEFFFFFF, %p40;\n" " fma.rn.f32 %f148, %f146, %f23, %f147;\n" " fma.rn.f32 %f150, %f23, %f22, %f100;\n" " fma.rn.f32 %f179, %f148, %f150, %f22;\n" " and.b32 %r101, %r23, 2;\n" " setp.eq.s32 %p42, %r101, 0;\n" " @%p42 bra $L__BB2_28;\n" "\n" " mov.f32 %f152, 0fBF800000;\n" " fma.rn.f32 %f179, %f179, %f152, %f100;\n" "\n" "$L__BB2_28:\n" " @%p33 bra $L__BB2_36;\n" "\n" " setp.eq.f32 %p44, %f18, 0f7F800000;\n" " @%p44 bra $L__BB2_35;\n" " bra.uni $L__BB2_30;\n" "\n" "$L__BB2_35:\n" " mul.rn.f32 %f180, %f16, %f100;\n" " bra.uni $L__BB2_36;\n" "\n" "$L__BB2_30:\n" " mov.b32 %r25, %f16;\n" " bfe.u32 %r103, %r25, 23, 8;\n" " add.s32 %r26, %r103, -128;\n" " shl.b32 %r104, %r25, 8;\n" " or.b32 %r27, %r104, -2147483648;\n" " shr.u32 %r28, %r26, 5;\n" " mov.u64 %rd69, 0;\n" " mov.u32 %r141, 0;\n" " mov.u64 %rd67, __cudart_i2opi_f;\n" " mov.u64 %rd68, %rd1;\n" "\n" "$L__BB2_31:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r105, [%rd67];\n" " mad.wide.u32 %rd40, %r105, %r27, %rd69;\n" " shr.u64 %rd69, %rd40, 32;\n" " st.local.u32 [%rd68], %rd40;\n" " add.s64 %rd68, %rd68, 4;\n" " add.s64 %rd67, %rd67, 4;\n" " add.s32 %r141, %r141, 1;\n" " setp.ne.s32 %p45, %r141, 6;\n" " @%p45 bra $L__BB2_31;\n" "\n" " st.local.u32 [%rd4], %rd69;\n" " mov.u32 %r106, 4;\n" " sub.s32 %r31, %r106, %r28;\n" " mov.u32 %r107, 6;\n" " sub.s32 %r108, %r107, %r28;\n" " mul.wide.s32 %rd41, %r108, 4;\n" " add.s64 %rd42, %rd1, %rd41;\n" " ld.local.u32 %r142, [%rd42];\n" " ld.local.u32 %r143, [%rd42+-4];\n" " and.b32 %r34, %r26, 31;\n" " setp.eq.s32 %p46, %r34, 0;\n" " @%p46 bra $L__BB2_34;\n" "\n" " mov.u32 %r109, 32;\n" " sub.s32 %r110, %r109, %r34;\n" " shr.u32 %r111, %r143, %r110;\n" " shl.b32 %r112, %r142, %r34;\n" " add.s32 %r142, %r111, %r112;\n" " mul.wide.s32 %rd43, %r31, 4;\n" " add.s64 %rd44, %rd1, %rd43;\n" " ld.local.u32 %r113, [%rd44];\n" " shr.u32 %r114, %r113, %r110;\n" " shl.b32 %r115, %r143, %r34;\n" " add.s32 %r143, %r114, %r115;\n" "\n" "$L__BB2_34:\n" " and.b32 %r116, %r25, -2147483648;\n" " shr.u32 %r117, %r143, 30;\n" " shl.b32 %r118, %r142, 2;\n" " or.b32 %r119, %r117, %r118;\n" " shr.u32 %r120, %r119, 31;\n" " shr.u32 %r121, %r142, 30;\n" " add.s32 %r122, %r120, %r121;\n" " neg.s32 %r123, %r122;\n" " setp.eq.s32 %p47, %r116, 0;\n" " selp.b32 %r144, %r122, %r123, %p47;\n" " setp.ne.s32 %p48, %r120, 0;\n" " xor.b32 %r124, %r116, -2147483648;\n" " selp.b32 %r125, %r124, %r116, %p48;\n" " selp.b32 %r126, -1, 0, %p48;\n" " xor.b32 %r127, %r119, %r126;\n" " shl.b32 %r128, %r143, 2;\n" " xor.b32 %r129, %r128, %r126;\n" " cvt.u64.u32 %rd45, %r127;\n" " shl.b64 %rd46, %rd45, 32;\n" " cvt.u64.u32 %rd47, %r129;\n" " or.b64 %rd48, %rd46, %rd47;\n" " cvt.rn.f64.s64 %fd3, %rd48;\n" " mul.f64 %fd4, %fd3, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f153, %fd4;\n" " setp.eq.s32 %p49, %r125, 0;\n" " neg.f32 %f154, %f153;\n" " selp.f32 %f180, %f153, %f154, %p49;\n" "\n" "$L__BB2_36:\n" " and.b32 %r41, %r144, 1;\n" " setp.eq.s32 %p50, %r41, 0;\n" " selp.f32 %f32, %f180, 0f3F800000, %p50;\n" " mul.rn.f32 %f33, %f180, %f180;\n" " mov.f32 %f181, 0fB94D4153;\n" " @%p50 bra $L__BB2_38;\n" "\n" " mov.f32 %f157, 0fBAB607ED;\n" " mov.f32 %f158, 0f37CBAC00;\n" " fma.rn.f32 %f181, %f158, %f33, %f157;\n" "\n" "$L__BB2_38:\n" " selp.f32 %f159, 0f3C0885E4, 0f3D2AAABB, %p50;\n" " fma.rn.f32 %f160, %f181, %f33, %f159;\n" " selp.f32 %f161, 0fBE2AAAA8, 0fBEFFFFFF, %p50;\n" " fma.rn.f32 %f162, %f160, %f33, %f161;\n" " fma.rn.f32 %f164, %f33, %f32, %f100;\n" " fma.rn.f32 %f182, %f162, %f164, %f32;\n" " and.b32 %r130, %r144, 2;\n" " setp.eq.s32 %p52, %r130, 0;\n" " @%p52 bra $L__BB2_40;\n" "\n" " mov.f32 %f166, 0fBF800000;\n" " fma.rn.f32 %f182, %f182, %f166, %f100;\n" "\n" "$L__BB2_40:\n" " ld.param.u64 %rd63, [fc_rope_neox_partial_pos_param_0];\n" " cvta.to.global.u64 %rd62, %rd63;\n" " mov.u32 %r136, %ctaid.y;\n" " ld.param.u32 %r135, [fc_rope_neox_partial_pos_param_1];\n" " mov.u32 %r134, %ctaid.x;\n" " cvt.s64.s32 %rd61, %r134;\n" " mul.wide.s32 %rd60, %r135, %r136;\n" " ld.param.u32 %r133, [fc_rope_neox_partial_pos_param_2];\n" " cvt.s64.s32 %rd59, %r133;\n" " add.s64 %rd58, %rd60, %rd61;\n" " mul.lo.s64 %rd57, %rd58, %rd59;\n" " mov.u32 %r132, %tid.x;\n" " cvt.s64.s32 %rd49, %r132;\n" " add.s64 %rd50, %rd57, %rd49;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd52, %rd62, %rd51;\n" " add.s32 %r131, %r3, %r132;\n" " cvt.s64.s32 %rd53, %r131;\n" " add.s64 %rd54, %rd57, %rd53;\n" " shl.b64 %rd55, %rd54, 2;\n" " add.s64 %rd56, %rd62, %rd55;\n" " ld.global.f32 %f167, [%rd52];\n" " mul.f32 %f168, %f179, %f167;\n" " ld.global.f32 %f169, [%rd56];\n" " mul.f32 %f170, %f182, %f169;\n" " sub.f32 %f171, %f168, %f170;\n" " st.global.f32 [%rd52], %f171;\n" " mul.f32 %f172, %f179, %f169;\n" " fma.rn.f32 %f173, %f182, %f167, %f172;\n" " st.global.f32 [%rd56], %f173;\n" "\n" "$L__BB2_41:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_MOE_ROUTING[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_moe_router_topk\n" "// _ZZ18fc_moe_router_topkE2rv has been demoted\n" "// _ZZ18fc_moe_router_topkE2ri has been demoted\n" "// _ZZ18fc_moe_router_topkE7swt_sum has been demoted\n" "// _ZZ21fc_moe_router_topk_nxE2rv has been demoted\n" "// _ZZ21fc_moe_router_topk_nxE2ri has been demoted\n" "// _ZZ21fc_moe_router_topk_nxE7swt_sum has been demoted\n" ".extern .shared .align 16 .b8 sdata[];\n" "\n" ".visible .entry fc_moe_router_topk(\n" " .param .u64 fc_moe_router_topk_param_0,\n" " .param .u64 fc_moe_router_topk_param_1,\n" " .param .u64 fc_moe_router_topk_param_2,\n" " .param .u32 fc_moe_router_topk_param_3,\n" " .param .u32 fc_moe_router_topk_param_4,\n" " .param .f32 fc_moe_router_topk_param_5\n" ")\n" "{\n" " .reg .pred %p<62>;\n" " .reg .f32 %f<182>;\n" " .reg .b32 %r<174>;\n" " .reg .b64 %rd<67>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ18fc_moe_router_topkE2rv[1024];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ18fc_moe_router_topkE2ri[1024];\n" " // demoted variable\n" " .shared .align 4 .f32 _ZZ18fc_moe_router_topkE7swt_sum;\n" "\n" " ld.param.u64 %rd22, [fc_moe_router_topk_param_0];\n" " ld.param.u64 %rd21, [fc_moe_router_topk_param_1];\n" " ld.param.u64 %rd23, [fc_moe_router_topk_param_2];\n" " ld.param.u32 %r79, [fc_moe_router_topk_param_3];\n" " ld.param.u32 %r80, [fc_moe_router_topk_param_4];\n" " ld.param.f32 %f32, [fc_moe_router_topk_param_5];\n" " cvta.to.global.u64 %rd1, %rd22;\n" " cvta.to.global.u64 %rd2, %rd23;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r2, %tid.x;\n" " setp.ge.s32 %p1, %r2, %r79;\n" " mov.f32 %f171, 0fF149F2CA;\n" " @%p1 bra $L__BB0_7;\n" "\n" " not.b32 %r81, %r2;\n" " add.s32 %r82, %r81, %r79;\n" " div.u32 %r3, %r82, %r1;\n" " add.s32 %r83, %r3, 1;\n" " and.b32 %r147, %r83, 3;\n" " setp.eq.s32 %p2, %r147, 0;\n" " mov.f32 %f171, 0fF149F2CA;\n" " mov.u32 %r148, %r2;\n" " @%p2 bra $L__BB0_4;\n" "\n" " mul.wide.s32 %rd24, %r2, 4;\n" " add.s64 %rd63, %rd1, %rd24;\n" " mul.wide.s32 %rd4, %r1, 4;\n" " mov.f32 %f171, 0fF149F2CA;\n" " mov.u32 %r148, %r2;\n" "\n" "$L__BB0_3:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f37, [%rd63];\n" " setp.gt.f32 %p3, %f37, %f171;\n" " selp.f32 %f171, %f37, %f171, %p3;\n" " add.s32 %r148, %r148, %r1;\n" " add.s64 %rd63, %rd63, %rd4;\n" " add.s32 %r147, %r147, -1;\n" " setp.ne.s32 %p4, %r147, 0;\n" " @%p4 bra $L__BB0_3;\n" "\n" "$L__BB0_4:\n" " setp.lt.u32 %p5, %r3, 3;\n" " @%p5 bra $L__BB0_7;\n" "\n" " shl.b32 %r10, %r1, 2;\n" " cvt.s64.s32 %rd27, %r10;\n" "\n" "$L__BB0_6:\n" " mul.wide.s32 %rd25, %r148, 4;\n" " add.s64 %rd26, %rd1, %rd25;\n" " ld.global.f32 %f38, [%rd26];\n" " setp.gt.f32 %p6, %f38, %f171;\n" " selp.f32 %f39, %f38, %f171, %p6;\n" " add.s64 %rd28, %rd26, %rd27;\n" " ld.global.f32 %f40, [%rd28];\n" " setp.gt.f32 %p7, %f40, %f39;\n" " selp.f32 %f41, %f40, %f39, %p7;\n" " add.s32 %r84, %r148, %r1;\n" " add.s32 %r85, %r84, %r1;\n" " add.s64 %rd29, %rd28, %rd27;\n" " ld.global.f32 %f42, [%rd29];\n" " setp.gt.f32 %p8, %f42, %f41;\n" " selp.f32 %f43, %f42, %f41, %p8;\n" " add.s32 %r86, %r85, %r1;\n" " add.s64 %rd30, %rd29, %rd27;\n" " ld.global.f32 %f44, [%rd30];\n" " setp.gt.f32 %p9, %f44, %f43;\n" " selp.f32 %f171, %f44, %f43, %p9;\n" " add.s32 %r148, %r86, %r1;\n" " setp.lt.s32 %p10, %r148, %r79;\n" " @%p10 bra $L__BB0_6;\n" "\n" "$L__BB0_7:\n" " shl.b32 %r87, %r2, 2;\n" " mov.u32 %r88, sdata;\n" " add.s32 %r13, %r88, %r87;\n" " st.shared.f32 [%r13], %f171;\n" " bar.sync 0;\n" " shr.u32 %r89, %r1, 31;\n" " add.s32 %r90, %r1, %r89;\n" " shr.s32 %r14, %r90, 1;\n" " setp.lt.s32 %p11, %r1, 2;\n" " @%p11 bra $L__BB0_13;\n" "\n" " mov.u32 %r150, %r14;\n" "\n" "$L__BB0_9:\n" " mov.u32 %r15, %r150;\n" " setp.ge.s32 %p12, %r2, %r15;\n" " @%p12 bra $L__BB0_12;\n" "\n" " shl.b32 %r91, %r15, 2;\n" " add.s32 %r92, %r13, %r91;\n" " ld.shared.f32 %f45, [%r13];\n" " ld.shared.f32 %f8, [%r92];\n" " setp.leu.f32 %p13, %f8, %f45;\n" " @%p13 bra $L__BB0_12;\n" "\n" " st.shared.f32 [%r13], %f8;\n" "\n" "$L__BB0_12:\n" " bar.sync 0;\n" " shr.s32 %r150, %r15, 1;\n" " setp.gt.s32 %p14, %r15, 1;\n" " @%p14 bra $L__BB0_9;\n" "\n" "$L__BB0_13:\n" " mov.f32 %f176, 0f00000000;\n" " ld.shared.f32 %f9, [sdata];\n" " @%p1 bra $L__BB0_20;\n" "\n" " not.b32 %r93, %r2;\n" " add.s32 %r94, %r93, %r79;\n" " div.u32 %r17, %r94, %r1;\n" " add.s32 %r95, %r17, 1;\n" " and.b32 %r152, %r95, 3;\n" " setp.eq.s32 %p16, %r152, 0;\n" " mov.f32 %f176, 0f00000000;\n" " mov.u32 %r153, %r2;\n" " @%p16 bra $L__BB0_17;\n" "\n" " mul.wide.s32 %rd31, %r2, 4;\n" " add.s64 %rd64, %rd1, %rd31;\n" " mul.wide.s32 %rd8, %r1, 4;\n" " mov.f32 %f176, 0f00000000;\n" " mov.f32 %f52, 0f3F000000;\n" " mov.f32 %f53, 0f3BBB989D;\n" " mov.u32 %r153, %r2;\n" "\n" "$L__BB0_16:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f50, [%rd64];\n" " sub.f32 %f51, %f50, %f9;\n" " fma.rn.f32 %f54, %f51, %f53, %f52;\n" " mov.f32 %f55, 0f3FB8AA3B;\n" " mov.f32 %f56, 0f437C0000;\n" " cvt.sat.f32.f32 %f57, %f54;\n" " mov.f32 %f58, 0f4B400001;\n" " fma.rm.f32 %f59, %f57, %f56, %f58;\n" " add.f32 %f60, %f59, 0fCB40007F;\n" " neg.f32 %f61, %f60;\n" " fma.rn.f32 %f62, %f51, %f55, %f61;\n" " mov.f32 %f63, 0f32A57060;\n" " fma.rn.f32 %f64, %f51, %f63, %f62;\n" " mov.b32 %r96, %f59;\n" " shl.b32 %r97, %r96, 23;\n" " mov.b32 %f65, %r97;\n" " ex2.approx.ftz.f32 %f66, %f64;\n" " mul.f32 %f67, %f66, %f65;\n" " st.global.f32 [%rd64], %f67;\n" " add.f32 %f176, %f176, %f67;\n" " add.s32 %r153, %r153, %r1;\n" " add.s64 %rd64, %rd64, %rd8;\n" " add.s32 %r152, %r152, -1;\n" " setp.ne.s32 %p17, %r152, 0;\n" " @%p17 bra $L__BB0_16;\n" "\n" "$L__BB0_17:\n" " setp.lt.u32 %p18, %r17, 3;\n" " @%p18 bra $L__BB0_20;\n" "\n" " shl.b32 %r24, %r1, 2;\n" " mov.f32 %f70, 0f3F000000;\n" " mov.f32 %f71, 0f3BBB989D;\n" " cvt.s64.s32 %rd34, %r24;\n" "\n" "$L__BB0_19:\n" " mul.wide.s32 %rd32, %r153, 4;\n" " add.s64 %rd33, %rd1, %rd32;\n" " ld.global.f32 %f68, [%rd33];\n" " sub.f32 %f69, %f68, %f9;\n" " fma.rn.f32 %f72, %f69, %f71, %f70;\n" " mov.f32 %f73, 0f3FB8AA3B;\n" " mov.f32 %f74, 0f437C0000;\n" " cvt.sat.f32.f32 %f75, %f72;\n" " mov.f32 %f76, 0f4B400001;\n" " fma.rm.f32 %f77, %f75, %f74, %f76;\n" " add.f32 %f78, %f77, 0fCB40007F;\n" " neg.f32 %f79, %f78;\n" " fma.rn.f32 %f80, %f69, %f73, %f79;\n" " mov.f32 %f81, 0f32A57060;\n" " fma.rn.f32 %f82, %f69, %f81, %f80;\n" " mov.b32 %r98, %f77;\n" " shl.b32 %r99, %r98, 23;\n" " mov.b32 %f83, %r99;\n" " ex2.approx.ftz.f32 %f84, %f82;\n" " mul.f32 %f85, %f84, %f83;\n" " st.global.f32 [%rd33], %f85;\n" " add.f32 %f86, %f176, %f85;\n" " add.s64 %rd35, %rd33, %rd34;\n" " ld.global.f32 %f87, [%rd35];\n" " sub.f32 %f88, %f87, %f9;\n" " fma.rn.f32 %f89, %f88, %f71, %f70;\n" " cvt.sat.f32.f32 %f90, %f89;\n" " fma.rm.f32 %f91, %f90, %f74, %f76;\n" " add.f32 %f92, %f91, 0fCB40007F;\n" " neg.f32 %f93, %f92;\n" " fma.rn.f32 %f94, %f88, %f73, %f93;\n" " fma.rn.f32 %f95, %f88, %f81, %f94;\n" " mov.b32 %r100, %f91;\n" " shl.b32 %r101, %r100, 23;\n" " mov.b32 %f96, %r101;\n" " ex2.approx.ftz.f32 %f97, %f95;\n" " mul.f32 %f98, %f97, %f96;\n" " st.global.f32 [%rd35], %f98;\n" " add.f32 %f99, %f86, %f98;\n" " add.s32 %r102, %r153, %r1;\n" " add.s32 %r103, %r102, %r1;\n" " add.s64 %rd36, %rd35, %rd34;\n" " ld.global.f32 %f100, [%rd36];\n" " sub.f32 %f101, %f100, %f9;\n" " fma.rn.f32 %f102, %f101, %f71, %f70;\n" " cvt.sat.f32.f32 %f103, %f102;\n" " fma.rm.f32 %f104, %f103, %f74, %f76;\n" " add.f32 %f105, %f104, 0fCB40007F;\n" " neg.f32 %f106, %f105;\n" " fma.rn.f32 %f107, %f101, %f73, %f106;\n" " fma.rn.f32 %f108, %f101, %f81, %f107;\n" " mov.b32 %r104, %f104;\n" " shl.b32 %r105, %r104, 23;\n" " mov.b32 %f109, %r105;\n" " ex2.approx.ftz.f32 %f110, %f108;\n" " mul.f32 %f111, %f110, %f109;\n" " st.global.f32 [%rd36], %f111;\n" " add.f32 %f112, %f99, %f111;\n" " add.s32 %r106, %r103, %r1;\n" " add.s64 %rd37, %rd36, %rd34;\n" " ld.global.f32 %f113, [%rd37];\n" " sub.f32 %f114, %f113, %f9;\n" " fma.rn.f32 %f115, %f114, %f71, %f70;\n" " cvt.sat.f32.f32 %f116, %f115;\n" " fma.rm.f32 %f117, %f116, %f74, %f76;\n" " add.f32 %f118, %f117, 0fCB40007F;\n" " neg.f32 %f119, %f118;\n" " fma.rn.f32 %f120, %f114, %f73, %f119;\n" " fma.rn.f32 %f121, %f114, %f81, %f120;\n" " mov.b32 %r107, %f117;\n" " shl.b32 %r108, %r107, 23;\n" " mov.b32 %f122, %r108;\n" " ex2.approx.ftz.f32 %f123, %f121;\n" " mul.f32 %f124, %f123, %f122;\n" " st.global.f32 [%rd37], %f124;\n" " add.f32 %f176, %f112, %f124;\n" " add.s32 %r153, %r106, %r1;\n" " setp.lt.s32 %p19, %r153, %r79;\n" " @%p19 bra $L__BB0_19;\n" "\n" "$L__BB0_20:\n" " shl.b32 %r109, %r1, 2;\n" " add.s32 %r27, %r88, %r109;\n" " add.s32 %r28, %r27, %r87;\n" " st.shared.f32 [%r28], %f176;\n" " bar.sync 0;\n" " @%p11 bra $L__BB0_25;\n" "\n" " mov.u32 %r155, %r14;\n" "\n" "$L__BB0_22:\n" " setp.ge.s32 %p21, %r2, %r155;\n" " @%p21 bra $L__BB0_24;\n" "\n" " shl.b32 %r112, %r155, 2;\n" " add.s32 %r113, %r28, %r112;\n" " ld.shared.f32 %f125, [%r28];\n" " ld.shared.f32 %f126, [%r113];\n" " add.f32 %f127, %f126, %f125;\n" " st.shared.f32 [%r28], %f127;\n" "\n" "$L__BB0_24:\n" " bar.sync 0;\n" " shr.s32 %r30, %r155, 1;\n" " setp.gt.s32 %p22, %r155, 1;\n" " mov.u32 %r155, %r30;\n" " @%p22 bra $L__BB0_22;\n" "\n" "$L__BB0_25:\n" " ld.shared.f32 %f128, [%r27];\n" " rcp.rn.f32 %f17, %f128;\n" " @%p1 bra $L__BB0_32;\n" "\n" " not.b32 %r114, %r2;\n" " add.s32 %r115, %r114, %r79;\n" " div.u32 %r31, %r115, %r1;\n" " add.s32 %r116, %r31, 1;\n" " and.b32 %r157, %r116, 3;\n" " setp.eq.s32 %p24, %r157, 0;\n" " mov.u32 %r158, %r2;\n" " @%p24 bra $L__BB0_29;\n" "\n" " mul.wide.s32 %rd38, %r2, 4;\n" " add.s64 %rd65, %rd1, %rd38;\n" " mul.wide.s32 %rd12, %r1, 4;\n" " mov.u32 %r158, %r2;\n" "\n" "$L__BB0_28:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f129, [%rd65];\n" " mul.f32 %f130, %f17, %f129;\n" " st.global.f32 [%rd65], %f130;\n" " add.s32 %r158, %r158, %r1;\n" " add.s64 %rd65, %rd65, %rd12;\n" " add.s32 %r157, %r157, -1;\n" " setp.ne.s32 %p25, %r157, 0;\n" " @%p25 bra $L__BB0_28;\n" "\n" "$L__BB0_29:\n" " setp.lt.u32 %p26, %r31, 3;\n" " @%p26 bra $L__BB0_32;\n" "\n" " cvt.s64.s32 %rd41, %r109;\n" "\n" "$L__BB0_31:\n" " mul.wide.s32 %rd39, %r158, 4;\n" " add.s64 %rd40, %rd1, %rd39;\n" " ld.global.f32 %f131, [%rd40];\n" " mul.f32 %f132, %f17, %f131;\n" " st.global.f32 [%rd40], %f132;\n" " add.s64 %rd42, %rd40, %rd41;\n" " ld.global.f32 %f133, [%rd42];\n" " mul.f32 %f134, %f17, %f133;\n" " st.global.f32 [%rd42], %f134;\n" " add.s32 %r117, %r158, %r1;\n" " add.s32 %r118, %r117, %r1;\n" " add.s64 %rd43, %rd42, %rd41;\n" " ld.global.f32 %f135, [%rd43];\n" " mul.f32 %f136, %f17, %f135;\n" " st.global.f32 [%rd43], %f136;\n" " add.s32 %r119, %r118, %r1;\n" " add.s64 %rd44, %rd43, %rd41;\n" " ld.global.f32 %f137, [%rd44];\n" " mul.f32 %f138, %f17, %f137;\n" " st.global.f32 [%rd44], %f138;\n" " add.s32 %r158, %r119, %r1;\n" " setp.lt.s32 %p27, %r158, %r79;\n" " @%p27 bra $L__BB0_31;\n" "\n" "$L__BB0_32:\n" " bar.sync 0;\n" " setp.ne.s32 %p28, %r2, 0;\n" " @%p28 bra $L__BB0_34;\n" "\n" " mov.u32 %r120, 0;\n" " st.shared.u32 [_ZZ18fc_moe_router_topkE7swt_sum], %r120;\n" "\n" "$L__BB0_34:\n" " setp.lt.s32 %p29, %r80, 1;\n" " @%p29 bra $L__BB0_58;\n" "\n" " cvta.to.global.u64 %rd15, %rd21;\n" " mov.u32 %r123, _ZZ18fc_moe_router_topkE2rv;\n" " add.s32 %r41, %r123, %r87;\n" " mov.u32 %r124, _ZZ18fc_moe_router_topkE2ri;\n" " add.s32 %r42, %r124, %r87;\n" " not.b32 %r125, %r2;\n" " add.s32 %r126, %r125, %r79;\n" " div.u32 %r43, %r126, %r1;\n" " add.s32 %r127, %r43, 1;\n" " and.b32 %r44, %r127, 3;\n" " sub.s32 %r45, %r127, %r44;\n" " mov.u32 %r160, 0;\n" " mov.f32 %f139, 0fBF800000;\n" " setp.lt.u32 %p31, %r43, 3;\n" " setp.eq.s32 %p37, %r44, 0;\n" " setp.eq.s32 %p38, %r44, 1;\n" " mul.wide.s32 %rd52, %r1, 4;\n" " cvt.s64.s32 %rd47, %r109;\n" "\n" "$L__BB0_36:\n" " mov.u32 %r167, -1;\n" " mov.f32 %f180, %f139;\n" " @%p1 bra $L__BB0_44;\n" "\n" " mov.u32 %r167, -1;\n" " mov.u32 %r165, %r2;\n" " mov.f32 %f180, %f139;\n" " @%p31 bra $L__BB0_40;\n" "\n" " mov.f32 %f180, 0fBF800000;\n" " mov.u32 %r167, -1;\n" " mov.u32 %r165, %r2;\n" " mov.u32 %r163, %r45;\n" "\n" "$L__BB0_39:\n" " mul.wide.s32 %rd45, %r165, 4;\n" " add.s64 %rd46, %rd1, %rd45;\n" " ld.global.f32 %f143, [%rd46];\n" " setp.gt.f32 %p32, %f143, %f180;\n" " selp.f32 %f144, %f143, %f180, %p32;\n" " selp.b32 %r132, %r165, %r167, %p32;\n" " add.s64 %rd48, %rd46, %rd47;\n" " ld.global.f32 %f145, [%rd48];\n" " setp.gt.f32 %p33, %f145, %f144;\n" " selp.f32 %f146, %f145, %f144, %p33;\n" " add.s32 %r133, %r165, %r1;\n" " selp.b32 %r134, %r133, %r132, %p33;\n" " add.s32 %r135, %r133, %r1;\n" " add.s64 %rd49, %rd48, %rd47;\n" " ld.global.f32 %f147, [%rd49];\n" " setp.gt.f32 %p34, %f147, %f146;\n" " selp.f32 %f148, %f147, %f146, %p34;\n" " selp.b32 %r136, %r135, %r134, %p34;\n" " add.s32 %r137, %r135, %r1;\n" " add.s64 %rd50, %rd49, %rd47;\n" " ld.global.f32 %f149, [%rd50];\n" " setp.gt.f32 %p35, %f149, %f148;\n" " selp.f32 %f180, %f149, %f148, %p35;\n" " selp.b32 %r167, %r137, %r136, %p35;\n" " add.s32 %r165, %r137, %r1;\n" " add.s32 %r163, %r163, -4;\n" " setp.ne.s32 %p36, %r163, 0;\n" " @%p36 bra $L__BB0_39;\n" "\n" "$L__BB0_40:\n" " @%p37 bra $L__BB0_44;\n" "\n" " mul.wide.s32 %rd51, %r165, 4;\n" " add.s64 %rd16, %rd1, %rd51;\n" " ld.global.f32 %f150, [%rd16];\n" " setp.gt.f32 %p39, %f150, %f180;\n" " selp.f32 %f180, %f150, %f180, %p39;\n" " selp.b32 %r167, %r165, %r167, %p39;\n" " @%p38 bra $L__BB0_44;\n" "\n" " add.s32 %r58, %r165, %r1;\n" " setp.eq.s32 %p40, %r44, 2;\n" " add.s64 %rd17, %rd16, %rd52;\n" " ld.global.f32 %f151, [%rd17];\n" " setp.gt.f32 %p41, %f151, %f180;\n" " selp.f32 %f180, %f151, %f180, %p41;\n" " selp.b32 %r167, %r58, %r167, %p41;\n" " @%p40 bra $L__BB0_44;\n" "\n" " add.s64 %rd54, %rd17, %rd52;\n" " ld.global.f32 %f152, [%rd54];\n" " setp.gt.f32 %p42, %f152, %f180;\n" " selp.f32 %f180, %f152, %f180, %p42;\n" " add.s32 %r138, %r58, %r1;\n" " selp.b32 %r167, %r138, %r167, %p42;\n" "\n" "$L__BB0_44:\n" " st.shared.f32 [%r41], %f180;\n" " st.shared.u32 [%r42], %r167;\n" " bar.sync 0;\n" " @%p11 bra $L__BB0_54;\n" "\n" " mov.u32 %r168, %r14;\n" "\n" "$L__BB0_46:\n" " mov.u32 %r62, %r168;\n" " setp.ge.s32 %p44, %r2, %r62;\n" " @%p44 bra $L__BB0_53;\n" "\n" " shl.b32 %r139, %r62, 2;\n" " add.s32 %r140, %r41, %r139;\n" " ld.shared.f32 %f26, [%r41];\n" " ld.shared.f32 %f27, [%r140];\n" " setp.gt.f32 %p45, %f27, %f26;\n" " add.s32 %r63, %r42, %r139;\n" " @%p45 bra $L__BB0_51;\n" " bra.uni $L__BB0_48;\n" "\n" "$L__BB0_51:\n" " ld.shared.u32 %r169, [%r63];\n" "\n" "$L__BB0_52:\n" " st.shared.f32 [%r41], %f27;\n" " st.shared.u32 [%r42], %r169;\n" " bra.uni $L__BB0_53;\n" "\n" "$L__BB0_48:\n" " setp.neu.f32 %p46, %f27, %f26;\n" " @%p46 bra $L__BB0_53;\n" "\n" " ld.shared.u32 %r169, [%r63];\n" " setp.lt.s32 %p47, %r169, 0;\n" " @%p47 bra $L__BB0_53;\n" "\n" " ld.shared.u32 %r141, [%r42];\n" " setp.lt.s32 %p48, %r141, 0;\n" " setp.lt.s32 %p49, %r169, %r141;\n" " or.pred %p50, %p48, %p49;\n" " @%p50 bra $L__BB0_52;\n" "\n" "$L__BB0_53:\n" " bar.sync 0;\n" " shr.s32 %r168, %r62, 1;\n" " setp.gt.s32 %p51, %r62, 1;\n" " @%p51 bra $L__BB0_46;\n" "\n" "$L__BB0_54:\n" " @%p28 bra $L__BB0_57;\n" "\n" " ld.shared.u32 %r68, [_ZZ18fc_moe_router_topkE2ri];\n" " mul.wide.s32 %rd55, %r160, 4;\n" " add.s64 %rd56, %rd15, %rd55;\n" " st.global.u32 [%rd56], %r68;\n" " ld.shared.f32 %f153, [_ZZ18fc_moe_router_topkE2rv];\n" " add.s64 %rd57, %rd2, %rd55;\n" " st.global.f32 [%rd57], %f153;\n" " ld.shared.f32 %f154, [_ZZ18fc_moe_router_topkE7swt_sum];\n" " add.f32 %f155, %f153, %f154;\n" " st.shared.f32 [_ZZ18fc_moe_router_topkE7swt_sum], %f155;\n" " setp.lt.s32 %p53, %r68, 0;\n" " @%p53 bra $L__BB0_57;\n" "\n" " mul.wide.s32 %rd58, %r68, 4;\n" " add.s64 %rd59, %rd1, %rd58;\n" " mov.u32 %r142, -1082130432;\n" " st.global.u32 [%rd59], %r142;\n" "\n" "$L__BB0_57:\n" " bar.sync 0;\n" " add.s32 %r160, %r160, 1;\n" " setp.lt.s32 %p54, %r160, %r80;\n" " @%p54 bra $L__BB0_36;\n" "\n" "$L__BB0_58:\n" " @%p28 bra $L__BB0_68;\n" " bra.uni $L__BB0_59;\n" "\n" "$L__BB0_68:\n" " ret;\n" "\n" "$L__BB0_59:\n" " ld.shared.f32 %f28, [_ZZ18fc_moe_router_topkE7swt_sum];\n" " setp.leu.f32 %p56, %f28, 0f00000000;\n" " mov.f32 %f181, 0f00000000;\n" " @%p56 bra $L__BB0_61;\n" "\n" " rcp.rn.f32 %f181, %f28;\n" "\n" "$L__BB0_61:\n" " mul.f32 %f31, %f181, %f32;\n" " @%p29 bra $L__BB0_68;\n" "\n" " add.s32 %r144, %r80, -1;\n" " and.b32 %r173, %r80, 3;\n" " setp.lt.u32 %p58, %r144, 3;\n" " mov.u32 %r172, 0;\n" " @%p58 bra $L__BB0_65;\n" "\n" " sub.s32 %r171, %r80, %r173;\n" " mov.u32 %r172, 0;\n" "\n" "$L__BB0_64:\n" " mul.wide.s32 %rd60, %r172, 4;\n" " add.s64 %rd61, %rd2, %rd60;\n" " ld.global.f32 %f157, [%rd61];\n" " mul.f32 %f158, %f31, %f157;\n" " st.global.f32 [%rd61], %f158;\n" " ld.global.f32 %f159, [%rd61+4];\n" " mul.f32 %f160, %f31, %f159;\n" " st.global.f32 [%rd61+4], %f160;\n" " ld.global.f32 %f161, [%rd61+8];\n" " mul.f32 %f162, %f31, %f161;\n" " st.global.f32 [%rd61+8], %f162;\n" " ld.global.f32 %f163, [%rd61+12];\n" " mul.f32 %f164, %f31, %f163;\n" " st.global.f32 [%rd61+12], %f164;\n" " add.s32 %r172, %r172, 4;\n" " add.s32 %r171, %r171, -4;\n" " setp.ne.s32 %p59, %r171, 0;\n" " @%p59 bra $L__BB0_64;\n" "\n" "$L__BB0_65:\n" " setp.eq.s32 %p60, %r173, 0;\n" " @%p60 bra $L__BB0_68;\n" "\n" " mul.wide.s32 %rd62, %r172, 4;\n" " add.s64 %rd66, %rd2, %rd62;\n" "\n" "$L__BB0_67:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f165, [%rd66];\n" " mul.f32 %f166, %f31, %f165;\n" " st.global.f32 [%rd66], %f166;\n" " add.s64 %rd66, %rd66, 4;\n" " add.s32 %r173, %r173, -1;\n" " setp.ne.s32 %p61, %r173, 0;\n" " @%p61 bra $L__BB0_67;\n" " bra.uni $L__BB0_68;\n" "\n" "}\n" " // .globl fc_moe_router_topk_nx\n" ".visible .entry fc_moe_router_topk_nx(\n" " .param .u64 fc_moe_router_topk_nx_param_0,\n" " .param .u64 fc_moe_router_topk_nx_param_1,\n" " .param .u64 fc_moe_router_topk_nx_param_2,\n" " .param .u32 fc_moe_router_topk_nx_param_3,\n" " .param .u32 fc_moe_router_topk_nx_param_4,\n" " .param .f32 fc_moe_router_topk_nx_param_5\n" ")\n" "{\n" " .reg .pred %p<62>;\n" " .reg .f32 %f<182>;\n" " .reg .b32 %r<171>;\n" " .reg .b64 %rd<97>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ21fc_moe_router_topk_nxE2rv[1024];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ21fc_moe_router_topk_nxE2ri[1024];\n" " // demoted variable\n" " .shared .align 4 .f32 _ZZ21fc_moe_router_topk_nxE7swt_sum;\n" "\n" " ld.param.u64 %rd26, [fc_moe_router_topk_nx_param_0];\n" " ld.param.u64 %rd25, [fc_moe_router_topk_nx_param_1];\n" " ld.param.u64 %rd27, [fc_moe_router_topk_nx_param_2];\n" " ld.param.u32 %r75, [fc_moe_router_topk_nx_param_3];\n" " ld.param.u32 %r76, [fc_moe_router_topk_nx_param_4];\n" " ld.param.f32 %f32, [fc_moe_router_topk_nx_param_5];\n" " cvta.to.global.u64 %rd1, %rd26;\n" " cvta.to.global.u64 %rd2, %rd27;\n" " mov.u32 %r77, %ctaid.x;\n" " mul.wide.s32 %rd3, %r75, %r77;\n" " mul.wide.s32 %rd4, %r76, %r77;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r2, %tid.x;\n" " setp.ge.s32 %p1, %r2, %r75;\n" " mov.f32 %f171, 0fF149F2CA;\n" " @%p1 bra $L__BB1_7;\n" "\n" " not.b32 %r78, %r2;\n" " add.s32 %r79, %r78, %r75;\n" " div.u32 %r3, %r79, %r1;\n" " add.s32 %r80, %r3, 1;\n" " and.b32 %r144, %r80, 3;\n" " setp.eq.s32 %p2, %r144, 0;\n" " mov.f32 %f171, 0fF149F2CA;\n" " mov.u32 %r145, %r2;\n" " @%p2 bra $L__BB1_4;\n" "\n" " cvt.s64.s32 %rd28, %r2;\n" " add.s64 %rd29, %rd3, %rd28;\n" " shl.b64 %rd30, %rd29, 2;\n" " add.s64 %rd93, %rd1, %rd30;\n" " mul.wide.s32 %rd6, %r1, 4;\n" " mov.f32 %f171, 0fF149F2CA;\n" " mov.u32 %r145, %r2;\n" "\n" "$L__BB1_3:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f37, [%rd93];\n" " setp.gt.f32 %p3, %f37, %f171;\n" " selp.f32 %f171, %f37, %f171, %p3;\n" " add.s32 %r145, %r145, %r1;\n" " add.s64 %rd93, %rd93, %rd6;\n" " add.s32 %r144, %r144, -1;\n" " setp.ne.s32 %p4, %r144, 0;\n" " @%p4 bra $L__BB1_3;\n" "\n" "$L__BB1_4:\n" " setp.lt.u32 %p5, %r3, 3;\n" " @%p5 bra $L__BB1_7;\n" "\n" " mul.wide.s32 %rd9, %r1, 4;\n" "\n" "$L__BB1_6:\n" " cvt.s64.s32 %rd31, %r145;\n" " add.s64 %rd32, %rd3, %rd31;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd34, %rd1, %rd33;\n" " ld.global.f32 %f38, [%rd34];\n" " setp.gt.f32 %p6, %f38, %f171;\n" " selp.f32 %f39, %f38, %f171, %p6;\n" " add.s64 %rd35, %rd34, %rd9;\n" " ld.global.f32 %f40, [%rd35];\n" " setp.gt.f32 %p7, %f40, %f39;\n" " selp.f32 %f41, %f40, %f39, %p7;\n" " add.s32 %r81, %r145, %r1;\n" " add.s32 %r82, %r81, %r1;\n" " add.s64 %rd36, %rd35, %rd9;\n" " ld.global.f32 %f42, [%rd36];\n" " setp.gt.f32 %p8, %f42, %f41;\n" " selp.f32 %f43, %f42, %f41, %p8;\n" " add.s32 %r83, %r82, %r1;\n" " add.s64 %rd37, %rd36, %rd9;\n" " ld.global.f32 %f44, [%rd37];\n" " setp.gt.f32 %p9, %f44, %f43;\n" " selp.f32 %f171, %f44, %f43, %p9;\n" " add.s32 %r145, %r83, %r1;\n" " setp.lt.s32 %p10, %r145, %r75;\n" " @%p10 bra $L__BB1_6;\n" "\n" "$L__BB1_7:\n" " shl.b32 %r84, %r2, 2;\n" " mov.u32 %r85, sdata;\n" " add.s32 %r12, %r85, %r84;\n" " st.shared.f32 [%r12], %f171;\n" " bar.sync 0;\n" " shr.u32 %r86, %r1, 31;\n" " add.s32 %r87, %r1, %r86;\n" " shr.s32 %r13, %r87, 1;\n" " setp.lt.s32 %p11, %r1, 2;\n" " @%p11 bra $L__BB1_13;\n" "\n" " mov.u32 %r147, %r13;\n" "\n" "$L__BB1_9:\n" " mov.u32 %r14, %r147;\n" " setp.ge.s32 %p12, %r2, %r14;\n" " @%p12 bra $L__BB1_12;\n" "\n" " shl.b32 %r88, %r14, 2;\n" " add.s32 %r89, %r12, %r88;\n" " ld.shared.f32 %f45, [%r12];\n" " ld.shared.f32 %f8, [%r89];\n" " setp.leu.f32 %p13, %f8, %f45;\n" " @%p13 bra $L__BB1_12;\n" "\n" " st.shared.f32 [%r12], %f8;\n" "\n" "$L__BB1_12:\n" " bar.sync 0;\n" " shr.s32 %r147, %r14, 1;\n" " setp.gt.s32 %p14, %r14, 1;\n" " @%p14 bra $L__BB1_9;\n" "\n" "$L__BB1_13:\n" " mov.f32 %f176, 0f00000000;\n" " ld.shared.f32 %f9, [sdata];\n" " @%p1 bra $L__BB1_20;\n" "\n" " not.b32 %r90, %r2;\n" " add.s32 %r91, %r90, %r75;\n" " div.u32 %r16, %r91, %r1;\n" " add.s32 %r92, %r16, 1;\n" " and.b32 %r149, %r92, 3;\n" " setp.eq.s32 %p16, %r149, 0;\n" " mov.f32 %f176, 0f00000000;\n" " mov.u32 %r150, %r2;\n" " @%p16 bra $L__BB1_17;\n" "\n" " cvt.s64.s32 %rd38, %r2;\n" " add.s64 %rd39, %rd3, %rd38;\n" " shl.b64 %rd40, %rd39, 2;\n" " add.s64 %rd94, %rd1, %rd40;\n" " mul.wide.s32 %rd11, %r1, 4;\n" " mov.f32 %f176, 0f00000000;\n" " mov.f32 %f52, 0f3F000000;\n" " mov.f32 %f53, 0f3BBB989D;\n" " mov.u32 %r150, %r2;\n" "\n" "$L__BB1_16:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f50, [%rd94];\n" " sub.f32 %f51, %f50, %f9;\n" " fma.rn.f32 %f54, %f51, %f53, %f52;\n" " mov.f32 %f55, 0f3FB8AA3B;\n" " mov.f32 %f56, 0f437C0000;\n" " cvt.sat.f32.f32 %f57, %f54;\n" " mov.f32 %f58, 0f4B400001;\n" " fma.rm.f32 %f59, %f57, %f56, %f58;\n" " add.f32 %f60, %f59, 0fCB40007F;\n" " neg.f32 %f61, %f60;\n" " fma.rn.f32 %f62, %f51, %f55, %f61;\n" " mov.f32 %f63, 0f32A57060;\n" " fma.rn.f32 %f64, %f51, %f63, %f62;\n" " mov.b32 %r93, %f59;\n" " shl.b32 %r94, %r93, 23;\n" " mov.b32 %f65, %r94;\n" " ex2.approx.ftz.f32 %f66, %f64;\n" " mul.f32 %f67, %f66, %f65;\n" " st.global.f32 [%rd94], %f67;\n" " add.f32 %f176, %f176, %f67;\n" " add.s32 %r150, %r150, %r1;\n" " add.s64 %rd94, %rd94, %rd11;\n" " add.s32 %r149, %r149, -1;\n" " setp.ne.s32 %p17, %r149, 0;\n" " @%p17 bra $L__BB1_16;\n" "\n" "$L__BB1_17:\n" " setp.lt.u32 %p18, %r16, 3;\n" " @%p18 bra $L__BB1_20;\n" "\n" " mul.wide.s32 %rd14, %r1, 4;\n" " mov.f32 %f70, 0f3F000000;\n" " mov.f32 %f71, 0f3BBB989D;\n" "\n" "$L__BB1_19:\n" " cvt.s64.s32 %rd41, %r150;\n" " add.s64 %rd42, %rd3, %rd41;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd44, %rd1, %rd43;\n" " ld.global.f32 %f68, [%rd44];\n" " sub.f32 %f69, %f68, %f9;\n" " fma.rn.f32 %f72, %f69, %f71, %f70;\n" " mov.f32 %f73, 0f3FB8AA3B;\n" " mov.f32 %f74, 0f437C0000;\n" " cvt.sat.f32.f32 %f75, %f72;\n" " mov.f32 %f76, 0f4B400001;\n" " fma.rm.f32 %f77, %f75, %f74, %f76;\n" " add.f32 %f78, %f77, 0fCB40007F;\n" " neg.f32 %f79, %f78;\n" " fma.rn.f32 %f80, %f69, %f73, %f79;\n" " mov.f32 %f81, 0f32A57060;\n" " fma.rn.f32 %f82, %f69, %f81, %f80;\n" " mov.b32 %r95, %f77;\n" " shl.b32 %r96, %r95, 23;\n" " mov.b32 %f83, %r96;\n" " ex2.approx.ftz.f32 %f84, %f82;\n" " mul.f32 %f85, %f84, %f83;\n" " st.global.f32 [%rd44], %f85;\n" " add.f32 %f86, %f176, %f85;\n" " add.s64 %rd45, %rd44, %rd14;\n" " ld.global.f32 %f87, [%rd45];\n" " sub.f32 %f88, %f87, %f9;\n" " fma.rn.f32 %f89, %f88, %f71, %f70;\n" " cvt.sat.f32.f32 %f90, %f89;\n" " fma.rm.f32 %f91, %f90, %f74, %f76;\n" " add.f32 %f92, %f91, 0fCB40007F;\n" " neg.f32 %f93, %f92;\n" " fma.rn.f32 %f94, %f88, %f73, %f93;\n" " fma.rn.f32 %f95, %f88, %f81, %f94;\n" " mov.b32 %r97, %f91;\n" " shl.b32 %r98, %r97, 23;\n" " mov.b32 %f96, %r98;\n" " ex2.approx.ftz.f32 %f97, %f95;\n" " mul.f32 %f98, %f97, %f96;\n" " st.global.f32 [%rd45], %f98;\n" " add.f32 %f99, %f86, %f98;\n" " add.s32 %r99, %r150, %r1;\n" " add.s32 %r100, %r99, %r1;\n" " add.s64 %rd46, %rd45, %rd14;\n" " ld.global.f32 %f100, [%rd46];\n" " sub.f32 %f101, %f100, %f9;\n" " fma.rn.f32 %f102, %f101, %f71, %f70;\n" " cvt.sat.f32.f32 %f103, %f102;\n" " fma.rm.f32 %f104, %f103, %f74, %f76;\n" " add.f32 %f105, %f104, 0fCB40007F;\n" " neg.f32 %f106, %f105;\n" " fma.rn.f32 %f107, %f101, %f73, %f106;\n" " fma.rn.f32 %f108, %f101, %f81, %f107;\n" " mov.b32 %r101, %f104;\n" " shl.b32 %r102, %r101, 23;\n" " mov.b32 %f109, %r102;\n" " ex2.approx.ftz.f32 %f110, %f108;\n" " mul.f32 %f111, %f110, %f109;\n" " st.global.f32 [%rd46], %f111;\n" " add.f32 %f112, %f99, %f111;\n" " add.s32 %r103, %r100, %r1;\n" " add.s64 %rd47, %rd46, %rd14;\n" " ld.global.f32 %f113, [%rd47];\n" " sub.f32 %f114, %f113, %f9;\n" " fma.rn.f32 %f115, %f114, %f71, %f70;\n" " cvt.sat.f32.f32 %f116, %f115;\n" " fma.rm.f32 %f117, %f116, %f74, %f76;\n" " add.f32 %f118, %f117, 0fCB40007F;\n" " neg.f32 %f119, %f118;\n" " fma.rn.f32 %f120, %f114, %f73, %f119;\n" " fma.rn.f32 %f121, %f114, %f81, %f120;\n" " mov.b32 %r104, %f117;\n" " shl.b32 %r105, %r104, 23;\n" " mov.b32 %f122, %r105;\n" " ex2.approx.ftz.f32 %f123, %f121;\n" " mul.f32 %f124, %f123, %f122;\n" " st.global.f32 [%rd47], %f124;\n" " add.f32 %f176, %f112, %f124;\n" " add.s32 %r150, %r103, %r1;\n" " setp.lt.s32 %p19, %r150, %r75;\n" " @%p19 bra $L__BB1_19;\n" "\n" "$L__BB1_20:\n" " shl.b32 %r106, %r1, 2;\n" " add.s32 %r25, %r85, %r106;\n" " add.s32 %r26, %r25, %r84;\n" " st.shared.f32 [%r26], %f176;\n" " bar.sync 0;\n" " @%p11 bra $L__BB1_25;\n" "\n" " mov.u32 %r152, %r13;\n" "\n" "$L__BB1_22:\n" " setp.ge.s32 %p21, %r2, %r152;\n" " @%p21 bra $L__BB1_24;\n" "\n" " shl.b32 %r109, %r152, 2;\n" " add.s32 %r110, %r26, %r109;\n" " ld.shared.f32 %f125, [%r26];\n" " ld.shared.f32 %f126, [%r110];\n" " add.f32 %f127, %f126, %f125;\n" " st.shared.f32 [%r26], %f127;\n" "\n" "$L__BB1_24:\n" " bar.sync 0;\n" " shr.s32 %r28, %r152, 1;\n" " setp.gt.s32 %p22, %r152, 1;\n" " mov.u32 %r152, %r28;\n" " @%p22 bra $L__BB1_22;\n" "\n" "$L__BB1_25:\n" " ld.shared.f32 %f128, [%r25];\n" " rcp.rn.f32 %f17, %f128;\n" " @%p1 bra $L__BB1_32;\n" "\n" " not.b32 %r111, %r2;\n" " add.s32 %r112, %r111, %r75;\n" " div.u32 %r29, %r112, %r1;\n" " add.s32 %r113, %r29, 1;\n" " and.b32 %r154, %r113, 3;\n" " setp.eq.s32 %p24, %r154, 0;\n" " mov.u32 %r155, %r2;\n" " @%p24 bra $L__BB1_29;\n" "\n" " cvt.s64.s32 %rd48, %r2;\n" " add.s64 %rd49, %rd3, %rd48;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd95, %rd1, %rd50;\n" " mul.wide.s32 %rd16, %r1, 4;\n" " mov.u32 %r155, %r2;\n" "\n" "$L__BB1_28:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f129, [%rd95];\n" " mul.f32 %f130, %f17, %f129;\n" " st.global.f32 [%rd95], %f130;\n" " add.s32 %r155, %r155, %r1;\n" " add.s64 %rd95, %rd95, %rd16;\n" " add.s32 %r154, %r154, -1;\n" " setp.ne.s32 %p25, %r154, 0;\n" " @%p25 bra $L__BB1_28;\n" "\n" "$L__BB1_29:\n" " setp.lt.u32 %p26, %r29, 3;\n" " @%p26 bra $L__BB1_32;\n" "\n" " mul.wide.s32 %rd19, %r1, 4;\n" "\n" "$L__BB1_31:\n" " cvt.s64.s32 %rd51, %r155;\n" " add.s64 %rd52, %rd3, %rd51;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd54, %rd1, %rd53;\n" " ld.global.f32 %f131, [%rd54];\n" " mul.f32 %f132, %f17, %f131;\n" " st.global.f32 [%rd54], %f132;\n" " add.s64 %rd55, %rd54, %rd19;\n" " ld.global.f32 %f133, [%rd55];\n" " mul.f32 %f134, %f17, %f133;\n" " st.global.f32 [%rd55], %f134;\n" " add.s32 %r114, %r155, %r1;\n" " add.s32 %r115, %r114, %r1;\n" " add.s64 %rd56, %rd55, %rd19;\n" " ld.global.f32 %f135, [%rd56];\n" " mul.f32 %f136, %f17, %f135;\n" " st.global.f32 [%rd56], %f136;\n" " add.s32 %r116, %r115, %r1;\n" " add.s64 %rd57, %rd56, %rd19;\n" " ld.global.f32 %f137, [%rd57];\n" " mul.f32 %f138, %f17, %f137;\n" " st.global.f32 [%rd57], %f138;\n" " add.s32 %r155, %r116, %r1;\n" " setp.lt.s32 %p27, %r155, %r75;\n" " @%p27 bra $L__BB1_31;\n" "\n" "$L__BB1_32:\n" " bar.sync 0;\n" " setp.ne.s32 %p28, %r2, 0;\n" " @%p28 bra $L__BB1_34;\n" "\n" " mov.u32 %r117, 0;\n" " st.shared.u32 [_ZZ21fc_moe_router_topk_nxE7swt_sum], %r117;\n" "\n" "$L__BB1_34:\n" " setp.lt.s32 %p29, %r76, 1;\n" " @%p29 bra $L__BB1_58;\n" "\n" " mov.u32 %r120, _ZZ21fc_moe_router_topk_nxE2rv;\n" " add.s32 %r38, %r120, %r84;\n" " mov.u32 %r121, _ZZ21fc_moe_router_topk_nxE2ri;\n" " add.s32 %r39, %r121, %r84;\n" " not.b32 %r122, %r2;\n" " add.s32 %r123, %r122, %r75;\n" " div.u32 %r40, %r123, %r1;\n" " add.s32 %r124, %r40, 1;\n" " and.b32 %r41, %r124, 3;\n" " sub.s32 %r42, %r124, %r41;\n" " mul.wide.s32 %rd20, %r1, 4;\n" " cvta.to.global.u64 %rd21, %rd25;\n" " mov.u32 %r157, 0;\n" " mov.f32 %f139, 0fBF800000;\n" " setp.lt.u32 %p31, %r40, 3;\n" " setp.eq.s32 %p37, %r41, 0;\n" " setp.eq.s32 %p38, %r41, 1;\n" "\n" "$L__BB1_36:\n" " mov.u32 %r164, -1;\n" " mov.f32 %f180, %f139;\n" " @%p1 bra $L__BB1_44;\n" "\n" " mov.u32 %r164, -1;\n" " mov.u32 %r162, %r2;\n" " mov.f32 %f180, %f139;\n" " @%p31 bra $L__BB1_40;\n" "\n" " mov.f32 %f180, 0fBF800000;\n" " mov.u32 %r164, -1;\n" " mov.u32 %r162, %r2;\n" " mov.u32 %r160, %r42;\n" "\n" "$L__BB1_39:\n" " cvt.s64.s32 %rd58, %r162;\n" " add.s64 %rd59, %rd3, %rd58;\n" " shl.b64 %rd60, %rd59, 2;\n" " add.s64 %rd61, %rd1, %rd60;\n" " ld.global.f32 %f143, [%rd61];\n" " setp.gt.f32 %p32, %f143, %f180;\n" " selp.f32 %f144, %f143, %f180, %p32;\n" " selp.b32 %r129, %r162, %r164, %p32;\n" " add.s64 %rd62, %rd61, %rd20;\n" " ld.global.f32 %f145, [%rd62];\n" " setp.gt.f32 %p33, %f145, %f144;\n" " selp.f32 %f146, %f145, %f144, %p33;\n" " add.s32 %r130, %r162, %r1;\n" " selp.b32 %r131, %r130, %r129, %p33;\n" " add.s32 %r132, %r130, %r1;\n" " add.s64 %rd63, %rd62, %rd20;\n" " ld.global.f32 %f147, [%rd63];\n" " setp.gt.f32 %p34, %f147, %f146;\n" " selp.f32 %f148, %f147, %f146, %p34;\n" " selp.b32 %r133, %r132, %r131, %p34;\n" " add.s32 %r134, %r132, %r1;\n" " add.s64 %rd64, %rd63, %rd20;\n" " ld.global.f32 %f149, [%rd64];\n" " setp.gt.f32 %p35, %f149, %f148;\n" " selp.f32 %f180, %f149, %f148, %p35;\n" " selp.b32 %r164, %r134, %r133, %p35;\n" " add.s32 %r162, %r134, %r1;\n" " add.s32 %r160, %r160, -4;\n" " setp.ne.s32 %p36, %r160, 0;\n" " @%p36 bra $L__BB1_39;\n" "\n" "$L__BB1_40:\n" " @%p37 bra $L__BB1_44;\n" "\n" " cvt.s64.s32 %rd65, %r162;\n" " add.s64 %rd66, %rd3, %rd65;\n" " shl.b64 %rd67, %rd66, 2;\n" " add.s64 %rd68, %rd1, %rd67;\n" " ld.global.f32 %f150, [%rd68];\n" " setp.gt.f32 %p39, %f150, %f180;\n" " selp.f32 %f180, %f150, %f180, %p39;\n" " selp.b32 %r164, %r162, %r164, %p39;\n" " @%p38 bra $L__BB1_44;\n" "\n" " add.s32 %r54, %r162, %r1;\n" " setp.eq.s32 %p40, %r41, 2;\n" " cvt.s64.s32 %rd69, %r54;\n" " add.s64 %rd70, %rd3, %rd69;\n" " shl.b64 %rd71, %rd70, 2;\n" " add.s64 %rd72, %rd1, %rd71;\n" " ld.global.f32 %f151, [%rd72];\n" " setp.gt.f32 %p41, %f151, %f180;\n" " selp.f32 %f180, %f151, %f180, %p41;\n" " selp.b32 %r164, %r54, %r164, %p41;\n" " @%p40 bra $L__BB1_44;\n" "\n" " add.s32 %r135, %r54, %r1;\n" " cvt.s64.s32 %rd73, %r135;\n" " add.s64 %rd74, %rd3, %rd73;\n" " shl.b64 %rd75, %rd74, 2;\n" " add.s64 %rd76, %rd1, %rd75;\n" " ld.global.f32 %f152, [%rd76];\n" " setp.gt.f32 %p42, %f152, %f180;\n" " selp.f32 %f180, %f152, %f180, %p42;\n" " selp.b32 %r164, %r135, %r164, %p42;\n" "\n" "$L__BB1_44:\n" " st.shared.f32 [%r38], %f180;\n" " st.shared.u32 [%r39], %r164;\n" " bar.sync 0;\n" " @%p11 bra $L__BB1_54;\n" "\n" " mov.u32 %r165, %r13;\n" "\n" "$L__BB1_46:\n" " mov.u32 %r58, %r165;\n" " setp.ge.s32 %p44, %r2, %r58;\n" " @%p44 bra $L__BB1_53;\n" "\n" " shl.b32 %r136, %r58, 2;\n" " add.s32 %r137, %r38, %r136;\n" " ld.shared.f32 %f26, [%r38];\n" " ld.shared.f32 %f27, [%r137];\n" " setp.gt.f32 %p45, %f27, %f26;\n" " add.s32 %r59, %r39, %r136;\n" " @%p45 bra $L__BB1_51;\n" " bra.uni $L__BB1_48;\n" "\n" "$L__BB1_51:\n" " ld.shared.u32 %r166, [%r59];\n" "\n" "$L__BB1_52:\n" " st.shared.f32 [%r38], %f27;\n" " st.shared.u32 [%r39], %r166;\n" " bra.uni $L__BB1_53;\n" "\n" "$L__BB1_48:\n" " setp.neu.f32 %p46, %f27, %f26;\n" " @%p46 bra $L__BB1_53;\n" "\n" " ld.shared.u32 %r166, [%r59];\n" " setp.lt.s32 %p47, %r166, 0;\n" " @%p47 bra $L__BB1_53;\n" "\n" " ld.shared.u32 %r138, [%r39];\n" " setp.lt.s32 %p48, %r138, 0;\n" " setp.lt.s32 %p49, %r166, %r138;\n" " or.pred %p50, %p48, %p49;\n" " @%p50 bra $L__BB1_52;\n" "\n" "$L__BB1_53:\n" " bar.sync 0;\n" " shr.s32 %r165, %r58, 1;\n" " setp.gt.s32 %p51, %r58, 1;\n" " @%p51 bra $L__BB1_46;\n" "\n" "$L__BB1_54:\n" " @%p28 bra $L__BB1_57;\n" "\n" " ld.shared.u32 %r64, [_ZZ21fc_moe_router_topk_nxE2ri];\n" " cvt.s64.s32 %rd77, %r157;\n" " add.s64 %rd78, %rd4, %rd77;\n" " shl.b64 %rd79, %rd78, 2;\n" " add.s64 %rd80, %rd21, %rd79;\n" " st.global.u32 [%rd80], %r64;\n" " ld.shared.f32 %f153, [_ZZ21fc_moe_router_topk_nxE2rv];\n" " add.s64 %rd81, %rd2, %rd79;\n" " st.global.f32 [%rd81], %f153;\n" " ld.shared.f32 %f154, [_ZZ21fc_moe_router_topk_nxE7swt_sum];\n" " add.f32 %f155, %f153, %f154;\n" " st.shared.f32 [_ZZ21fc_moe_router_topk_nxE7swt_sum], %f155;\n" " setp.lt.s32 %p53, %r64, 0;\n" " @%p53 bra $L__BB1_57;\n" "\n" " cvt.s64.s32 %rd82, %r64;\n" " add.s64 %rd83, %rd3, %rd82;\n" " shl.b64 %rd84, %rd83, 2;\n" " add.s64 %rd85, %rd1, %rd84;\n" " mov.u32 %r139, -1082130432;\n" " st.global.u32 [%rd85], %r139;\n" "\n" "$L__BB1_57:\n" " bar.sync 0;\n" " add.s32 %r157, %r157, 1;\n" " setp.lt.s32 %p54, %r157, %r76;\n" " @%p54 bra $L__BB1_36;\n" "\n" "$L__BB1_58:\n" " @%p28 bra $L__BB1_68;\n" " bra.uni $L__BB1_59;\n" "\n" "$L__BB1_68:\n" " ret;\n" "\n" "$L__BB1_59:\n" " ld.shared.f32 %f28, [_ZZ21fc_moe_router_topk_nxE7swt_sum];\n" " setp.leu.f32 %p56, %f28, 0f00000000;\n" " mov.f32 %f181, 0f00000000;\n" " @%p56 bra $L__BB1_61;\n" "\n" " rcp.rn.f32 %f181, %f28;\n" "\n" "$L__BB1_61:\n" " mul.f32 %f31, %f181, %f32;\n" " @%p29 bra $L__BB1_68;\n" "\n" " add.s32 %r141, %r76, -1;\n" " and.b32 %r170, %r76, 3;\n" " setp.lt.u32 %p58, %r141, 3;\n" " mov.u32 %r169, 0;\n" " @%p58 bra $L__BB1_65;\n" "\n" " sub.s32 %r168, %r76, %r170;\n" " mov.u32 %r169, 0;\n" "\n" "$L__BB1_64:\n" " cvt.s64.s32 %rd86, %r169;\n" " add.s64 %rd87, %rd4, %rd86;\n" " shl.b64 %rd88, %rd87, 2;\n" " add.s64 %rd89, %rd2, %rd88;\n" " ld.global.f32 %f157, [%rd89];\n" " mul.f32 %f158, %f31, %f157;\n" " st.global.f32 [%rd89], %f158;\n" " ld.global.f32 %f159, [%rd89+4];\n" " mul.f32 %f160, %f31, %f159;\n" " st.global.f32 [%rd89+4], %f160;\n" " ld.global.f32 %f161, [%rd89+8];\n" " mul.f32 %f162, %f31, %f161;\n" " st.global.f32 [%rd89+8], %f162;\n" " ld.global.f32 %f163, [%rd89+12];\n" " mul.f32 %f164, %f31, %f163;\n" " st.global.f32 [%rd89+12], %f164;\n" " add.s32 %r169, %r169, 4;\n" " add.s32 %r168, %r168, -4;\n" " setp.ne.s32 %p59, %r168, 0;\n" " @%p59 bra $L__BB1_64;\n" "\n" "$L__BB1_65:\n" " setp.eq.s32 %p60, %r170, 0;\n" " @%p60 bra $L__BB1_68;\n" "\n" " cvt.s64.s32 %rd90, %r169;\n" " add.s64 %rd91, %rd4, %rd90;\n" " shl.b64 %rd92, %rd91, 2;\n" " add.s64 %rd96, %rd2, %rd92;\n" "\n" "$L__BB1_67:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f165, [%rd96];\n" " mul.f32 %f166, %f31, %f165;\n" " st.global.f32 [%rd96], %f166;\n" " add.s64 %rd96, %rd96, 4;\n" " add.s32 %r170, %r170, -1;\n" " setp.ne.s32 %p61, %r170, 0;\n" " @%p61 bra $L__BB1_67;\n" " bra.uni $L__BB1_68;\n" "\n" "}\n" " // .globl fc_moe_weighted_add\n" ".visible .entry fc_moe_weighted_add(\n" " .param .u64 fc_moe_weighted_add_param_0,\n" " .param .u64 fc_moe_weighted_add_param_1,\n" " .param .f32 fc_moe_weighted_add_param_2,\n" " .param .u32 fc_moe_weighted_add_param_3\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<5>;\n" " .reg .b32 %r<6>;\n" " .reg .b64 %rd<8>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_moe_weighted_add_param_0];\n" " ld.param.u64 %rd2, [fc_moe_weighted_add_param_1];\n" " ld.param.f32 %f1, [fc_moe_weighted_add_param_2];\n" " ld.param.u32 %r2, [fc_moe_weighted_add_param_3];\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r1, %r3, %r4, %r5;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB2_2;\n" "\n" " cvta.to.global.u64 %rd3, %rd1;\n" " mul.wide.s32 %rd4, %r1, 4;\n" " add.s64 %rd5, %rd3, %rd4;\n" " ld.global.f32 %f2, [%rd5];\n" " cvta.to.global.u64 %rd6, %rd2;\n" " add.s64 %rd7, %rd6, %rd4;\n" " ld.global.f32 %f3, [%rd7];\n" " fma.rn.f32 %f4, %f2, %f1, %f3;\n" " st.global.f32 [%rd7], %f4;\n" "\n" "$L__BB2_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_shared_gate\n" ".visible .entry fc_moe_shared_gate(\n" " .param .u64 fc_moe_shared_gate_param_0,\n" " .param .f32 fc_moe_shared_gate_param_1,\n" " .param .u32 fc_moe_shared_gate_param_2\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<22>;\n" " .reg .b32 %r<8>;\n" " .reg .b64 %rd<5>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_moe_shared_gate_param_0];\n" " ld.param.f32 %f1, [fc_moe_shared_gate_param_1];\n" " ld.param.u32 %r2, [fc_moe_shared_gate_param_2];\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r1, %r3, %r4, %r5;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB3_2;\n" "\n" " cvta.to.global.u64 %rd2, %rd1;\n" " neg.f32 %f2, %f1;\n" " mov.f32 %f3, 0f3F000000;\n" " mov.f32 %f4, 0f3BBB989D;\n" " fma.rn.f32 %f5, %f2, %f4, %f3;\n" " mov.f32 %f6, 0f3FB8AA3B;\n" " mov.f32 %f7, 0f437C0000;\n" " cvt.sat.f32.f32 %f8, %f5;\n" " mov.f32 %f9, 0f4B400001;\n" " fma.rm.f32 %f10, %f8, %f7, %f9;\n" " add.f32 %f11, %f10, 0fCB40007F;\n" " neg.f32 %f12, %f11;\n" " fma.rn.f32 %f13, %f2, %f6, %f12;\n" " mov.f32 %f14, 0f32A57060;\n" " fma.rn.f32 %f15, %f2, %f14, %f13;\n" " mov.b32 %r6, %f10;\n" " shl.b32 %r7, %r6, 23;\n" " mov.b32 %f16, %r7;\n" " ex2.approx.ftz.f32 %f17, %f15;\n" " fma.rn.f32 %f18, %f17, %f16, 0f3F800000;\n" " rcp.rn.f32 %f19, %f18;\n" " mul.wide.s32 %rd3, %r1, 4;\n" " add.s64 %rd4, %rd2, %rd3;\n" " ld.global.f32 %f20, [%rd4];\n" " mul.f32 %f21, %f20, %f19;\n" " st.global.f32 [%rd4], %f21;\n" "\n" "$L__BB3_2:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_MOE_EXPERT_GROUP[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_moe_gateup_q3k\n" "// _ZZ15fc_moe_down_q3kE4sred has been demoted\n" "// _ZZ15fc_moe_down_q4kE4sred has been demoted\n" "// _ZZ15fc_moe_down_q6kE4sred has been demoted\n" "// _ZZ15fc_moe_down_f16E4sred has been demoted\n" "// _ZZ16fc_moe_down_bf16E4sred has been demoted\n" "// _ZZ19fc_moe_down_f16_vecE4sred has been demoted\n" "// _ZZ20fc_moe_down_bf16_vecE4sred has been demoted\n" "// _ZZ18fc_moe_down_q3k_nxE4sred has been demoted\n" "// _ZZ18fc_moe_down_q4k_nxE4sred has been demoted\n" "// _ZZ18fc_moe_down_q6k_nxE4sred has been demoted\n" "// _ZZ18fc_moe_down_f16_nxE4sred has been demoted\n" "// _ZZ19fc_moe_down_bf16_nxE4sred has been demoted\n" "// _ZZ22fc_moe_down_f16_vec_nxE4sred has been demoted\n" "// _ZZ23fc_moe_down_bf16_vec_nxE4sred has been demoted\n" "// _ZZ27fc_moe_gateup_q3k_dedup_shxE4sh_x has been demoted\n" "// _ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj has been demoted\n" "// _ZZ27fc_moe_gateup_q3k_dedup_shxE6sh_tok has been demoted\n" "// _ZZ21fc_moe_dedup_build_wlE8sh_wbase has been demoted\n" "// _ZZ28fc_moe_gateup_q3k_dedup_shxcE4sh_x has been demoted\n" "// _ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj has been demoted\n" "// _ZZ28fc_moe_gateup_q3k_dedup_shxcE6sh_tok has been demoted\n" ".extern .shared .align 16 .b8 sh_x[];\n" ".extern .shared .align 16 .b8 sh_x4[];\n" ".extern .shared .align 16 .b8 sdata[];\n" "\n" ".visible .entry fc_moe_gateup_q3k(\n" " .param .u64 fc_moe_gateup_q3k_param_0,\n" " .param .u64 fc_moe_gateup_q3k_param_1,\n" " .param .u64 fc_moe_gateup_q3k_param_2,\n" " .param .u64 fc_moe_gateup_q3k_param_3,\n" " .param .u64 fc_moe_gateup_q3k_param_4,\n" " .param .u32 fc_moe_gateup_q3k_param_5,\n" " .param .u32 fc_moe_gateup_q3k_param_6,\n" " .param .u32 fc_moe_gateup_q3k_param_7\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .f32 %f<189>;\n" " .reg .b32 %r<146>;\n" " .reg .b64 %rd<71>;\n" "\n" "\n" " ld.param.u64 %rd23, [fc_moe_gateup_q3k_param_0];\n" " ld.param.u64 %rd24, [fc_moe_gateup_q3k_param_1];\n" " ld.param.u64 %rd25, [fc_moe_gateup_q3k_param_2];\n" " ld.param.u64 %rd28, [fc_moe_gateup_q3k_param_3];\n" " ld.param.u64 %rd26, [fc_moe_gateup_q3k_param_4];\n" " ld.param.u32 %r28, [fc_moe_gateup_q3k_param_5];\n" " ld.param.u32 %r29, [fc_moe_gateup_q3k_param_6];\n" " ld.param.u32 %r30, [fc_moe_gateup_q3k_param_7];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r32, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r33, %ctaid.x;\n" " shl.b32 %r34, %r33, 3;\n" " add.s32 %r35, %r34, %r32;\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " mul.wide.s32 %rd29, %r36, 4;\n" " add.s64 %rd27, %rd28, %rd29;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd27];\n" " // end inline asm\n" " cvt.s64.s32 %rd2, %r28;\n" " cvt.s64.s32 %rd3, %r35;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f185, 0f00000000;\n" " mov.f32 %f186, %f185;\n" " @%p1 bra $L__BB0_15;\n" "\n" " cvt.s64.s32 %rd30, %r30;\n" " cvt.s64.s32 %rd31, %r31;\n" " mul.lo.s64 %rd32, %rd31, %rd2;\n" " add.s64 %rd33, %rd32, %rd3;\n" " mul.lo.s64 %rd34, %rd33, %rd30;\n" " cvt.u64.u32 %rd4, %r2;\n" " shr.u32 %r39, %r2, 4;\n" " cvt.u64.u32 %rd5, %r39;\n" " mov.u32 %r40, -2;\n" " sub.s32 %r4, %r40, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd6, %r1;\n" " shl.b32 %r41, %r1, 2;\n" " mov.u32 %r42, sh_x;\n" " add.s32 %r6, %r42, %r41;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd7, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd8, %r8;\n" " add.s32 %r9, %r1, 768;\n" " cvta.to.global.u64 %rd35, %rd24;\n" " add.s64 %rd70, %rd35, %rd34;\n" " cvta.to.global.u64 %rd36, %rd23;\n" " add.s64 %rd69, %rd36, %rd34;\n" " mov.f32 %f186, 0f00000000;\n" " mov.u32 %r139, 0;\n" " shl.b64 %rd53, %rd4, 2;\n" " shl.b64 %rd56, %rd5, 3;\n" " mov.u32 %r140, %r139;\n" " mov.f32 %f185, %f186;\n" "\n" "$L__BB0_2:\n" " add.s32 %r43, %r139, %r5;\n" " max.s32 %r44, %r43, -2049;\n" " sub.s32 %r12, %r4, %r44;\n" " sub.s32 %r45, %r29, %r140;\n" " min.s32 %r13, %r45, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB0_10;\n" "\n" " shr.u32 %r46, %r12, 8;\n" " add.s32 %r47, %r46, 1;\n" " cvt.s64.s32 %rd13, %r140;\n" " and.b32 %r14, %r47, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r141, %r1;\n" " @%p3 bra $L__BB0_7;\n" "\n" " add.s64 %rd38, %rd6, %rd13;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd37, %rd25, %rd39;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd37];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r141, %r7;\n" " @%p4 bra $L__BB0_7;\n" "\n" " add.s64 %rd41, %rd7, %rd13;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd40, %rd25, %rd42;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd40];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r141, %r8;\n" " @%p5 bra $L__BB0_7;\n" "\n" " add.s64 %rd44, %rd8, %rd13;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd43, %rd25, %rd45;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd43];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r141, %r9;\n" "\n" "$L__BB0_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB0_10;\n" "\n" " cvt.s64.s32 %rd46, %r141;\n" " add.s64 %rd47, %rd46, %rd13;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd66, %rd25, %rd48;\n" " shl.b32 %r48, %r141, 2;\n" " mov.u32 %r49, sh_x;\n" " add.s32 %r50, %r49, %r48;\n" " add.s32 %r142, %r50, 2048;\n" "\n" "$L__BB0_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd66];\n" " // end inline asm\n" " st.shared.f32 [%r142+-2048], %f20;\n" " add.s64 %rd50, %rd66, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd50];\n" " // end inline asm\n" " st.shared.f32 [%r142+-1024], %f21;\n" " add.s64 %rd51, %rd66, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd51];\n" " // end inline asm\n" " st.shared.f32 [%r142], %f22;\n" " add.s64 %rd52, %rd66, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd52];\n" " // end inline asm\n" " st.shared.f32 [%r142+1024], %f23;\n" " add.s64 %rd66, %rd66, 4096;\n" " add.s32 %r142, %r142, 4096;\n" " add.s32 %r141, %r141, 1024;\n" " setp.lt.s32 %p7, %r141, %r13;\n" " @%p7 bra $L__BB0_9;\n" "\n" "$L__BB0_10:\n" " cvt.u32.u64 %r51, %rd3;\n" " setp.ge.s32 %p8, %r51, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB0_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB0_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r144, 0;\n" " mov.u32 %r145, %r42;\n" "\n" "$L__BB0_13:\n" " add.s64 %rd54, %rd69, %rd53;\n" " add.s64 %rd55, %rd70, %rd53;\n" " add.s64 %rd57, %rd69, %rd56;\n" " ld.global.v2.u32 {%r54, %r55}, [%rd57+128];\n" " add.s64 %rd58, %rd70, %rd56;\n" " ld.global.v2.u32 {%r58, %r59}, [%rd58+128];\n" " ld.global.u32 %r62, [%rd54];\n" " shr.u32 %r63, %r62, 16;\n" " ld.global.u32 %r64, [%rd55];\n" " shr.u32 %r65, %r64, 16;\n" " and.b32 %r66, %r62, 15;\n" " or.b32 %r67, %r66, 1258291200;\n" " mov.b32 %f24, %r67;\n" " mov.f32 %f25, 0fCB000004;\n" " mov.f32 %f26, 0f3F800000;\n" " fma.rn.f32 %f27, %f24, %f26, %f25;\n" " and.b32 %r68, %r54, 255;\n" " cvt.rn.f32.u32 %f28, %r68;\n" " mov.f32 %f29, 0f42000000;\n" " sub.rn.f32 %f30, %f28, %f29;\n" " and.b32 %r69, %r64, 15;\n" " or.b32 %r70, %r69, 1258291200;\n" " mov.b32 %f31, %r70;\n" " fma.rn.f32 %f32, %f31, %f26, %f25;\n" " and.b32 %r71, %r58, 255;\n" " cvt.rn.f32.u32 %f33, %r71;\n" " sub.rn.f32 %f34, %f33, %f29;\n" " mul.rn.f32 %f35, %f30, %f27;\n" " shl.b32 %r72, %r2, 2;\n" " add.s32 %r73, %r145, %r72;\n" " ld.shared.f32 %f36, [%r73];\n" " mov.f32 %f37, 0f00000000;\n" " fma.rn.f32 %f38, %f35, %f36, %f37;\n" " mul.rn.f32 %f39, %f34, %f32;\n" " fma.rn.f32 %f40, %f39, %f36, %f37;\n" " and.b32 %r74, %r62, 240;\n" " or.b32 %r75, %r74, 1258291200;\n" " mov.b32 %f41, %r75;\n" " mov.f32 %f42, 0fC9000040;\n" " mov.f32 %f43, 0f3D800000;\n" " fma.rn.f32 %f44, %f41, %f43, %f42;\n" " bfe.u32 %r76, %r54, 8, 8;\n" " cvt.rn.f32.u32 %f45, %r76;\n" " sub.rn.f32 %f46, %f45, %f29;\n" " and.b32 %r77, %r64, 240;\n" " or.b32 %r78, %r77, 1258291200;\n" " mov.b32 %f47, %r78;\n" " fma.rn.f32 %f48, %f47, %f43, %f42;\n" " bfe.u32 %r79, %r58, 8, 8;\n" " cvt.rn.f32.u32 %f49, %r79;\n" " sub.rn.f32 %f50, %f49, %f29;\n" " mul.rn.f32 %f51, %f46, %f44;\n" " ld.shared.f32 %f52, [%r73+128];\n" " fma.rn.f32 %f53, %f51, %f52, %f38;\n" " mul.rn.f32 %f54, %f50, %f48;\n" " fma.rn.f32 %f55, %f54, %f52, %f40;\n" " and.b32 %r80, %r62, 3840;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f56, %r81;\n" " mov.f32 %f57, 0fC7000400;\n" " mov.f32 %f58, 0f3B800000;\n" " fma.rn.f32 %f59, %f56, %f58, %f57;\n" " bfe.u32 %r82, %r54, 16, 8;\n" " cvt.rn.f32.u32 %f60, %r82;\n" " sub.rn.f32 %f61, %f60, %f29;\n" " and.b32 %r83, %r64, 3840;\n" " or.b32 %r84, %r83, 1258291200;\n" " mov.b32 %f62, %r84;\n" " fma.rn.f32 %f63, %f62, %f58, %f57;\n" " bfe.u32 %r85, %r58, 16, 8;\n" " cvt.rn.f32.u32 %f64, %r85;\n" " sub.rn.f32 %f65, %f64, %f29;\n" " mul.rn.f32 %f66, %f61, %f59;\n" " ld.shared.f32 %f67, [%r73+256];\n" " fma.rn.f32 %f68, %f66, %f67, %f53;\n" " mul.rn.f32 %f69, %f65, %f63;\n" " fma.rn.f32 %f70, %f69, %f67, %f55;\n" " and.b32 %r86, %r62, 61440;\n" " or.b32 %r87, %r86, 1258291200;\n" " mov.b32 %f71, %r87;\n" " mov.f32 %f72, 0fC5004000;\n" " mov.f32 %f73, 0f39800000;\n" " fma.rn.f32 %f74, %f71, %f73, %f72;\n" " shr.u32 %r88, %r54, 24;\n" " cvt.rn.f32.u32 %f75, %r88;\n" " sub.rn.f32 %f76, %f75, %f29;\n" " and.b32 %r89, %r64, 61440;\n" " or.b32 %r90, %r89, 1258291200;\n" " mov.b32 %f77, %r90;\n" " fma.rn.f32 %f78, %f77, %f73, %f72;\n" " shr.u32 %r91, %r58, 24;\n" " cvt.rn.f32.u32 %f79, %r91;\n" " sub.rn.f32 %f80, %f79, %f29;\n" " mul.rn.f32 %f81, %f76, %f74;\n" " ld.shared.f32 %f82, [%r73+384];\n" " fma.rn.f32 %f83, %f81, %f82, %f68;\n" " mul.rn.f32 %f84, %f80, %f78;\n" " fma.rn.f32 %f85, %f84, %f82, %f70;\n" " and.b32 %r92, %r62, 983040;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f86, %r93;\n" " mov.f32 %f87, 0fC3040000;\n" " mov.f32 %f88, 0f37800000;\n" " fma.rn.f32 %f89, %f86, %f88, %f87;\n" " and.b32 %r94, %r55, 255;\n" " cvt.rn.f32.u32 %f90, %r94;\n" " sub.rn.f32 %f91, %f90, %f29;\n" " and.b32 %r95, %r64, 983040;\n" " or.b32 %r96, %r95, 1258291200;\n" " mov.b32 %f92, %r96;\n" " fma.rn.f32 %f93, %f92, %f88, %f87;\n" " and.b32 %r97, %r59, 255;\n" " cvt.rn.f32.u32 %f94, %r97;\n" " sub.rn.f32 %f95, %f94, %f29;\n" " mul.rn.f32 %f96, %f91, %f89;\n" " ld.shared.f32 %f97, [%r73+512];\n" " fma.rn.f32 %f98, %f96, %f97, %f83;\n" " mul.rn.f32 %f99, %f95, %f93;\n" " fma.rn.f32 %f100, %f99, %f97, %f85;\n" " and.b32 %r98, %r63, 240;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f101, %r99;\n" " fma.rn.f32 %f102, %f101, %f43, %f42;\n" " bfe.u32 %r100, %r55, 8, 8;\n" " cvt.rn.f32.u32 %f103, %r100;\n" " sub.rn.f32 %f104, %f103, %f29;\n" " and.b32 %r101, %r65, 240;\n" " or.b32 %r102, %r101, 1258291200;\n" " mov.b32 %f105, %r102;\n" " fma.rn.f32 %f106, %f105, %f43, %f42;\n" " bfe.u32 %r103, %r59, 8, 8;\n" " cvt.rn.f32.u32 %f107, %r103;\n" " sub.rn.f32 %f108, %f107, %f29;\n" " mul.rn.f32 %f109, %f104, %f102;\n" " ld.shared.f32 %f110, [%r73+640];\n" " fma.rn.f32 %f111, %f109, %f110, %f98;\n" " mul.rn.f32 %f112, %f108, %f106;\n" " fma.rn.f32 %f113, %f112, %f110, %f100;\n" " and.b32 %r104, %r63, 3840;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f114, %r105;\n" " fma.rn.f32 %f115, %f114, %f58, %f57;\n" " bfe.u32 %r106, %r55, 16, 8;\n" " cvt.rn.f32.u32 %f116, %r106;\n" " sub.rn.f32 %f117, %f116, %f29;\n" " and.b32 %r107, %r65, 3840;\n" " or.b32 %r108, %r107, 1258291200;\n" " mov.b32 %f118, %r108;\n" " fma.rn.f32 %f119, %f118, %f58, %f57;\n" " bfe.u32 %r109, %r59, 16, 8;\n" " cvt.rn.f32.u32 %f120, %r109;\n" " sub.rn.f32 %f121, %f120, %f29;\n" " mul.rn.f32 %f122, %f117, %f115;\n" " ld.shared.f32 %f123, [%r73+768];\n" " fma.rn.f32 %f124, %f122, %f123, %f111;\n" " mul.rn.f32 %f125, %f121, %f119;\n" " fma.rn.f32 %f126, %f125, %f123, %f113;\n" " and.b32 %r110, %r63, 61440;\n" " or.b32 %r111, %r110, 1258291200;\n" " mov.b32 %f127, %r111;\n" " fma.rn.f32 %f128, %f127, %f73, %f72;\n" " shr.u32 %r112, %r55, 24;\n" " cvt.rn.f32.u32 %f129, %r112;\n" " sub.rn.f32 %f130, %f129, %f29;\n" " and.b32 %r113, %r65, 61440;\n" " or.b32 %r114, %r113, 1258291200;\n" " mov.b32 %f131, %r114;\n" " fma.rn.f32 %f132, %f131, %f73, %f72;\n" " shr.u32 %r115, %r59, 24;\n" " cvt.rn.f32.u32 %f133, %r115;\n" " sub.rn.f32 %f134, %f133, %f29;\n" " mul.rn.f32 %f135, %f130, %f128;\n" " ld.shared.f32 %f136, [%r73+896];\n" " fma.rn.f32 %f137, %f135, %f136, %f124;\n" " mul.rn.f32 %f138, %f134, %f132;\n" " fma.rn.f32 %f139, %f138, %f136, %f126;\n" " ld.global.f32 %f140, [%rd69+144];\n" " fma.rn.f32 %f185, %f140, %f137, %f185;\n" " ld.global.f32 %f141, [%rd70+144];\n" " fma.rn.f32 %f186, %f141, %f139, %f186;\n" " add.s64 %rd69, %rd69, 152;\n" " add.s64 %rd70, %rd70, 152;\n" " add.s32 %r145, %r145, 1024;\n" " add.s32 %r144, %r144, 1;\n" " setp.lt.s32 %p10, %r144, %r21;\n" " @%p10 bra $L__BB0_13;\n" "\n" "$L__BB0_14:\n" " add.s32 %r140, %r140, 2048;\n" " setp.lt.s32 %p11, %r140, %r29;\n" " add.s32 %r139, %r139, 2048;\n" " @%p11 bra $L__BB0_2;\n" "\n" "$L__BB0_15:\n" " cvt.u32.u64 %r136, %rd3;\n" " setp.ge.s32 %p12, %r136, %r28;\n" " mov.u32 %r126, 16;\n" " mov.u32 %r135, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f142, %f185, %r126, %r135;\n" " // end inline asm\n" " add.f32 %f145, %f185, %f142;\n" " mov.u32 %r128, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f144, %f145, %r128, %r135;\n" " // end inline asm\n" " add.f32 %f147, %f145, %f144;\n" " mov.u32 %r130, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f146, %f147, %r130, %r135;\n" " // end inline asm\n" " add.f32 %f149, %f147, %f146;\n" " mov.u32 %r132, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f148, %f149, %r132, %r135;\n" " // end inline asm\n" " add.f32 %f151, %f149, %f148;\n" " mov.u32 %r134, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f150, %f151, %r134, %r135;\n" " // end inline asm\n" " add.f32 %f11, %f151, %f150;\n" " // begin inline asm\n" " shfl.down.b32 %f152, %f186, %r126, %r135;\n" " // end inline asm\n" " add.f32 %f155, %f186, %f152;\n" " // begin inline asm\n" " shfl.down.b32 %f154, %f155, %r128, %r135;\n" " // end inline asm\n" " add.f32 %f157, %f155, %f154;\n" " // begin inline asm\n" " shfl.down.b32 %f156, %f157, %r130, %r135;\n" " // end inline asm\n" " add.f32 %f159, %f157, %f156;\n" " // begin inline asm\n" " shfl.down.b32 %f158, %f159, %r132, %r135;\n" " // end inline asm\n" " add.f32 %f161, %f159, %f158;\n" " // begin inline asm\n" " shfl.down.b32 %f160, %f161, %r134, %r135;\n" " // end inline asm\n" " add.f32 %f12, %f161, %f160;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB0_17;\n" "\n" " neg.f32 %f162, %f11;\n" " mov.f32 %f163, 0f3F000000;\n" " mov.f32 %f164, 0f3BBB989D;\n" " fma.rn.f32 %f165, %f162, %f164, %f163;\n" " mov.f32 %f166, 0f3FB8AA3B;\n" " mov.f32 %f167, 0f437C0000;\n" " cvt.sat.f32.f32 %f168, %f165;\n" " mov.f32 %f169, 0f4B400001;\n" " fma.rm.f32 %f170, %f168, %f167, %f169;\n" " add.f32 %f171, %f170, 0fCB40007F;\n" " neg.f32 %f172, %f171;\n" " fma.rn.f32 %f173, %f162, %f166, %f172;\n" " mov.f32 %f174, 0f32A57060;\n" " fma.rn.f32 %f175, %f162, %f174, %f173;\n" " mov.b32 %r137, %f170;\n" " shl.b32 %r138, %r137, 23;\n" " mov.b32 %f176, %r138;\n" " ex2.approx.ftz.f32 %f177, %f175;\n" " fma.rn.f32 %f178, %f177, %f176, 0f3F800000;\n" " div.rn.f32 %f179, %f11, %f178;\n" " mul.f32 %f180, %f12, %f179;\n" " mul.lo.s64 %rd59, %rd2, %rd1;\n" " add.s64 %rd60, %rd59, %rd3;\n" " cvta.to.global.u64 %rd61, %rd26;\n" " shl.b64 %rd62, %rd60, 2;\n" " add.s64 %rd63, %rd61, %rd62;\n" " st.global.f32 [%rd63], %f180;\n" "\n" "$L__BB0_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_q4k\n" ".visible .entry fc_moe_gateup_q4k(\n" " .param .u64 fc_moe_gateup_q4k_param_0,\n" " .param .u64 fc_moe_gateup_q4k_param_1,\n" " .param .u64 fc_moe_gateup_q4k_param_2,\n" " .param .u64 fc_moe_gateup_q4k_param_3,\n" " .param .u64 fc_moe_gateup_q4k_param_4,\n" " .param .u32 fc_moe_gateup_q4k_param_5,\n" " .param .u32 fc_moe_gateup_q4k_param_6,\n" " .param .u32 fc_moe_gateup_q4k_param_7\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .f32 %f<242>;\n" " .reg .b32 %r<169>;\n" " .reg .b64 %rd<67>;\n" "\n" "\n" " ld.param.u64 %rd22, [fc_moe_gateup_q4k_param_0];\n" " ld.param.u64 %rd23, [fc_moe_gateup_q4k_param_1];\n" " ld.param.u64 %rd24, [fc_moe_gateup_q4k_param_2];\n" " ld.param.u64 %rd27, [fc_moe_gateup_q4k_param_3];\n" " ld.param.u64 %rd25, [fc_moe_gateup_q4k_param_4];\n" " ld.param.u32 %r28, [fc_moe_gateup_q4k_param_5];\n" " ld.param.u32 %r29, [fc_moe_gateup_q4k_param_6];\n" " ld.param.u32 %r30, [fc_moe_gateup_q4k_param_7];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r32, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r33, %ctaid.x;\n" " shl.b32 %r34, %r33, 3;\n" " add.s32 %r35, %r34, %r32;\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " mul.wide.s32 %rd28, %r36, 4;\n" " add.s64 %rd26, %rd27, %rd28;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd26];\n" " // end inline asm\n" " cvt.s64.s32 %rd2, %r28;\n" " cvt.s64.s32 %rd3, %r35;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f238, 0f00000000;\n" " mov.f32 %f239, %f238;\n" " @%p1 bra $L__BB1_15;\n" "\n" " cvt.s64.s32 %rd29, %r30;\n" " cvt.s64.s32 %rd30, %r31;\n" " mul.lo.s64 %rd31, %rd30, %rd2;\n" " add.s64 %rd32, %rd31, %rd3;\n" " mul.lo.s64 %rd33, %rd32, %rd29;\n" " cvt.u64.u32 %rd4, %r2;\n" " mov.u32 %r39, -2;\n" " sub.s32 %r4, %r39, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd5, %r1;\n" " shl.b32 %r40, %r1, 2;\n" " mov.u32 %r41, sh_x;\n" " add.s32 %r6, %r41, %r40;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd6, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd7, %r8;\n" " add.s32 %r9, %r1, 768;\n" " cvta.to.global.u64 %rd34, %rd23;\n" " add.s64 %rd66, %rd34, %rd33;\n" " cvta.to.global.u64 %rd35, %rd22;\n" " add.s64 %rd65, %rd35, %rd33;\n" " mov.f32 %f239, 0f00000000;\n" " mov.u32 %r162, 0;\n" " shl.b64 %rd52, %rd4, 2;\n" " mov.u32 %r163, %r162;\n" " mov.f32 %f238, %f239;\n" "\n" "$L__BB1_2:\n" " add.s32 %r42, %r162, %r5;\n" " max.s32 %r43, %r42, -2049;\n" " sub.s32 %r12, %r4, %r43;\n" " sub.s32 %r44, %r29, %r163;\n" " min.s32 %r13, %r44, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB1_10;\n" "\n" " shr.u32 %r45, %r12, 8;\n" " add.s32 %r46, %r45, 1;\n" " cvt.s64.s32 %rd12, %r163;\n" " and.b32 %r14, %r46, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r164, %r1;\n" " @%p3 bra $L__BB1_7;\n" "\n" " add.s64 %rd37, %rd5, %rd12;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd36, %rd24, %rd38;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd36];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r164, %r7;\n" " @%p4 bra $L__BB1_7;\n" "\n" " add.s64 %rd40, %rd6, %rd12;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd39, %rd24, %rd41;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r164, %r8;\n" " @%p5 bra $L__BB1_7;\n" "\n" " add.s64 %rd43, %rd7, %rd12;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd24, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r164, %r9;\n" "\n" "$L__BB1_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB1_10;\n" "\n" " cvt.s64.s32 %rd45, %r164;\n" " add.s64 %rd46, %rd45, %rd12;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd62, %rd24, %rd47;\n" " shl.b32 %r47, %r164, 2;\n" " mov.u32 %r48, sh_x;\n" " add.s32 %r49, %r48, %r47;\n" " add.s32 %r165, %r49, 2048;\n" "\n" "$L__BB1_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd62];\n" " // end inline asm\n" " st.shared.f32 [%r165+-2048], %f20;\n" " add.s64 %rd49, %rd62, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r165+-1024], %f21;\n" " add.s64 %rd50, %rd62, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd50];\n" " // end inline asm\n" " st.shared.f32 [%r165], %f22;\n" " add.s64 %rd51, %rd62, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd51];\n" " // end inline asm\n" " st.shared.f32 [%r165+1024], %f23;\n" " add.s64 %rd62, %rd62, 4096;\n" " add.s32 %r165, %r165, 4096;\n" " add.s32 %r164, %r164, 1024;\n" " setp.lt.s32 %p7, %r164, %r13;\n" " @%p7 bra $L__BB1_9;\n" "\n" "$L__BB1_10:\n" " cvt.u32.u64 %r50, %rd3;\n" " setp.ge.s32 %p8, %r50, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB1_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB1_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r167, 0;\n" " mov.u32 %r168, %r41;\n" "\n" "$L__BB1_13:\n" " add.s64 %rd53, %rd65, %rd52;\n" " add.s64 %rd54, %rd66, %rd52;\n" " ld.global.v2.u32 {%r53, %r54}, [%rd65+128];\n" " ld.global.v2.u32 {%r57, %r58}, [%rd66+128];\n" " ld.global.v2.u32 {%r61, %r62}, [%rd65+136];\n" " ld.global.v2.u32 {%r65, %r66}, [%rd66+136];\n" " ld.global.v2.f32 {%f24, %f25}, [%rd65+144];\n" " ld.global.v2.f32 {%f28, %f29}, [%rd66+144];\n" " ld.global.u32 %r69, [%rd53];\n" " shr.u32 %r70, %r69, 16;\n" " ld.global.u32 %r71, [%rd54];\n" " shr.u32 %r72, %r71, 16;\n" " and.b32 %r73, %r53, 255;\n" " cvt.rn.f32.u32 %f32, %r73;\n" " mul.rn.f32 %f33, %f24, %f32;\n" " and.b32 %r74, %r69, 15;\n" " or.b32 %r75, %r74, 1258291200;\n" " mov.b32 %f34, %r75;\n" " mov.f32 %f35, 0fCB000000;\n" " mov.f32 %f36, 0f3F800000;\n" " fma.rn.f32 %f37, %f34, %f36, %f35;\n" " mul.rn.f32 %f38, %f33, %f37;\n" " and.b32 %r76, %r61, 255;\n" " cvt.rn.f32.u32 %f39, %r76;\n" " mul.rn.f32 %f40, %f25, %f39;\n" " sub.rn.f32 %f41, %f38, %f40;\n" " and.b32 %r77, %r57, 255;\n" " cvt.rn.f32.u32 %f42, %r77;\n" " mul.rn.f32 %f43, %f28, %f42;\n" " and.b32 %r78, %r71, 15;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f44, %r79;\n" " fma.rn.f32 %f45, %f44, %f36, %f35;\n" " mul.rn.f32 %f46, %f43, %f45;\n" " and.b32 %r80, %r65, 255;\n" " cvt.rn.f32.u32 %f47, %r80;\n" " mul.rn.f32 %f48, %f29, %f47;\n" " sub.rn.f32 %f49, %f46, %f48;\n" " shl.b32 %r81, %r2, 2;\n" " add.s32 %r82, %r168, %r81;\n" " ld.shared.f32 %f50, [%r82];\n" " mov.f32 %f51, 0f00000000;\n" " fma.rn.f32 %f52, %f41, %f50, %f51;\n" " fma.rn.f32 %f53, %f49, %f50, %f51;\n" " bfe.u32 %r83, %r53, 8, 8;\n" " cvt.rn.f32.u32 %f54, %r83;\n" " mul.rn.f32 %f55, %f24, %f54;\n" " and.b32 %r84, %r69, 240;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f56, %r85;\n" " mov.f32 %f57, 0fC9000000;\n" " mov.f32 %f58, 0f3D800000;\n" " fma.rn.f32 %f59, %f56, %f58, %f57;\n" " mul.rn.f32 %f60, %f55, %f59;\n" " bfe.u32 %r86, %r61, 8, 8;\n" " cvt.rn.f32.u32 %f61, %r86;\n" " mul.rn.f32 %f62, %f25, %f61;\n" " sub.rn.f32 %f63, %f60, %f62;\n" " bfe.u32 %r87, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f64, %r87;\n" " mul.rn.f32 %f65, %f28, %f64;\n" " and.b32 %r88, %r71, 240;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f66, %r89;\n" " fma.rn.f32 %f67, %f66, %f58, %f57;\n" " mul.rn.f32 %f68, %f65, %f67;\n" " bfe.u32 %r90, %r65, 8, 8;\n" " cvt.rn.f32.u32 %f69, %r90;\n" " mul.rn.f32 %f70, %f29, %f69;\n" " sub.rn.f32 %f71, %f68, %f70;\n" " ld.shared.f32 %f72, [%r82+128];\n" " fma.rn.f32 %f73, %f63, %f72, %f52;\n" " fma.rn.f32 %f74, %f71, %f72, %f53;\n" " bfe.u32 %r91, %r53, 16, 8;\n" " cvt.rn.f32.u32 %f75, %r91;\n" " mul.rn.f32 %f76, %f24, %f75;\n" " and.b32 %r92, %r69, 3840;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f77, %r93;\n" " mov.f32 %f78, 0fC7000000;\n" " mov.f32 %f79, 0f3B800000;\n" " fma.rn.f32 %f80, %f77, %f79, %f78;\n" " mul.rn.f32 %f81, %f76, %f80;\n" " bfe.u32 %r94, %r61, 16, 8;\n" " cvt.rn.f32.u32 %f82, %r94;\n" " mul.rn.f32 %f83, %f25, %f82;\n" " sub.rn.f32 %f84, %f81, %f83;\n" " bfe.u32 %r95, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f85, %r95;\n" " mul.rn.f32 %f86, %f28, %f85;\n" " and.b32 %r96, %r71, 3840;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f87, %r97;\n" " fma.rn.f32 %f88, %f87, %f79, %f78;\n" " mul.rn.f32 %f89, %f86, %f88;\n" " bfe.u32 %r98, %r65, 16, 8;\n" " cvt.rn.f32.u32 %f90, %r98;\n" " mul.rn.f32 %f91, %f29, %f90;\n" " sub.rn.f32 %f92, %f89, %f91;\n" " ld.shared.f32 %f93, [%r82+256];\n" " fma.rn.f32 %f94, %f84, %f93, %f73;\n" " fma.rn.f32 %f95, %f92, %f93, %f74;\n" " shr.u32 %r99, %r53, 24;\n" " cvt.rn.f32.u32 %f96, %r99;\n" " mul.rn.f32 %f97, %f24, %f96;\n" " and.b32 %r100, %r69, 61440;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f98, %r101;\n" " mov.f32 %f99, 0fC5000000;\n" " mov.f32 %f100, 0f39800000;\n" " fma.rn.f32 %f101, %f98, %f100, %f99;\n" " mul.rn.f32 %f102, %f97, %f101;\n" " shr.u32 %r102, %r61, 24;\n" " cvt.rn.f32.u32 %f103, %r102;\n" " mul.rn.f32 %f104, %f25, %f103;\n" " sub.rn.f32 %f105, %f102, %f104;\n" " shr.u32 %r103, %r57, 24;\n" " cvt.rn.f32.u32 %f106, %r103;\n" " mul.rn.f32 %f107, %f28, %f106;\n" " and.b32 %r104, %r71, 61440;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f108, %r105;\n" " fma.rn.f32 %f109, %f108, %f100, %f99;\n" " mul.rn.f32 %f110, %f107, %f109;\n" " shr.u32 %r106, %r65, 24;\n" " cvt.rn.f32.u32 %f111, %r106;\n" " mul.rn.f32 %f112, %f29, %f111;\n" " sub.rn.f32 %f113, %f110, %f112;\n" " ld.shared.f32 %f114, [%r82+384];\n" " fma.rn.f32 %f115, %f105, %f114, %f94;\n" " fma.rn.f32 %f116, %f113, %f114, %f95;\n" " and.b32 %r107, %r54, 255;\n" " cvt.rn.f32.u32 %f117, %r107;\n" " mul.rn.f32 %f118, %f24, %f117;\n" " and.b32 %r108, %r69, 983040;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f119, %r109;\n" " mov.f32 %f120, 0fC3000000;\n" " mov.f32 %f121, 0f37800000;\n" " fma.rn.f32 %f122, %f119, %f121, %f120;\n" " mul.rn.f32 %f123, %f118, %f122;\n" " and.b32 %r110, %r62, 255;\n" " cvt.rn.f32.u32 %f124, %r110;\n" " mul.rn.f32 %f125, %f25, %f124;\n" " sub.rn.f32 %f126, %f123, %f125;\n" " and.b32 %r111, %r58, 255;\n" " cvt.rn.f32.u32 %f127, %r111;\n" " mul.rn.f32 %f128, %f28, %f127;\n" " and.b32 %r112, %r71, 983040;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f129, %r113;\n" " fma.rn.f32 %f130, %f129, %f121, %f120;\n" " mul.rn.f32 %f131, %f128, %f130;\n" " and.b32 %r114, %r66, 255;\n" " cvt.rn.f32.u32 %f132, %r114;\n" " mul.rn.f32 %f133, %f29, %f132;\n" " sub.rn.f32 %f134, %f131, %f133;\n" " ld.shared.f32 %f135, [%r82+512];\n" " fma.rn.f32 %f136, %f126, %f135, %f115;\n" " fma.rn.f32 %f137, %f134, %f135, %f116;\n" " bfe.u32 %r115, %r54, 8, 8;\n" " cvt.rn.f32.u32 %f138, %r115;\n" " mul.rn.f32 %f139, %f24, %f138;\n" " and.b32 %r116, %r70, 240;\n" " or.b32 %r117, %r116, 1258291200;\n" " mov.b32 %f140, %r117;\n" " fma.rn.f32 %f141, %f140, %f58, %f57;\n" " mul.rn.f32 %f142, %f139, %f141;\n" " bfe.u32 %r118, %r62, 8, 8;\n" " cvt.rn.f32.u32 %f143, %r118;\n" " mul.rn.f32 %f144, %f25, %f143;\n" " sub.rn.f32 %f145, %f142, %f144;\n" " bfe.u32 %r119, %r58, 8, 8;\n" " cvt.rn.f32.u32 %f146, %r119;\n" " mul.rn.f32 %f147, %f28, %f146;\n" " and.b32 %r120, %r72, 240;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f148, %r121;\n" " fma.rn.f32 %f149, %f148, %f58, %f57;\n" " mul.rn.f32 %f150, %f147, %f149;\n" " bfe.u32 %r122, %r66, 8, 8;\n" " cvt.rn.f32.u32 %f151, %r122;\n" " mul.rn.f32 %f152, %f29, %f151;\n" " sub.rn.f32 %f153, %f150, %f152;\n" " ld.shared.f32 %f154, [%r82+640];\n" " fma.rn.f32 %f155, %f145, %f154, %f136;\n" " fma.rn.f32 %f156, %f153, %f154, %f137;\n" " bfe.u32 %r123, %r54, 16, 8;\n" " cvt.rn.f32.u32 %f157, %r123;\n" " mul.rn.f32 %f158, %f24, %f157;\n" " and.b32 %r124, %r70, 3840;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f159, %r125;\n" " fma.rn.f32 %f160, %f159, %f79, %f78;\n" " mul.rn.f32 %f161, %f158, %f160;\n" " bfe.u32 %r126, %r62, 16, 8;\n" " cvt.rn.f32.u32 %f162, %r126;\n" " mul.rn.f32 %f163, %f25, %f162;\n" " sub.rn.f32 %f164, %f161, %f163;\n" " bfe.u32 %r127, %r58, 16, 8;\n" " cvt.rn.f32.u32 %f165, %r127;\n" " mul.rn.f32 %f166, %f28, %f165;\n" " and.b32 %r128, %r72, 3840;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f167, %r129;\n" " fma.rn.f32 %f168, %f167, %f79, %f78;\n" " mul.rn.f32 %f169, %f166, %f168;\n" " bfe.u32 %r130, %r66, 16, 8;\n" " cvt.rn.f32.u32 %f170, %r130;\n" " mul.rn.f32 %f171, %f29, %f170;\n" " sub.rn.f32 %f172, %f169, %f171;\n" " ld.shared.f32 %f173, [%r82+768];\n" " fma.rn.f32 %f174, %f164, %f173, %f155;\n" " fma.rn.f32 %f175, %f172, %f173, %f156;\n" " shr.u32 %r131, %r54, 24;\n" " cvt.rn.f32.u32 %f176, %r131;\n" " mul.rn.f32 %f177, %f24, %f176;\n" " and.b32 %r132, %r70, 61440;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f178, %r133;\n" " fma.rn.f32 %f179, %f178, %f100, %f99;\n" " mul.rn.f32 %f180, %f177, %f179;\n" " shr.u32 %r134, %r62, 24;\n" " cvt.rn.f32.u32 %f181, %r134;\n" " mul.rn.f32 %f182, %f25, %f181;\n" " sub.rn.f32 %f183, %f180, %f182;\n" " shr.u32 %r135, %r58, 24;\n" " cvt.rn.f32.u32 %f184, %r135;\n" " mul.rn.f32 %f185, %f28, %f184;\n" " and.b32 %r136, %r72, 61440;\n" " or.b32 %r137, %r136, 1258291200;\n" " mov.b32 %f186, %r137;\n" " fma.rn.f32 %f187, %f186, %f100, %f99;\n" " mul.rn.f32 %f188, %f185, %f187;\n" " shr.u32 %r138, %r66, 24;\n" " cvt.rn.f32.u32 %f189, %r138;\n" " mul.rn.f32 %f190, %f29, %f189;\n" " sub.rn.f32 %f191, %f188, %f190;\n" " ld.shared.f32 %f192, [%r82+896];\n" " fma.rn.f32 %f193, %f183, %f192, %f174;\n" " fma.rn.f32 %f194, %f191, %f192, %f175;\n" " add.f32 %f238, %f238, %f193;\n" " add.f32 %f239, %f239, %f194;\n" " add.s64 %rd65, %rd65, 152;\n" " add.s64 %rd66, %rd66, 152;\n" " add.s32 %r168, %r168, 1024;\n" " add.s32 %r167, %r167, 1;\n" " setp.lt.s32 %p10, %r167, %r21;\n" " @%p10 bra $L__BB1_13;\n" "\n" "$L__BB1_14:\n" " add.s32 %r163, %r163, 2048;\n" " setp.lt.s32 %p11, %r163, %r29;\n" " add.s32 %r162, %r162, 2048;\n" " @%p11 bra $L__BB1_2;\n" "\n" "$L__BB1_15:\n" " cvt.u32.u64 %r159, %rd3;\n" " setp.ge.s32 %p12, %r159, %r28;\n" " mov.u32 %r149, 16;\n" " mov.u32 %r158, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f238, %r149, %r158;\n" " // end inline asm\n" " add.f32 %f198, %f238, %f195;\n" " mov.u32 %r151, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f197, %f198, %r151, %r158;\n" " // end inline asm\n" " add.f32 %f200, %f198, %f197;\n" " mov.u32 %r153, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f199, %f200, %r153, %r158;\n" " // end inline asm\n" " add.f32 %f202, %f200, %f199;\n" " mov.u32 %r155, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f201, %f202, %r155, %r158;\n" " // end inline asm\n" " add.f32 %f204, %f202, %f201;\n" " mov.u32 %r157, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f203, %f204, %r157, %r158;\n" " // end inline asm\n" " add.f32 %f11, %f204, %f203;\n" " // begin inline asm\n" " shfl.down.b32 %f205, %f239, %r149, %r158;\n" " // end inline asm\n" " add.f32 %f208, %f239, %f205;\n" " // begin inline asm\n" " shfl.down.b32 %f207, %f208, %r151, %r158;\n" " // end inline asm\n" " add.f32 %f210, %f208, %f207;\n" " // begin inline asm\n" " shfl.down.b32 %f209, %f210, %r153, %r158;\n" " // end inline asm\n" " add.f32 %f212, %f210, %f209;\n" " // begin inline asm\n" " shfl.down.b32 %f211, %f212, %r155, %r158;\n" " // end inline asm\n" " add.f32 %f214, %f212, %f211;\n" " // begin inline asm\n" " shfl.down.b32 %f213, %f214, %r157, %r158;\n" " // end inline asm\n" " add.f32 %f12, %f214, %f213;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB1_17;\n" "\n" " neg.f32 %f215, %f11;\n" " mov.f32 %f216, 0f3F000000;\n" " mov.f32 %f217, 0f3BBB989D;\n" " fma.rn.f32 %f218, %f215, %f217, %f216;\n" " mov.f32 %f219, 0f3FB8AA3B;\n" " mov.f32 %f220, 0f437C0000;\n" " cvt.sat.f32.f32 %f221, %f218;\n" " mov.f32 %f222, 0f4B400001;\n" " fma.rm.f32 %f223, %f221, %f220, %f222;\n" " add.f32 %f224, %f223, 0fCB40007F;\n" " neg.f32 %f225, %f224;\n" " fma.rn.f32 %f226, %f215, %f219, %f225;\n" " mov.f32 %f227, 0f32A57060;\n" " fma.rn.f32 %f228, %f215, %f227, %f226;\n" " mov.b32 %r160, %f223;\n" " shl.b32 %r161, %r160, 23;\n" " mov.b32 %f229, %r161;\n" " ex2.approx.ftz.f32 %f230, %f228;\n" " fma.rn.f32 %f231, %f230, %f229, 0f3F800000;\n" " div.rn.f32 %f232, %f11, %f231;\n" " mul.f32 %f233, %f12, %f232;\n" " mul.lo.s64 %rd55, %rd2, %rd1;\n" " add.s64 %rd56, %rd55, %rd3;\n" " cvta.to.global.u64 %rd57, %rd25;\n" " shl.b64 %rd58, %rd56, 2;\n" " add.s64 %rd59, %rd57, %rd58;\n" " st.global.f32 [%rd59], %f233;\n" "\n" "$L__BB1_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_f16\n" ".visible .entry fc_moe_gateup_f16(\n" " .param .u64 fc_moe_gateup_f16_param_0,\n" " .param .u64 fc_moe_gateup_f16_param_1,\n" " .param .u64 fc_moe_gateup_f16_param_2,\n" " .param .u64 fc_moe_gateup_f16_param_3,\n" " .param .u64 fc_moe_gateup_f16_param_4,\n" " .param .u32 fc_moe_gateup_f16_param_5,\n" " .param .u32 fc_moe_gateup_f16_param_6,\n" " .param .u32 fc_moe_gateup_f16_param_7\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .b16 %rs<17>;\n" " .reg .f32 %f<112>;\n" " .reg .b32 %r<85>;\n" " .reg .b64 %rd<67>;\n" "\n" "\n" " ld.param.u64 %rd22, [fc_moe_gateup_f16_param_0];\n" " ld.param.u64 %rd23, [fc_moe_gateup_f16_param_1];\n" " ld.param.u64 %rd24, [fc_moe_gateup_f16_param_2];\n" " ld.param.u64 %rd27, [fc_moe_gateup_f16_param_3];\n" " ld.param.u64 %rd25, [fc_moe_gateup_f16_param_4];\n" " ld.param.u32 %r28, [fc_moe_gateup_f16_param_5];\n" " ld.param.u32 %r29, [fc_moe_gateup_f16_param_6];\n" " ld.param.u32 %r30, [fc_moe_gateup_f16_param_7];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r32, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r33, %ctaid.x;\n" " shl.b32 %r34, %r33, 3;\n" " add.s32 %r35, %r34, %r32;\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " mul.wide.s32 %rd28, %r36, 4;\n" " add.s64 %rd26, %rd27, %rd28;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd26];\n" " // end inline asm\n" " cvt.s64.s32 %rd2, %r28;\n" " cvt.s64.s32 %rd3, %r35;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f108, 0f00000000;\n" " mov.f32 %f109, %f108;\n" " @%p1 bra $L__BB2_15;\n" "\n" " cvt.s64.s32 %rd29, %r30;\n" " cvt.s64.s32 %rd30, %r31;\n" " mul.lo.s64 %rd31, %rd30, %rd2;\n" " add.s64 %rd32, %rd31, %rd3;\n" " mul.lo.s64 %rd33, %rd32, %rd29;\n" " mov.u32 %r39, -2;\n" " sub.s32 %r4, %r39, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd4, %r1;\n" " shl.b32 %r40, %r1, 2;\n" " mov.u32 %r41, sh_x;\n" " add.s32 %r6, %r41, %r40;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd5, %r7;\n" " cvt.u64.u32 %rd6, %r2;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd7, %r8;\n" " add.s32 %r9, %r1, 768;\n" " cvta.to.global.u64 %rd34, %rd23;\n" " add.s64 %rd66, %rd34, %rd33;\n" " cvta.to.global.u64 %rd35, %rd22;\n" " add.s64 %rd65, %rd35, %rd33;\n" " mov.f32 %f109, 0f00000000;\n" " mov.u32 %r78, 0;\n" " shl.b64 %rd52, %rd6, 1;\n" " mov.u32 %r79, %r78;\n" " mov.f32 %f108, %f109;\n" "\n" "$L__BB2_2:\n" " add.s32 %r42, %r78, %r5;\n" " max.s32 %r43, %r42, -2049;\n" " sub.s32 %r12, %r4, %r43;\n" " sub.s32 %r44, %r29, %r79;\n" " min.s32 %r13, %r44, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB2_10;\n" "\n" " shr.u32 %r45, %r12, 8;\n" " add.s32 %r46, %r45, 1;\n" " cvt.s64.s32 %rd12, %r79;\n" " and.b32 %r14, %r46, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r80, %r1;\n" " @%p3 bra $L__BB2_7;\n" "\n" " add.s64 %rd37, %rd4, %rd12;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd36, %rd24, %rd38;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd36];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r80, %r7;\n" " @%p4 bra $L__BB2_7;\n" "\n" " add.s64 %rd40, %rd5, %rd12;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd39, %rd24, %rd41;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r80, %r8;\n" " @%p5 bra $L__BB2_7;\n" "\n" " add.s64 %rd43, %rd7, %rd12;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd24, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r80, %r9;\n" "\n" "$L__BB2_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB2_10;\n" "\n" " cvt.s64.s32 %rd45, %r80;\n" " add.s64 %rd46, %rd45, %rd12;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd62, %rd24, %rd47;\n" " shl.b32 %r47, %r80, 2;\n" " mov.u32 %r48, sh_x;\n" " add.s32 %r49, %r48, %r47;\n" " add.s32 %r81, %r49, 2048;\n" "\n" "$L__BB2_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd62];\n" " // end inline asm\n" " st.shared.f32 [%r81+-2048], %f20;\n" " add.s64 %rd49, %rd62, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r81+-1024], %f21;\n" " add.s64 %rd50, %rd62, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd50];\n" " // end inline asm\n" " st.shared.f32 [%r81], %f22;\n" " add.s64 %rd51, %rd62, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd51];\n" " // end inline asm\n" " st.shared.f32 [%r81+1024], %f23;\n" " add.s64 %rd62, %rd62, 4096;\n" " add.s32 %r81, %r81, 4096;\n" " add.s32 %r80, %r80, 1024;\n" " setp.lt.s32 %p7, %r80, %r13;\n" " @%p7 bra $L__BB2_9;\n" "\n" "$L__BB2_10:\n" " cvt.u32.u64 %r50, %rd3;\n" " setp.ge.s32 %p8, %r50, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB2_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB2_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r83, 0;\n" " mov.u32 %r84, %r41;\n" "\n" "$L__BB2_13:\n" " shl.b32 %r53, %r2, 2;\n" " add.s32 %r54, %r84, %r53;\n" " ld.shared.f32 %f40, [%r54];\n" " add.s64 %rd53, %rd65, %rd52;\n" " ld.global.u16 %rs1, [%rd53];\n" " // begin inline asm\n" " cvt.f32.f16 %f24, %rs1;\n" " // end inline asm\n" " mov.f32 %f41, 0f00000000;\n" " fma.rn.f32 %f42, %f24, %f40, %f41;\n" " add.s64 %rd54, %rd66, %rd52;\n" " ld.global.u16 %rs2, [%rd54];\n" " // begin inline asm\n" " cvt.f32.f16 %f25, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f43, %f25, %f40, %f41;\n" " ld.shared.f32 %f44, [%r54+128];\n" " ld.global.u16 %rs3, [%rd53+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f26, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f45, %f26, %f44, %f42;\n" " ld.global.u16 %rs4, [%rd54+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f27, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f46, %f27, %f44, %f43;\n" " ld.shared.f32 %f47, [%r54+256];\n" " ld.global.u16 %rs5, [%rd53+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f28, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f48, %f28, %f47, %f45;\n" " ld.global.u16 %rs6, [%rd54+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f29, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f49, %f29, %f47, %f46;\n" " ld.shared.f32 %f50, [%r54+384];\n" " ld.global.u16 %rs7, [%rd53+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f30, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f51, %f30, %f50, %f48;\n" " ld.global.u16 %rs8, [%rd54+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f31, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f52, %f31, %f50, %f49;\n" " ld.shared.f32 %f53, [%r54+512];\n" " ld.global.u16 %rs9, [%rd53+256];\n" " // begin inline asm\n" " cvt.f32.f16 %f32, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f54, %f32, %f53, %f51;\n" " ld.global.u16 %rs10, [%rd54+256];\n" " // begin inline asm\n" " cvt.f32.f16 %f33, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f55, %f33, %f53, %f52;\n" " ld.shared.f32 %f56, [%r54+640];\n" " ld.global.u16 %rs11, [%rd53+320];\n" " // begin inline asm\n" " cvt.f32.f16 %f34, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f57, %f34, %f56, %f54;\n" " ld.global.u16 %rs12, [%rd54+320];\n" " // begin inline asm\n" " cvt.f32.f16 %f35, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f58, %f35, %f56, %f55;\n" " ld.shared.f32 %f59, [%r54+768];\n" " ld.global.u16 %rs13, [%rd53+384];\n" " // begin inline asm\n" " cvt.f32.f16 %f36, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f60, %f36, %f59, %f57;\n" " ld.global.u16 %rs14, [%rd54+384];\n" " // begin inline asm\n" " cvt.f32.f16 %f37, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f61, %f37, %f59, %f58;\n" " ld.shared.f32 %f62, [%r54+896];\n" " ld.global.u16 %rs15, [%rd53+448];\n" " // begin inline asm\n" " cvt.f32.f16 %f38, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f63, %f38, %f62, %f60;\n" " ld.global.u16 %rs16, [%rd54+448];\n" " // begin inline asm\n" " cvt.f32.f16 %f39, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f64, %f39, %f62, %f61;\n" " add.f32 %f108, %f108, %f63;\n" " add.f32 %f109, %f109, %f64;\n" " add.s64 %rd65, %rd65, 512;\n" " add.s64 %rd66, %rd66, 512;\n" " add.s32 %r84, %r84, 1024;\n" " add.s32 %r83, %r83, 1;\n" " setp.lt.s32 %p10, %r83, %r21;\n" " @%p10 bra $L__BB2_13;\n" "\n" "$L__BB2_14:\n" " add.s32 %r79, %r79, 2048;\n" " setp.lt.s32 %p11, %r79, %r29;\n" " add.s32 %r78, %r78, 2048;\n" " @%p11 bra $L__BB2_2;\n" "\n" "$L__BB2_15:\n" " cvt.u32.u64 %r75, %rd3;\n" " setp.ge.s32 %p12, %r75, %r28;\n" " mov.u32 %r65, 16;\n" " mov.u32 %r74, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f65, %f108, %r65, %r74;\n" " // end inline asm\n" " add.f32 %f68, %f108, %f65;\n" " mov.u32 %r67, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f67, %f68, %r67, %r74;\n" " // end inline asm\n" " add.f32 %f70, %f68, %f67;\n" " mov.u32 %r69, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f69, %f70, %r69, %r74;\n" " // end inline asm\n" " add.f32 %f72, %f70, %f69;\n" " mov.u32 %r71, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f71, %f72, %r71, %r74;\n" " // end inline asm\n" " add.f32 %f74, %f72, %f71;\n" " mov.u32 %r73, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f73, %f74, %r73, %r74;\n" " // end inline asm\n" " add.f32 %f11, %f74, %f73;\n" " // begin inline asm\n" " shfl.down.b32 %f75, %f109, %r65, %r74;\n" " // end inline asm\n" " add.f32 %f78, %f109, %f75;\n" " // begin inline asm\n" " shfl.down.b32 %f77, %f78, %r67, %r74;\n" " // end inline asm\n" " add.f32 %f80, %f78, %f77;\n" " // begin inline asm\n" " shfl.down.b32 %f79, %f80, %r69, %r74;\n" " // end inline asm\n" " add.f32 %f82, %f80, %f79;\n" " // begin inline asm\n" " shfl.down.b32 %f81, %f82, %r71, %r74;\n" " // end inline asm\n" " add.f32 %f84, %f82, %f81;\n" " // begin inline asm\n" " shfl.down.b32 %f83, %f84, %r73, %r74;\n" " // end inline asm\n" " add.f32 %f12, %f84, %f83;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB2_17;\n" "\n" " neg.f32 %f85, %f11;\n" " mov.f32 %f86, 0f3F000000;\n" " mov.f32 %f87, 0f3BBB989D;\n" " fma.rn.f32 %f88, %f85, %f87, %f86;\n" " mov.f32 %f89, 0f3FB8AA3B;\n" " mov.f32 %f90, 0f437C0000;\n" " cvt.sat.f32.f32 %f91, %f88;\n" " mov.f32 %f92, 0f4B400001;\n" " fma.rm.f32 %f93, %f91, %f90, %f92;\n" " add.f32 %f94, %f93, 0fCB40007F;\n" " neg.f32 %f95, %f94;\n" " fma.rn.f32 %f96, %f85, %f89, %f95;\n" " mov.f32 %f97, 0f32A57060;\n" " fma.rn.f32 %f98, %f85, %f97, %f96;\n" " mov.b32 %r76, %f93;\n" " shl.b32 %r77, %r76, 23;\n" " mov.b32 %f99, %r77;\n" " ex2.approx.ftz.f32 %f100, %f98;\n" " fma.rn.f32 %f101, %f100, %f99, 0f3F800000;\n" " div.rn.f32 %f102, %f11, %f101;\n" " mul.f32 %f103, %f12, %f102;\n" " mul.lo.s64 %rd55, %rd2, %rd1;\n" " add.s64 %rd56, %rd55, %rd3;\n" " cvta.to.global.u64 %rd57, %rd25;\n" " shl.b64 %rd58, %rd56, 2;\n" " add.s64 %rd59, %rd57, %rd58;\n" " st.global.f32 [%rd59], %f103;\n" "\n" "$L__BB2_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_bf16\n" ".visible .entry fc_moe_gateup_bf16(\n" " .param .u64 fc_moe_gateup_bf16_param_0,\n" " .param .u64 fc_moe_gateup_bf16_param_1,\n" " .param .u64 fc_moe_gateup_bf16_param_2,\n" " .param .u64 fc_moe_gateup_bf16_param_3,\n" " .param .u64 fc_moe_gateup_bf16_param_4,\n" " .param .u32 fc_moe_gateup_bf16_param_5,\n" " .param .u32 fc_moe_gateup_bf16_param_6,\n" " .param .u32 fc_moe_gateup_bf16_param_7\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .f32 %f<112>;\n" " .reg .b32 %r<117>;\n" " .reg .b64 %rd<67>;\n" "\n" "\n" " ld.param.u64 %rd22, [fc_moe_gateup_bf16_param_0];\n" " ld.param.u64 %rd23, [fc_moe_gateup_bf16_param_1];\n" " ld.param.u64 %rd24, [fc_moe_gateup_bf16_param_2];\n" " ld.param.u64 %rd27, [fc_moe_gateup_bf16_param_3];\n" " ld.param.u64 %rd25, [fc_moe_gateup_bf16_param_4];\n" " ld.param.u32 %r28, [fc_moe_gateup_bf16_param_5];\n" " ld.param.u32 %r29, [fc_moe_gateup_bf16_param_6];\n" " ld.param.u32 %r30, [fc_moe_gateup_bf16_param_7];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r32, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r33, %ctaid.x;\n" " shl.b32 %r34, %r33, 3;\n" " add.s32 %r35, %r34, %r32;\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " mul.wide.s32 %rd28, %r36, 4;\n" " add.s64 %rd26, %rd27, %rd28;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd26];\n" " // end inline asm\n" " cvt.s64.s32 %rd2, %r28;\n" " cvt.s64.s32 %rd3, %r35;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f108, 0f00000000;\n" " mov.f32 %f109, %f108;\n" " @%p1 bra $L__BB3_15;\n" "\n" " cvt.s64.s32 %rd29, %r30;\n" " cvt.s64.s32 %rd30, %r31;\n" " mul.lo.s64 %rd31, %rd30, %rd2;\n" " add.s64 %rd32, %rd31, %rd3;\n" " mul.lo.s64 %rd33, %rd32, %rd29;\n" " mov.u32 %r39, -2;\n" " sub.s32 %r4, %r39, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd4, %r1;\n" " shl.b32 %r40, %r1, 2;\n" " mov.u32 %r41, sh_x;\n" " add.s32 %r6, %r41, %r40;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd5, %r7;\n" " cvt.u64.u32 %rd6, %r2;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd7, %r8;\n" " add.s32 %r9, %r1, 768;\n" " cvta.to.global.u64 %rd34, %rd23;\n" " add.s64 %rd66, %rd34, %rd33;\n" " cvta.to.global.u64 %rd35, %rd22;\n" " add.s64 %rd65, %rd35, %rd33;\n" " mov.f32 %f109, 0f00000000;\n" " mov.u32 %r110, 0;\n" " shl.b64 %rd52, %rd6, 1;\n" " mov.u32 %r111, %r110;\n" " mov.f32 %f108, %f109;\n" "\n" "$L__BB3_2:\n" " add.s32 %r42, %r110, %r5;\n" " max.s32 %r43, %r42, -2049;\n" " sub.s32 %r12, %r4, %r43;\n" " sub.s32 %r44, %r29, %r111;\n" " min.s32 %r13, %r44, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB3_10;\n" "\n" " shr.u32 %r45, %r12, 8;\n" " add.s32 %r46, %r45, 1;\n" " cvt.s64.s32 %rd12, %r111;\n" " and.b32 %r14, %r46, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r112, %r1;\n" " @%p3 bra $L__BB3_7;\n" "\n" " add.s64 %rd37, %rd4, %rd12;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd36, %rd24, %rd38;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd36];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r112, %r7;\n" " @%p4 bra $L__BB3_7;\n" "\n" " add.s64 %rd40, %rd5, %rd12;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd39, %rd24, %rd41;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd39];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r112, %r8;\n" " @%p5 bra $L__BB3_7;\n" "\n" " add.s64 %rd43, %rd7, %rd12;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd24, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r112, %r9;\n" "\n" "$L__BB3_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB3_10;\n" "\n" " cvt.s64.s32 %rd45, %r112;\n" " add.s64 %rd46, %rd45, %rd12;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd62, %rd24, %rd47;\n" " shl.b32 %r47, %r112, 2;\n" " mov.u32 %r48, sh_x;\n" " add.s32 %r49, %r48, %r47;\n" " add.s32 %r113, %r49, 2048;\n" "\n" "$L__BB3_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd62];\n" " // end inline asm\n" " st.shared.f32 [%r113+-2048], %f20;\n" " add.s64 %rd49, %rd62, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r113+-1024], %f21;\n" " add.s64 %rd50, %rd62, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd50];\n" " // end inline asm\n" " st.shared.f32 [%r113], %f22;\n" " add.s64 %rd51, %rd62, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd51];\n" " // end inline asm\n" " st.shared.f32 [%r113+1024], %f23;\n" " add.s64 %rd62, %rd62, 4096;\n" " add.s32 %r113, %r113, 4096;\n" " add.s32 %r112, %r112, 1024;\n" " setp.lt.s32 %p7, %r112, %r13;\n" " @%p7 bra $L__BB3_9;\n" "\n" "$L__BB3_10:\n" " cvt.u32.u64 %r50, %rd3;\n" " setp.ge.s32 %p8, %r50, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB3_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB3_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r115, 0;\n" " mov.u32 %r116, %r41;\n" "\n" "$L__BB3_13:\n" " add.s64 %rd53, %rd65, %rd52;\n" " ld.global.u16 %r53, [%rd53];\n" " shl.b32 %r54, %r53, 16;\n" " mov.b32 %f24, %r54;\n" " shl.b32 %r55, %r2, 2;\n" " add.s32 %r56, %r116, %r55;\n" " ld.shared.f32 %f25, [%r56];\n" " mov.f32 %f26, 0f00000000;\n" " fma.rn.f32 %f27, %f24, %f25, %f26;\n" " add.s64 %rd54, %rd66, %rd52;\n" " ld.global.u16 %r57, [%rd54];\n" " shl.b32 %r58, %r57, 16;\n" " mov.b32 %f28, %r58;\n" " fma.rn.f32 %f29, %f28, %f25, %f26;\n" " ld.global.u16 %r59, [%rd53+64];\n" " shl.b32 %r60, %r59, 16;\n" " mov.b32 %f30, %r60;\n" " ld.shared.f32 %f31, [%r56+128];\n" " fma.rn.f32 %f32, %f30, %f31, %f27;\n" " ld.global.u16 %r61, [%rd54+64];\n" " shl.b32 %r62, %r61, 16;\n" " mov.b32 %f33, %r62;\n" " fma.rn.f32 %f34, %f33, %f31, %f29;\n" " ld.global.u16 %r63, [%rd53+128];\n" " shl.b32 %r64, %r63, 16;\n" " mov.b32 %f35, %r64;\n" " ld.shared.f32 %f36, [%r56+256];\n" " fma.rn.f32 %f37, %f35, %f36, %f32;\n" " ld.global.u16 %r65, [%rd54+128];\n" " shl.b32 %r66, %r65, 16;\n" " mov.b32 %f38, %r66;\n" " fma.rn.f32 %f39, %f38, %f36, %f34;\n" " ld.global.u16 %r67, [%rd53+192];\n" " shl.b32 %r68, %r67, 16;\n" " mov.b32 %f40, %r68;\n" " ld.shared.f32 %f41, [%r56+384];\n" " fma.rn.f32 %f42, %f40, %f41, %f37;\n" " ld.global.u16 %r69, [%rd54+192];\n" " shl.b32 %r70, %r69, 16;\n" " mov.b32 %f43, %r70;\n" " fma.rn.f32 %f44, %f43, %f41, %f39;\n" " ld.global.u16 %r71, [%rd53+256];\n" " shl.b32 %r72, %r71, 16;\n" " mov.b32 %f45, %r72;\n" " ld.shared.f32 %f46, [%r56+512];\n" " fma.rn.f32 %f47, %f45, %f46, %f42;\n" " ld.global.u16 %r73, [%rd54+256];\n" " shl.b32 %r74, %r73, 16;\n" " mov.b32 %f48, %r74;\n" " fma.rn.f32 %f49, %f48, %f46, %f44;\n" " ld.global.u16 %r75, [%rd53+320];\n" " shl.b32 %r76, %r75, 16;\n" " mov.b32 %f50, %r76;\n" " ld.shared.f32 %f51, [%r56+640];\n" " fma.rn.f32 %f52, %f50, %f51, %f47;\n" " ld.global.u16 %r77, [%rd54+320];\n" " shl.b32 %r78, %r77, 16;\n" " mov.b32 %f53, %r78;\n" " fma.rn.f32 %f54, %f53, %f51, %f49;\n" " ld.global.u16 %r79, [%rd53+384];\n" " shl.b32 %r80, %r79, 16;\n" " mov.b32 %f55, %r80;\n" " ld.shared.f32 %f56, [%r56+768];\n" " fma.rn.f32 %f57, %f55, %f56, %f52;\n" " ld.global.u16 %r81, [%rd54+384];\n" " shl.b32 %r82, %r81, 16;\n" " mov.b32 %f58, %r82;\n" " fma.rn.f32 %f59, %f58, %f56, %f54;\n" " ld.global.u16 %r83, [%rd53+448];\n" " shl.b32 %r84, %r83, 16;\n" " mov.b32 %f60, %r84;\n" " ld.shared.f32 %f61, [%r56+896];\n" " fma.rn.f32 %f62, %f60, %f61, %f57;\n" " ld.global.u16 %r85, [%rd54+448];\n" " shl.b32 %r86, %r85, 16;\n" " mov.b32 %f63, %r86;\n" " fma.rn.f32 %f64, %f63, %f61, %f59;\n" " add.f32 %f108, %f108, %f62;\n" " add.f32 %f109, %f109, %f64;\n" " add.s64 %rd65, %rd65, 512;\n" " add.s64 %rd66, %rd66, 512;\n" " add.s32 %r116, %r116, 1024;\n" " add.s32 %r115, %r115, 1;\n" " setp.lt.s32 %p10, %r115, %r21;\n" " @%p10 bra $L__BB3_13;\n" "\n" "$L__BB3_14:\n" " add.s32 %r111, %r111, 2048;\n" " setp.lt.s32 %p11, %r111, %r29;\n" " add.s32 %r110, %r110, 2048;\n" " @%p11 bra $L__BB3_2;\n" "\n" "$L__BB3_15:\n" " cvt.u32.u64 %r107, %rd3;\n" " setp.ge.s32 %p12, %r107, %r28;\n" " mov.u32 %r97, 16;\n" " mov.u32 %r106, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f65, %f108, %r97, %r106;\n" " // end inline asm\n" " add.f32 %f68, %f108, %f65;\n" " mov.u32 %r99, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f67, %f68, %r99, %r106;\n" " // end inline asm\n" " add.f32 %f70, %f68, %f67;\n" " mov.u32 %r101, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f69, %f70, %r101, %r106;\n" " // end inline asm\n" " add.f32 %f72, %f70, %f69;\n" " mov.u32 %r103, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f71, %f72, %r103, %r106;\n" " // end inline asm\n" " add.f32 %f74, %f72, %f71;\n" " mov.u32 %r105, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f73, %f74, %r105, %r106;\n" " // end inline asm\n" " add.f32 %f11, %f74, %f73;\n" " // begin inline asm\n" " shfl.down.b32 %f75, %f109, %r97, %r106;\n" " // end inline asm\n" " add.f32 %f78, %f109, %f75;\n" " // begin inline asm\n" " shfl.down.b32 %f77, %f78, %r99, %r106;\n" " // end inline asm\n" " add.f32 %f80, %f78, %f77;\n" " // begin inline asm\n" " shfl.down.b32 %f79, %f80, %r101, %r106;\n" " // end inline asm\n" " add.f32 %f82, %f80, %f79;\n" " // begin inline asm\n" " shfl.down.b32 %f81, %f82, %r103, %r106;\n" " // end inline asm\n" " add.f32 %f84, %f82, %f81;\n" " // begin inline asm\n" " shfl.down.b32 %f83, %f84, %r105, %r106;\n" " // end inline asm\n" " add.f32 %f12, %f84, %f83;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB3_17;\n" "\n" " neg.f32 %f85, %f11;\n" " mov.f32 %f86, 0f3F000000;\n" " mov.f32 %f87, 0f3BBB989D;\n" " fma.rn.f32 %f88, %f85, %f87, %f86;\n" " mov.f32 %f89, 0f3FB8AA3B;\n" " mov.f32 %f90, 0f437C0000;\n" " cvt.sat.f32.f32 %f91, %f88;\n" " mov.f32 %f92, 0f4B400001;\n" " fma.rm.f32 %f93, %f91, %f90, %f92;\n" " add.f32 %f94, %f93, 0fCB40007F;\n" " neg.f32 %f95, %f94;\n" " fma.rn.f32 %f96, %f85, %f89, %f95;\n" " mov.f32 %f97, 0f32A57060;\n" " fma.rn.f32 %f98, %f85, %f97, %f96;\n" " mov.b32 %r108, %f93;\n" " shl.b32 %r109, %r108, 23;\n" " mov.b32 %f99, %r109;\n" " ex2.approx.ftz.f32 %f100, %f98;\n" " fma.rn.f32 %f101, %f100, %f99, 0f3F800000;\n" " div.rn.f32 %f102, %f11, %f101;\n" " mul.f32 %f103, %f12, %f102;\n" " mul.lo.s64 %rd55, %rd2, %rd1;\n" " add.s64 %rd56, %rd55, %rd3;\n" " cvta.to.global.u64 %rd57, %rd25;\n" " shl.b64 %rd58, %rd56, 2;\n" " add.s64 %rd59, %rd57, %rd58;\n" " st.global.f32 [%rd59], %f103;\n" "\n" "$L__BB3_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_f16_vec\n" ".visible .entry fc_moe_gateup_f16_vec(\n" " .param .u64 fc_moe_gateup_f16_vec_param_0,\n" " .param .u64 fc_moe_gateup_f16_vec_param_1,\n" " .param .u64 fc_moe_gateup_f16_vec_param_2,\n" " .param .u64 fc_moe_gateup_f16_vec_param_3,\n" " .param .u64 fc_moe_gateup_f16_vec_param_4,\n" " .param .u32 fc_moe_gateup_f16_vec_param_5,\n" " .param .u32 fc_moe_gateup_f16_vec_param_6,\n" " .param .u32 fc_moe_gateup_f16_vec_param_7\n" ")\n" "{\n" " .reg .pred %p<17>;\n" " .reg .b16 %rs<49>;\n" " .reg .f32 %f<230>;\n" " .reg .b32 %r<164>;\n" " .reg .b64 %rd<81>;\n" "\n" "\n" " ld.param.u64 %rd28, [fc_moe_gateup_f16_vec_param_0];\n" " ld.param.u64 %rd29, [fc_moe_gateup_f16_vec_param_1];\n" " ld.param.u64 %rd30, [fc_moe_gateup_f16_vec_param_2];\n" " ld.param.u64 %rd33, [fc_moe_gateup_f16_vec_param_3];\n" " ld.param.u64 %rd31, [fc_moe_gateup_f16_vec_param_4];\n" " ld.param.u32 %r31, [fc_moe_gateup_f16_vec_param_5];\n" " ld.param.u32 %r32, [fc_moe_gateup_f16_vec_param_6];\n" " ld.param.u32 %r33, [fc_moe_gateup_f16_vec_param_7];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r35, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r36, %ctaid.x;\n" " shl.b32 %r37, %r36, 3;\n" " add.s32 %r38, %r37, %r35;\n" " mov.u32 %r39, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r39;\n" " mul.wide.s32 %rd34, %r39, 4;\n" " add.s64 %rd32, %rd33, %rd34;\n" " // begin inline asm\n" " ld.global.nc.s32 %r34, [%rd32];\n" " // end inline asm\n" " cvt.s64.s32 %rd2, %r31;\n" " cvt.s64.s32 %rd3, %r38;\n" " setp.lt.s32 %p1, %r32, 1;\n" " mov.f32 %f226, 0f00000000;\n" " mov.f32 %f227, %f226;\n" " @%p1 bra $L__BB4_18;\n" "\n" " cvt.s64.s32 %rd35, %r33;\n" " cvt.s64.s32 %rd36, %r34;\n" " mul.lo.s64 %rd37, %rd36, %rd2;\n" " add.s64 %rd38, %rd37, %rd3;\n" " mul.lo.s64 %rd39, %rd38, %rd35;\n" " cvt.u64.u32 %rd4, %r2;\n" " mov.u32 %r42, -2;\n" " sub.s32 %r4, %r42, %r1;\n" " not.b32 %r5, %r32;\n" " cvt.s64.s32 %rd5, %r1;\n" " shl.b32 %r43, %r1, 2;\n" " mov.u32 %r44, sh_x4;\n" " add.s32 %r6, %r44, %r43;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd6, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd7, %r8;\n" " add.s32 %r9, %r1, 768;\n" " cvta.to.global.u64 %rd40, %rd29;\n" " add.s64 %rd80, %rd40, %rd39;\n" " cvta.to.global.u64 %rd41, %rd28;\n" " add.s64 %rd79, %rd41, %rd39;\n" " mov.f32 %f227, 0f00000000;\n" " mov.u32 %r156, 0;\n" " mov.u32 %r157, %r156;\n" " mov.f32 %f226, %f227;\n" "\n" "$L__BB4_2:\n" " add.s32 %r45, %r156, %r5;\n" " max.s32 %r46, %r45, -2049;\n" " sub.s32 %r12, %r4, %r46;\n" " sub.s32 %r47, %r32, %r157;\n" " min.s32 %r13, %r47, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB4_10;\n" "\n" " shr.u32 %r48, %r12, 8;\n" " add.s32 %r49, %r48, 1;\n" " cvt.s64.s32 %rd12, %r157;\n" " and.b32 %r14, %r49, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r158, %r1;\n" " @%p3 bra $L__BB4_7;\n" "\n" " add.s64 %rd43, %rd5, %rd12;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd30, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f23;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r158, %r7;\n" " @%p4 bra $L__BB4_7;\n" "\n" " add.s64 %rd46, %rd6, %rd12;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd45, %rd30, %rd47;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd45];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f24;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r158, %r8;\n" " @%p5 bra $L__BB4_7;\n" "\n" " add.s64 %rd49, %rd7, %rd12;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd48, %rd30, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f25;\n" " mov.u32 %r158, %r9;\n" "\n" "$L__BB4_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB4_10;\n" "\n" " cvt.s64.s32 %rd51, %r158;\n" " add.s64 %rd52, %rd51, %rd12;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd72, %rd30, %rd53;\n" " shl.b32 %r50, %r158, 2;\n" " mov.u32 %r51, sh_x4;\n" " add.s32 %r52, %r51, %r50;\n" " add.s32 %r159, %r52, 2048;\n" "\n" "$L__BB4_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd72];\n" " // end inline asm\n" " st.shared.f32 [%r159+-2048], %f26;\n" " add.s64 %rd55, %rd72, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd55];\n" " // end inline asm\n" " st.shared.f32 [%r159+-1024], %f27;\n" " add.s64 %rd56, %rd72, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd56];\n" " // end inline asm\n" " st.shared.f32 [%r159], %f28;\n" " add.s64 %rd57, %rd72, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd57];\n" " // end inline asm\n" " st.shared.f32 [%r159+1024], %f29;\n" " add.s64 %rd72, %rd72, 4096;\n" " add.s32 %r159, %r159, 4096;\n" " add.s32 %r158, %r158, 1024;\n" " setp.lt.s32 %p7, %r158, %r13;\n" " @%p7 bra $L__BB4_9;\n" "\n" "$L__BB4_10:\n" " cvt.u32.u64 %r53, %rd3;\n" " setp.ge.s32 %p8, %r53, %r31;\n" " bar.sync 0;\n" " @%p8 bra $L__BB4_17;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB4_17;\n" "\n" " shr.s32 %r55, %r13, 8;\n" " max.s32 %r21, %r55, 1;\n" " and.b32 %r22, %r21, 1;\n" " setp.lt.s32 %p10, %r13, 512;\n" " mov.u32 %r163, %r44;\n" " @%p10 bra $L__BB4_15;\n" "\n" " sub.s32 %r162, %r21, %r22;\n" " mov.u32 %r163, sh_x4;\n" "\n" "$L__BB4_14:\n" " shl.b64 %rd59, %rd4, 3;\n" " add.s64 %rd60, %rd79, %rd59;\n" " ld.global.v2.u32 {%r57, %r58}, [%rd60];\n" " add.s64 %rd61, %rd80, %rd59;\n" " ld.global.v2.u32 {%r61, %r62}, [%rd61];\n" " shl.b32 %r65, %r2, 4;\n" " add.s32 %r66, %r163, %r65;\n" " ld.shared.v4.f32 {%f63, %f64, %f65, %f66}, [%r66];\n" " cvt.u16.u32 %rs1, %r57;\n" " // begin inline asm\n" " cvt.f32.f16 %f31, %rs1;\n" " // end inline asm\n" " mov.f32 %f71, 0f00000000;\n" " fma.rn.f32 %f72, %f31, %f63, %f71;\n" " cvt.u16.u32 %rs2, %r61;\n" " // begin inline asm\n" " cvt.f32.f16 %f32, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f73, %f32, %f63, %f71;\n" " shr.u32 %r67, %r57, 16;\n" " cvt.u16.u32 %rs3, %r67;\n" " // begin inline asm\n" " cvt.f32.f16 %f33, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f74, %f33, %f64, %f72;\n" " shr.u32 %r68, %r61, 16;\n" " cvt.u16.u32 %rs4, %r68;\n" " // begin inline asm\n" " cvt.f32.f16 %f34, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f75, %f34, %f64, %f73;\n" " cvt.u16.u32 %rs5, %r58;\n" " // begin inline asm\n" " cvt.f32.f16 %f35, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f76, %f35, %f65, %f74;\n" " cvt.u16.u32 %rs6, %r62;\n" " // begin inline asm\n" " cvt.f32.f16 %f36, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f77, %f36, %f65, %f75;\n" " shr.u32 %r69, %r58, 16;\n" " cvt.u16.u32 %rs7, %r69;\n" " // begin inline asm\n" " cvt.f32.f16 %f37, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f78, %f37, %f66, %f76;\n" " shr.u32 %r70, %r62, 16;\n" " cvt.u16.u32 %rs8, %r70;\n" " // begin inline asm\n" " cvt.f32.f16 %f38, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f79, %f38, %f66, %f77;\n" " ld.global.v2.u32 {%r71, %r72}, [%rd60+256];\n" " ld.global.v2.u32 {%r75, %r76}, [%rd61+256];\n" " ld.shared.v4.f32 {%f80, %f81, %f82, %f83}, [%r66+512];\n" " cvt.u16.u32 %rs9, %r71;\n" " // begin inline asm\n" " cvt.f32.f16 %f39, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f88, %f39, %f80, %f78;\n" " cvt.u16.u32 %rs10, %r75;\n" " // begin inline asm\n" " cvt.f32.f16 %f40, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f89, %f40, %f80, %f79;\n" " shr.u32 %r79, %r71, 16;\n" " cvt.u16.u32 %rs11, %r79;\n" " // begin inline asm\n" " cvt.f32.f16 %f41, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f90, %f41, %f81, %f88;\n" " shr.u32 %r80, %r75, 16;\n" " cvt.u16.u32 %rs12, %r80;\n" " // begin inline asm\n" " cvt.f32.f16 %f42, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f91, %f42, %f81, %f89;\n" " cvt.u16.u32 %rs13, %r72;\n" " // begin inline asm\n" " cvt.f32.f16 %f43, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f92, %f43, %f82, %f90;\n" " cvt.u16.u32 %rs14, %r76;\n" " // begin inline asm\n" " cvt.f32.f16 %f44, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f93, %f44, %f82, %f91;\n" " shr.u32 %r81, %r72, 16;\n" " cvt.u16.u32 %rs15, %r81;\n" " // begin inline asm\n" " cvt.f32.f16 %f45, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f94, %f45, %f83, %f92;\n" " shr.u32 %r82, %r76, 16;\n" " cvt.u16.u32 %rs16, %r82;\n" " // begin inline asm\n" " cvt.f32.f16 %f46, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f95, %f46, %f83, %f93;\n" " add.f32 %f96, %f226, %f94;\n" " add.f32 %f97, %f227, %f95;\n" " ld.global.v2.u32 {%r83, %r84}, [%rd60+512];\n" " ld.global.v2.u32 {%r87, %r88}, [%rd61+512];\n" " ld.shared.v4.f32 {%f98, %f99, %f100, %f101}, [%r66+1024];\n" " cvt.u16.u32 %rs17, %r83;\n" " // begin inline asm\n" " cvt.f32.f16 %f47, %rs17;\n" " // end inline asm\n" " fma.rn.f32 %f106, %f47, %f98, %f71;\n" " cvt.u16.u32 %rs18, %r87;\n" " // begin inline asm\n" " cvt.f32.f16 %f48, %rs18;\n" " // end inline asm\n" " fma.rn.f32 %f107, %f48, %f98, %f71;\n" " shr.u32 %r91, %r83, 16;\n" " cvt.u16.u32 %rs19, %r91;\n" " // begin inline asm\n" " cvt.f32.f16 %f49, %rs19;\n" " // end inline asm\n" " fma.rn.f32 %f108, %f49, %f99, %f106;\n" " shr.u32 %r92, %r87, 16;\n" " cvt.u16.u32 %rs20, %r92;\n" " // begin inline asm\n" " cvt.f32.f16 %f50, %rs20;\n" " // end inline asm\n" " fma.rn.f32 %f109, %f50, %f99, %f107;\n" " cvt.u16.u32 %rs21, %r84;\n" " // begin inline asm\n" " cvt.f32.f16 %f51, %rs21;\n" " // end inline asm\n" " fma.rn.f32 %f110, %f51, %f100, %f108;\n" " cvt.u16.u32 %rs22, %r88;\n" " // begin inline asm\n" " cvt.f32.f16 %f52, %rs22;\n" " // end inline asm\n" " fma.rn.f32 %f111, %f52, %f100, %f109;\n" " shr.u32 %r93, %r84, 16;\n" " cvt.u16.u32 %rs23, %r93;\n" " // begin inline asm\n" " cvt.f32.f16 %f53, %rs23;\n" " // end inline asm\n" " fma.rn.f32 %f112, %f53, %f101, %f110;\n" " shr.u32 %r94, %r88, 16;\n" " cvt.u16.u32 %rs24, %r94;\n" " // begin inline asm\n" " cvt.f32.f16 %f54, %rs24;\n" " // end inline asm\n" " fma.rn.f32 %f113, %f54, %f101, %f111;\n" " ld.global.v2.u32 {%r95, %r96}, [%rd60+768];\n" " ld.global.v2.u32 {%r99, %r100}, [%rd61+768];\n" " ld.shared.v4.f32 {%f114, %f115, %f116, %f117}, [%r66+1536];\n" " cvt.u16.u32 %rs25, %r95;\n" " // begin inline asm\n" " cvt.f32.f16 %f55, %rs25;\n" " // end inline asm\n" " fma.rn.f32 %f122, %f55, %f114, %f112;\n" " cvt.u16.u32 %rs26, %r99;\n" " // begin inline asm\n" " cvt.f32.f16 %f56, %rs26;\n" " // end inline asm\n" " fma.rn.f32 %f123, %f56, %f114, %f113;\n" " shr.u32 %r103, %r95, 16;\n" " cvt.u16.u32 %rs27, %r103;\n" " // begin inline asm\n" " cvt.f32.f16 %f57, %rs27;\n" " // end inline asm\n" " fma.rn.f32 %f124, %f57, %f115, %f122;\n" " shr.u32 %r104, %r99, 16;\n" " cvt.u16.u32 %rs28, %r104;\n" " // begin inline asm\n" " cvt.f32.f16 %f58, %rs28;\n" " // end inline asm\n" " fma.rn.f32 %f125, %f58, %f115, %f123;\n" " cvt.u16.u32 %rs29, %r96;\n" " // begin inline asm\n" " cvt.f32.f16 %f59, %rs29;\n" " // end inline asm\n" " fma.rn.f32 %f126, %f59, %f116, %f124;\n" " cvt.u16.u32 %rs30, %r100;\n" " // begin inline asm\n" " cvt.f32.f16 %f60, %rs30;\n" " // end inline asm\n" " fma.rn.f32 %f127, %f60, %f116, %f125;\n" " shr.u32 %r105, %r96, 16;\n" " cvt.u16.u32 %rs31, %r105;\n" " // begin inline asm\n" " cvt.f32.f16 %f61, %rs31;\n" " // end inline asm\n" " fma.rn.f32 %f128, %f61, %f117, %f126;\n" " shr.u32 %r106, %r100, 16;\n" " cvt.u16.u32 %rs32, %r106;\n" " // begin inline asm\n" " cvt.f32.f16 %f62, %rs32;\n" " // end inline asm\n" " fma.rn.f32 %f129, %f62, %f117, %f127;\n" " add.f32 %f226, %f96, %f128;\n" " add.f32 %f227, %f97, %f129;\n" " add.s64 %rd79, %rd79, 1024;\n" " add.s64 %rd80, %rd80, 1024;\n" " add.s32 %r163, %r163, 2048;\n" " add.s32 %r162, %r162, -2;\n" " setp.ne.s32 %p11, %r162, 0;\n" " @%p11 bra $L__BB4_14;\n" "\n" "$L__BB4_15:\n" " setp.eq.s32 %p12, %r22, 0;\n" " @%p12 bra $L__BB4_17;\n" "\n" " shl.b64 %rd62, %rd4, 3;\n" " add.s64 %rd63, %rd79, %rd62;\n" " ld.global.v2.u32 {%r107, %r108}, [%rd63];\n" " add.s64 %rd64, %rd80, %rd62;\n" " ld.global.v2.u32 {%r111, %r112}, [%rd64];\n" " shl.b32 %r115, %r2, 4;\n" " add.s32 %r116, %r163, %r115;\n" " ld.shared.v4.f32 {%f146, %f147, %f148, %f149}, [%r116];\n" " cvt.u16.u32 %rs33, %r107;\n" " // begin inline asm\n" " cvt.f32.f16 %f130, %rs33;\n" " // end inline asm\n" " mov.f32 %f154, 0f00000000;\n" " fma.rn.f32 %f155, %f130, %f146, %f154;\n" " cvt.u16.u32 %rs34, %r111;\n" " // begin inline asm\n" " cvt.f32.f16 %f131, %rs34;\n" " // end inline asm\n" " fma.rn.f32 %f156, %f131, %f146, %f154;\n" " shr.u32 %r117, %r107, 16;\n" " cvt.u16.u32 %rs35, %r117;\n" " // begin inline asm\n" " cvt.f32.f16 %f132, %rs35;\n" " // end inline asm\n" " fma.rn.f32 %f157, %f132, %f147, %f155;\n" " shr.u32 %r118, %r111, 16;\n" " cvt.u16.u32 %rs36, %r118;\n" " // begin inline asm\n" " cvt.f32.f16 %f133, %rs36;\n" " // end inline asm\n" " fma.rn.f32 %f158, %f133, %f147, %f156;\n" " cvt.u16.u32 %rs37, %r108;\n" " // begin inline asm\n" " cvt.f32.f16 %f134, %rs37;\n" " // end inline asm\n" " fma.rn.f32 %f159, %f134, %f148, %f157;\n" " cvt.u16.u32 %rs38, %r112;\n" " // begin inline asm\n" " cvt.f32.f16 %f135, %rs38;\n" " // end inline asm\n" " fma.rn.f32 %f160, %f135, %f148, %f158;\n" " shr.u32 %r119, %r108, 16;\n" " cvt.u16.u32 %rs39, %r119;\n" " // begin inline asm\n" " cvt.f32.f16 %f136, %rs39;\n" " // end inline asm\n" " fma.rn.f32 %f161, %f136, %f149, %f159;\n" " shr.u32 %r120, %r112, 16;\n" " cvt.u16.u32 %rs40, %r120;\n" " // begin inline asm\n" " cvt.f32.f16 %f137, %rs40;\n" " // end inline asm\n" " fma.rn.f32 %f162, %f137, %f149, %f160;\n" " ld.global.v2.u32 {%r121, %r122}, [%rd63+256];\n" " ld.global.v2.u32 {%r125, %r126}, [%rd64+256];\n" " ld.shared.v4.f32 {%f163, %f164, %f165, %f166}, [%r116+512];\n" " cvt.u16.u32 %rs41, %r121;\n" " // begin inline asm\n" " cvt.f32.f16 %f138, %rs41;\n" " // end inline asm\n" " fma.rn.f32 %f171, %f138, %f163, %f161;\n" " cvt.u16.u32 %rs42, %r125;\n" " // begin inline asm\n" " cvt.f32.f16 %f139, %rs42;\n" " // end inline asm\n" " fma.rn.f32 %f172, %f139, %f163, %f162;\n" " shr.u32 %r129, %r121, 16;\n" " cvt.u16.u32 %rs43, %r129;\n" " // begin inline asm\n" " cvt.f32.f16 %f140, %rs43;\n" " // end inline asm\n" " fma.rn.f32 %f173, %f140, %f164, %f171;\n" " shr.u32 %r130, %r125, 16;\n" " cvt.u16.u32 %rs44, %r130;\n" " // begin inline asm\n" " cvt.f32.f16 %f141, %rs44;\n" " // end inline asm\n" " fma.rn.f32 %f174, %f141, %f164, %f172;\n" " cvt.u16.u32 %rs45, %r122;\n" " // begin inline asm\n" " cvt.f32.f16 %f142, %rs45;\n" " // end inline asm\n" " fma.rn.f32 %f175, %f142, %f165, %f173;\n" " cvt.u16.u32 %rs46, %r126;\n" " // begin inline asm\n" " cvt.f32.f16 %f143, %rs46;\n" " // end inline asm\n" " fma.rn.f32 %f176, %f143, %f165, %f174;\n" " shr.u32 %r131, %r122, 16;\n" " cvt.u16.u32 %rs47, %r131;\n" " // begin inline asm\n" " cvt.f32.f16 %f144, %rs47;\n" " // end inline asm\n" " fma.rn.f32 %f177, %f144, %f166, %f175;\n" " shr.u32 %r132, %r126, 16;\n" " cvt.u16.u32 %rs48, %r132;\n" " // begin inline asm\n" " cvt.f32.f16 %f145, %rs48;\n" " // end inline asm\n" " fma.rn.f32 %f178, %f145, %f166, %f176;\n" " add.f32 %f226, %f226, %f177;\n" " add.f32 %f227, %f227, %f178;\n" " add.s64 %rd80, %rd80, 512;\n" " add.s64 %rd79, %rd79, 512;\n" "\n" "$L__BB4_17:\n" " add.s32 %r157, %r157, 2048;\n" " setp.lt.s32 %p13, %r157, %r32;\n" " add.s32 %r156, %r156, 2048;\n" " @%p13 bra $L__BB4_2;\n" "\n" "$L__BB4_18:\n" " cvt.u32.u64 %r153, %rd3;\n" " setp.ge.s32 %p14, %r153, %r31;\n" " mov.u32 %r143, 16;\n" " mov.u32 %r152, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f179, %f226, %r143, %r152;\n" " // end inline asm\n" " add.f32 %f182, %f226, %f179;\n" " mov.u32 %r145, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f181, %f182, %r145, %r152;\n" " // end inline asm\n" " add.f32 %f184, %f182, %f181;\n" " mov.u32 %r147, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f183, %f184, %r147, %r152;\n" " // end inline asm\n" " add.f32 %f186, %f184, %f183;\n" " mov.u32 %r149, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f185, %f186, %r149, %r152;\n" " // end inline asm\n" " add.f32 %f188, %f186, %f185;\n" " mov.u32 %r151, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f187, %f188, %r151, %r152;\n" " // end inline asm\n" " add.f32 %f17, %f188, %f187;\n" " // begin inline asm\n" " shfl.down.b32 %f189, %f227, %r143, %r152;\n" " // end inline asm\n" " add.f32 %f192, %f227, %f189;\n" " // begin inline asm\n" " shfl.down.b32 %f191, %f192, %r145, %r152;\n" " // end inline asm\n" " add.f32 %f194, %f192, %f191;\n" " // begin inline asm\n" " shfl.down.b32 %f193, %f194, %r147, %r152;\n" " // end inline asm\n" " add.f32 %f196, %f194, %f193;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f196, %r149, %r152;\n" " // end inline asm\n" " add.f32 %f198, %f196, %f195;\n" " // begin inline asm\n" " shfl.down.b32 %f197, %f198, %r151, %r152;\n" " // end inline asm\n" " add.f32 %f18, %f198, %f197;\n" " setp.ne.s32 %p15, %r2, 0;\n" " or.pred %p16, %p15, %p14;\n" " @%p16 bra $L__BB4_20;\n" "\n" " neg.f32 %f199, %f17;\n" " mov.f32 %f200, 0f3F000000;\n" " mov.f32 %f201, 0f3BBB989D;\n" " fma.rn.f32 %f202, %f199, %f201, %f200;\n" " mov.f32 %f203, 0f3FB8AA3B;\n" " mov.f32 %f204, 0f437C0000;\n" " cvt.sat.f32.f32 %f205, %f202;\n" " mov.f32 %f206, 0f4B400001;\n" " fma.rm.f32 %f207, %f205, %f204, %f206;\n" " add.f32 %f208, %f207, 0fCB40007F;\n" " neg.f32 %f209, %f208;\n" " fma.rn.f32 %f210, %f199, %f203, %f209;\n" " mov.f32 %f211, 0f32A57060;\n" " fma.rn.f32 %f212, %f199, %f211, %f210;\n" " mov.b32 %r154, %f207;\n" " shl.b32 %r155, %r154, 23;\n" " mov.b32 %f213, %r155;\n" " ex2.approx.ftz.f32 %f214, %f212;\n" " fma.rn.f32 %f215, %f214, %f213, 0f3F800000;\n" " div.rn.f32 %f216, %f17, %f215;\n" " mul.f32 %f217, %f18, %f216;\n" " mul.lo.s64 %rd65, %rd2, %rd1;\n" " add.s64 %rd66, %rd65, %rd3;\n" " cvta.to.global.u64 %rd67, %rd31;\n" " shl.b64 %rd68, %rd66, 2;\n" " add.s64 %rd69, %rd67, %rd68;\n" " st.global.f32 [%rd69], %f217;\n" "\n" "$L__BB4_20:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_bf16_vec\n" ".visible .entry fc_moe_gateup_bf16_vec(\n" " .param .u64 fc_moe_gateup_bf16_vec_param_0,\n" " .param .u64 fc_moe_gateup_bf16_vec_param_1,\n" " .param .u64 fc_moe_gateup_bf16_vec_param_2,\n" " .param .u64 fc_moe_gateup_bf16_vec_param_3,\n" " .param .u64 fc_moe_gateup_bf16_vec_param_4,\n" " .param .u32 fc_moe_gateup_bf16_vec_param_5,\n" " .param .u32 fc_moe_gateup_bf16_vec_param_6,\n" " .param .u32 fc_moe_gateup_bf16_vec_param_7\n" ")\n" "{\n" " .reg .pred %p<17>;\n" " .reg .f32 %f<230>;\n" " .reg .b32 %r<188>;\n" " .reg .b64 %rd<81>;\n" "\n" "\n" " ld.param.u64 %rd28, [fc_moe_gateup_bf16_vec_param_0];\n" " ld.param.u64 %rd29, [fc_moe_gateup_bf16_vec_param_1];\n" " ld.param.u64 %rd30, [fc_moe_gateup_bf16_vec_param_2];\n" " ld.param.u64 %rd33, [fc_moe_gateup_bf16_vec_param_3];\n" " ld.param.u64 %rd31, [fc_moe_gateup_bf16_vec_param_4];\n" " ld.param.u32 %r31, [fc_moe_gateup_bf16_vec_param_5];\n" " ld.param.u32 %r32, [fc_moe_gateup_bf16_vec_param_6];\n" " ld.param.u32 %r33, [fc_moe_gateup_bf16_vec_param_7];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r35, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r36, %ctaid.x;\n" " shl.b32 %r37, %r36, 3;\n" " add.s32 %r38, %r37, %r35;\n" " mov.u32 %r39, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r39;\n" " mul.wide.s32 %rd34, %r39, 4;\n" " add.s64 %rd32, %rd33, %rd34;\n" " // begin inline asm\n" " ld.global.nc.s32 %r34, [%rd32];\n" " // end inline asm\n" " cvt.s64.s32 %rd2, %r31;\n" " cvt.s64.s32 %rd3, %r38;\n" " setp.lt.s32 %p1, %r32, 1;\n" " mov.f32 %f226, 0f00000000;\n" " mov.f32 %f227, %f226;\n" " @%p1 bra $L__BB5_18;\n" "\n" " cvt.s64.s32 %rd35, %r33;\n" " cvt.s64.s32 %rd36, %r34;\n" " mul.lo.s64 %rd37, %rd36, %rd2;\n" " add.s64 %rd38, %rd37, %rd3;\n" " mul.lo.s64 %rd39, %rd38, %rd35;\n" " cvt.u64.u32 %rd4, %r2;\n" " mov.u32 %r42, -2;\n" " sub.s32 %r4, %r42, %r1;\n" " not.b32 %r5, %r32;\n" " cvt.s64.s32 %rd5, %r1;\n" " shl.b32 %r43, %r1, 2;\n" " mov.u32 %r44, sh_x4;\n" " add.s32 %r6, %r44, %r43;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd6, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd7, %r8;\n" " add.s32 %r9, %r1, 768;\n" " cvta.to.global.u64 %rd40, %rd29;\n" " add.s64 %rd80, %rd40, %rd39;\n" " cvta.to.global.u64 %rd41, %rd28;\n" " add.s64 %rd79, %rd41, %rd39;\n" " mov.f32 %f227, 0f00000000;\n" " mov.u32 %r180, 0;\n" " mov.u32 %r181, %r180;\n" " mov.f32 %f226, %f227;\n" "\n" "$L__BB5_2:\n" " add.s32 %r45, %r180, %r5;\n" " max.s32 %r46, %r45, -2049;\n" " sub.s32 %r12, %r4, %r46;\n" " sub.s32 %r47, %r32, %r181;\n" " min.s32 %r13, %r47, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB5_10;\n" "\n" " shr.u32 %r48, %r12, 8;\n" " add.s32 %r49, %r48, 1;\n" " cvt.s64.s32 %rd12, %r181;\n" " and.b32 %r14, %r49, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r182, %r1;\n" " @%p3 bra $L__BB5_7;\n" "\n" " add.s64 %rd43, %rd5, %rd12;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd30, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f23;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r182, %r7;\n" " @%p4 bra $L__BB5_7;\n" "\n" " add.s64 %rd46, %rd6, %rd12;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd45, %rd30, %rd47;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd45];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f24;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r182, %r8;\n" " @%p5 bra $L__BB5_7;\n" "\n" " add.s64 %rd49, %rd7, %rd12;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd48, %rd30, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f25;\n" " mov.u32 %r182, %r9;\n" "\n" "$L__BB5_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB5_10;\n" "\n" " cvt.s64.s32 %rd51, %r182;\n" " add.s64 %rd52, %rd51, %rd12;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd72, %rd30, %rd53;\n" " shl.b32 %r50, %r182, 2;\n" " mov.u32 %r51, sh_x4;\n" " add.s32 %r52, %r51, %r50;\n" " add.s32 %r183, %r52, 2048;\n" "\n" "$L__BB5_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd72];\n" " // end inline asm\n" " st.shared.f32 [%r183+-2048], %f26;\n" " add.s64 %rd55, %rd72, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd55];\n" " // end inline asm\n" " st.shared.f32 [%r183+-1024], %f27;\n" " add.s64 %rd56, %rd72, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd56];\n" " // end inline asm\n" " st.shared.f32 [%r183], %f28;\n" " add.s64 %rd57, %rd72, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd57];\n" " // end inline asm\n" " st.shared.f32 [%r183+1024], %f29;\n" " add.s64 %rd72, %rd72, 4096;\n" " add.s32 %r183, %r183, 4096;\n" " add.s32 %r182, %r182, 1024;\n" " setp.lt.s32 %p7, %r182, %r13;\n" " @%p7 bra $L__BB5_9;\n" "\n" "$L__BB5_10:\n" " cvt.u32.u64 %r53, %rd3;\n" " setp.ge.s32 %p8, %r53, %r31;\n" " bar.sync 0;\n" " @%p8 bra $L__BB5_17;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB5_17;\n" "\n" " shr.s32 %r55, %r13, 8;\n" " max.s32 %r21, %r55, 1;\n" " and.b32 %r22, %r21, 1;\n" " setp.lt.s32 %p10, %r13, 512;\n" " mov.u32 %r187, %r44;\n" " @%p10 bra $L__BB5_15;\n" "\n" " sub.s32 %r186, %r21, %r22;\n" " mov.u32 %r187, sh_x4;\n" "\n" "$L__BB5_14:\n" " shl.b64 %rd59, %rd4, 3;\n" " add.s64 %rd60, %rd79, %rd59;\n" " ld.global.v2.u32 {%r57, %r58}, [%rd60];\n" " add.s64 %rd61, %rd80, %rd59;\n" " ld.global.v2.u32 {%r61, %r62}, [%rd61];\n" " shl.b32 %r65, %r2, 4;\n" " add.s32 %r66, %r187, %r65;\n" " ld.shared.v4.f32 {%f31, %f32, %f33, %f34}, [%r66];\n" " shl.b32 %r67, %r57, 16;\n" " mov.b32 %f39, %r67;\n" " mov.f32 %f40, 0f00000000;\n" " fma.rn.f32 %f41, %f39, %f31, %f40;\n" " shl.b32 %r68, %r61, 16;\n" " mov.b32 %f42, %r68;\n" " fma.rn.f32 %f43, %f42, %f31, %f40;\n" " and.b32 %r69, %r57, -65536;\n" " mov.b32 %f44, %r69;\n" " fma.rn.f32 %f45, %f44, %f32, %f41;\n" " and.b32 %r70, %r61, -65536;\n" " mov.b32 %f46, %r70;\n" " fma.rn.f32 %f47, %f46, %f32, %f43;\n" " shl.b32 %r71, %r58, 16;\n" " mov.b32 %f48, %r71;\n" " fma.rn.f32 %f49, %f48, %f33, %f45;\n" " shl.b32 %r72, %r62, 16;\n" " mov.b32 %f50, %r72;\n" " fma.rn.f32 %f51, %f50, %f33, %f47;\n" " and.b32 %r73, %r58, -65536;\n" " mov.b32 %f52, %r73;\n" " fma.rn.f32 %f53, %f52, %f34, %f49;\n" " and.b32 %r74, %r62, -65536;\n" " mov.b32 %f54, %r74;\n" " fma.rn.f32 %f55, %f54, %f34, %f51;\n" " ld.global.v2.u32 {%r75, %r76}, [%rd60+256];\n" " ld.global.v2.u32 {%r79, %r80}, [%rd61+256];\n" " ld.shared.v4.f32 {%f56, %f57, %f58, %f59}, [%r66+512];\n" " shl.b32 %r83, %r75, 16;\n" " mov.b32 %f64, %r83;\n" " fma.rn.f32 %f65, %f64, %f56, %f53;\n" " shl.b32 %r84, %r79, 16;\n" " mov.b32 %f66, %r84;\n" " fma.rn.f32 %f67, %f66, %f56, %f55;\n" " and.b32 %r85, %r75, -65536;\n" " mov.b32 %f68, %r85;\n" " fma.rn.f32 %f69, %f68, %f57, %f65;\n" " and.b32 %r86, %r79, -65536;\n" " mov.b32 %f70, %r86;\n" " fma.rn.f32 %f71, %f70, %f57, %f67;\n" " shl.b32 %r87, %r76, 16;\n" " mov.b32 %f72, %r87;\n" " fma.rn.f32 %f73, %f72, %f58, %f69;\n" " shl.b32 %r88, %r80, 16;\n" " mov.b32 %f74, %r88;\n" " fma.rn.f32 %f75, %f74, %f58, %f71;\n" " and.b32 %r89, %r76, -65536;\n" " mov.b32 %f76, %r89;\n" " fma.rn.f32 %f77, %f76, %f59, %f73;\n" " and.b32 %r90, %r80, -65536;\n" " mov.b32 %f78, %r90;\n" " fma.rn.f32 %f79, %f78, %f59, %f75;\n" " add.f32 %f80, %f226, %f77;\n" " add.f32 %f81, %f227, %f79;\n" " ld.global.v2.u32 {%r91, %r92}, [%rd60+512];\n" " ld.global.v2.u32 {%r95, %r96}, [%rd61+512];\n" " ld.shared.v4.f32 {%f82, %f83, %f84, %f85}, [%r66+1024];\n" " shl.b32 %r99, %r91, 16;\n" " mov.b32 %f90, %r99;\n" " fma.rn.f32 %f91, %f90, %f82, %f40;\n" " shl.b32 %r100, %r95, 16;\n" " mov.b32 %f92, %r100;\n" " fma.rn.f32 %f93, %f92, %f82, %f40;\n" " and.b32 %r101, %r91, -65536;\n" " mov.b32 %f94, %r101;\n" " fma.rn.f32 %f95, %f94, %f83, %f91;\n" " and.b32 %r102, %r95, -65536;\n" " mov.b32 %f96, %r102;\n" " fma.rn.f32 %f97, %f96, %f83, %f93;\n" " shl.b32 %r103, %r92, 16;\n" " mov.b32 %f98, %r103;\n" " fma.rn.f32 %f99, %f98, %f84, %f95;\n" " shl.b32 %r104, %r96, 16;\n" " mov.b32 %f100, %r104;\n" " fma.rn.f32 %f101, %f100, %f84, %f97;\n" " and.b32 %r105, %r92, -65536;\n" " mov.b32 %f102, %r105;\n" " fma.rn.f32 %f103, %f102, %f85, %f99;\n" " and.b32 %r106, %r96, -65536;\n" " mov.b32 %f104, %r106;\n" " fma.rn.f32 %f105, %f104, %f85, %f101;\n" " ld.global.v2.u32 {%r107, %r108}, [%rd60+768];\n" " ld.global.v2.u32 {%r111, %r112}, [%rd61+768];\n" " ld.shared.v4.f32 {%f106, %f107, %f108, %f109}, [%r66+1536];\n" " shl.b32 %r115, %r107, 16;\n" " mov.b32 %f114, %r115;\n" " fma.rn.f32 %f115, %f114, %f106, %f103;\n" " shl.b32 %r116, %r111, 16;\n" " mov.b32 %f116, %r116;\n" " fma.rn.f32 %f117, %f116, %f106, %f105;\n" " and.b32 %r117, %r107, -65536;\n" " mov.b32 %f118, %r117;\n" " fma.rn.f32 %f119, %f118, %f107, %f115;\n" " and.b32 %r118, %r111, -65536;\n" " mov.b32 %f120, %r118;\n" " fma.rn.f32 %f121, %f120, %f107, %f117;\n" " shl.b32 %r119, %r108, 16;\n" " mov.b32 %f122, %r119;\n" " fma.rn.f32 %f123, %f122, %f108, %f119;\n" " shl.b32 %r120, %r112, 16;\n" " mov.b32 %f124, %r120;\n" " fma.rn.f32 %f125, %f124, %f108, %f121;\n" " and.b32 %r121, %r108, -65536;\n" " mov.b32 %f126, %r121;\n" " fma.rn.f32 %f127, %f126, %f109, %f123;\n" " and.b32 %r122, %r112, -65536;\n" " mov.b32 %f128, %r122;\n" " fma.rn.f32 %f129, %f128, %f109, %f125;\n" " add.f32 %f226, %f80, %f127;\n" " add.f32 %f227, %f81, %f129;\n" " add.s64 %rd79, %rd79, 1024;\n" " add.s64 %rd80, %rd80, 1024;\n" " add.s32 %r187, %r187, 2048;\n" " add.s32 %r186, %r186, -2;\n" " setp.ne.s32 %p11, %r186, 0;\n" " @%p11 bra $L__BB5_14;\n" "\n" "$L__BB5_15:\n" " setp.eq.s32 %p12, %r22, 0;\n" " @%p12 bra $L__BB5_17;\n" "\n" " shl.b64 %rd62, %rd4, 3;\n" " add.s64 %rd63, %rd79, %rd62;\n" " ld.global.v2.u32 {%r123, %r124}, [%rd63];\n" " add.s64 %rd64, %rd80, %rd62;\n" " ld.global.v2.u32 {%r127, %r128}, [%rd64];\n" " shl.b32 %r131, %r2, 4;\n" " add.s32 %r132, %r187, %r131;\n" " ld.shared.v4.f32 {%f130, %f131, %f132, %f133}, [%r132];\n" " shl.b32 %r133, %r123, 16;\n" " mov.b32 %f138, %r133;\n" " mov.f32 %f139, 0f00000000;\n" " fma.rn.f32 %f140, %f138, %f130, %f139;\n" " shl.b32 %r134, %r127, 16;\n" " mov.b32 %f141, %r134;\n" " fma.rn.f32 %f142, %f141, %f130, %f139;\n" " and.b32 %r135, %r123, -65536;\n" " mov.b32 %f143, %r135;\n" " fma.rn.f32 %f144, %f143, %f131, %f140;\n" " and.b32 %r136, %r127, -65536;\n" " mov.b32 %f145, %r136;\n" " fma.rn.f32 %f146, %f145, %f131, %f142;\n" " shl.b32 %r137, %r124, 16;\n" " mov.b32 %f147, %r137;\n" " fma.rn.f32 %f148, %f147, %f132, %f144;\n" " shl.b32 %r138, %r128, 16;\n" " mov.b32 %f149, %r138;\n" " fma.rn.f32 %f150, %f149, %f132, %f146;\n" " and.b32 %r139, %r124, -65536;\n" " mov.b32 %f151, %r139;\n" " fma.rn.f32 %f152, %f151, %f133, %f148;\n" " and.b32 %r140, %r128, -65536;\n" " mov.b32 %f153, %r140;\n" " fma.rn.f32 %f154, %f153, %f133, %f150;\n" " ld.global.v2.u32 {%r141, %r142}, [%rd63+256];\n" " ld.global.v2.u32 {%r145, %r146}, [%rd64+256];\n" " ld.shared.v4.f32 {%f155, %f156, %f157, %f158}, [%r132+512];\n" " shl.b32 %r149, %r141, 16;\n" " mov.b32 %f163, %r149;\n" " fma.rn.f32 %f164, %f163, %f155, %f152;\n" " shl.b32 %r150, %r145, 16;\n" " mov.b32 %f165, %r150;\n" " fma.rn.f32 %f166, %f165, %f155, %f154;\n" " and.b32 %r151, %r141, -65536;\n" " mov.b32 %f167, %r151;\n" " fma.rn.f32 %f168, %f167, %f156, %f164;\n" " and.b32 %r152, %r145, -65536;\n" " mov.b32 %f169, %r152;\n" " fma.rn.f32 %f170, %f169, %f156, %f166;\n" " shl.b32 %r153, %r142, 16;\n" " mov.b32 %f171, %r153;\n" " fma.rn.f32 %f172, %f171, %f157, %f168;\n" " shl.b32 %r154, %r146, 16;\n" " mov.b32 %f173, %r154;\n" " fma.rn.f32 %f174, %f173, %f157, %f170;\n" " and.b32 %r155, %r142, -65536;\n" " mov.b32 %f175, %r155;\n" " fma.rn.f32 %f176, %f175, %f158, %f172;\n" " and.b32 %r156, %r146, -65536;\n" " mov.b32 %f177, %r156;\n" " fma.rn.f32 %f178, %f177, %f158, %f174;\n" " add.f32 %f226, %f226, %f176;\n" " add.f32 %f227, %f227, %f178;\n" " add.s64 %rd80, %rd80, 512;\n" " add.s64 %rd79, %rd79, 512;\n" "\n" "$L__BB5_17:\n" " add.s32 %r181, %r181, 2048;\n" " setp.lt.s32 %p13, %r181, %r32;\n" " add.s32 %r180, %r180, 2048;\n" " @%p13 bra $L__BB5_2;\n" "\n" "$L__BB5_18:\n" " cvt.u32.u64 %r177, %rd3;\n" " setp.ge.s32 %p14, %r177, %r31;\n" " mov.u32 %r167, 16;\n" " mov.u32 %r176, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f179, %f226, %r167, %r176;\n" " // end inline asm\n" " add.f32 %f182, %f226, %f179;\n" " mov.u32 %r169, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f181, %f182, %r169, %r176;\n" " // end inline asm\n" " add.f32 %f184, %f182, %f181;\n" " mov.u32 %r171, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f183, %f184, %r171, %r176;\n" " // end inline asm\n" " add.f32 %f186, %f184, %f183;\n" " mov.u32 %r173, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f185, %f186, %r173, %r176;\n" " // end inline asm\n" " add.f32 %f188, %f186, %f185;\n" " mov.u32 %r175, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f187, %f188, %r175, %r176;\n" " // end inline asm\n" " add.f32 %f17, %f188, %f187;\n" " // begin inline asm\n" " shfl.down.b32 %f189, %f227, %r167, %r176;\n" " // end inline asm\n" " add.f32 %f192, %f227, %f189;\n" " // begin inline asm\n" " shfl.down.b32 %f191, %f192, %r169, %r176;\n" " // end inline asm\n" " add.f32 %f194, %f192, %f191;\n" " // begin inline asm\n" " shfl.down.b32 %f193, %f194, %r171, %r176;\n" " // end inline asm\n" " add.f32 %f196, %f194, %f193;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f196, %r173, %r176;\n" " // end inline asm\n" " add.f32 %f198, %f196, %f195;\n" " // begin inline asm\n" " shfl.down.b32 %f197, %f198, %r175, %r176;\n" " // end inline asm\n" " add.f32 %f18, %f198, %f197;\n" " setp.ne.s32 %p15, %r2, 0;\n" " or.pred %p16, %p15, %p14;\n" " @%p16 bra $L__BB5_20;\n" "\n" " neg.f32 %f199, %f17;\n" " mov.f32 %f200, 0f3F000000;\n" " mov.f32 %f201, 0f3BBB989D;\n" " fma.rn.f32 %f202, %f199, %f201, %f200;\n" " mov.f32 %f203, 0f3FB8AA3B;\n" " mov.f32 %f204, 0f437C0000;\n" " cvt.sat.f32.f32 %f205, %f202;\n" " mov.f32 %f206, 0f4B400001;\n" " fma.rm.f32 %f207, %f205, %f204, %f206;\n" " add.f32 %f208, %f207, 0fCB40007F;\n" " neg.f32 %f209, %f208;\n" " fma.rn.f32 %f210, %f199, %f203, %f209;\n" " mov.f32 %f211, 0f32A57060;\n" " fma.rn.f32 %f212, %f199, %f211, %f210;\n" " mov.b32 %r178, %f207;\n" " shl.b32 %r179, %r178, 23;\n" " mov.b32 %f213, %r179;\n" " ex2.approx.ftz.f32 %f214, %f212;\n" " fma.rn.f32 %f215, %f214, %f213, 0f3F800000;\n" " div.rn.f32 %f216, %f17, %f215;\n" " mul.f32 %f217, %f18, %f216;\n" " mul.lo.s64 %rd65, %rd2, %rd1;\n" " add.s64 %rd66, %rd65, %rd3;\n" " cvta.to.global.u64 %rd67, %rd31;\n" " shl.b64 %rd68, %rd66, 2;\n" " add.s64 %rd69, %rd67, %rd68;\n" " st.global.f32 [%rd69], %f217;\n" "\n" "$L__BB5_20:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_q3k\n" ".visible .entry fc_moe_down_q3k(\n" " .param .u64 fc_moe_down_q3k_param_0,\n" " .param .u64 fc_moe_down_q3k_param_1,\n" " .param .u64 fc_moe_down_q3k_param_2,\n" " .param .u64 fc_moe_down_q3k_param_3,\n" " .param .u64 fc_moe_down_q3k_param_4,\n" " .param .u32 fc_moe_down_q3k_param_5,\n" " .param .u32 fc_moe_down_q3k_param_6,\n" " .param .u32 fc_moe_down_q3k_param_7,\n" " .param .u32 fc_moe_down_q3k_param_8,\n" " .param .u32 fc_moe_down_q3k_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<194>;\n" " .reg .b32 %r<151>;\n" " .reg .b64 %rd<61>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ15fc_moe_down_q3kE4sred[128];\n" "\n" " ld.param.u64 %rd25, [fc_moe_down_q3k_param_0];\n" " ld.param.u64 %rd26, [fc_moe_down_q3k_param_1];\n" " ld.param.u64 %rd27, [fc_moe_down_q3k_param_2];\n" " ld.param.u64 %rd28, [fc_moe_down_q3k_param_3];\n" " ld.param.u64 %rd29, [fc_moe_down_q3k_param_4];\n" " ld.param.u32 %r30, [fc_moe_down_q3k_param_5];\n" " ld.param.u32 %r31, [fc_moe_down_q3k_param_6];\n" " ld.param.u32 %r32, [fc_moe_down_q3k_param_7];\n" " ld.param.u32 %r33, [fc_moe_down_q3k_param_8];\n" " ld.param.u32 %r34, [fc_moe_down_q3k_param_9];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r35, %ntid.x;\n" " shr.u32 %r36, %r35, 5;\n" " div.u32 %r4, %r36, %r34;\n" " div.s32 %r37, %r2, %r34;\n" " mul.lo.s32 %r38, %r37, %r34;\n" " sub.s32 %r144, %r2, %r38;\n" " mov.u32 %r39, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r39;\n" " add.s32 %r40, %r6, %r37;\n" " shl.b32 %r7, %r40, 1;\n" " setp.ge.s32 %p1, %r7, %r31;\n" " setp.ge.s32 %p2, %r144, %r30;\n" " mov.f32 %f187, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f188, %f187;\n" " @%p3 bra $L__BB6_6;\n" "\n" " cvt.s64.s32 %rd1, %r33;\n" " cvt.s64.s32 %rd2, %r7;\n" " cvt.s64.s32 %rd3, %r32;\n" " shr.u32 %r41, %r3, 4;\n" " cvt.u64.u32 %rd4, %r41;\n" " cvt.u64.u32 %rd5, %r3;\n" " mul.wide.s32 %rd7, %r3, 4;\n" " add.s64 %rd6, %rd26, %rd7;\n" " mul.wide.s32 %rd30, %r41, 8;\n" " or.b64 %rd8, %rd30, 128;\n" " shr.s32 %r8, %r32, 8;\n" " cvt.s64.s32 %rd9, %r31;\n" " add.s32 %r9, %r7, 1;\n" " cvta.to.global.u64 %rd10, %rd25;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r9, %r31;\n" " setp.lt.s32 %p5, %r32, 256;\n" " shl.b64 %rd48, %rd5, 2;\n" " shl.b64 %rd50, %rd4, 3;\n" " mov.f32 %f188, %f25;\n" " mov.f32 %f187, %f25;\n" "\n" "$L__BB6_2:\n" " cvt.s64.s32 %rd11, %r144;\n" " mul.wide.s32 %rd32, %r144, 4;\n" " add.s64 %rd31, %rd27, %rd32;\n" " // begin inline asm\n" " ld.global.nc.s32 %r42, [%rd31];\n" " // end inline asm\n" " cvt.s64.s32 %rd33, %r42;\n" " mul.lo.s64 %rd34, %rd33, %rd9;\n" " add.s64 %rd35, %rd34, %rd2;\n" " mul.lo.s64 %rd36, %rd35, %rd1;\n" " add.s64 %rd60, %rd10, %rd36;\n" " add.s64 %rd37, %rd60, %rd1;\n" " selp.b64 %rd59, %rd37, %rd60, %p4;\n" " mov.f32 %f185, %f25;\n" " mov.f32 %f186, %f25;\n" " @%p5 bra $L__BB6_5;\n" "\n" " mul.lo.s64 %rd38, %rd3, %rd11;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd58, %rd6, %rd39;\n" " add.s64 %rd57, %rd59, %rd7;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r145, 0;\n" " mov.f32 %f186, %f29;\n" " mov.f32 %f185, %f29;\n" "\n" "$L__BB6_4:\n" " add.s64 %rd49, %rd60, %rd48;\n" " add.s64 %rd51, %rd60, %rd50;\n" " ld.global.v2.u32 {%r44, %r45}, [%rd51+128];\n" " add.s64 %rd52, %rd59, %rd8;\n" " ld.global.v2.u32 {%r48, %r49}, [%rd52];\n" " ld.global.u32 %r52, [%rd49];\n" " shr.u32 %r53, %r52, 16;\n" " ld.global.u32 %r54, [%rd57];\n" " shr.u32 %r55, %r54, 16;\n" " ld.global.f32 %f38, [%rd59+144];\n" " ld.global.f32 %f39, [%rd60+144];\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd58];\n" " // end inline asm\n" " and.b32 %r56, %r52, 15;\n" " or.b32 %r57, %r56, 1258291200;\n" " mov.b32 %f40, %r57;\n" " mov.f32 %f41, 0fCB000004;\n" " mov.f32 %f42, 0f3F800000;\n" " fma.rn.f32 %f43, %f40, %f42, %f41;\n" " and.b32 %r58, %r44, 255;\n" " cvt.rn.f32.u32 %f44, %r58;\n" " mov.f32 %f45, 0f42000000;\n" " sub.rn.f32 %f46, %f44, %f45;\n" " and.b32 %r59, %r54, 15;\n" " or.b32 %r60, %r59, 1258291200;\n" " mov.b32 %f47, %r60;\n" " fma.rn.f32 %f48, %f47, %f42, %f41;\n" " and.b32 %r61, %r48, 255;\n" " cvt.rn.f32.u32 %f49, %r61;\n" " sub.rn.f32 %f50, %f49, %f45;\n" " mul.rn.f32 %f51, %f46, %f43;\n" " fma.rn.f32 %f53, %f51, %f30, %f29;\n" " mul.rn.f32 %f54, %f50, %f48;\n" " fma.rn.f32 %f55, %f54, %f30, %f29;\n" " add.s64 %rd41, %rd58, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd41];\n" " // end inline asm\n" " and.b32 %r62, %r52, 240;\n" " or.b32 %r63, %r62, 1258291200;\n" " mov.b32 %f56, %r63;\n" " mov.f32 %f57, 0fC9000040;\n" " mov.f32 %f58, 0f3D800000;\n" " fma.rn.f32 %f59, %f56, %f58, %f57;\n" " bfe.u32 %r64, %r44, 8, 8;\n" " cvt.rn.f32.u32 %f60, %r64;\n" " sub.rn.f32 %f61, %f60, %f45;\n" " and.b32 %r65, %r54, 240;\n" " or.b32 %r66, %r65, 1258291200;\n" " mov.b32 %f62, %r66;\n" " fma.rn.f32 %f63, %f62, %f58, %f57;\n" " bfe.u32 %r67, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f64, %r67;\n" " sub.rn.f32 %f65, %f64, %f45;\n" " mul.rn.f32 %f66, %f61, %f59;\n" " fma.rn.f32 %f67, %f66, %f31, %f53;\n" " mul.rn.f32 %f68, %f65, %f63;\n" " fma.rn.f32 %f69, %f68, %f31, %f55;\n" " add.s64 %rd42, %rd58, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd42];\n" " // end inline asm\n" " and.b32 %r68, %r52, 3840;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f70, %r69;\n" " mov.f32 %f71, 0fC7000400;\n" " mov.f32 %f72, 0f3B800000;\n" " fma.rn.f32 %f73, %f70, %f72, %f71;\n" " bfe.u32 %r70, %r44, 16, 8;\n" " cvt.rn.f32.u32 %f74, %r70;\n" " sub.rn.f32 %f75, %f74, %f45;\n" " and.b32 %r71, %r54, 3840;\n" " or.b32 %r72, %r71, 1258291200;\n" " mov.b32 %f76, %r72;\n" " fma.rn.f32 %f77, %f76, %f72, %f71;\n" " bfe.u32 %r73, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f78, %r73;\n" " sub.rn.f32 %f79, %f78, %f45;\n" " mul.rn.f32 %f80, %f75, %f73;\n" " fma.rn.f32 %f81, %f80, %f32, %f67;\n" " mul.rn.f32 %f82, %f79, %f77;\n" " fma.rn.f32 %f83, %f82, %f32, %f69;\n" " add.s64 %rd43, %rd58, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd43];\n" " // end inline asm\n" " and.b32 %r74, %r52, 61440;\n" " or.b32 %r75, %r74, 1258291200;\n" " mov.b32 %f84, %r75;\n" " mov.f32 %f85, 0fC5004000;\n" " mov.f32 %f86, 0f39800000;\n" " fma.rn.f32 %f87, %f84, %f86, %f85;\n" " shr.u32 %r76, %r44, 24;\n" " cvt.rn.f32.u32 %f88, %r76;\n" " sub.rn.f32 %f89, %f88, %f45;\n" " and.b32 %r77, %r54, 61440;\n" " or.b32 %r78, %r77, 1258291200;\n" " mov.b32 %f90, %r78;\n" " fma.rn.f32 %f91, %f90, %f86, %f85;\n" " shr.u32 %r79, %r48, 24;\n" " cvt.rn.f32.u32 %f92, %r79;\n" " sub.rn.f32 %f93, %f92, %f45;\n" " mul.rn.f32 %f94, %f89, %f87;\n" " fma.rn.f32 %f95, %f94, %f33, %f81;\n" " mul.rn.f32 %f96, %f93, %f91;\n" " fma.rn.f32 %f97, %f96, %f33, %f83;\n" " add.s64 %rd44, %rd58, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd44];\n" " // end inline asm\n" " and.b32 %r80, %r52, 983040;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f98, %r81;\n" " mov.f32 %f99, 0fC3040000;\n" " mov.f32 %f100, 0f37800000;\n" " fma.rn.f32 %f101, %f98, %f100, %f99;\n" " and.b32 %r82, %r45, 255;\n" " cvt.rn.f32.u32 %f102, %r82;\n" " sub.rn.f32 %f103, %f102, %f45;\n" " and.b32 %r83, %r54, 983040;\n" " or.b32 %r84, %r83, 1258291200;\n" " mov.b32 %f104, %r84;\n" " fma.rn.f32 %f105, %f104, %f100, %f99;\n" " and.b32 %r85, %r49, 255;\n" " cvt.rn.f32.u32 %f106, %r85;\n" " sub.rn.f32 %f107, %f106, %f45;\n" " mul.rn.f32 %f108, %f103, %f101;\n" " fma.rn.f32 %f109, %f108, %f34, %f95;\n" " mul.rn.f32 %f110, %f107, %f105;\n" " fma.rn.f32 %f111, %f110, %f34, %f97;\n" " add.s64 %rd45, %rd58, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd45];\n" " // end inline asm\n" " and.b32 %r86, %r53, 240;\n" " or.b32 %r87, %r86, 1258291200;\n" " mov.b32 %f112, %r87;\n" " fma.rn.f32 %f113, %f112, %f58, %f57;\n" " bfe.u32 %r88, %r45, 8, 8;\n" " cvt.rn.f32.u32 %f114, %r88;\n" " sub.rn.f32 %f115, %f114, %f45;\n" " and.b32 %r89, %r55, 240;\n" " or.b32 %r90, %r89, 1258291200;\n" " mov.b32 %f116, %r90;\n" " fma.rn.f32 %f117, %f116, %f58, %f57;\n" " bfe.u32 %r91, %r49, 8, 8;\n" " cvt.rn.f32.u32 %f118, %r91;\n" " sub.rn.f32 %f119, %f118, %f45;\n" " mul.rn.f32 %f120, %f115, %f113;\n" " fma.rn.f32 %f121, %f120, %f35, %f109;\n" " mul.rn.f32 %f122, %f119, %f117;\n" " fma.rn.f32 %f123, %f122, %f35, %f111;\n" " add.s64 %rd46, %rd58, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd46];\n" " // end inline asm\n" " and.b32 %r92, %r53, 3840;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f124, %r93;\n" " fma.rn.f32 %f125, %f124, %f72, %f71;\n" " bfe.u32 %r94, %r45, 16, 8;\n" " cvt.rn.f32.u32 %f126, %r94;\n" " sub.rn.f32 %f127, %f126, %f45;\n" " and.b32 %r95, %r55, 3840;\n" " or.b32 %r96, %r95, 1258291200;\n" " mov.b32 %f128, %r96;\n" " fma.rn.f32 %f129, %f128, %f72, %f71;\n" " bfe.u32 %r97, %r49, 16, 8;\n" " cvt.rn.f32.u32 %f130, %r97;\n" " sub.rn.f32 %f131, %f130, %f45;\n" " mul.rn.f32 %f132, %f127, %f125;\n" " fma.rn.f32 %f133, %f132, %f36, %f121;\n" " mul.rn.f32 %f134, %f131, %f129;\n" " fma.rn.f32 %f135, %f134, %f36, %f123;\n" " add.s64 %rd47, %rd58, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd47];\n" " // end inline asm\n" " and.b32 %r98, %r53, 61440;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f136, %r99;\n" " fma.rn.f32 %f137, %f136, %f86, %f85;\n" " shr.u32 %r100, %r45, 24;\n" " cvt.rn.f32.u32 %f138, %r100;\n" " sub.rn.f32 %f139, %f138, %f45;\n" " and.b32 %r101, %r55, 61440;\n" " or.b32 %r102, %r101, 1258291200;\n" " mov.b32 %f140, %r102;\n" " fma.rn.f32 %f141, %f140, %f86, %f85;\n" " shr.u32 %r103, %r49, 24;\n" " cvt.rn.f32.u32 %f142, %r103;\n" " sub.rn.f32 %f143, %f142, %f45;\n" " mul.rn.f32 %f144, %f139, %f137;\n" " fma.rn.f32 %f145, %f144, %f37, %f133;\n" " mul.rn.f32 %f146, %f143, %f141;\n" " fma.rn.f32 %f147, %f146, %f37, %f135;\n" " fma.rn.f32 %f185, %f39, %f145, %f185;\n" " fma.rn.f32 %f186, %f38, %f147, %f186;\n" " add.s64 %rd60, %rd60, 152;\n" " add.s64 %rd59, %rd59, 152;\n" " add.s64 %rd58, %rd58, 1024;\n" " add.s64 %rd57, %rd57, 152;\n" " add.s32 %r145, %r145, 1;\n" " setp.lt.s32 %p6, %r145, %r8;\n" " @%p6 bra $L__BB6_4;\n" "\n" "$L__BB6_5:\n" " shl.b64 %rd54, %rd11, 2;\n" " add.s64 %rd53, %rd28, %rd54;\n" " // begin inline asm\n" " ld.global.nc.f32 %f148, [%rd53];\n" " // end inline asm\n" " fma.rn.f32 %f187, %f148, %f185, %f187;\n" " fma.rn.f32 %f188, %f148, %f186, %f188;\n" " cvt.u32.u64 %r104, %rd11;\n" " add.s32 %r144, %r104, %r34;\n" " setp.lt.s32 %p7, %r144, %r30;\n" " @%p7 bra $L__BB6_2;\n" "\n" "$L__BB6_6:\n" " cvta.to.global.u64 %rd24, %rd29;\n" " mov.u32 %r115, 16;\n" " mov.u32 %r124, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f149, %f187, %r115, %r124;\n" " // end inline asm\n" " add.f32 %f152, %f187, %f149;\n" " mov.u32 %r117, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f151, %f152, %r117, %r124;\n" " // end inline asm\n" " add.f32 %f154, %f152, %f151;\n" " mov.u32 %r119, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f153, %f154, %r119, %r124;\n" " // end inline asm\n" " add.f32 %f156, %f154, %f153;\n" " mov.u32 %r121, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f155, %f156, %r121, %r124;\n" " // end inline asm\n" " add.f32 %f158, %f156, %f155;\n" " mov.u32 %r123, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f157, %f158, %r123, %r124;\n" " // end inline asm\n" " add.f32 %f13, %f158, %f157;\n" " // begin inline asm\n" " shfl.down.b32 %f159, %f188, %r115, %r124;\n" " // end inline asm\n" " add.f32 %f162, %f188, %f159;\n" " // begin inline asm\n" " shfl.down.b32 %f161, %f162, %r117, %r124;\n" " // end inline asm\n" " add.f32 %f164, %f162, %f161;\n" " // begin inline asm\n" " shfl.down.b32 %f163, %f164, %r119, %r124;\n" " // end inline asm\n" " add.f32 %f166, %f164, %f163;\n" " // begin inline asm\n" " shfl.down.b32 %f165, %f166, %r121, %r124;\n" " // end inline asm\n" " add.f32 %f168, %f166, %f165;\n" " // begin inline asm\n" " shfl.down.b32 %f167, %f168, %r123, %r124;\n" " // end inline asm\n" " add.f32 %f14, %f168, %f167;\n" " setp.ne.s32 %p8, %r3, 0;\n" " @%p8 bra $L__BB6_8;\n" "\n" " shl.b32 %r125, %r2, 3;\n" " mov.u32 %r126, _ZZ15fc_moe_down_q3kE4sred;\n" " add.s32 %r127, %r126, %r125;\n" " st.shared.f32 [%r127], %f13;\n" " st.shared.f32 [%r127+4], %f14;\n" "\n" "$L__BB6_8:\n" " bar.sync 0;\n" " shl.b32 %r128, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r128;\n" " @%p9 bra $L__BB6_18;\n" "\n" " shr.u32 %r14, %r1, 1;\n" " add.s32 %r129, %r6, %r14;\n" " and.b32 %r15, %r1, 1;\n" " bfi.b32 %r16, %r129, %r15, 1, 31;\n" " setp.ge.s32 %p10, %r16, %r31;\n" " @%p10 bra $L__BB6_18;\n" "\n" " setp.lt.s32 %p11, %r34, 1;\n" " mov.f32 %f193, 0f00000000;\n" " @%p11 bra $L__BB6_17;\n" "\n" " mul.lo.s32 %r17, %r14, %r34;\n" " and.b32 %r150, %r34, 3;\n" " add.s32 %r131, %r34, -1;\n" " setp.lt.u32 %p12, %r131, 3;\n" " mov.f32 %f193, 0f00000000;\n" " mov.u32 %r148, 0;\n" " @%p12 bra $L__BB6_14;\n" "\n" " sub.s32 %r147, %r34, %r150;\n" " mov.f32 %f193, 0f00000000;\n" " mov.u32 %r148, 0;\n" "\n" "$L__BB6_13:\n" " add.s32 %r133, %r148, %r17;\n" " shl.b32 %r134, %r133, 3;\n" " mov.u32 %r135, _ZZ15fc_moe_down_q3kE4sred;\n" " add.s32 %r136, %r135, %r134;\n" " shl.b32 %r137, %r15, 2;\n" " add.s32 %r138, %r136, %r137;\n" " ld.shared.f32 %f173, [%r138];\n" " add.f32 %f174, %f193, %f173;\n" " ld.shared.f32 %f175, [%r138+8];\n" " add.f32 %f176, %f174, %f175;\n" " ld.shared.f32 %f177, [%r138+16];\n" " add.f32 %f178, %f176, %f177;\n" " ld.shared.f32 %f179, [%r138+24];\n" " add.f32 %f193, %f178, %f179;\n" " add.s32 %r148, %r148, 4;\n" " add.s32 %r147, %r147, -4;\n" " setp.ne.s32 %p13, %r147, 0;\n" " @%p13 bra $L__BB6_13;\n" "\n" "$L__BB6_14:\n" " setp.eq.s32 %p14, %r150, 0;\n" " @%p14 bra $L__BB6_17;\n" "\n" " add.s32 %r139, %r148, %r17;\n" " shl.b32 %r140, %r139, 3;\n" " mov.u32 %r141, _ZZ15fc_moe_down_q3kE4sred;\n" " add.s32 %r142, %r141, %r140;\n" " shl.b32 %r143, %r15, 2;\n" " add.s32 %r149, %r142, %r143;\n" "\n" "$L__BB6_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f180, [%r149];\n" " add.f32 %f193, %f193, %f180;\n" " add.s32 %r149, %r149, 8;\n" " add.s32 %r150, %r150, -1;\n" " setp.ne.s32 %p15, %r150, 0;\n" " @%p15 bra $L__BB6_16;\n" "\n" "$L__BB6_17:\n" " mul.wide.s32 %rd55, %r16, 4;\n" " add.s64 %rd56, %rd24, %rd55;\n" " st.global.f32 [%rd56], %f193;\n" "\n" "$L__BB6_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_q4k\n" ".visible .entry fc_moe_down_q4k(\n" " .param .u64 fc_moe_down_q4k_param_0,\n" " .param .u64 fc_moe_down_q4k_param_1,\n" " .param .u64 fc_moe_down_q4k_param_2,\n" " .param .u64 fc_moe_down_q4k_param_3,\n" " .param .u64 fc_moe_down_q4k_param_4,\n" " .param .u32 fc_moe_down_q4k_param_5,\n" " .param .u32 fc_moe_down_q4k_param_6,\n" " .param .u32 fc_moe_down_q4k_param_7,\n" " .param .u32 fc_moe_down_q4k_param_8,\n" " .param .u32 fc_moe_down_q4k_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<247>;\n" " .reg .b32 %r<174>;\n" " .reg .b64 %rd<55>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ15fc_moe_down_q4kE4sred[128];\n" "\n" " ld.param.u64 %rd23, [fc_moe_down_q4k_param_0];\n" " ld.param.u64 %rd24, [fc_moe_down_q4k_param_1];\n" " ld.param.u64 %rd25, [fc_moe_down_q4k_param_2];\n" " ld.param.u64 %rd26, [fc_moe_down_q4k_param_3];\n" " ld.param.u64 %rd27, [fc_moe_down_q4k_param_4];\n" " ld.param.u32 %r30, [fc_moe_down_q4k_param_5];\n" " ld.param.u32 %r31, [fc_moe_down_q4k_param_6];\n" " ld.param.u32 %r32, [fc_moe_down_q4k_param_7];\n" " ld.param.u32 %r33, [fc_moe_down_q4k_param_8];\n" " ld.param.u32 %r34, [fc_moe_down_q4k_param_9];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r35, %ntid.x;\n" " shr.u32 %r36, %r35, 5;\n" " div.u32 %r4, %r36, %r34;\n" " div.s32 %r37, %r2, %r34;\n" " mul.lo.s32 %r38, %r37, %r34;\n" " sub.s32 %r167, %r2, %r38;\n" " mov.u32 %r39, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r39;\n" " add.s32 %r40, %r6, %r37;\n" " shl.b32 %r7, %r40, 1;\n" " setp.ge.s32 %p1, %r7, %r31;\n" " setp.ge.s32 %p2, %r167, %r30;\n" " mov.f32 %f240, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f241, %f240;\n" " @%p3 bra $L__BB7_6;\n" "\n" " cvt.s64.s32 %rd1, %r33;\n" " cvt.s64.s32 %rd2, %r7;\n" " cvt.s64.s32 %rd3, %r32;\n" " cvt.u64.u32 %rd4, %r3;\n" " mul.wide.s32 %rd6, %r3, 4;\n" " add.s64 %rd5, %rd24, %rd6;\n" " shr.s32 %r8, %r32, 8;\n" " cvt.s64.s32 %rd7, %r31;\n" " add.s32 %r9, %r7, 1;\n" " cvta.to.global.u64 %rd8, %rd23;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r9, %r31;\n" " setp.lt.s32 %p5, %r32, 256;\n" " shl.b64 %rd45, %rd4, 2;\n" " mov.f32 %f241, %f25;\n" " mov.f32 %f240, %f25;\n" "\n" "$L__BB7_2:\n" " cvt.s64.s32 %rd9, %r167;\n" " mul.wide.s32 %rd29, %r167, 4;\n" " add.s64 %rd28, %rd25, %rd29;\n" " // begin inline asm\n" " ld.global.nc.s32 %r41, [%rd28];\n" " // end inline asm\n" " cvt.s64.s32 %rd30, %r41;\n" " mul.lo.s64 %rd31, %rd30, %rd7;\n" " add.s64 %rd32, %rd31, %rd2;\n" " mul.lo.s64 %rd33, %rd32, %rd1;\n" " add.s64 %rd54, %rd8, %rd33;\n" " add.s64 %rd34, %rd54, %rd1;\n" " selp.b64 %rd53, %rd34, %rd54, %p4;\n" " mov.f32 %f238, %f25;\n" " mov.f32 %f239, %f25;\n" " @%p5 bra $L__BB7_5;\n" "\n" " mul.lo.s64 %rd35, %rd3, %rd9;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd52, %rd5, %rd36;\n" " add.s64 %rd51, %rd53, %rd6;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r168, 0;\n" " mov.f32 %f239, %f29;\n" " mov.f32 %f238, %f29;\n" "\n" "$L__BB7_4:\n" " add.s64 %rd46, %rd54, %rd45;\n" " ld.global.v2.u32 {%r43, %r44}, [%rd54+128];\n" " ld.global.v2.u32 {%r47, %r48}, [%rd53+128];\n" " ld.global.v2.u32 {%r51, %r52}, [%rd54+136];\n" " ld.global.v2.u32 {%r55, %r56}, [%rd53+136];\n" " ld.global.v2.f32 {%f38, %f39}, [%rd54+144];\n" " ld.global.v2.f32 {%f42, %f43}, [%rd53+144];\n" " ld.global.u32 %r59, [%rd46];\n" " shr.u32 %r60, %r59, 16;\n" " ld.global.u32 %r61, [%rd51];\n" " shr.u32 %r62, %r61, 16;\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd52];\n" " // end inline asm\n" " and.b32 %r63, %r43, 255;\n" " cvt.rn.f32.u32 %f46, %r63;\n" " mul.rn.f32 %f47, %f38, %f46;\n" " and.b32 %r64, %r59, 15;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f48, %r65;\n" " mov.f32 %f49, 0fCB000000;\n" " mov.f32 %f50, 0f3F800000;\n" " fma.rn.f32 %f51, %f48, %f50, %f49;\n" " mul.rn.f32 %f52, %f47, %f51;\n" " and.b32 %r66, %r51, 255;\n" " cvt.rn.f32.u32 %f53, %r66;\n" " mul.rn.f32 %f54, %f39, %f53;\n" " sub.rn.f32 %f55, %f52, %f54;\n" " and.b32 %r67, %r47, 255;\n" " cvt.rn.f32.u32 %f56, %r67;\n" " mul.rn.f32 %f57, %f42, %f56;\n" " and.b32 %r68, %r61, 15;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f58, %r69;\n" " fma.rn.f32 %f59, %f58, %f50, %f49;\n" " mul.rn.f32 %f60, %f57, %f59;\n" " and.b32 %r70, %r55, 255;\n" " cvt.rn.f32.u32 %f61, %r70;\n" " mul.rn.f32 %f62, %f43, %f61;\n" " sub.rn.f32 %f63, %f60, %f62;\n" " fma.rn.f32 %f65, %f55, %f30, %f29;\n" " fma.rn.f32 %f66, %f63, %f30, %f29;\n" " add.s64 %rd38, %rd52, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd38];\n" " // end inline asm\n" " bfe.u32 %r71, %r43, 8, 8;\n" " cvt.rn.f32.u32 %f67, %r71;\n" " mul.rn.f32 %f68, %f38, %f67;\n" " and.b32 %r72, %r59, 240;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f69, %r73;\n" " mov.f32 %f70, 0fC9000000;\n" " mov.f32 %f71, 0f3D800000;\n" " fma.rn.f32 %f72, %f69, %f71, %f70;\n" " mul.rn.f32 %f73, %f68, %f72;\n" " bfe.u32 %r74, %r51, 8, 8;\n" " cvt.rn.f32.u32 %f74, %r74;\n" " mul.rn.f32 %f75, %f39, %f74;\n" " sub.rn.f32 %f76, %f73, %f75;\n" " bfe.u32 %r75, %r47, 8, 8;\n" " cvt.rn.f32.u32 %f77, %r75;\n" " mul.rn.f32 %f78, %f42, %f77;\n" " and.b32 %r76, %r61, 240;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f79, %r77;\n" " fma.rn.f32 %f80, %f79, %f71, %f70;\n" " mul.rn.f32 %f81, %f78, %f80;\n" " bfe.u32 %r78, %r55, 8, 8;\n" " cvt.rn.f32.u32 %f82, %r78;\n" " mul.rn.f32 %f83, %f43, %f82;\n" " sub.rn.f32 %f84, %f81, %f83;\n" " fma.rn.f32 %f85, %f76, %f31, %f65;\n" " fma.rn.f32 %f86, %f84, %f31, %f66;\n" " add.s64 %rd39, %rd52, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd39];\n" " // end inline asm\n" " bfe.u32 %r79, %r43, 16, 8;\n" " cvt.rn.f32.u32 %f87, %r79;\n" " mul.rn.f32 %f88, %f38, %f87;\n" " and.b32 %r80, %r59, 3840;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f89, %r81;\n" " mov.f32 %f90, 0fC7000000;\n" " mov.f32 %f91, 0f3B800000;\n" " fma.rn.f32 %f92, %f89, %f91, %f90;\n" " mul.rn.f32 %f93, %f88, %f92;\n" " bfe.u32 %r82, %r51, 16, 8;\n" " cvt.rn.f32.u32 %f94, %r82;\n" " mul.rn.f32 %f95, %f39, %f94;\n" " sub.rn.f32 %f96, %f93, %f95;\n" " bfe.u32 %r83, %r47, 16, 8;\n" " cvt.rn.f32.u32 %f97, %r83;\n" " mul.rn.f32 %f98, %f42, %f97;\n" " and.b32 %r84, %r61, 3840;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f99, %r85;\n" " fma.rn.f32 %f100, %f99, %f91, %f90;\n" " mul.rn.f32 %f101, %f98, %f100;\n" " bfe.u32 %r86, %r55, 16, 8;\n" " cvt.rn.f32.u32 %f102, %r86;\n" " mul.rn.f32 %f103, %f43, %f102;\n" " sub.rn.f32 %f104, %f101, %f103;\n" " fma.rn.f32 %f105, %f96, %f32, %f85;\n" " fma.rn.f32 %f106, %f104, %f32, %f86;\n" " add.s64 %rd40, %rd52, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd40];\n" " // end inline asm\n" " shr.u32 %r87, %r43, 24;\n" " cvt.rn.f32.u32 %f107, %r87;\n" " mul.rn.f32 %f108, %f38, %f107;\n" " and.b32 %r88, %r59, 61440;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f109, %r89;\n" " mov.f32 %f110, 0fC5000000;\n" " mov.f32 %f111, 0f39800000;\n" " fma.rn.f32 %f112, %f109, %f111, %f110;\n" " mul.rn.f32 %f113, %f108, %f112;\n" " shr.u32 %r90, %r51, 24;\n" " cvt.rn.f32.u32 %f114, %r90;\n" " mul.rn.f32 %f115, %f39, %f114;\n" " sub.rn.f32 %f116, %f113, %f115;\n" " shr.u32 %r91, %r47, 24;\n" " cvt.rn.f32.u32 %f117, %r91;\n" " mul.rn.f32 %f118, %f42, %f117;\n" " and.b32 %r92, %r61, 61440;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f119, %r93;\n" " fma.rn.f32 %f120, %f119, %f111, %f110;\n" " mul.rn.f32 %f121, %f118, %f120;\n" " shr.u32 %r94, %r55, 24;\n" " cvt.rn.f32.u32 %f122, %r94;\n" " mul.rn.f32 %f123, %f43, %f122;\n" " sub.rn.f32 %f124, %f121, %f123;\n" " fma.rn.f32 %f125, %f116, %f33, %f105;\n" " fma.rn.f32 %f126, %f124, %f33, %f106;\n" " add.s64 %rd41, %rd52, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd41];\n" " // end inline asm\n" " and.b32 %r95, %r44, 255;\n" " cvt.rn.f32.u32 %f127, %r95;\n" " mul.rn.f32 %f128, %f38, %f127;\n" " and.b32 %r96, %r59, 983040;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f129, %r97;\n" " mov.f32 %f130, 0fC3000000;\n" " mov.f32 %f131, 0f37800000;\n" " fma.rn.f32 %f132, %f129, %f131, %f130;\n" " mul.rn.f32 %f133, %f128, %f132;\n" " and.b32 %r98, %r52, 255;\n" " cvt.rn.f32.u32 %f134, %r98;\n" " mul.rn.f32 %f135, %f39, %f134;\n" " sub.rn.f32 %f136, %f133, %f135;\n" " and.b32 %r99, %r48, 255;\n" " cvt.rn.f32.u32 %f137, %r99;\n" " mul.rn.f32 %f138, %f42, %f137;\n" " and.b32 %r100, %r61, 983040;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f139, %r101;\n" " fma.rn.f32 %f140, %f139, %f131, %f130;\n" " mul.rn.f32 %f141, %f138, %f140;\n" " and.b32 %r102, %r56, 255;\n" " cvt.rn.f32.u32 %f142, %r102;\n" " mul.rn.f32 %f143, %f43, %f142;\n" " sub.rn.f32 %f144, %f141, %f143;\n" " fma.rn.f32 %f145, %f136, %f34, %f125;\n" " fma.rn.f32 %f146, %f144, %f34, %f126;\n" " add.s64 %rd42, %rd52, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd42];\n" " // end inline asm\n" " bfe.u32 %r103, %r44, 8, 8;\n" " cvt.rn.f32.u32 %f147, %r103;\n" " mul.rn.f32 %f148, %f38, %f147;\n" " and.b32 %r104, %r60, 240;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f149, %r105;\n" " fma.rn.f32 %f150, %f149, %f71, %f70;\n" " mul.rn.f32 %f151, %f148, %f150;\n" " bfe.u32 %r106, %r52, 8, 8;\n" " cvt.rn.f32.u32 %f152, %r106;\n" " mul.rn.f32 %f153, %f39, %f152;\n" " sub.rn.f32 %f154, %f151, %f153;\n" " bfe.u32 %r107, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f155, %r107;\n" " mul.rn.f32 %f156, %f42, %f155;\n" " and.b32 %r108, %r62, 240;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f157, %r109;\n" " fma.rn.f32 %f158, %f157, %f71, %f70;\n" " mul.rn.f32 %f159, %f156, %f158;\n" " bfe.u32 %r110, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f160, %r110;\n" " mul.rn.f32 %f161, %f43, %f160;\n" " sub.rn.f32 %f162, %f159, %f161;\n" " fma.rn.f32 %f163, %f154, %f35, %f145;\n" " fma.rn.f32 %f164, %f162, %f35, %f146;\n" " add.s64 %rd43, %rd52, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd43];\n" " // end inline asm\n" " bfe.u32 %r111, %r44, 16, 8;\n" " cvt.rn.f32.u32 %f165, %r111;\n" " mul.rn.f32 %f166, %f38, %f165;\n" " and.b32 %r112, %r60, 3840;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f167, %r113;\n" " fma.rn.f32 %f168, %f167, %f91, %f90;\n" " mul.rn.f32 %f169, %f166, %f168;\n" " bfe.u32 %r114, %r52, 16, 8;\n" " cvt.rn.f32.u32 %f170, %r114;\n" " mul.rn.f32 %f171, %f39, %f170;\n" " sub.rn.f32 %f172, %f169, %f171;\n" " bfe.u32 %r115, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f173, %r115;\n" " mul.rn.f32 %f174, %f42, %f173;\n" " and.b32 %r116, %r62, 3840;\n" " or.b32 %r117, %r116, 1258291200;\n" " mov.b32 %f175, %r117;\n" " fma.rn.f32 %f176, %f175, %f91, %f90;\n" " mul.rn.f32 %f177, %f174, %f176;\n" " bfe.u32 %r118, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f178, %r118;\n" " mul.rn.f32 %f179, %f43, %f178;\n" " sub.rn.f32 %f180, %f177, %f179;\n" " fma.rn.f32 %f181, %f172, %f36, %f163;\n" " fma.rn.f32 %f182, %f180, %f36, %f164;\n" " add.s64 %rd44, %rd52, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd44];\n" " // end inline asm\n" " shr.u32 %r119, %r44, 24;\n" " cvt.rn.f32.u32 %f183, %r119;\n" " mul.rn.f32 %f184, %f38, %f183;\n" " and.b32 %r120, %r60, 61440;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f185, %r121;\n" " fma.rn.f32 %f186, %f185, %f111, %f110;\n" " mul.rn.f32 %f187, %f184, %f186;\n" " shr.u32 %r122, %r52, 24;\n" " cvt.rn.f32.u32 %f188, %r122;\n" " mul.rn.f32 %f189, %f39, %f188;\n" " sub.rn.f32 %f190, %f187, %f189;\n" " shr.u32 %r123, %r48, 24;\n" " cvt.rn.f32.u32 %f191, %r123;\n" " mul.rn.f32 %f192, %f42, %f191;\n" " and.b32 %r124, %r62, 61440;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f193, %r125;\n" " fma.rn.f32 %f194, %f193, %f111, %f110;\n" " mul.rn.f32 %f195, %f192, %f194;\n" " shr.u32 %r126, %r56, 24;\n" " cvt.rn.f32.u32 %f196, %r126;\n" " mul.rn.f32 %f197, %f43, %f196;\n" " sub.rn.f32 %f198, %f195, %f197;\n" " fma.rn.f32 %f199, %f190, %f37, %f181;\n" " fma.rn.f32 %f200, %f198, %f37, %f182;\n" " add.f32 %f238, %f238, %f199;\n" " add.f32 %f239, %f239, %f200;\n" " add.s64 %rd54, %rd54, 152;\n" " add.s64 %rd53, %rd53, 152;\n" " add.s64 %rd52, %rd52, 1024;\n" " add.s64 %rd51, %rd51, 152;\n" " add.s32 %r168, %r168, 1;\n" " setp.lt.s32 %p6, %r168, %r8;\n" " @%p6 bra $L__BB7_4;\n" "\n" "$L__BB7_5:\n" " shl.b64 %rd48, %rd9, 2;\n" " add.s64 %rd47, %rd26, %rd48;\n" " // begin inline asm\n" " ld.global.nc.f32 %f201, [%rd47];\n" " // end inline asm\n" " fma.rn.f32 %f240, %f201, %f238, %f240;\n" " fma.rn.f32 %f241, %f201, %f239, %f241;\n" " cvt.u32.u64 %r127, %rd9;\n" " add.s32 %r167, %r127, %r34;\n" " setp.lt.s32 %p7, %r167, %r30;\n" " @%p7 bra $L__BB7_2;\n" "\n" "$L__BB7_6:\n" " cvta.to.global.u64 %rd22, %rd27;\n" " mov.u32 %r138, 16;\n" " mov.u32 %r147, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f202, %f240, %r138, %r147;\n" " // end inline asm\n" " add.f32 %f205, %f240, %f202;\n" " mov.u32 %r140, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f204, %f205, %r140, %r147;\n" " // end inline asm\n" " add.f32 %f207, %f205, %f204;\n" " mov.u32 %r142, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f206, %f207, %r142, %r147;\n" " // end inline asm\n" " add.f32 %f209, %f207, %f206;\n" " mov.u32 %r144, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f208, %f209, %r144, %r147;\n" " // end inline asm\n" " add.f32 %f211, %f209, %f208;\n" " mov.u32 %r146, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f210, %f211, %r146, %r147;\n" " // end inline asm\n" " add.f32 %f13, %f211, %f210;\n" " // begin inline asm\n" " shfl.down.b32 %f212, %f241, %r138, %r147;\n" " // end inline asm\n" " add.f32 %f215, %f241, %f212;\n" " // begin inline asm\n" " shfl.down.b32 %f214, %f215, %r140, %r147;\n" " // end inline asm\n" " add.f32 %f217, %f215, %f214;\n" " // begin inline asm\n" " shfl.down.b32 %f216, %f217, %r142, %r147;\n" " // end inline asm\n" " add.f32 %f219, %f217, %f216;\n" " // begin inline asm\n" " shfl.down.b32 %f218, %f219, %r144, %r147;\n" " // end inline asm\n" " add.f32 %f221, %f219, %f218;\n" " // begin inline asm\n" " shfl.down.b32 %f220, %f221, %r146, %r147;\n" " // end inline asm\n" " add.f32 %f14, %f221, %f220;\n" " setp.ne.s32 %p8, %r3, 0;\n" " @%p8 bra $L__BB7_8;\n" "\n" " shl.b32 %r148, %r2, 3;\n" " mov.u32 %r149, _ZZ15fc_moe_down_q4kE4sred;\n" " add.s32 %r150, %r149, %r148;\n" " st.shared.f32 [%r150], %f13;\n" " st.shared.f32 [%r150+4], %f14;\n" "\n" "$L__BB7_8:\n" " bar.sync 0;\n" " shl.b32 %r151, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r151;\n" " @%p9 bra $L__BB7_18;\n" "\n" " shr.u32 %r14, %r1, 1;\n" " add.s32 %r152, %r6, %r14;\n" " and.b32 %r15, %r1, 1;\n" " bfi.b32 %r16, %r152, %r15, 1, 31;\n" " setp.ge.s32 %p10, %r16, %r31;\n" " @%p10 bra $L__BB7_18;\n" "\n" " setp.lt.s32 %p11, %r34, 1;\n" " mov.f32 %f246, 0f00000000;\n" " @%p11 bra $L__BB7_17;\n" "\n" " mul.lo.s32 %r17, %r14, %r34;\n" " and.b32 %r173, %r34, 3;\n" " add.s32 %r154, %r34, -1;\n" " setp.lt.u32 %p12, %r154, 3;\n" " mov.f32 %f246, 0f00000000;\n" " mov.u32 %r171, 0;\n" " @%p12 bra $L__BB7_14;\n" "\n" " sub.s32 %r170, %r34, %r173;\n" " mov.f32 %f246, 0f00000000;\n" " mov.u32 %r171, 0;\n" "\n" "$L__BB7_13:\n" " add.s32 %r156, %r171, %r17;\n" " shl.b32 %r157, %r156, 3;\n" " mov.u32 %r158, _ZZ15fc_moe_down_q4kE4sred;\n" " add.s32 %r159, %r158, %r157;\n" " shl.b32 %r160, %r15, 2;\n" " add.s32 %r161, %r159, %r160;\n" " ld.shared.f32 %f226, [%r161];\n" " add.f32 %f227, %f246, %f226;\n" " ld.shared.f32 %f228, [%r161+8];\n" " add.f32 %f229, %f227, %f228;\n" " ld.shared.f32 %f230, [%r161+16];\n" " add.f32 %f231, %f229, %f230;\n" " ld.shared.f32 %f232, [%r161+24];\n" " add.f32 %f246, %f231, %f232;\n" " add.s32 %r171, %r171, 4;\n" " add.s32 %r170, %r170, -4;\n" " setp.ne.s32 %p13, %r170, 0;\n" " @%p13 bra $L__BB7_13;\n" "\n" "$L__BB7_14:\n" " setp.eq.s32 %p14, %r173, 0;\n" " @%p14 bra $L__BB7_17;\n" "\n" " add.s32 %r162, %r171, %r17;\n" " shl.b32 %r163, %r162, 3;\n" " mov.u32 %r164, _ZZ15fc_moe_down_q4kE4sred;\n" " add.s32 %r165, %r164, %r163;\n" " shl.b32 %r166, %r15, 2;\n" " add.s32 %r172, %r165, %r166;\n" "\n" "$L__BB7_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f233, [%r172];\n" " add.f32 %f246, %f246, %f233;\n" " add.s32 %r172, %r172, 8;\n" " add.s32 %r173, %r173, -1;\n" " setp.ne.s32 %p15, %r173, 0;\n" " @%p15 bra $L__BB7_16;\n" "\n" "$L__BB7_17:\n" " mul.wide.s32 %rd49, %r16, 4;\n" " add.s64 %rd50, %rd22, %rd49;\n" " st.global.f32 [%rd50], %f246;\n" "\n" "$L__BB7_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_q6k\n" ".visible .entry fc_moe_down_q6k(\n" " .param .u64 fc_moe_down_q6k_param_0,\n" " .param .u64 fc_moe_down_q6k_param_1,\n" " .param .u64 fc_moe_down_q6k_param_2,\n" " .param .u64 fc_moe_down_q6k_param_3,\n" " .param .u64 fc_moe_down_q6k_param_4,\n" " .param .u32 fc_moe_down_q6k_param_5,\n" " .param .u32 fc_moe_down_q6k_param_6,\n" " .param .u32 fc_moe_down_q6k_param_7,\n" " .param .u32 fc_moe_down_q6k_param_8,\n" " .param .u32 fc_moe_down_q6k_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<255>;\n" " .reg .b32 %r<185>;\n" " .reg .b64 %rd<69>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ15fc_moe_down_q6kE4sred[128];\n" "\n" " ld.param.u64 %rd29, [fc_moe_down_q6k_param_0];\n" " ld.param.u64 %rd30, [fc_moe_down_q6k_param_1];\n" " ld.param.u64 %rd31, [fc_moe_down_q6k_param_2];\n" " ld.param.u64 %rd32, [fc_moe_down_q6k_param_3];\n" " ld.param.u64 %rd33, [fc_moe_down_q6k_param_4];\n" " ld.param.u32 %r30, [fc_moe_down_q6k_param_5];\n" " ld.param.u32 %r31, [fc_moe_down_q6k_param_6];\n" " ld.param.u32 %r32, [fc_moe_down_q6k_param_7];\n" " ld.param.u32 %r33, [fc_moe_down_q6k_param_8];\n" " ld.param.u32 %r34, [fc_moe_down_q6k_param_9];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r35, %ntid.x;\n" " shr.u32 %r36, %r35, 5;\n" " div.u32 %r4, %r36, %r34;\n" " div.s32 %r37, %r2, %r34;\n" " mul.lo.s32 %r38, %r37, %r34;\n" " sub.s32 %r178, %r2, %r38;\n" " mov.u32 %r39, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r39;\n" " add.s32 %r40, %r6, %r37;\n" " shl.b32 %r7, %r40, 1;\n" " setp.ge.s32 %p1, %r7, %r31;\n" " setp.ge.s32 %p2, %r178, %r30;\n" " mov.f32 %f248, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f249, %f248;\n" " @%p3 bra $L__BB8_6;\n" "\n" " cvt.s64.s32 %rd1, %r33;\n" " cvt.s64.s32 %rd2, %r7;\n" " cvt.s64.s32 %rd3, %r32;\n" " shr.u32 %r41, %r3, 4;\n" " cvt.u64.u32 %rd4, %r41;\n" " cvt.u64.u32 %rd5, %r3;\n" " mul.wide.s32 %rd7, %r3, 4;\n" " add.s64 %rd6, %rd30, %rd7;\n" " mul.wide.s32 %rd34, %r3, 2;\n" " or.b64 %rd8, %rd34, 128;\n" " mul.wide.s32 %rd35, %r41, 8;\n" " or.b64 %rd9, %rd35, 192;\n" " shr.s32 %r8, %r32, 8;\n" " cvt.s64.s32 %rd10, %r31;\n" " add.s32 %r9, %r7, 1;\n" " cvta.to.global.u64 %rd11, %rd29;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r9, %r31;\n" " setp.lt.s32 %p5, %r32, 256;\n" " shl.b64 %rd57, %rd4, 3;\n" " mov.f32 %f249, %f25;\n" " mov.f32 %f248, %f25;\n" "\n" "$L__BB8_2:\n" " cvt.s64.s32 %rd12, %r178;\n" " mul.wide.s32 %rd37, %r178, 4;\n" " add.s64 %rd36, %rd31, %rd37;\n" " // begin inline asm\n" " ld.global.nc.s32 %r42, [%rd36];\n" " // end inline asm\n" " cvt.s64.s32 %rd38, %r42;\n" " mul.lo.s64 %rd39, %rd38, %rd10;\n" " add.s64 %rd40, %rd39, %rd2;\n" " mul.lo.s64 %rd41, %rd40, %rd1;\n" " add.s64 %rd68, %rd11, %rd41;\n" " add.s64 %rd42, %rd68, %rd1;\n" " selp.b64 %rd67, %rd42, %rd68, %p4;\n" " mov.f32 %f246, %f25;\n" " mov.f32 %f247, %f25;\n" " @%p5 bra $L__BB8_5;\n" "\n" " mul.lo.s64 %rd43, %rd3, %rd12;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd66, %rd6, %rd44;\n" " add.s64 %rd65, %rd67, %rd7;\n" " add.s64 %rd64, %rd67, %rd8;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r179, 0;\n" " mov.f32 %f247, %f29;\n" " mov.f32 %f246, %f29;\n" "\n" "$L__BB8_4:\n" " shl.b64 %rd53, %rd5, 2;\n" " add.s64 %rd54, %rd68, %rd53;\n" " shl.b64 %rd55, %rd5, 1;\n" " add.s64 %rd56, %rd68, %rd55;\n" " ld.global.u16 %r44, [%rd56+128];\n" " ld.global.u16 %r45, [%rd64];\n" " ld.global.f32 %f38, [%rd68+208];\n" " ld.global.f32 %f39, [%rd67+208];\n" " add.s64 %rd58, %rd68, %rd57;\n" " ld.global.v2.u32 {%r46, %r47}, [%rd58+192];\n" " add.s64 %rd59, %rd67, %rd9;\n" " ld.global.v2.u32 {%r50, %r51}, [%rd59];\n" " ld.global.u32 %r54, [%rd54];\n" " shr.u32 %r55, %r54, 16;\n" " ld.global.u32 %r56, [%rd65];\n" " shr.u32 %r57, %r56, 16;\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd66];\n" " // end inline asm\n" " and.b32 %r58, %r54, 15;\n" " or.b32 %r59, %r58, 1258291200;\n" " mov.b32 %f40, %r59;\n" " mov.f32 %f41, 0fCB000020;\n" " mov.f32 %f42, 0f3F800000;\n" " fma.rn.f32 %f43, %f40, %f42, %f41;\n" " and.b32 %r60, %r44, 3;\n" " or.b32 %r61, %r60, 1258291200;\n" " mov.b32 %f44, %r61;\n" " mov.f32 %f45, 0fCD000000;\n" " mov.f32 %f46, 0f41800000;\n" " fma.rn.f32 %f47, %f44, %f46, %f45;\n" " add.rn.f32 %f48, %f43, %f47;\n" " and.b32 %r62, %r56, 15;\n" " or.b32 %r63, %r62, 1258291200;\n" " mov.b32 %f49, %r63;\n" " fma.rn.f32 %f50, %f49, %f42, %f41;\n" " and.b32 %r64, %r45, 3;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f51, %r65;\n" " fma.rn.f32 %f52, %f51, %f46, %f45;\n" " add.rn.f32 %f53, %f50, %f52;\n" " and.b32 %r66, %r46, 255;\n" " cvt.rn.f32.u32 %f54, %r66;\n" " mov.f32 %f55, 0f43000000;\n" " sub.rn.f32 %f56, %f54, %f55;\n" " and.b32 %r67, %r50, 255;\n" " cvt.rn.f32.u32 %f57, %r67;\n" " sub.rn.f32 %f58, %f57, %f55;\n" " mul.rn.f32 %f59, %f56, %f48;\n" " fma.rn.f32 %f61, %f59, %f30, %f29;\n" " mul.rn.f32 %f62, %f58, %f53;\n" " fma.rn.f32 %f63, %f62, %f30, %f29;\n" " add.s64 %rd46, %rd66, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd46];\n" " // end inline asm\n" " and.b32 %r68, %r54, 3840;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f64, %r69;\n" " mov.f32 %f65, 0fC7002000;\n" " mov.f32 %f66, 0f3B800000;\n" " fma.rn.f32 %f67, %f64, %f66, %f65;\n" " and.b32 %r70, %r44, 12;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f68, %r71;\n" " mov.f32 %f69, 0fCC000000;\n" " mov.f32 %f70, 0f40800000;\n" " fma.rn.f32 %f71, %f68, %f70, %f69;\n" " add.rn.f32 %f72, %f67, %f71;\n" " and.b32 %r72, %r56, 3840;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f73, %r73;\n" " fma.rn.f32 %f74, %f73, %f66, %f65;\n" " and.b32 %r74, %r45, 12;\n" " or.b32 %r75, %r74, 1258291200;\n" " mov.b32 %f75, %r75;\n" " fma.rn.f32 %f76, %f75, %f70, %f69;\n" " add.rn.f32 %f77, %f74, %f76;\n" " bfe.u32 %r76, %r46, 8, 8;\n" " cvt.rn.f32.u32 %f78, %r76;\n" " sub.rn.f32 %f79, %f78, %f55;\n" " bfe.u32 %r77, %r50, 8, 8;\n" " cvt.rn.f32.u32 %f80, %r77;\n" " sub.rn.f32 %f81, %f80, %f55;\n" " mul.rn.f32 %f82, %f79, %f72;\n" " fma.rn.f32 %f83, %f82, %f31, %f61;\n" " mul.rn.f32 %f84, %f81, %f77;\n" " fma.rn.f32 %f85, %f84, %f31, %f63;\n" " add.s64 %rd47, %rd66, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd47];\n" " // end inline asm\n" " and.b32 %r78, %r54, 240;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f86, %r79;\n" " mov.f32 %f87, 0fC9000200;\n" " mov.f32 %f88, 0f3D800000;\n" " fma.rn.f32 %f89, %f86, %f88, %f87;\n" " and.b32 %r80, %r44, 48;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f90, %r81;\n" " mov.f32 %f91, 0fCB000000;\n" " fma.rn.f32 %f92, %f90, %f42, %f91;\n" " add.rn.f32 %f93, %f89, %f92;\n" " and.b32 %r82, %r56, 240;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f94, %r83;\n" " fma.rn.f32 %f95, %f94, %f88, %f87;\n" " and.b32 %r84, %r45, 48;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f96, %r85;\n" " fma.rn.f32 %f97, %f96, %f42, %f91;\n" " add.rn.f32 %f98, %f95, %f97;\n" " bfe.u32 %r86, %r46, 16, 8;\n" " cvt.rn.f32.u32 %f99, %r86;\n" " sub.rn.f32 %f100, %f99, %f55;\n" " bfe.u32 %r87, %r50, 16, 8;\n" " cvt.rn.f32.u32 %f101, %r87;\n" " sub.rn.f32 %f102, %f101, %f55;\n" " mul.rn.f32 %f103, %f100, %f93;\n" " fma.rn.f32 %f104, %f103, %f32, %f83;\n" " mul.rn.f32 %f105, %f102, %f98;\n" " fma.rn.f32 %f106, %f105, %f32, %f85;\n" " add.s64 %rd48, %rd66, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd48];\n" " // end inline asm\n" " and.b32 %r88, %r54, 61440;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f107, %r89;\n" " mov.f32 %f108, 0fC5020000;\n" " mov.f32 %f109, 0f39800000;\n" " fma.rn.f32 %f110, %f107, %f109, %f108;\n" " and.b32 %r90, %r44, 192;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f111, %r91;\n" " mov.f32 %f112, 0fCA000000;\n" " mov.f32 %f113, 0f3E800000;\n" " fma.rn.f32 %f114, %f111, %f113, %f112;\n" " add.rn.f32 %f115, %f110, %f114;\n" " and.b32 %r92, %r56, 61440;\n" " or.b32 %r93, %r92, 1258291200;\n" " mov.b32 %f116, %r93;\n" " fma.rn.f32 %f117, %f116, %f109, %f108;\n" " and.b32 %r94, %r45, 192;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f118, %r95;\n" " fma.rn.f32 %f119, %f118, %f113, %f112;\n" " add.rn.f32 %f120, %f117, %f119;\n" " shr.u32 %r96, %r46, 24;\n" " cvt.rn.f32.u32 %f121, %r96;\n" " sub.rn.f32 %f122, %f121, %f55;\n" " shr.u32 %r97, %r50, 24;\n" " cvt.rn.f32.u32 %f123, %r97;\n" " sub.rn.f32 %f124, %f123, %f55;\n" " mul.rn.f32 %f125, %f122, %f115;\n" " fma.rn.f32 %f126, %f125, %f33, %f104;\n" " mul.rn.f32 %f127, %f124, %f120;\n" " fma.rn.f32 %f128, %f127, %f33, %f106;\n" " add.s64 %rd49, %rd66, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd49];\n" " // end inline asm\n" " and.b32 %r98, %r54, 983040;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f129, %r99;\n" " mov.f32 %f130, 0fC3200000;\n" " mov.f32 %f131, 0f37800000;\n" " fma.rn.f32 %f132, %f129, %f131, %f130;\n" " and.b32 %r100, %r44, 768;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f133, %r101;\n" " mov.f32 %f134, 0fC9000000;\n" " fma.rn.f32 %f135, %f133, %f88, %f134;\n" " add.rn.f32 %f136, %f132, %f135;\n" " and.b32 %r102, %r56, 983040;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f137, %r103;\n" " fma.rn.f32 %f138, %f137, %f131, %f130;\n" " and.b32 %r104, %r45, 768;\n" " or.b32 %r105, %r104, 1258291200;\n" " mov.b32 %f139, %r105;\n" " fma.rn.f32 %f140, %f139, %f88, %f134;\n" " add.rn.f32 %f141, %f138, %f140;\n" " and.b32 %r106, %r47, 255;\n" " cvt.rn.f32.u32 %f142, %r106;\n" " sub.rn.f32 %f143, %f142, %f55;\n" " and.b32 %r107, %r51, 255;\n" " cvt.rn.f32.u32 %f144, %r107;\n" " sub.rn.f32 %f145, %f144, %f55;\n" " mul.rn.f32 %f146, %f143, %f136;\n" " fma.rn.f32 %f147, %f146, %f34, %f126;\n" " mul.rn.f32 %f148, %f145, %f141;\n" " fma.rn.f32 %f149, %f148, %f34, %f128;\n" " add.s64 %rd50, %rd66, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd50];\n" " // end inline asm\n" " and.b32 %r108, %r55, 3840;\n" " or.b32 %r109, %r108, 1258291200;\n" " mov.b32 %f150, %r109;\n" " fma.rn.f32 %f151, %f150, %f66, %f65;\n" " and.b32 %r110, %r44, 3072;\n" " or.b32 %r111, %r110, 1258291200;\n" " mov.b32 %f152, %r111;\n" " mov.f32 %f153, 0fC8000000;\n" " mov.f32 %f154, 0f3C800000;\n" " fma.rn.f32 %f155, %f152, %f154, %f153;\n" " add.rn.f32 %f156, %f151, %f155;\n" " and.b32 %r112, %r57, 3840;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f157, %r113;\n" " fma.rn.f32 %f158, %f157, %f66, %f65;\n" " and.b32 %r114, %r45, 3072;\n" " or.b32 %r115, %r114, 1258291200;\n" " mov.b32 %f159, %r115;\n" " fma.rn.f32 %f160, %f159, %f154, %f153;\n" " add.rn.f32 %f161, %f158, %f160;\n" " bfe.u32 %r116, %r47, 8, 8;\n" " cvt.rn.f32.u32 %f162, %r116;\n" " sub.rn.f32 %f163, %f162, %f55;\n" " bfe.u32 %r117, %r51, 8, 8;\n" " cvt.rn.f32.u32 %f164, %r117;\n" " sub.rn.f32 %f165, %f164, %f55;\n" " mul.rn.f32 %f166, %f163, %f156;\n" " fma.rn.f32 %f167, %f166, %f35, %f147;\n" " mul.rn.f32 %f168, %f165, %f161;\n" " fma.rn.f32 %f169, %f168, %f35, %f149;\n" " add.s64 %rd51, %rd66, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd51];\n" " // end inline asm\n" " and.b32 %r118, %r55, 240;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f170, %r119;\n" " fma.rn.f32 %f171, %f170, %f88, %f87;\n" " and.b32 %r120, %r44, 12288;\n" " or.b32 %r121, %r120, 1258291200;\n" " mov.b32 %f172, %r121;\n" " mov.f32 %f173, 0fC7000000;\n" " fma.rn.f32 %f174, %f172, %f66, %f173;\n" " add.rn.f32 %f175, %f171, %f174;\n" " and.b32 %r122, %r57, 240;\n" " or.b32 %r123, %r122, 1258291200;\n" " mov.b32 %f176, %r123;\n" " fma.rn.f32 %f177, %f176, %f88, %f87;\n" " and.b32 %r124, %r45, 12288;\n" " or.b32 %r125, %r124, 1258291200;\n" " mov.b32 %f178, %r125;\n" " fma.rn.f32 %f179, %f178, %f66, %f173;\n" " add.rn.f32 %f180, %f177, %f179;\n" " bfe.u32 %r126, %r47, 16, 8;\n" " cvt.rn.f32.u32 %f181, %r126;\n" " sub.rn.f32 %f182, %f181, %f55;\n" " bfe.u32 %r127, %r51, 16, 8;\n" " cvt.rn.f32.u32 %f183, %r127;\n" " sub.rn.f32 %f184, %f183, %f55;\n" " mul.rn.f32 %f185, %f182, %f175;\n" " fma.rn.f32 %f186, %f185, %f36, %f167;\n" " mul.rn.f32 %f187, %f184, %f180;\n" " fma.rn.f32 %f188, %f187, %f36, %f169;\n" " add.s64 %rd52, %rd66, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd52];\n" " // end inline asm\n" " and.b32 %r128, %r55, 61440;\n" " or.b32 %r129, %r128, 1258291200;\n" " mov.b32 %f189, %r129;\n" " fma.rn.f32 %f190, %f189, %f109, %f108;\n" " and.b32 %r130, %r44, 49152;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f191, %r131;\n" " mov.f32 %f192, 0fC6000000;\n" " mov.f32 %f193, 0f3A800000;\n" " fma.rn.f32 %f194, %f191, %f193, %f192;\n" " add.rn.f32 %f195, %f190, %f194;\n" " and.b32 %r132, %r57, 61440;\n" " or.b32 %r133, %r132, 1258291200;\n" " mov.b32 %f196, %r133;\n" " fma.rn.f32 %f197, %f196, %f109, %f108;\n" " and.b32 %r134, %r45, 49152;\n" " or.b32 %r135, %r134, 1258291200;\n" " mov.b32 %f198, %r135;\n" " fma.rn.f32 %f199, %f198, %f193, %f192;\n" " add.rn.f32 %f200, %f197, %f199;\n" " shr.u32 %r136, %r47, 24;\n" " cvt.rn.f32.u32 %f201, %r136;\n" " sub.rn.f32 %f202, %f201, %f55;\n" " shr.u32 %r137, %r51, 24;\n" " cvt.rn.f32.u32 %f203, %r137;\n" " sub.rn.f32 %f204, %f203, %f55;\n" " mul.rn.f32 %f205, %f202, %f195;\n" " fma.rn.f32 %f206, %f205, %f37, %f186;\n" " mul.rn.f32 %f207, %f204, %f200;\n" " fma.rn.f32 %f208, %f207, %f37, %f188;\n" " fma.rn.f32 %f246, %f38, %f206, %f246;\n" " fma.rn.f32 %f247, %f39, %f208, %f247;\n" " add.s64 %rd68, %rd68, 216;\n" " add.s64 %rd67, %rd67, 216;\n" " add.s64 %rd66, %rd66, 1024;\n" " add.s64 %rd65, %rd65, 216;\n" " add.s64 %rd64, %rd64, 216;\n" " add.s32 %r179, %r179, 1;\n" " setp.lt.s32 %p6, %r179, %r8;\n" " @%p6 bra $L__BB8_4;\n" "\n" "$L__BB8_5:\n" " shl.b64 %rd61, %rd12, 2;\n" " add.s64 %rd60, %rd32, %rd61;\n" " // begin inline asm\n" " ld.global.nc.f32 %f209, [%rd60];\n" " // end inline asm\n" " fma.rn.f32 %f248, %f209, %f246, %f248;\n" " fma.rn.f32 %f249, %f209, %f247, %f249;\n" " cvt.u32.u64 %r138, %rd12;\n" " add.s32 %r178, %r138, %r34;\n" " setp.lt.s32 %p7, %r178, %r30;\n" " @%p7 bra $L__BB8_2;\n" "\n" "$L__BB8_6:\n" " cvta.to.global.u64 %rd28, %rd33;\n" " mov.u32 %r149, 16;\n" " mov.u32 %r158, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f210, %f248, %r149, %r158;\n" " // end inline asm\n" " add.f32 %f213, %f248, %f210;\n" " mov.u32 %r151, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f212, %f213, %r151, %r158;\n" " // end inline asm\n" " add.f32 %f215, %f213, %f212;\n" " mov.u32 %r153, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f214, %f215, %r153, %r158;\n" " // end inline asm\n" " add.f32 %f217, %f215, %f214;\n" " mov.u32 %r155, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f216, %f217, %r155, %r158;\n" " // end inline asm\n" " add.f32 %f219, %f217, %f216;\n" " mov.u32 %r157, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f218, %f219, %r157, %r158;\n" " // end inline asm\n" " add.f32 %f13, %f219, %f218;\n" " // begin inline asm\n" " shfl.down.b32 %f220, %f249, %r149, %r158;\n" " // end inline asm\n" " add.f32 %f223, %f249, %f220;\n" " // begin inline asm\n" " shfl.down.b32 %f222, %f223, %r151, %r158;\n" " // end inline asm\n" " add.f32 %f225, %f223, %f222;\n" " // begin inline asm\n" " shfl.down.b32 %f224, %f225, %r153, %r158;\n" " // end inline asm\n" " add.f32 %f227, %f225, %f224;\n" " // begin inline asm\n" " shfl.down.b32 %f226, %f227, %r155, %r158;\n" " // end inline asm\n" " add.f32 %f229, %f227, %f226;\n" " // begin inline asm\n" " shfl.down.b32 %f228, %f229, %r157, %r158;\n" " // end inline asm\n" " add.f32 %f14, %f229, %f228;\n" " setp.ne.s32 %p8, %r3, 0;\n" " @%p8 bra $L__BB8_8;\n" "\n" " shl.b32 %r159, %r2, 3;\n" " mov.u32 %r160, _ZZ15fc_moe_down_q6kE4sred;\n" " add.s32 %r161, %r160, %r159;\n" " st.shared.f32 [%r161], %f13;\n" " st.shared.f32 [%r161+4], %f14;\n" "\n" "$L__BB8_8:\n" " bar.sync 0;\n" " shl.b32 %r162, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r162;\n" " @%p9 bra $L__BB8_18;\n" "\n" " shr.u32 %r14, %r1, 1;\n" " add.s32 %r163, %r6, %r14;\n" " and.b32 %r15, %r1, 1;\n" " bfi.b32 %r16, %r163, %r15, 1, 31;\n" " setp.ge.s32 %p10, %r16, %r31;\n" " @%p10 bra $L__BB8_18;\n" "\n" " setp.lt.s32 %p11, %r34, 1;\n" " mov.f32 %f254, 0f00000000;\n" " @%p11 bra $L__BB8_17;\n" "\n" " mul.lo.s32 %r17, %r14, %r34;\n" " and.b32 %r184, %r34, 3;\n" " add.s32 %r165, %r34, -1;\n" " setp.lt.u32 %p12, %r165, 3;\n" " mov.f32 %f254, 0f00000000;\n" " mov.u32 %r182, 0;\n" " @%p12 bra $L__BB8_14;\n" "\n" " sub.s32 %r181, %r34, %r184;\n" " mov.f32 %f254, 0f00000000;\n" " mov.u32 %r182, 0;\n" "\n" "$L__BB8_13:\n" " add.s32 %r167, %r182, %r17;\n" " shl.b32 %r168, %r167, 3;\n" " mov.u32 %r169, _ZZ15fc_moe_down_q6kE4sred;\n" " add.s32 %r170, %r169, %r168;\n" " shl.b32 %r171, %r15, 2;\n" " add.s32 %r172, %r170, %r171;\n" " ld.shared.f32 %f234, [%r172];\n" " add.f32 %f235, %f254, %f234;\n" " ld.shared.f32 %f236, [%r172+8];\n" " add.f32 %f237, %f235, %f236;\n" " ld.shared.f32 %f238, [%r172+16];\n" " add.f32 %f239, %f237, %f238;\n" " ld.shared.f32 %f240, [%r172+24];\n" " add.f32 %f254, %f239, %f240;\n" " add.s32 %r182, %r182, 4;\n" " add.s32 %r181, %r181, -4;\n" " setp.ne.s32 %p13, %r181, 0;\n" " @%p13 bra $L__BB8_13;\n" "\n" "$L__BB8_14:\n" " setp.eq.s32 %p14, %r184, 0;\n" " @%p14 bra $L__BB8_17;\n" "\n" " add.s32 %r173, %r182, %r17;\n" " shl.b32 %r174, %r173, 3;\n" " mov.u32 %r175, _ZZ15fc_moe_down_q6kE4sred;\n" " add.s32 %r176, %r175, %r174;\n" " shl.b32 %r177, %r15, 2;\n" " add.s32 %r183, %r176, %r177;\n" "\n" "$L__BB8_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f241, [%r183];\n" " add.f32 %f254, %f254, %f241;\n" " add.s32 %r183, %r183, 8;\n" " add.s32 %r184, %r184, -1;\n" " setp.ne.s32 %p15, %r184, 0;\n" " @%p15 bra $L__BB8_16;\n" "\n" "$L__BB8_17:\n" " mul.wide.s32 %rd62, %r16, 4;\n" " add.s64 %rd63, %rd28, %rd62;\n" " st.global.f32 [%rd63], %f254;\n" "\n" "$L__BB8_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_f16\n" ".visible .entry fc_moe_down_f16(\n" " .param .u64 fc_moe_down_f16_param_0,\n" " .param .u64 fc_moe_down_f16_param_1,\n" " .param .u64 fc_moe_down_f16_param_2,\n" " .param .u64 fc_moe_down_f16_param_3,\n" " .param .u64 fc_moe_down_f16_param_4,\n" " .param .u32 fc_moe_down_f16_param_5,\n" " .param .u32 fc_moe_down_f16_param_6,\n" " .param .u32 fc_moe_down_f16_param_7,\n" " .param .u32 fc_moe_down_f16_param_8,\n" " .param .u32 fc_moe_down_f16_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .b16 %rs<17>;\n" " .reg .f32 %f<117>;\n" " .reg .b32 %r<91>;\n" " .reg .b64 %rd<54>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ15fc_moe_down_f16E4sred[128];\n" "\n" " ld.param.u64 %rd20, [fc_moe_down_f16_param_0];\n" " ld.param.u64 %rd21, [fc_moe_down_f16_param_1];\n" " ld.param.u64 %rd22, [fc_moe_down_f16_param_2];\n" " ld.param.u64 %rd23, [fc_moe_down_f16_param_3];\n" " ld.param.u64 %rd24, [fc_moe_down_f16_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_f16_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_f16_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_f16_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_f16_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_f16_param_9];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r36, %ntid.x;\n" " shr.u32 %r37, %r36, 5;\n" " div.u32 %r4, %r37, %r35;\n" " div.s32 %r38, %r2, %r35;\n" " mul.lo.s32 %r39, %r38, %r35;\n" " sub.s32 %r84, %r2, %r39;\n" " mov.u32 %r40, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r40;\n" " add.s32 %r41, %r6, %r38;\n" " shl.b32 %r7, %r41, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r84, %r31;\n" " mov.f32 %f110, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f111, %f110;\n" " @%p3 bra $L__BB9_6;\n" "\n" " cvt.s64.s32 %rd1, %r34;\n" " cvt.s64.s32 %rd2, %r7;\n" " cvt.s64.s32 %rd3, %r33;\n" " cvt.u64.u32 %rd4, %r3;\n" " mul.wide.s32 %rd25, %r3, 4;\n" " add.s64 %rd5, %rd21, %rd25;\n" " mul.wide.s32 %rd26, %r3, 2;\n" " or.b64 %rd6, %rd26, 256;\n" " shr.s32 %r8, %r33, 8;\n" " cvt.s64.s32 %rd7, %r32;\n" " add.s32 %r9, %r7, 1;\n" " cvta.to.global.u64 %rd8, %rd20;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r33, 256;\n" " setp.lt.s32 %p5, %r9, %r32;\n" " shl.b64 %rd45, %rd4, 1;\n" " mov.f32 %f111, %f25;\n" " mov.f32 %f110, %f25;\n" "\n" "$L__BB9_2:\n" " cvt.s64.s32 %rd9, %r84;\n" " mul.wide.s32 %rd28, %r84, 4;\n" " add.s64 %rd27, %rd22, %rd28;\n" " // begin inline asm\n" " ld.global.nc.s32 %r42, [%rd27];\n" " // end inline asm\n" " mov.f32 %f108, %f25;\n" " mov.f32 %f109, %f25;\n" " @%p4 bra $L__BB9_5;\n" "\n" " cvt.s64.s32 %rd29, %r42;\n" " mul.lo.s64 %rd30, %rd29, %rd7;\n" " add.s64 %rd31, %rd30, %rd2;\n" " mul.lo.s64 %rd32, %rd31, %rd1;\n" " add.s64 %rd53, %rd8, %rd32;\n" " mul.lo.s64 %rd33, %rd3, %rd9;\n" " shl.b64 %rd34, %rd33, 2;\n" " add.s64 %rd52, %rd5, %rd34;\n" " add.s64 %rd35, %rd53, %rd1;\n" " selp.b64 %rd36, %rd35, %rd53, %p5;\n" " add.s64 %rd51, %rd36, %rd6;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r85, 0;\n" " mov.f32 %f109, %f29;\n" " mov.f32 %f108, %f29;\n" "\n" "$L__BB9_4:\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd52];\n" " // end inline asm\n" " add.s64 %rd46, %rd53, %rd45;\n" " ld.global.u16 %rs1, [%rd46];\n" " // begin inline asm\n" " cvt.f32.f16 %f31, %rs1;\n" " // end inline asm\n" " fma.rn.f32 %f55, %f31, %f30, %f29;\n" " ld.global.u16 %rs2, [%rd51+-256];\n" " // begin inline asm\n" " cvt.f32.f16 %f32, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f56, %f32, %f30, %f29;\n" " add.s64 %rd38, %rd52, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd38];\n" " // end inline asm\n" " ld.global.u16 %rs3, [%rd46+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f34, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f57, %f34, %f33, %f55;\n" " ld.global.u16 %rs4, [%rd51+-192];\n" " // begin inline asm\n" " cvt.f32.f16 %f35, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f58, %f35, %f33, %f56;\n" " add.s64 %rd39, %rd52, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd39];\n" " // end inline asm\n" " ld.global.u16 %rs5, [%rd46+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f37, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f59, %f37, %f36, %f57;\n" " ld.global.u16 %rs6, [%rd51+-128];\n" " // begin inline asm\n" " cvt.f32.f16 %f38, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f60, %f38, %f36, %f58;\n" " add.s64 %rd40, %rd52, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f39, [%rd40];\n" " // end inline asm\n" " ld.global.u16 %rs7, [%rd46+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f40, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f61, %f40, %f39, %f59;\n" " ld.global.u16 %rs8, [%rd51+-64];\n" " // begin inline asm\n" " cvt.f32.f16 %f41, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f62, %f41, %f39, %f60;\n" " add.s64 %rd41, %rd52, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f42, [%rd41];\n" " // end inline asm\n" " ld.global.u16 %rs9, [%rd46+256];\n" " // begin inline asm\n" " cvt.f32.f16 %f43, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f63, %f43, %f42, %f61;\n" " ld.global.u16 %rs10, [%rd51];\n" " // begin inline asm\n" " cvt.f32.f16 %f44, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f64, %f44, %f42, %f62;\n" " add.s64 %rd42, %rd52, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f45, [%rd42];\n" " // end inline asm\n" " ld.global.u16 %rs11, [%rd46+320];\n" " // begin inline asm\n" " cvt.f32.f16 %f46, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f65, %f46, %f45, %f63;\n" " ld.global.u16 %rs12, [%rd51+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f47, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f66, %f47, %f45, %f64;\n" " add.s64 %rd43, %rd52, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f48, [%rd43];\n" " // end inline asm\n" " ld.global.u16 %rs13, [%rd46+384];\n" " // begin inline asm\n" " cvt.f32.f16 %f49, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f67, %f49, %f48, %f65;\n" " ld.global.u16 %rs14, [%rd51+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f50, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f68, %f50, %f48, %f66;\n" " add.s64 %rd44, %rd52, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f51, [%rd44];\n" " // end inline asm\n" " ld.global.u16 %rs15, [%rd46+448];\n" " // begin inline asm\n" " cvt.f32.f16 %f52, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f69, %f52, %f51, %f67;\n" " ld.global.u16 %rs16, [%rd51+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f53, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f70, %f53, %f51, %f68;\n" " add.f32 %f108, %f108, %f69;\n" " add.f32 %f109, %f109, %f70;\n" " add.s64 %rd53, %rd53, 512;\n" " add.s64 %rd52, %rd52, 1024;\n" " add.s64 %rd51, %rd51, 512;\n" " add.s32 %r85, %r85, 1;\n" " setp.lt.s32 %p6, %r85, %r8;\n" " @%p6 bra $L__BB9_4;\n" "\n" "$L__BB9_5:\n" " shl.b64 %rd48, %rd9, 2;\n" " add.s64 %rd47, %rd23, %rd48;\n" " // begin inline asm\n" " ld.global.nc.f32 %f71, [%rd47];\n" " // end inline asm\n" " fma.rn.f32 %f110, %f71, %f108, %f110;\n" " fma.rn.f32 %f111, %f71, %f109, %f111;\n" " cvt.u32.u64 %r44, %rd9;\n" " add.s32 %r84, %r44, %r35;\n" " setp.lt.s32 %p7, %r84, %r31;\n" " @%p7 bra $L__BB9_2;\n" "\n" "$L__BB9_6:\n" " cvta.to.global.u64 %rd19, %rd24;\n" " mov.u32 %r55, 16;\n" " mov.u32 %r64, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f72, %f110, %r55, %r64;\n" " // end inline asm\n" " add.f32 %f75, %f110, %f72;\n" " mov.u32 %r57, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f74, %f75, %r57, %r64;\n" " // end inline asm\n" " add.f32 %f77, %f75, %f74;\n" " mov.u32 %r59, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f76, %f77, %r59, %r64;\n" " // end inline asm\n" " add.f32 %f79, %f77, %f76;\n" " mov.u32 %r61, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f78, %f79, %r61, %r64;\n" " // end inline asm\n" " add.f32 %f81, %f79, %f78;\n" " mov.u32 %r63, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f80, %f81, %r63, %r64;\n" " // end inline asm\n" " add.f32 %f13, %f81, %f80;\n" " // begin inline asm\n" " shfl.down.b32 %f82, %f111, %r55, %r64;\n" " // end inline asm\n" " add.f32 %f85, %f111, %f82;\n" " // begin inline asm\n" " shfl.down.b32 %f84, %f85, %r57, %r64;\n" " // end inline asm\n" " add.f32 %f87, %f85, %f84;\n" " // begin inline asm\n" " shfl.down.b32 %f86, %f87, %r59, %r64;\n" " // end inline asm\n" " add.f32 %f89, %f87, %f86;\n" " // begin inline asm\n" " shfl.down.b32 %f88, %f89, %r61, %r64;\n" " // end inline asm\n" " add.f32 %f91, %f89, %f88;\n" " // begin inline asm\n" " shfl.down.b32 %f90, %f91, %r63, %r64;\n" " // end inline asm\n" " add.f32 %f14, %f91, %f90;\n" " setp.ne.s32 %p8, %r3, 0;\n" " @%p8 bra $L__BB9_8;\n" "\n" " shl.b32 %r65, %r2, 3;\n" " mov.u32 %r66, _ZZ15fc_moe_down_f16E4sred;\n" " add.s32 %r67, %r66, %r65;\n" " st.shared.f32 [%r67], %f13;\n" " st.shared.f32 [%r67+4], %f14;\n" "\n" "$L__BB9_8:\n" " bar.sync 0;\n" " shl.b32 %r68, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r68;\n" " @%p9 bra $L__BB9_18;\n" "\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r69, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r69, %r16, 1, 31;\n" " setp.ge.s32 %p10, %r17, %r32;\n" " @%p10 bra $L__BB9_18;\n" "\n" " setp.lt.s32 %p11, %r35, 1;\n" " mov.f32 %f116, 0f00000000;\n" " @%p11 bra $L__BB9_17;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r90, %r35, 3;\n" " add.s32 %r71, %r35, -1;\n" " setp.lt.u32 %p12, %r71, 3;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r88, 0;\n" " @%p12 bra $L__BB9_14;\n" "\n" " sub.s32 %r87, %r35, %r90;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r88, 0;\n" "\n" "$L__BB9_13:\n" " add.s32 %r73, %r88, %r18;\n" " shl.b32 %r74, %r73, 3;\n" " mov.u32 %r75, _ZZ15fc_moe_down_f16E4sred;\n" " add.s32 %r76, %r75, %r74;\n" " shl.b32 %r77, %r16, 2;\n" " add.s32 %r78, %r76, %r77;\n" " ld.shared.f32 %f96, [%r78];\n" " add.f32 %f97, %f116, %f96;\n" " ld.shared.f32 %f98, [%r78+8];\n" " add.f32 %f99, %f97, %f98;\n" " ld.shared.f32 %f100, [%r78+16];\n" " add.f32 %f101, %f99, %f100;\n" " ld.shared.f32 %f102, [%r78+24];\n" " add.f32 %f116, %f101, %f102;\n" " add.s32 %r88, %r88, 4;\n" " add.s32 %r87, %r87, -4;\n" " setp.ne.s32 %p13, %r87, 0;\n" " @%p13 bra $L__BB9_13;\n" "\n" "$L__BB9_14:\n" " setp.eq.s32 %p14, %r90, 0;\n" " @%p14 bra $L__BB9_17;\n" "\n" " add.s32 %r79, %r88, %r18;\n" " shl.b32 %r80, %r79, 3;\n" " mov.u32 %r81, _ZZ15fc_moe_down_f16E4sred;\n" " add.s32 %r82, %r81, %r80;\n" " shl.b32 %r83, %r16, 2;\n" " add.s32 %r89, %r82, %r83;\n" "\n" "$L__BB9_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f103, [%r89];\n" " add.f32 %f116, %f116, %f103;\n" " add.s32 %r89, %r89, 8;\n" " add.s32 %r90, %r90, -1;\n" " setp.ne.s32 %p15, %r90, 0;\n" " @%p15 bra $L__BB9_16;\n" "\n" "$L__BB9_17:\n" " mul.wide.s32 %rd49, %r17, 4;\n" " add.s64 %rd50, %rd19, %rd49;\n" " st.global.f32 [%rd50], %f116;\n" "\n" "$L__BB9_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_bf16\n" ".visible .entry fc_moe_down_bf16(\n" " .param .u64 fc_moe_down_bf16_param_0,\n" " .param .u64 fc_moe_down_bf16_param_1,\n" " .param .u64 fc_moe_down_bf16_param_2,\n" " .param .u64 fc_moe_down_bf16_param_3,\n" " .param .u64 fc_moe_down_bf16_param_4,\n" " .param .u32 fc_moe_down_bf16_param_5,\n" " .param .u32 fc_moe_down_bf16_param_6,\n" " .param .u32 fc_moe_down_bf16_param_7,\n" " .param .u32 fc_moe_down_bf16_param_8,\n" " .param .u32 fc_moe_down_bf16_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<117>;\n" " .reg .b32 %r<123>;\n" " .reg .b64 %rd<54>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ16fc_moe_down_bf16E4sred[128];\n" "\n" " ld.param.u64 %rd20, [fc_moe_down_bf16_param_0];\n" " ld.param.u64 %rd21, [fc_moe_down_bf16_param_1];\n" " ld.param.u64 %rd22, [fc_moe_down_bf16_param_2];\n" " ld.param.u64 %rd23, [fc_moe_down_bf16_param_3];\n" " ld.param.u64 %rd24, [fc_moe_down_bf16_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_bf16_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_bf16_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_bf16_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_bf16_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_bf16_param_9];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r36, %ntid.x;\n" " shr.u32 %r37, %r36, 5;\n" " div.u32 %r4, %r37, %r35;\n" " div.s32 %r38, %r2, %r35;\n" " mul.lo.s32 %r39, %r38, %r35;\n" " sub.s32 %r116, %r2, %r39;\n" " mov.u32 %r40, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r40;\n" " add.s32 %r41, %r6, %r38;\n" " shl.b32 %r7, %r41, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r116, %r31;\n" " mov.f32 %f110, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f111, %f110;\n" " @%p3 bra $L__BB10_6;\n" "\n" " cvt.s64.s32 %rd1, %r34;\n" " cvt.s64.s32 %rd2, %r7;\n" " cvt.s64.s32 %rd3, %r33;\n" " cvt.u64.u32 %rd4, %r3;\n" " mul.wide.s32 %rd25, %r3, 4;\n" " add.s64 %rd5, %rd21, %rd25;\n" " mul.wide.s32 %rd26, %r3, 2;\n" " or.b64 %rd6, %rd26, 256;\n" " shr.s32 %r8, %r33, 8;\n" " cvt.s64.s32 %rd7, %r32;\n" " add.s32 %r9, %r7, 1;\n" " cvta.to.global.u64 %rd8, %rd20;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r33, 256;\n" " setp.lt.s32 %p5, %r9, %r32;\n" " shl.b64 %rd45, %rd4, 1;\n" " mov.f32 %f111, %f25;\n" " mov.f32 %f110, %f25;\n" "\n" "$L__BB10_2:\n" " cvt.s64.s32 %rd9, %r116;\n" " mul.wide.s32 %rd28, %r116, 4;\n" " add.s64 %rd27, %rd22, %rd28;\n" " // begin inline asm\n" " ld.global.nc.s32 %r42, [%rd27];\n" " // end inline asm\n" " mov.f32 %f108, %f25;\n" " mov.f32 %f109, %f25;\n" " @%p4 bra $L__BB10_5;\n" "\n" " cvt.s64.s32 %rd29, %r42;\n" " mul.lo.s64 %rd30, %rd29, %rd7;\n" " add.s64 %rd31, %rd30, %rd2;\n" " mul.lo.s64 %rd32, %rd31, %rd1;\n" " add.s64 %rd53, %rd8, %rd32;\n" " mul.lo.s64 %rd33, %rd3, %rd9;\n" " shl.b64 %rd34, %rd33, 2;\n" " add.s64 %rd52, %rd5, %rd34;\n" " add.s64 %rd35, %rd53, %rd1;\n" " selp.b64 %rd36, %rd35, %rd53, %p5;\n" " add.s64 %rd51, %rd36, %rd6;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r117, 0;\n" " mov.f32 %f109, %f29;\n" " mov.f32 %f108, %f29;\n" "\n" "$L__BB10_4:\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd52];\n" " // end inline asm\n" " add.s64 %rd46, %rd53, %rd45;\n" " ld.global.u16 %r44, [%rd46];\n" " shl.b32 %r45, %r44, 16;\n" " mov.b32 %f38, %r45;\n" " fma.rn.f32 %f40, %f38, %f30, %f29;\n" " ld.global.u16 %r46, [%rd51+-256];\n" " shl.b32 %r47, %r46, 16;\n" " mov.b32 %f41, %r47;\n" " fma.rn.f32 %f42, %f41, %f30, %f29;\n" " add.s64 %rd38, %rd52, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd38];\n" " // end inline asm\n" " ld.global.u16 %r48, [%rd46+64];\n" " shl.b32 %r49, %r48, 16;\n" " mov.b32 %f43, %r49;\n" " fma.rn.f32 %f44, %f43, %f31, %f40;\n" " ld.global.u16 %r50, [%rd51+-192];\n" " shl.b32 %r51, %r50, 16;\n" " mov.b32 %f45, %r51;\n" " fma.rn.f32 %f46, %f45, %f31, %f42;\n" " add.s64 %rd39, %rd52, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd39];\n" " // end inline asm\n" " ld.global.u16 %r52, [%rd46+128];\n" " shl.b32 %r53, %r52, 16;\n" " mov.b32 %f47, %r53;\n" " fma.rn.f32 %f48, %f47, %f32, %f44;\n" " ld.global.u16 %r54, [%rd51+-128];\n" " shl.b32 %r55, %r54, 16;\n" " mov.b32 %f49, %r55;\n" " fma.rn.f32 %f50, %f49, %f32, %f46;\n" " add.s64 %rd40, %rd52, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd40];\n" " // end inline asm\n" " ld.global.u16 %r56, [%rd46+192];\n" " shl.b32 %r57, %r56, 16;\n" " mov.b32 %f51, %r57;\n" " fma.rn.f32 %f52, %f51, %f33, %f48;\n" " ld.global.u16 %r58, [%rd51+-64];\n" " shl.b32 %r59, %r58, 16;\n" " mov.b32 %f53, %r59;\n" " fma.rn.f32 %f54, %f53, %f33, %f50;\n" " add.s64 %rd41, %rd52, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd41];\n" " // end inline asm\n" " ld.global.u16 %r60, [%rd46+256];\n" " shl.b32 %r61, %r60, 16;\n" " mov.b32 %f55, %r61;\n" " fma.rn.f32 %f56, %f55, %f34, %f52;\n" " ld.global.u16 %r62, [%rd51];\n" " shl.b32 %r63, %r62, 16;\n" " mov.b32 %f57, %r63;\n" " fma.rn.f32 %f58, %f57, %f34, %f54;\n" " add.s64 %rd42, %rd52, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd42];\n" " // end inline asm\n" " ld.global.u16 %r64, [%rd46+320];\n" " shl.b32 %r65, %r64, 16;\n" " mov.b32 %f59, %r65;\n" " fma.rn.f32 %f60, %f59, %f35, %f56;\n" " ld.global.u16 %r66, [%rd51+64];\n" " shl.b32 %r67, %r66, 16;\n" " mov.b32 %f61, %r67;\n" " fma.rn.f32 %f62, %f61, %f35, %f58;\n" " add.s64 %rd43, %rd52, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd43];\n" " // end inline asm\n" " ld.global.u16 %r68, [%rd46+384];\n" " shl.b32 %r69, %r68, 16;\n" " mov.b32 %f63, %r69;\n" " fma.rn.f32 %f64, %f63, %f36, %f60;\n" " ld.global.u16 %r70, [%rd51+128];\n" " shl.b32 %r71, %r70, 16;\n" " mov.b32 %f65, %r71;\n" " fma.rn.f32 %f66, %f65, %f36, %f62;\n" " add.s64 %rd44, %rd52, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd44];\n" " // end inline asm\n" " ld.global.u16 %r72, [%rd46+448];\n" " shl.b32 %r73, %r72, 16;\n" " mov.b32 %f67, %r73;\n" " fma.rn.f32 %f68, %f67, %f37, %f64;\n" " ld.global.u16 %r74, [%rd51+192];\n" " shl.b32 %r75, %r74, 16;\n" " mov.b32 %f69, %r75;\n" " fma.rn.f32 %f70, %f69, %f37, %f66;\n" " add.f32 %f108, %f108, %f68;\n" " add.f32 %f109, %f109, %f70;\n" " add.s64 %rd53, %rd53, 512;\n" " add.s64 %rd52, %rd52, 1024;\n" " add.s64 %rd51, %rd51, 512;\n" " add.s32 %r117, %r117, 1;\n" " setp.lt.s32 %p6, %r117, %r8;\n" " @%p6 bra $L__BB10_4;\n" "\n" "$L__BB10_5:\n" " shl.b64 %rd48, %rd9, 2;\n" " add.s64 %rd47, %rd23, %rd48;\n" " // begin inline asm\n" " ld.global.nc.f32 %f71, [%rd47];\n" " // end inline asm\n" " fma.rn.f32 %f110, %f71, %f108, %f110;\n" " fma.rn.f32 %f111, %f71, %f109, %f111;\n" " cvt.u32.u64 %r76, %rd9;\n" " add.s32 %r116, %r76, %r35;\n" " setp.lt.s32 %p7, %r116, %r31;\n" " @%p7 bra $L__BB10_2;\n" "\n" "$L__BB10_6:\n" " cvta.to.global.u64 %rd19, %rd24;\n" " mov.u32 %r87, 16;\n" " mov.u32 %r96, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f72, %f110, %r87, %r96;\n" " // end inline asm\n" " add.f32 %f75, %f110, %f72;\n" " mov.u32 %r89, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f74, %f75, %r89, %r96;\n" " // end inline asm\n" " add.f32 %f77, %f75, %f74;\n" " mov.u32 %r91, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f76, %f77, %r91, %r96;\n" " // end inline asm\n" " add.f32 %f79, %f77, %f76;\n" " mov.u32 %r93, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f78, %f79, %r93, %r96;\n" " // end inline asm\n" " add.f32 %f81, %f79, %f78;\n" " mov.u32 %r95, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f80, %f81, %r95, %r96;\n" " // end inline asm\n" " add.f32 %f13, %f81, %f80;\n" " // begin inline asm\n" " shfl.down.b32 %f82, %f111, %r87, %r96;\n" " // end inline asm\n" " add.f32 %f85, %f111, %f82;\n" " // begin inline asm\n" " shfl.down.b32 %f84, %f85, %r89, %r96;\n" " // end inline asm\n" " add.f32 %f87, %f85, %f84;\n" " // begin inline asm\n" " shfl.down.b32 %f86, %f87, %r91, %r96;\n" " // end inline asm\n" " add.f32 %f89, %f87, %f86;\n" " // begin inline asm\n" " shfl.down.b32 %f88, %f89, %r93, %r96;\n" " // end inline asm\n" " add.f32 %f91, %f89, %f88;\n" " // begin inline asm\n" " shfl.down.b32 %f90, %f91, %r95, %r96;\n" " // end inline asm\n" " add.f32 %f14, %f91, %f90;\n" " setp.ne.s32 %p8, %r3, 0;\n" " @%p8 bra $L__BB10_8;\n" "\n" " shl.b32 %r97, %r2, 3;\n" " mov.u32 %r98, _ZZ16fc_moe_down_bf16E4sred;\n" " add.s32 %r99, %r98, %r97;\n" " st.shared.f32 [%r99], %f13;\n" " st.shared.f32 [%r99+4], %f14;\n" "\n" "$L__BB10_8:\n" " bar.sync 0;\n" " shl.b32 %r100, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r100;\n" " @%p9 bra $L__BB10_18;\n" "\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r101, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r101, %r16, 1, 31;\n" " setp.ge.s32 %p10, %r17, %r32;\n" " @%p10 bra $L__BB10_18;\n" "\n" " setp.lt.s32 %p11, %r35, 1;\n" " mov.f32 %f116, 0f00000000;\n" " @%p11 bra $L__BB10_17;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r122, %r35, 3;\n" " add.s32 %r103, %r35, -1;\n" " setp.lt.u32 %p12, %r103, 3;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r120, 0;\n" " @%p12 bra $L__BB10_14;\n" "\n" " sub.s32 %r119, %r35, %r122;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r120, 0;\n" "\n" "$L__BB10_13:\n" " add.s32 %r105, %r120, %r18;\n" " shl.b32 %r106, %r105, 3;\n" " mov.u32 %r107, _ZZ16fc_moe_down_bf16E4sred;\n" " add.s32 %r108, %r107, %r106;\n" " shl.b32 %r109, %r16, 2;\n" " add.s32 %r110, %r108, %r109;\n" " ld.shared.f32 %f96, [%r110];\n" " add.f32 %f97, %f116, %f96;\n" " ld.shared.f32 %f98, [%r110+8];\n" " add.f32 %f99, %f97, %f98;\n" " ld.shared.f32 %f100, [%r110+16];\n" " add.f32 %f101, %f99, %f100;\n" " ld.shared.f32 %f102, [%r110+24];\n" " add.f32 %f116, %f101, %f102;\n" " add.s32 %r120, %r120, 4;\n" " add.s32 %r119, %r119, -4;\n" " setp.ne.s32 %p13, %r119, 0;\n" " @%p13 bra $L__BB10_13;\n" "\n" "$L__BB10_14:\n" " setp.eq.s32 %p14, %r122, 0;\n" " @%p14 bra $L__BB10_17;\n" "\n" " add.s32 %r111, %r120, %r18;\n" " shl.b32 %r112, %r111, 3;\n" " mov.u32 %r113, _ZZ16fc_moe_down_bf16E4sred;\n" " add.s32 %r114, %r113, %r112;\n" " shl.b32 %r115, %r16, 2;\n" " add.s32 %r121, %r114, %r115;\n" "\n" "$L__BB10_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f103, [%r121];\n" " add.f32 %f116, %f116, %f103;\n" " add.s32 %r121, %r121, 8;\n" " add.s32 %r122, %r122, -1;\n" " setp.ne.s32 %p15, %r122, 0;\n" " @%p15 bra $L__BB10_16;\n" "\n" "$L__BB10_17:\n" " mul.wide.s32 %rd49, %r17, 4;\n" " add.s64 %rd50, %rd19, %rd49;\n" " st.global.f32 [%rd50], %f116;\n" "\n" "$L__BB10_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_f16_vec\n" ".visible .entry fc_moe_down_f16_vec(\n" " .param .u64 fc_moe_down_f16_vec_param_0,\n" " .param .u64 fc_moe_down_f16_vec_param_1,\n" " .param .u64 fc_moe_down_f16_vec_param_2,\n" " .param .u64 fc_moe_down_f16_vec_param_3,\n" " .param .u64 fc_moe_down_f16_vec_param_4,\n" " .param .u32 fc_moe_down_f16_vec_param_5,\n" " .param .u32 fc_moe_down_f16_vec_param_6,\n" " .param .u32 fc_moe_down_f16_vec_param_7,\n" " .param .u32 fc_moe_down_f16_vec_param_8,\n" " .param .u32 fc_moe_down_f16_vec_param_9\n" ")\n" "{\n" " .reg .pred %p<18>;\n" " .reg .b16 %rs<49>;\n" " .reg .f32 %f<213>;\n" " .reg .b32 %r<164>;\n" " .reg .b64 %rd<59>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ19fc_moe_down_f16_vecE4sred[128];\n" "\n" " ld.param.u64 %rd21, [fc_moe_down_f16_vec_param_0];\n" " ld.param.u64 %rd22, [fc_moe_down_f16_vec_param_1];\n" " ld.param.u64 %rd23, [fc_moe_down_f16_vec_param_2];\n" " ld.param.u64 %rd24, [fc_moe_down_f16_vec_param_3];\n" " ld.param.u64 %rd25, [fc_moe_down_f16_vec_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_f16_vec_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_f16_vec_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_f16_vec_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_f16_vec_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_f16_vec_param_9];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r36, %ntid.x;\n" " shr.u32 %r37, %r36, 5;\n" " div.u32 %r4, %r37, %r35;\n" " div.s32 %r38, %r2, %r35;\n" " mul.lo.s32 %r39, %r38, %r35;\n" " sub.s32 %r157, %r2, %r39;\n" " mov.u32 %r40, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r40;\n" " add.s32 %r41, %r6, %r38;\n" " shl.b32 %r7, %r41, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r157, %r31;\n" " mov.f32 %f206, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f207, %f206;\n" " @%p3 bra $L__BB11_9;\n" "\n" " cvt.s64.s32 %rd1, %r34;\n" " shr.s32 %r42, %r33, 8;\n" " cvt.s64.s32 %rd2, %r7;\n" " cvt.s64.s32 %rd3, %r33;\n" " cvt.u64.u32 %rd4, %r3;\n" " max.s32 %r43, %r42, 1;\n" " and.b32 %r8, %r43, 1;\n" " sub.s32 %r9, %r43, %r8;\n" " cvt.s64.s32 %rd5, %r32;\n" " add.s32 %r10, %r7, 1;\n" " cvta.to.global.u64 %rd6, %rd21;\n" " mov.f32 %f31, 0f00000000;\n" " setp.lt.s32 %p4, %r10, %r32;\n" " setp.lt.s32 %p5, %r33, 256;\n" " setp.lt.s32 %p6, %r33, 512;\n" " setp.eq.s32 %p8, %r8, 0;\n" " mov.f32 %f207, %f31;\n" " mov.f32 %f206, %f31;\n" "\n" "$L__BB11_2:\n" " cvt.s64.s32 %rd7, %r157;\n" " mul.wide.s32 %rd27, %r157, 4;\n" " add.s64 %rd26, %rd23, %rd27;\n" " // begin inline asm\n" " ld.global.nc.s32 %r44, [%rd26];\n" " // end inline asm\n" " cvt.s64.s32 %rd28, %r44;\n" " mul.lo.s64 %rd29, %rd28, %rd5;\n" " add.s64 %rd30, %rd29, %rd2;\n" " mul.lo.s64 %rd31, %rd30, %rd1;\n" " add.s64 %rd58, %rd6, %rd31;\n" " add.s64 %rd32, %rd58, %rd1;\n" " selp.b64 %rd57, %rd32, %rd58, %p4;\n" " mov.f32 %f200, %f31;\n" " mov.f32 %f201, %f31;\n" " @%p5 bra $L__BB11_8;\n" "\n" " mul.lo.s64 %rd33, %rd7, %rd3;\n" " shl.b64 %rd34, %rd33, 2;\n" " add.s64 %rd56, %rd22, %rd34;\n" " mov.f32 %f201, 0f00000000;\n" " mov.f32 %f200, %f201;\n" " @%p6 bra $L__BB11_6;\n" "\n" " mov.f32 %f38, 0f00000000;\n" " mov.f32 %f201, %f38;\n" " mov.f32 %f200, %f38;\n" " mov.u32 %r158, %r9;\n" "\n" "$L__BB11_5:\n" " shl.b64 %rd39, %rd4, 3;\n" " add.s64 %rd40, %rd58, %rd39;\n" " ld.global.v2.u32 {%r45, %r46}, [%rd40];\n" " add.s64 %rd41, %rd57, %rd39;\n" " ld.global.v2.u32 {%r49, %r50}, [%rd41];\n" " shl.b64 %rd42, %rd4, 4;\n" " add.s64 %rd35, %rd56, %rd42;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f39,%f40,%f41,%f42}, [%rd35];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r45;\n" " // begin inline asm\n" " cvt.f32.f16 %f43, %rs1;\n" " // end inline asm\n" " fma.rn.f32 %f88, %f43, %f39, %f38;\n" " cvt.u16.u32 %rs2, %r49;\n" " // begin inline asm\n" " cvt.f32.f16 %f44, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f89, %f44, %f39, %f38;\n" " shr.u32 %r53, %r45, 16;\n" " cvt.u16.u32 %rs3, %r53;\n" " // begin inline asm\n" " cvt.f32.f16 %f45, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f90, %f45, %f40, %f88;\n" " shr.u32 %r54, %r49, 16;\n" " cvt.u16.u32 %rs4, %r54;\n" " // begin inline asm\n" " cvt.f32.f16 %f46, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f91, %f46, %f40, %f89;\n" " cvt.u16.u32 %rs5, %r46;\n" " // begin inline asm\n" " cvt.f32.f16 %f47, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f92, %f47, %f41, %f90;\n" " cvt.u16.u32 %rs6, %r50;\n" " // begin inline asm\n" " cvt.f32.f16 %f48, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f93, %f48, %f41, %f91;\n" " shr.u32 %r55, %r46, 16;\n" " cvt.u16.u32 %rs7, %r55;\n" " // begin inline asm\n" " cvt.f32.f16 %f49, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f94, %f49, %f42, %f92;\n" " shr.u32 %r56, %r50, 16;\n" " cvt.u16.u32 %rs8, %r56;\n" " // begin inline asm\n" " cvt.f32.f16 %f50, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f95, %f50, %f42, %f93;\n" " ld.global.v2.u32 {%r57, %r58}, [%rd40+256];\n" " ld.global.v2.u32 {%r61, %r62}, [%rd41+256];\n" " add.s64 %rd36, %rd35, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f51,%f52,%f53,%f54}, [%rd36];\n" " // end inline asm\n" " cvt.u16.u32 %rs9, %r57;\n" " // begin inline asm\n" " cvt.f32.f16 %f55, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f96, %f55, %f51, %f94;\n" " cvt.u16.u32 %rs10, %r61;\n" " // begin inline asm\n" " cvt.f32.f16 %f56, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f97, %f56, %f51, %f95;\n" " shr.u32 %r65, %r57, 16;\n" " cvt.u16.u32 %rs11, %r65;\n" " // begin inline asm\n" " cvt.f32.f16 %f57, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f98, %f57, %f52, %f96;\n" " shr.u32 %r66, %r61, 16;\n" " cvt.u16.u32 %rs12, %r66;\n" " // begin inline asm\n" " cvt.f32.f16 %f58, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f99, %f58, %f52, %f97;\n" " cvt.u16.u32 %rs13, %r58;\n" " // begin inline asm\n" " cvt.f32.f16 %f59, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f100, %f59, %f53, %f98;\n" " cvt.u16.u32 %rs14, %r62;\n" " // begin inline asm\n" " cvt.f32.f16 %f60, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f101, %f60, %f53, %f99;\n" " shr.u32 %r67, %r58, 16;\n" " cvt.u16.u32 %rs15, %r67;\n" " // begin inline asm\n" " cvt.f32.f16 %f61, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f102, %f61, %f54, %f100;\n" " shr.u32 %r68, %r62, 16;\n" " cvt.u16.u32 %rs16, %r68;\n" " // begin inline asm\n" " cvt.f32.f16 %f62, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f103, %f62, %f54, %f101;\n" " add.f32 %f104, %f200, %f102;\n" " add.f32 %f105, %f201, %f103;\n" " ld.global.v2.u32 {%r69, %r70}, [%rd40+512];\n" " ld.global.v2.u32 {%r73, %r74}, [%rd41+512];\n" " add.s64 %rd37, %rd35, 1024;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f63,%f64,%f65,%f66}, [%rd37];\n" " // end inline asm\n" " cvt.u16.u32 %rs17, %r69;\n" " // begin inline asm\n" " cvt.f32.f16 %f67, %rs17;\n" " // end inline asm\n" " fma.rn.f32 %f106, %f67, %f63, %f38;\n" " cvt.u16.u32 %rs18, %r73;\n" " // begin inline asm\n" " cvt.f32.f16 %f68, %rs18;\n" " // end inline asm\n" " fma.rn.f32 %f107, %f68, %f63, %f38;\n" " shr.u32 %r77, %r69, 16;\n" " cvt.u16.u32 %rs19, %r77;\n" " // begin inline asm\n" " cvt.f32.f16 %f69, %rs19;\n" " // end inline asm\n" " fma.rn.f32 %f108, %f69, %f64, %f106;\n" " shr.u32 %r78, %r73, 16;\n" " cvt.u16.u32 %rs20, %r78;\n" " // begin inline asm\n" " cvt.f32.f16 %f70, %rs20;\n" " // end inline asm\n" " fma.rn.f32 %f109, %f70, %f64, %f107;\n" " cvt.u16.u32 %rs21, %r70;\n" " // begin inline asm\n" " cvt.f32.f16 %f71, %rs21;\n" " // end inline asm\n" " fma.rn.f32 %f110, %f71, %f65, %f108;\n" " cvt.u16.u32 %rs22, %r74;\n" " // begin inline asm\n" " cvt.f32.f16 %f72, %rs22;\n" " // end inline asm\n" " fma.rn.f32 %f111, %f72, %f65, %f109;\n" " shr.u32 %r79, %r70, 16;\n" " cvt.u16.u32 %rs23, %r79;\n" " // begin inline asm\n" " cvt.f32.f16 %f73, %rs23;\n" " // end inline asm\n" " fma.rn.f32 %f112, %f73, %f66, %f110;\n" " shr.u32 %r80, %r74, 16;\n" " cvt.u16.u32 %rs24, %r80;\n" " // begin inline asm\n" " cvt.f32.f16 %f74, %rs24;\n" " // end inline asm\n" " fma.rn.f32 %f113, %f74, %f66, %f111;\n" " ld.global.v2.u32 {%r81, %r82}, [%rd40+768];\n" " ld.global.v2.u32 {%r85, %r86}, [%rd41+768];\n" " add.s64 %rd38, %rd35, 1536;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f75,%f76,%f77,%f78}, [%rd38];\n" " // end inline asm\n" " cvt.u16.u32 %rs25, %r81;\n" " // begin inline asm\n" " cvt.f32.f16 %f79, %rs25;\n" " // end inline asm\n" " fma.rn.f32 %f114, %f79, %f75, %f112;\n" " cvt.u16.u32 %rs26, %r85;\n" " // begin inline asm\n" " cvt.f32.f16 %f80, %rs26;\n" " // end inline asm\n" " fma.rn.f32 %f115, %f80, %f75, %f113;\n" " shr.u32 %r89, %r81, 16;\n" " cvt.u16.u32 %rs27, %r89;\n" " // begin inline asm\n" " cvt.f32.f16 %f81, %rs27;\n" " // end inline asm\n" " fma.rn.f32 %f116, %f81, %f76, %f114;\n" " shr.u32 %r90, %r85, 16;\n" " cvt.u16.u32 %rs28, %r90;\n" " // begin inline asm\n" " cvt.f32.f16 %f82, %rs28;\n" " // end inline asm\n" " fma.rn.f32 %f117, %f82, %f76, %f115;\n" " cvt.u16.u32 %rs29, %r82;\n" " // begin inline asm\n" " cvt.f32.f16 %f83, %rs29;\n" " // end inline asm\n" " fma.rn.f32 %f118, %f83, %f77, %f116;\n" " cvt.u16.u32 %rs30, %r86;\n" " // begin inline asm\n" " cvt.f32.f16 %f84, %rs30;\n" " // end inline asm\n" " fma.rn.f32 %f119, %f84, %f77, %f117;\n" " shr.u32 %r91, %r82, 16;\n" " cvt.u16.u32 %rs31, %r91;\n" " // begin inline asm\n" " cvt.f32.f16 %f85, %rs31;\n" " // end inline asm\n" " fma.rn.f32 %f120, %f85, %f78, %f118;\n" " shr.u32 %r92, %r86, 16;\n" " cvt.u16.u32 %rs32, %r92;\n" " // begin inline asm\n" " cvt.f32.f16 %f86, %rs32;\n" " // end inline asm\n" " fma.rn.f32 %f121, %f86, %f78, %f119;\n" " add.f32 %f200, %f104, %f120;\n" " add.f32 %f201, %f105, %f121;\n" " add.s64 %rd58, %rd58, 1024;\n" " add.s64 %rd57, %rd57, 1024;\n" " add.s64 %rd56, %rd56, 2048;\n" " add.s32 %r158, %r158, -2;\n" " setp.ne.s32 %p7, %r158, 0;\n" " @%p7 bra $L__BB11_5;\n" "\n" "$L__BB11_6:\n" " @%p8 bra $L__BB11_8;\n" "\n" " shl.b64 %rd45, %rd4, 3;\n" " add.s64 %rd46, %rd58, %rd45;\n" " ld.global.v2.u32 {%r93, %r94}, [%rd46];\n" " add.s64 %rd47, %rd57, %rd45;\n" " ld.global.v2.u32 {%r97, %r98}, [%rd47];\n" " shl.b64 %rd48, %rd4, 4;\n" " add.s64 %rd43, %rd56, %rd48;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f122,%f123,%f124,%f125}, [%rd43];\n" " // end inline asm\n" " cvt.u16.u32 %rs33, %r93;\n" " // begin inline asm\n" " cvt.f32.f16 %f126, %rs33;\n" " // end inline asm\n" " mov.f32 %f146, 0f00000000;\n" " fma.rn.f32 %f147, %f126, %f122, %f146;\n" " cvt.u16.u32 %rs34, %r97;\n" " // begin inline asm\n" " cvt.f32.f16 %f127, %rs34;\n" " // end inline asm\n" " fma.rn.f32 %f148, %f127, %f122, %f146;\n" " shr.u32 %r101, %r93, 16;\n" " cvt.u16.u32 %rs35, %r101;\n" " // begin inline asm\n" " cvt.f32.f16 %f128, %rs35;\n" " // end inline asm\n" " fma.rn.f32 %f149, %f128, %f123, %f147;\n" " shr.u32 %r102, %r97, 16;\n" " cvt.u16.u32 %rs36, %r102;\n" " // begin inline asm\n" " cvt.f32.f16 %f129, %rs36;\n" " // end inline asm\n" " fma.rn.f32 %f150, %f129, %f123, %f148;\n" " cvt.u16.u32 %rs37, %r94;\n" " // begin inline asm\n" " cvt.f32.f16 %f130, %rs37;\n" " // end inline asm\n" " fma.rn.f32 %f151, %f130, %f124, %f149;\n" " cvt.u16.u32 %rs38, %r98;\n" " // begin inline asm\n" " cvt.f32.f16 %f131, %rs38;\n" " // end inline asm\n" " fma.rn.f32 %f152, %f131, %f124, %f150;\n" " shr.u32 %r103, %r94, 16;\n" " cvt.u16.u32 %rs39, %r103;\n" " // begin inline asm\n" " cvt.f32.f16 %f132, %rs39;\n" " // end inline asm\n" " fma.rn.f32 %f153, %f132, %f125, %f151;\n" " shr.u32 %r104, %r98, 16;\n" " cvt.u16.u32 %rs40, %r104;\n" " // begin inline asm\n" " cvt.f32.f16 %f133, %rs40;\n" " // end inline asm\n" " fma.rn.f32 %f154, %f133, %f125, %f152;\n" " ld.global.v2.u32 {%r105, %r106}, [%rd46+256];\n" " ld.global.v2.u32 {%r109, %r110}, [%rd47+256];\n" " add.s64 %rd44, %rd43, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f134,%f135,%f136,%f137}, [%rd44];\n" " // end inline asm\n" " cvt.u16.u32 %rs41, %r105;\n" " // begin inline asm\n" " cvt.f32.f16 %f138, %rs41;\n" " // end inline asm\n" " fma.rn.f32 %f155, %f138, %f134, %f153;\n" " cvt.u16.u32 %rs42, %r109;\n" " // begin inline asm\n" " cvt.f32.f16 %f139, %rs42;\n" " // end inline asm\n" " fma.rn.f32 %f156, %f139, %f134, %f154;\n" " shr.u32 %r113, %r105, 16;\n" " cvt.u16.u32 %rs43, %r113;\n" " // begin inline asm\n" " cvt.f32.f16 %f140, %rs43;\n" " // end inline asm\n" " fma.rn.f32 %f157, %f140, %f135, %f155;\n" " shr.u32 %r114, %r109, 16;\n" " cvt.u16.u32 %rs44, %r114;\n" " // begin inline asm\n" " cvt.f32.f16 %f141, %rs44;\n" " // end inline asm\n" " fma.rn.f32 %f158, %f141, %f135, %f156;\n" " cvt.u16.u32 %rs45, %r106;\n" " // begin inline asm\n" " cvt.f32.f16 %f142, %rs45;\n" " // end inline asm\n" " fma.rn.f32 %f159, %f142, %f136, %f157;\n" " cvt.u16.u32 %rs46, %r110;\n" " // begin inline asm\n" " cvt.f32.f16 %f143, %rs46;\n" " // end inline asm\n" " fma.rn.f32 %f160, %f143, %f136, %f158;\n" " shr.u32 %r115, %r106, 16;\n" " cvt.u16.u32 %rs47, %r115;\n" " // begin inline asm\n" " cvt.f32.f16 %f144, %rs47;\n" " // end inline asm\n" " fma.rn.f32 %f161, %f144, %f137, %f159;\n" " shr.u32 %r116, %r110, 16;\n" " cvt.u16.u32 %rs48, %r116;\n" " // begin inline asm\n" " cvt.f32.f16 %f145, %rs48;\n" " // end inline asm\n" " fma.rn.f32 %f162, %f145, %f137, %f160;\n" " add.f32 %f200, %f200, %f161;\n" " add.f32 %f201, %f201, %f162;\n" "\n" "$L__BB11_8:\n" " shl.b64 %rd50, %rd7, 2;\n" " add.s64 %rd49, %rd24, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f163, [%rd49];\n" " // end inline asm\n" " fma.rn.f32 %f206, %f163, %f200, %f206;\n" " fma.rn.f32 %f207, %f163, %f201, %f207;\n" " cvt.u32.u64 %r117, %rd7;\n" " add.s32 %r157, %r117, %r35;\n" " setp.lt.s32 %p9, %r157, %r31;\n" " @%p9 bra $L__BB11_2;\n" "\n" "$L__BB11_9:\n" " cvta.to.global.u64 %rd20, %rd25;\n" " mov.u32 %r128, 16;\n" " mov.u32 %r137, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f164, %f206, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f167, %f206, %f164;\n" " mov.u32 %r130, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f166, %f167, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f169, %f167, %f166;\n" " mov.u32 %r132, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f168, %f169, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f171, %f169, %f168;\n" " mov.u32 %r134, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f170, %f171, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f173, %f171, %f170;\n" " mov.u32 %r136, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f172, %f173, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f19, %f173, %f172;\n" " // begin inline asm\n" " shfl.down.b32 %f174, %f207, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f177, %f207, %f174;\n" " // begin inline asm\n" " shfl.down.b32 %f176, %f177, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f179, %f177, %f176;\n" " // begin inline asm\n" " shfl.down.b32 %f178, %f179, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f181, %f179, %f178;\n" " // begin inline asm\n" " shfl.down.b32 %f180, %f181, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f183, %f181, %f180;\n" " // begin inline asm\n" " shfl.down.b32 %f182, %f183, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f20, %f183, %f182;\n" " setp.ne.s32 %p10, %r3, 0;\n" " @%p10 bra $L__BB11_11;\n" "\n" " shl.b32 %r138, %r2, 3;\n" " mov.u32 %r139, _ZZ19fc_moe_down_f16_vecE4sred;\n" " add.s32 %r140, %r139, %r138;\n" " st.shared.f32 [%r140], %f19;\n" " st.shared.f32 [%r140+4], %f20;\n" "\n" "$L__BB11_11:\n" " bar.sync 0;\n" " shl.b32 %r141, %r4, 1;\n" " setp.ge.s32 %p11, %r1, %r141;\n" " @%p11 bra $L__BB11_21;\n" "\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r142, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r142, %r16, 1, 31;\n" " setp.ge.s32 %p12, %r17, %r32;\n" " @%p12 bra $L__BB11_21;\n" "\n" " setp.lt.s32 %p13, %r35, 1;\n" " mov.f32 %f212, 0f00000000;\n" " @%p13 bra $L__BB11_20;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r163, %r35, 3;\n" " add.s32 %r144, %r35, -1;\n" " setp.lt.u32 %p14, %r144, 3;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r161, 0;\n" " @%p14 bra $L__BB11_17;\n" "\n" " sub.s32 %r160, %r35, %r163;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r161, 0;\n" "\n" "$L__BB11_16:\n" " add.s32 %r146, %r161, %r18;\n" " shl.b32 %r147, %r146, 3;\n" " mov.u32 %r148, _ZZ19fc_moe_down_f16_vecE4sred;\n" " add.s32 %r149, %r148, %r147;\n" " shl.b32 %r150, %r16, 2;\n" " add.s32 %r151, %r149, %r150;\n" " ld.shared.f32 %f188, [%r151];\n" " add.f32 %f189, %f212, %f188;\n" " ld.shared.f32 %f190, [%r151+8];\n" " add.f32 %f191, %f189, %f190;\n" " ld.shared.f32 %f192, [%r151+16];\n" " add.f32 %f193, %f191, %f192;\n" " ld.shared.f32 %f194, [%r151+24];\n" " add.f32 %f212, %f193, %f194;\n" " add.s32 %r161, %r161, 4;\n" " add.s32 %r160, %r160, -4;\n" " setp.ne.s32 %p15, %r160, 0;\n" " @%p15 bra $L__BB11_16;\n" "\n" "$L__BB11_17:\n" " setp.eq.s32 %p16, %r163, 0;\n" " @%p16 bra $L__BB11_20;\n" "\n" " add.s32 %r152, %r161, %r18;\n" " shl.b32 %r153, %r152, 3;\n" " mov.u32 %r154, _ZZ19fc_moe_down_f16_vecE4sred;\n" " add.s32 %r155, %r154, %r153;\n" " shl.b32 %r156, %r16, 2;\n" " add.s32 %r162, %r155, %r156;\n" "\n" "$L__BB11_19:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f195, [%r162];\n" " add.f32 %f212, %f212, %f195;\n" " add.s32 %r162, %r162, 8;\n" " add.s32 %r163, %r163, -1;\n" " setp.ne.s32 %p17, %r163, 0;\n" " @%p17 bra $L__BB11_19;\n" "\n" "$L__BB11_20:\n" " mul.wide.s32 %rd51, %r17, 4;\n" " add.s64 %rd52, %rd20, %rd51;\n" " st.global.f32 [%rd52], %f212;\n" "\n" "$L__BB11_21:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_bf16_vec\n" ".visible .entry fc_moe_down_bf16_vec(\n" " .param .u64 fc_moe_down_bf16_vec_param_0,\n" " .param .u64 fc_moe_down_bf16_vec_param_1,\n" " .param .u64 fc_moe_down_bf16_vec_param_2,\n" " .param .u64 fc_moe_down_bf16_vec_param_3,\n" " .param .u64 fc_moe_down_bf16_vec_param_4,\n" " .param .u32 fc_moe_down_bf16_vec_param_5,\n" " .param .u32 fc_moe_down_bf16_vec_param_6,\n" " .param .u32 fc_moe_down_bf16_vec_param_7,\n" " .param .u32 fc_moe_down_bf16_vec_param_8,\n" " .param .u32 fc_moe_down_bf16_vec_param_9\n" ")\n" "{\n" " .reg .pred %p<18>;\n" " .reg .f32 %f<213>;\n" " .reg .b32 %r<188>;\n" " .reg .b64 %rd<59>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ20fc_moe_down_bf16_vecE4sred[128];\n" "\n" " ld.param.u64 %rd21, [fc_moe_down_bf16_vec_param_0];\n" " ld.param.u64 %rd22, [fc_moe_down_bf16_vec_param_1];\n" " ld.param.u64 %rd23, [fc_moe_down_bf16_vec_param_2];\n" " ld.param.u64 %rd24, [fc_moe_down_bf16_vec_param_3];\n" " ld.param.u64 %rd25, [fc_moe_down_bf16_vec_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_bf16_vec_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_bf16_vec_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_bf16_vec_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_bf16_vec_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_bf16_vec_param_9];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r36, %ntid.x;\n" " shr.u32 %r37, %r36, 5;\n" " div.u32 %r4, %r37, %r35;\n" " div.s32 %r38, %r2, %r35;\n" " mul.lo.s32 %r39, %r38, %r35;\n" " sub.s32 %r181, %r2, %r39;\n" " mov.u32 %r40, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r40;\n" " add.s32 %r41, %r6, %r38;\n" " shl.b32 %r7, %r41, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r181, %r31;\n" " mov.f32 %f206, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f207, %f206;\n" " @%p3 bra $L__BB12_9;\n" "\n" " cvt.s64.s32 %rd1, %r34;\n" " shr.s32 %r42, %r33, 8;\n" " cvt.s64.s32 %rd2, %r7;\n" " cvt.s64.s32 %rd3, %r33;\n" " cvt.u64.u32 %rd4, %r3;\n" " max.s32 %r43, %r42, 1;\n" " and.b32 %r8, %r43, 1;\n" " sub.s32 %r9, %r43, %r8;\n" " cvt.s64.s32 %rd5, %r32;\n" " add.s32 %r10, %r7, 1;\n" " cvta.to.global.u64 %rd6, %rd21;\n" " mov.f32 %f31, 0f00000000;\n" " setp.lt.s32 %p4, %r10, %r32;\n" " setp.lt.s32 %p5, %r33, 256;\n" " setp.lt.s32 %p6, %r33, 512;\n" " setp.eq.s32 %p8, %r8, 0;\n" " mov.f32 %f207, %f31;\n" " mov.f32 %f206, %f31;\n" "\n" "$L__BB12_2:\n" " cvt.s64.s32 %rd7, %r181;\n" " mul.wide.s32 %rd27, %r181, 4;\n" " add.s64 %rd26, %rd23, %rd27;\n" " // begin inline asm\n" " ld.global.nc.s32 %r44, [%rd26];\n" " // end inline asm\n" " cvt.s64.s32 %rd28, %r44;\n" " mul.lo.s64 %rd29, %rd28, %rd5;\n" " add.s64 %rd30, %rd29, %rd2;\n" " mul.lo.s64 %rd31, %rd30, %rd1;\n" " add.s64 %rd58, %rd6, %rd31;\n" " add.s64 %rd32, %rd58, %rd1;\n" " selp.b64 %rd57, %rd32, %rd58, %p4;\n" " mov.f32 %f200, %f31;\n" " mov.f32 %f201, %f31;\n" " @%p5 bra $L__BB12_8;\n" "\n" " mul.lo.s64 %rd33, %rd7, %rd3;\n" " shl.b64 %rd34, %rd33, 2;\n" " add.s64 %rd56, %rd22, %rd34;\n" " mov.f32 %f201, 0f00000000;\n" " mov.f32 %f200, %f201;\n" " @%p6 bra $L__BB12_6;\n" "\n" " mov.f32 %f38, 0f00000000;\n" " mov.f32 %f201, %f38;\n" " mov.f32 %f200, %f38;\n" " mov.u32 %r182, %r9;\n" "\n" "$L__BB12_5:\n" " shl.b64 %rd39, %rd4, 3;\n" " add.s64 %rd40, %rd58, %rd39;\n" " ld.global.v2.u32 {%r45, %r46}, [%rd40];\n" " add.s64 %rd41, %rd57, %rd39;\n" " ld.global.v2.u32 {%r49, %r50}, [%rd41];\n" " shl.b64 %rd42, %rd4, 4;\n" " add.s64 %rd35, %rd56, %rd42;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f39,%f40,%f41,%f42}, [%rd35];\n" " // end inline asm\n" " shl.b32 %r53, %r45, 16;\n" " mov.b32 %f55, %r53;\n" " fma.rn.f32 %f57, %f55, %f39, %f38;\n" " shl.b32 %r54, %r49, 16;\n" " mov.b32 %f58, %r54;\n" " fma.rn.f32 %f59, %f58, %f39, %f38;\n" " and.b32 %r55, %r45, -65536;\n" " mov.b32 %f60, %r55;\n" " fma.rn.f32 %f61, %f60, %f40, %f57;\n" " and.b32 %r56, %r49, -65536;\n" " mov.b32 %f62, %r56;\n" " fma.rn.f32 %f63, %f62, %f40, %f59;\n" " shl.b32 %r57, %r46, 16;\n" " mov.b32 %f64, %r57;\n" " fma.rn.f32 %f65, %f64, %f41, %f61;\n" " shl.b32 %r58, %r50, 16;\n" " mov.b32 %f66, %r58;\n" " fma.rn.f32 %f67, %f66, %f41, %f63;\n" " and.b32 %r59, %r46, -65536;\n" " mov.b32 %f68, %r59;\n" " fma.rn.f32 %f69, %f68, %f42, %f65;\n" " and.b32 %r60, %r50, -65536;\n" " mov.b32 %f70, %r60;\n" " fma.rn.f32 %f71, %f70, %f42, %f67;\n" " ld.global.v2.u32 {%r61, %r62}, [%rd40+256];\n" " ld.global.v2.u32 {%r65, %r66}, [%rd41+256];\n" " add.s64 %rd36, %rd35, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f43,%f44,%f45,%f46}, [%rd36];\n" " // end inline asm\n" " shl.b32 %r69, %r61, 16;\n" " mov.b32 %f72, %r69;\n" " fma.rn.f32 %f73, %f72, %f43, %f69;\n" " shl.b32 %r70, %r65, 16;\n" " mov.b32 %f74, %r70;\n" " fma.rn.f32 %f75, %f74, %f43, %f71;\n" " and.b32 %r71, %r61, -65536;\n" " mov.b32 %f76, %r71;\n" " fma.rn.f32 %f77, %f76, %f44, %f73;\n" " and.b32 %r72, %r65, -65536;\n" " mov.b32 %f78, %r72;\n" " fma.rn.f32 %f79, %f78, %f44, %f75;\n" " shl.b32 %r73, %r62, 16;\n" " mov.b32 %f80, %r73;\n" " fma.rn.f32 %f81, %f80, %f45, %f77;\n" " shl.b32 %r74, %r66, 16;\n" " mov.b32 %f82, %r74;\n" " fma.rn.f32 %f83, %f82, %f45, %f79;\n" " and.b32 %r75, %r62, -65536;\n" " mov.b32 %f84, %r75;\n" " fma.rn.f32 %f85, %f84, %f46, %f81;\n" " and.b32 %r76, %r66, -65536;\n" " mov.b32 %f86, %r76;\n" " fma.rn.f32 %f87, %f86, %f46, %f83;\n" " add.f32 %f88, %f200, %f85;\n" " add.f32 %f89, %f201, %f87;\n" " ld.global.v2.u32 {%r77, %r78}, [%rd40+512];\n" " ld.global.v2.u32 {%r81, %r82}, [%rd41+512];\n" " add.s64 %rd37, %rd35, 1024;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f47,%f48,%f49,%f50}, [%rd37];\n" " // end inline asm\n" " shl.b32 %r85, %r77, 16;\n" " mov.b32 %f90, %r85;\n" " fma.rn.f32 %f91, %f90, %f47, %f38;\n" " shl.b32 %r86, %r81, 16;\n" " mov.b32 %f92, %r86;\n" " fma.rn.f32 %f93, %f92, %f47, %f38;\n" " and.b32 %r87, %r77, -65536;\n" " mov.b32 %f94, %r87;\n" " fma.rn.f32 %f95, %f94, %f48, %f91;\n" " and.b32 %r88, %r81, -65536;\n" " mov.b32 %f96, %r88;\n" " fma.rn.f32 %f97, %f96, %f48, %f93;\n" " shl.b32 %r89, %r78, 16;\n" " mov.b32 %f98, %r89;\n" " fma.rn.f32 %f99, %f98, %f49, %f95;\n" " shl.b32 %r90, %r82, 16;\n" " mov.b32 %f100, %r90;\n" " fma.rn.f32 %f101, %f100, %f49, %f97;\n" " and.b32 %r91, %r78, -65536;\n" " mov.b32 %f102, %r91;\n" " fma.rn.f32 %f103, %f102, %f50, %f99;\n" " and.b32 %r92, %r82, -65536;\n" " mov.b32 %f104, %r92;\n" " fma.rn.f32 %f105, %f104, %f50, %f101;\n" " ld.global.v2.u32 {%r93, %r94}, [%rd40+768];\n" " ld.global.v2.u32 {%r97, %r98}, [%rd41+768];\n" " add.s64 %rd38, %rd35, 1536;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f51,%f52,%f53,%f54}, [%rd38];\n" " // end inline asm\n" " shl.b32 %r101, %r93, 16;\n" " mov.b32 %f106, %r101;\n" " fma.rn.f32 %f107, %f106, %f51, %f103;\n" " shl.b32 %r102, %r97, 16;\n" " mov.b32 %f108, %r102;\n" " fma.rn.f32 %f109, %f108, %f51, %f105;\n" " and.b32 %r103, %r93, -65536;\n" " mov.b32 %f110, %r103;\n" " fma.rn.f32 %f111, %f110, %f52, %f107;\n" " and.b32 %r104, %r97, -65536;\n" " mov.b32 %f112, %r104;\n" " fma.rn.f32 %f113, %f112, %f52, %f109;\n" " shl.b32 %r105, %r94, 16;\n" " mov.b32 %f114, %r105;\n" " fma.rn.f32 %f115, %f114, %f53, %f111;\n" " shl.b32 %r106, %r98, 16;\n" " mov.b32 %f116, %r106;\n" " fma.rn.f32 %f117, %f116, %f53, %f113;\n" " and.b32 %r107, %r94, -65536;\n" " mov.b32 %f118, %r107;\n" " fma.rn.f32 %f119, %f118, %f54, %f115;\n" " and.b32 %r108, %r98, -65536;\n" " mov.b32 %f120, %r108;\n" " fma.rn.f32 %f121, %f120, %f54, %f117;\n" " add.f32 %f200, %f88, %f119;\n" " add.f32 %f201, %f89, %f121;\n" " add.s64 %rd58, %rd58, 1024;\n" " add.s64 %rd57, %rd57, 1024;\n" " add.s64 %rd56, %rd56, 2048;\n" " add.s32 %r182, %r182, -2;\n" " setp.ne.s32 %p7, %r182, 0;\n" " @%p7 bra $L__BB12_5;\n" "\n" "$L__BB12_6:\n" " @%p8 bra $L__BB12_8;\n" "\n" " shl.b64 %rd45, %rd4, 3;\n" " add.s64 %rd46, %rd58, %rd45;\n" " ld.global.v2.u32 {%r109, %r110}, [%rd46];\n" " add.s64 %rd47, %rd57, %rd45;\n" " ld.global.v2.u32 {%r113, %r114}, [%rd47];\n" " shl.b64 %rd48, %rd4, 4;\n" " add.s64 %rd43, %rd56, %rd48;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f122,%f123,%f124,%f125}, [%rd43];\n" " // end inline asm\n" " shl.b32 %r117, %r109, 16;\n" " mov.b32 %f130, %r117;\n" " mov.f32 %f131, 0f00000000;\n" " fma.rn.f32 %f132, %f130, %f122, %f131;\n" " shl.b32 %r118, %r113, 16;\n" " mov.b32 %f133, %r118;\n" " fma.rn.f32 %f134, %f133, %f122, %f131;\n" " and.b32 %r119, %r109, -65536;\n" " mov.b32 %f135, %r119;\n" " fma.rn.f32 %f136, %f135, %f123, %f132;\n" " and.b32 %r120, %r113, -65536;\n" " mov.b32 %f137, %r120;\n" " fma.rn.f32 %f138, %f137, %f123, %f134;\n" " shl.b32 %r121, %r110, 16;\n" " mov.b32 %f139, %r121;\n" " fma.rn.f32 %f140, %f139, %f124, %f136;\n" " shl.b32 %r122, %r114, 16;\n" " mov.b32 %f141, %r122;\n" " fma.rn.f32 %f142, %f141, %f124, %f138;\n" " and.b32 %r123, %r110, -65536;\n" " mov.b32 %f143, %r123;\n" " fma.rn.f32 %f144, %f143, %f125, %f140;\n" " and.b32 %r124, %r114, -65536;\n" " mov.b32 %f145, %r124;\n" " fma.rn.f32 %f146, %f145, %f125, %f142;\n" " ld.global.v2.u32 {%r125, %r126}, [%rd46+256];\n" " ld.global.v2.u32 {%r129, %r130}, [%rd47+256];\n" " add.s64 %rd44, %rd43, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f126,%f127,%f128,%f129}, [%rd44];\n" " // end inline asm\n" " shl.b32 %r133, %r125, 16;\n" " mov.b32 %f147, %r133;\n" " fma.rn.f32 %f148, %f147, %f126, %f144;\n" " shl.b32 %r134, %r129, 16;\n" " mov.b32 %f149, %r134;\n" " fma.rn.f32 %f150, %f149, %f126, %f146;\n" " and.b32 %r135, %r125, -65536;\n" " mov.b32 %f151, %r135;\n" " fma.rn.f32 %f152, %f151, %f127, %f148;\n" " and.b32 %r136, %r129, -65536;\n" " mov.b32 %f153, %r136;\n" " fma.rn.f32 %f154, %f153, %f127, %f150;\n" " shl.b32 %r137, %r126, 16;\n" " mov.b32 %f155, %r137;\n" " fma.rn.f32 %f156, %f155, %f128, %f152;\n" " shl.b32 %r138, %r130, 16;\n" " mov.b32 %f157, %r138;\n" " fma.rn.f32 %f158, %f157, %f128, %f154;\n" " and.b32 %r139, %r126, -65536;\n" " mov.b32 %f159, %r139;\n" " fma.rn.f32 %f160, %f159, %f129, %f156;\n" " and.b32 %r140, %r130, -65536;\n" " mov.b32 %f161, %r140;\n" " fma.rn.f32 %f162, %f161, %f129, %f158;\n" " add.f32 %f200, %f200, %f160;\n" " add.f32 %f201, %f201, %f162;\n" "\n" "$L__BB12_8:\n" " shl.b64 %rd50, %rd7, 2;\n" " add.s64 %rd49, %rd24, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f163, [%rd49];\n" " // end inline asm\n" " fma.rn.f32 %f206, %f163, %f200, %f206;\n" " fma.rn.f32 %f207, %f163, %f201, %f207;\n" " cvt.u32.u64 %r141, %rd7;\n" " add.s32 %r181, %r141, %r35;\n" " setp.lt.s32 %p9, %r181, %r31;\n" " @%p9 bra $L__BB12_2;\n" "\n" "$L__BB12_9:\n" " cvta.to.global.u64 %rd20, %rd25;\n" " mov.u32 %r152, 16;\n" " mov.u32 %r161, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f164, %f206, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f167, %f206, %f164;\n" " mov.u32 %r154, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f166, %f167, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f169, %f167, %f166;\n" " mov.u32 %r156, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f168, %f169, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f171, %f169, %f168;\n" " mov.u32 %r158, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f170, %f171, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f173, %f171, %f170;\n" " mov.u32 %r160, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f172, %f173, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f19, %f173, %f172;\n" " // begin inline asm\n" " shfl.down.b32 %f174, %f207, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f177, %f207, %f174;\n" " // begin inline asm\n" " shfl.down.b32 %f176, %f177, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f179, %f177, %f176;\n" " // begin inline asm\n" " shfl.down.b32 %f178, %f179, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f181, %f179, %f178;\n" " // begin inline asm\n" " shfl.down.b32 %f180, %f181, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f183, %f181, %f180;\n" " // begin inline asm\n" " shfl.down.b32 %f182, %f183, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f20, %f183, %f182;\n" " setp.ne.s32 %p10, %r3, 0;\n" " @%p10 bra $L__BB12_11;\n" "\n" " shl.b32 %r162, %r2, 3;\n" " mov.u32 %r163, _ZZ20fc_moe_down_bf16_vecE4sred;\n" " add.s32 %r164, %r163, %r162;\n" " st.shared.f32 [%r164], %f19;\n" " st.shared.f32 [%r164+4], %f20;\n" "\n" "$L__BB12_11:\n" " bar.sync 0;\n" " shl.b32 %r165, %r4, 1;\n" " setp.ge.s32 %p11, %r1, %r165;\n" " @%p11 bra $L__BB12_21;\n" "\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r166, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r166, %r16, 1, 31;\n" " setp.ge.s32 %p12, %r17, %r32;\n" " @%p12 bra $L__BB12_21;\n" "\n" " setp.lt.s32 %p13, %r35, 1;\n" " mov.f32 %f212, 0f00000000;\n" " @%p13 bra $L__BB12_20;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r187, %r35, 3;\n" " add.s32 %r168, %r35, -1;\n" " setp.lt.u32 %p14, %r168, 3;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r185, 0;\n" " @%p14 bra $L__BB12_17;\n" "\n" " sub.s32 %r184, %r35, %r187;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r185, 0;\n" "\n" "$L__BB12_16:\n" " add.s32 %r170, %r185, %r18;\n" " shl.b32 %r171, %r170, 3;\n" " mov.u32 %r172, _ZZ20fc_moe_down_bf16_vecE4sred;\n" " add.s32 %r173, %r172, %r171;\n" " shl.b32 %r174, %r16, 2;\n" " add.s32 %r175, %r173, %r174;\n" " ld.shared.f32 %f188, [%r175];\n" " add.f32 %f189, %f212, %f188;\n" " ld.shared.f32 %f190, [%r175+8];\n" " add.f32 %f191, %f189, %f190;\n" " ld.shared.f32 %f192, [%r175+16];\n" " add.f32 %f193, %f191, %f192;\n" " ld.shared.f32 %f194, [%r175+24];\n" " add.f32 %f212, %f193, %f194;\n" " add.s32 %r185, %r185, 4;\n" " add.s32 %r184, %r184, -4;\n" " setp.ne.s32 %p15, %r184, 0;\n" " @%p15 bra $L__BB12_16;\n" "\n" "$L__BB12_17:\n" " setp.eq.s32 %p16, %r187, 0;\n" " @%p16 bra $L__BB12_20;\n" "\n" " add.s32 %r176, %r185, %r18;\n" " shl.b32 %r177, %r176, 3;\n" " mov.u32 %r178, _ZZ20fc_moe_down_bf16_vecE4sred;\n" " add.s32 %r179, %r178, %r177;\n" " shl.b32 %r180, %r16, 2;\n" " add.s32 %r186, %r179, %r180;\n" "\n" "$L__BB12_19:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f195, [%r186];\n" " add.f32 %f212, %f212, %f195;\n" " add.s32 %r186, %r186, 8;\n" " add.s32 %r187, %r187, -1;\n" " setp.ne.s32 %p17, %r187, 0;\n" " @%p17 bra $L__BB12_19;\n" "\n" "$L__BB12_20:\n" " mul.wide.s32 %rd51, %r17, 4;\n" " add.s64 %rd52, %rd20, %rd51;\n" " st.global.f32 [%rd52], %f212;\n" "\n" "$L__BB12_21:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_q3k_nx\n" ".visible .entry fc_moe_gateup_q3k_nx(\n" " .param .u64 fc_moe_gateup_q3k_nx_param_0,\n" " .param .u64 fc_moe_gateup_q3k_nx_param_1,\n" " .param .u64 fc_moe_gateup_q3k_nx_param_2,\n" " .param .u64 fc_moe_gateup_q3k_nx_param_3,\n" " .param .u64 fc_moe_gateup_q3k_nx_param_4,\n" " .param .u32 fc_moe_gateup_q3k_nx_param_5,\n" " .param .u32 fc_moe_gateup_q3k_nx_param_6,\n" " .param .u32 fc_moe_gateup_q3k_nx_param_7,\n" " .param .u32 fc_moe_gateup_q3k_nx_param_8\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .f32 %f<189>;\n" " .reg .b32 %r<148>;\n" " .reg .b64 %rd<77>;\n" "\n" "\n" " ld.param.u64 %rd26, [fc_moe_gateup_q3k_nx_param_0];\n" " ld.param.u64 %rd27, [fc_moe_gateup_q3k_nx_param_1];\n" " ld.param.u64 %rd28, [fc_moe_gateup_q3k_nx_param_2];\n" " ld.param.u64 %rd31, [fc_moe_gateup_q3k_nx_param_3];\n" " ld.param.u64 %rd29, [fc_moe_gateup_q3k_nx_param_4];\n" " ld.param.u32 %r32, [fc_moe_gateup_q3k_nx_param_5];\n" " ld.param.u32 %r28, [fc_moe_gateup_q3k_nx_param_6];\n" " ld.param.u32 %r29, [fc_moe_gateup_q3k_nx_param_7];\n" " ld.param.u32 %r30, [fc_moe_gateup_q3k_nx_param_8];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r33, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r34, %ctaid.x;\n" " shl.b32 %r35, %r34, 3;\n" " add.s32 %r36, %r35, %r33;\n" " mov.u32 %r37, %ctaid.z;\n" " mul.wide.s32 %rd1, %r29, %r37;\n" " mul.wide.s32 %rd32, %r32, %r37;\n" " mov.u32 %r38, %ctaid.y;\n" " cvt.s64.s32 %rd33, %r38;\n" " add.s64 %rd2, %rd32, %rd33;\n" " shl.b64 %rd34, %rd2, 2;\n" " add.s64 %rd30, %rd31, %rd34;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd30];\n" " // end inline asm\n" " cvt.s64.s32 %rd3, %r28;\n" " cvt.s64.s32 %rd4, %r36;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f185, 0f00000000;\n" " mov.f32 %f186, %f185;\n" " @%p1 bra $L__BB13_15;\n" "\n" " cvt.s64.s32 %rd35, %r30;\n" " cvt.s64.s32 %rd36, %r31;\n" " mul.lo.s64 %rd37, %rd36, %rd3;\n" " add.s64 %rd38, %rd37, %rd4;\n" " mul.lo.s64 %rd39, %rd38, %rd35;\n" " cvt.u64.u32 %rd5, %r2;\n" " shr.u32 %r41, %r2, 4;\n" " cvt.u64.u32 %rd6, %r41;\n" " mov.u32 %r42, -2;\n" " sub.s32 %r4, %r42, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd7, %r1;\n" " shl.b32 %r43, %r1, 2;\n" " mov.u32 %r44, sh_x;\n" " add.s32 %r6, %r44, %r43;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd8, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd9, %r8;\n" " add.s32 %r9, %r1, 768;\n" " shl.b64 %rd40, %rd1, 2;\n" " add.s64 %rd10, %rd28, %rd40;\n" " cvta.to.global.u64 %rd41, %rd27;\n" " add.s64 %rd76, %rd41, %rd39;\n" " cvta.to.global.u64 %rd42, %rd26;\n" " add.s64 %rd75, %rd42, %rd39;\n" " mov.f32 %f186, 0f00000000;\n" " mov.u32 %r141, 0;\n" " shl.b64 %rd59, %rd5, 2;\n" " shl.b64 %rd62, %rd6, 3;\n" " mov.u32 %r142, %r141;\n" " mov.f32 %f185, %f186;\n" "\n" "$L__BB13_2:\n" " add.s32 %r45, %r141, %r5;\n" " max.s32 %r46, %r45, -2049;\n" " sub.s32 %r12, %r4, %r46;\n" " sub.s32 %r47, %r29, %r142;\n" " min.s32 %r13, %r47, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB13_10;\n" "\n" " shr.u32 %r48, %r12, 8;\n" " add.s32 %r49, %r48, 1;\n" " cvt.s64.s32 %rd15, %r142;\n" " add.s64 %rd16, %rd1, %rd15;\n" " and.b32 %r14, %r49, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r143, %r1;\n" " @%p3 bra $L__BB13_7;\n" "\n" " add.s64 %rd44, %rd16, %rd7;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd43, %rd28, %rd45;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd43];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r143, %r7;\n" " @%p4 bra $L__BB13_7;\n" "\n" " add.s64 %rd47, %rd16, %rd8;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd46, %rd28, %rd48;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd46];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r143, %r8;\n" " @%p5 bra $L__BB13_7;\n" "\n" " add.s64 %rd50, %rd16, %rd9;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd49, %rd28, %rd51;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd49];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r143, %r9;\n" "\n" "$L__BB13_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB13_10;\n" "\n" " cvt.s64.s32 %rd52, %r143;\n" " add.s64 %rd53, %rd52, %rd15;\n" " shl.b64 %rd54, %rd53, 2;\n" " add.s64 %rd72, %rd10, %rd54;\n" " shl.b32 %r50, %r143, 2;\n" " mov.u32 %r51, sh_x;\n" " add.s32 %r52, %r51, %r50;\n" " add.s32 %r144, %r52, 2048;\n" "\n" "$L__BB13_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd72];\n" " // end inline asm\n" " st.shared.f32 [%r144+-2048], %f20;\n" " add.s64 %rd56, %rd72, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd56];\n" " // end inline asm\n" " st.shared.f32 [%r144+-1024], %f21;\n" " add.s64 %rd57, %rd72, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd57];\n" " // end inline asm\n" " st.shared.f32 [%r144], %f22;\n" " add.s64 %rd58, %rd72, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd58];\n" " // end inline asm\n" " st.shared.f32 [%r144+1024], %f23;\n" " add.s64 %rd72, %rd72, 4096;\n" " add.s32 %r144, %r144, 4096;\n" " add.s32 %r143, %r143, 1024;\n" " setp.lt.s32 %p7, %r143, %r13;\n" " @%p7 bra $L__BB13_9;\n" "\n" "$L__BB13_10:\n" " cvt.u32.u64 %r53, %rd4;\n" " setp.ge.s32 %p8, %r53, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB13_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB13_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r146, 0;\n" " mov.u32 %r147, %r44;\n" "\n" "$L__BB13_13:\n" " add.s64 %rd60, %rd75, %rd59;\n" " add.s64 %rd61, %rd76, %rd59;\n" " add.s64 %rd63, %rd75, %rd62;\n" " ld.global.v2.u32 {%r56, %r57}, [%rd63+128];\n" " add.s64 %rd64, %rd76, %rd62;\n" " ld.global.v2.u32 {%r60, %r61}, [%rd64+128];\n" " ld.global.u32 %r64, [%rd60];\n" " shr.u32 %r65, %r64, 16;\n" " ld.global.u32 %r66, [%rd61];\n" " shr.u32 %r67, %r66, 16;\n" " and.b32 %r68, %r64, 15;\n" " or.b32 %r69, %r68, 1258291200;\n" " mov.b32 %f24, %r69;\n" " mov.f32 %f25, 0fCB000004;\n" " mov.f32 %f26, 0f3F800000;\n" " fma.rn.f32 %f27, %f24, %f26, %f25;\n" " and.b32 %r70, %r56, 255;\n" " cvt.rn.f32.u32 %f28, %r70;\n" " mov.f32 %f29, 0f42000000;\n" " sub.rn.f32 %f30, %f28, %f29;\n" " and.b32 %r71, %r66, 15;\n" " or.b32 %r72, %r71, 1258291200;\n" " mov.b32 %f31, %r72;\n" " fma.rn.f32 %f32, %f31, %f26, %f25;\n" " and.b32 %r73, %r60, 255;\n" " cvt.rn.f32.u32 %f33, %r73;\n" " sub.rn.f32 %f34, %f33, %f29;\n" " mul.rn.f32 %f35, %f30, %f27;\n" " shl.b32 %r74, %r2, 2;\n" " add.s32 %r75, %r147, %r74;\n" " ld.shared.f32 %f36, [%r75];\n" " mov.f32 %f37, 0f00000000;\n" " fma.rn.f32 %f38, %f35, %f36, %f37;\n" " mul.rn.f32 %f39, %f34, %f32;\n" " fma.rn.f32 %f40, %f39, %f36, %f37;\n" " and.b32 %r76, %r64, 240;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f41, %r77;\n" " mov.f32 %f42, 0fC9000040;\n" " mov.f32 %f43, 0f3D800000;\n" " fma.rn.f32 %f44, %f41, %f43, %f42;\n" " bfe.u32 %r78, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f45, %r78;\n" " sub.rn.f32 %f46, %f45, %f29;\n" " and.b32 %r79, %r66, 240;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f47, %r80;\n" " fma.rn.f32 %f48, %f47, %f43, %f42;\n" " bfe.u32 %r81, %r60, 8, 8;\n" " cvt.rn.f32.u32 %f49, %r81;\n" " sub.rn.f32 %f50, %f49, %f29;\n" " mul.rn.f32 %f51, %f46, %f44;\n" " ld.shared.f32 %f52, [%r75+128];\n" " fma.rn.f32 %f53, %f51, %f52, %f38;\n" " mul.rn.f32 %f54, %f50, %f48;\n" " fma.rn.f32 %f55, %f54, %f52, %f40;\n" " and.b32 %r82, %r64, 3840;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f56, %r83;\n" " mov.f32 %f57, 0fC7000400;\n" " mov.f32 %f58, 0f3B800000;\n" " fma.rn.f32 %f59, %f56, %f58, %f57;\n" " bfe.u32 %r84, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f60, %r84;\n" " sub.rn.f32 %f61, %f60, %f29;\n" " and.b32 %r85, %r66, 3840;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f62, %r86;\n" " fma.rn.f32 %f63, %f62, %f58, %f57;\n" " bfe.u32 %r87, %r60, 16, 8;\n" " cvt.rn.f32.u32 %f64, %r87;\n" " sub.rn.f32 %f65, %f64, %f29;\n" " mul.rn.f32 %f66, %f61, %f59;\n" " ld.shared.f32 %f67, [%r75+256];\n" " fma.rn.f32 %f68, %f66, %f67, %f53;\n" " mul.rn.f32 %f69, %f65, %f63;\n" " fma.rn.f32 %f70, %f69, %f67, %f55;\n" " and.b32 %r88, %r64, 61440;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f71, %r89;\n" " mov.f32 %f72, 0fC5004000;\n" " mov.f32 %f73, 0f39800000;\n" " fma.rn.f32 %f74, %f71, %f73, %f72;\n" " shr.u32 %r90, %r56, 24;\n" " cvt.rn.f32.u32 %f75, %r90;\n" " sub.rn.f32 %f76, %f75, %f29;\n" " and.b32 %r91, %r66, 61440;\n" " or.b32 %r92, %r91, 1258291200;\n" " mov.b32 %f77, %r92;\n" " fma.rn.f32 %f78, %f77, %f73, %f72;\n" " shr.u32 %r93, %r60, 24;\n" " cvt.rn.f32.u32 %f79, %r93;\n" " sub.rn.f32 %f80, %f79, %f29;\n" " mul.rn.f32 %f81, %f76, %f74;\n" " ld.shared.f32 %f82, [%r75+384];\n" " fma.rn.f32 %f83, %f81, %f82, %f68;\n" " mul.rn.f32 %f84, %f80, %f78;\n" " fma.rn.f32 %f85, %f84, %f82, %f70;\n" " and.b32 %r94, %r64, 983040;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f86, %r95;\n" " mov.f32 %f87, 0fC3040000;\n" " mov.f32 %f88, 0f37800000;\n" " fma.rn.f32 %f89, %f86, %f88, %f87;\n" " and.b32 %r96, %r57, 255;\n" " cvt.rn.f32.u32 %f90, %r96;\n" " sub.rn.f32 %f91, %f90, %f29;\n" " and.b32 %r97, %r66, 983040;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f92, %r98;\n" " fma.rn.f32 %f93, %f92, %f88, %f87;\n" " and.b32 %r99, %r61, 255;\n" " cvt.rn.f32.u32 %f94, %r99;\n" " sub.rn.f32 %f95, %f94, %f29;\n" " mul.rn.f32 %f96, %f91, %f89;\n" " ld.shared.f32 %f97, [%r75+512];\n" " fma.rn.f32 %f98, %f96, %f97, %f83;\n" " mul.rn.f32 %f99, %f95, %f93;\n" " fma.rn.f32 %f100, %f99, %f97, %f85;\n" " and.b32 %r100, %r65, 240;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f101, %r101;\n" " fma.rn.f32 %f102, %f101, %f43, %f42;\n" " bfe.u32 %r102, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f103, %r102;\n" " sub.rn.f32 %f104, %f103, %f29;\n" " and.b32 %r103, %r67, 240;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f105, %r104;\n" " fma.rn.f32 %f106, %f105, %f43, %f42;\n" " bfe.u32 %r105, %r61, 8, 8;\n" " cvt.rn.f32.u32 %f107, %r105;\n" " sub.rn.f32 %f108, %f107, %f29;\n" " mul.rn.f32 %f109, %f104, %f102;\n" " ld.shared.f32 %f110, [%r75+640];\n" " fma.rn.f32 %f111, %f109, %f110, %f98;\n" " mul.rn.f32 %f112, %f108, %f106;\n" " fma.rn.f32 %f113, %f112, %f110, %f100;\n" " and.b32 %r106, %r65, 3840;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f114, %r107;\n" " fma.rn.f32 %f115, %f114, %f58, %f57;\n" " bfe.u32 %r108, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f116, %r108;\n" " sub.rn.f32 %f117, %f116, %f29;\n" " and.b32 %r109, %r67, 3840;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f118, %r110;\n" " fma.rn.f32 %f119, %f118, %f58, %f57;\n" " bfe.u32 %r111, %r61, 16, 8;\n" " cvt.rn.f32.u32 %f120, %r111;\n" " sub.rn.f32 %f121, %f120, %f29;\n" " mul.rn.f32 %f122, %f117, %f115;\n" " ld.shared.f32 %f123, [%r75+768];\n" " fma.rn.f32 %f124, %f122, %f123, %f111;\n" " mul.rn.f32 %f125, %f121, %f119;\n" " fma.rn.f32 %f126, %f125, %f123, %f113;\n" " and.b32 %r112, %r65, 61440;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f127, %r113;\n" " fma.rn.f32 %f128, %f127, %f73, %f72;\n" " shr.u32 %r114, %r57, 24;\n" " cvt.rn.f32.u32 %f129, %r114;\n" " sub.rn.f32 %f130, %f129, %f29;\n" " and.b32 %r115, %r67, 61440;\n" " or.b32 %r116, %r115, 1258291200;\n" " mov.b32 %f131, %r116;\n" " fma.rn.f32 %f132, %f131, %f73, %f72;\n" " shr.u32 %r117, %r61, 24;\n" " cvt.rn.f32.u32 %f133, %r117;\n" " sub.rn.f32 %f134, %f133, %f29;\n" " mul.rn.f32 %f135, %f130, %f128;\n" " ld.shared.f32 %f136, [%r75+896];\n" " fma.rn.f32 %f137, %f135, %f136, %f124;\n" " mul.rn.f32 %f138, %f134, %f132;\n" " fma.rn.f32 %f139, %f138, %f136, %f126;\n" " ld.global.f32 %f140, [%rd75+144];\n" " fma.rn.f32 %f185, %f140, %f137, %f185;\n" " ld.global.f32 %f141, [%rd76+144];\n" " fma.rn.f32 %f186, %f141, %f139, %f186;\n" " add.s64 %rd75, %rd75, 152;\n" " add.s64 %rd76, %rd76, 152;\n" " add.s32 %r147, %r147, 1024;\n" " add.s32 %r146, %r146, 1;\n" " setp.lt.s32 %p10, %r146, %r21;\n" " @%p10 bra $L__BB13_13;\n" "\n" "$L__BB13_14:\n" " add.s32 %r142, %r142, 2048;\n" " setp.lt.s32 %p11, %r142, %r29;\n" " add.s32 %r141, %r141, 2048;\n" " @%p11 bra $L__BB13_2;\n" "\n" "$L__BB13_15:\n" " cvt.u32.u64 %r138, %rd4;\n" " setp.ge.s32 %p12, %r138, %r28;\n" " mov.u32 %r128, 16;\n" " mov.u32 %r137, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f142, %f185, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f145, %f185, %f142;\n" " mov.u32 %r130, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f144, %f145, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f147, %f145, %f144;\n" " mov.u32 %r132, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f146, %f147, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f149, %f147, %f146;\n" " mov.u32 %r134, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f148, %f149, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f151, %f149, %f148;\n" " mov.u32 %r136, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f150, %f151, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f11, %f151, %f150;\n" " // begin inline asm\n" " shfl.down.b32 %f152, %f186, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f155, %f186, %f152;\n" " // begin inline asm\n" " shfl.down.b32 %f154, %f155, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f157, %f155, %f154;\n" " // begin inline asm\n" " shfl.down.b32 %f156, %f157, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f159, %f157, %f156;\n" " // begin inline asm\n" " shfl.down.b32 %f158, %f159, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f161, %f159, %f158;\n" " // begin inline asm\n" " shfl.down.b32 %f160, %f161, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f12, %f161, %f160;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB13_17;\n" "\n" " neg.f32 %f162, %f11;\n" " mov.f32 %f163, 0f3F000000;\n" " mov.f32 %f164, 0f3BBB989D;\n" " fma.rn.f32 %f165, %f162, %f164, %f163;\n" " mov.f32 %f166, 0f3FB8AA3B;\n" " mov.f32 %f167, 0f437C0000;\n" " cvt.sat.f32.f32 %f168, %f165;\n" " mov.f32 %f169, 0f4B400001;\n" " fma.rm.f32 %f170, %f168, %f167, %f169;\n" " add.f32 %f171, %f170, 0fCB40007F;\n" " neg.f32 %f172, %f171;\n" " fma.rn.f32 %f173, %f162, %f166, %f172;\n" " mov.f32 %f174, 0f32A57060;\n" " fma.rn.f32 %f175, %f162, %f174, %f173;\n" " mov.b32 %r139, %f170;\n" " shl.b32 %r140, %r139, 23;\n" " mov.b32 %f176, %r140;\n" " ex2.approx.ftz.f32 %f177, %f175;\n" " fma.rn.f32 %f178, %f177, %f176, 0f3F800000;\n" " div.rn.f32 %f179, %f11, %f178;\n" " mul.f32 %f180, %f12, %f179;\n" " mul.lo.s64 %rd65, %rd2, %rd3;\n" " add.s64 %rd66, %rd65, %rd4;\n" " cvta.to.global.u64 %rd67, %rd29;\n" " shl.b64 %rd68, %rd66, 2;\n" " add.s64 %rd69, %rd67, %rd68;\n" " st.global.f32 [%rd69], %f180;\n" "\n" "$L__BB13_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_q4k_nx\n" ".visible .entry fc_moe_gateup_q4k_nx(\n" " .param .u64 fc_moe_gateup_q4k_nx_param_0,\n" " .param .u64 fc_moe_gateup_q4k_nx_param_1,\n" " .param .u64 fc_moe_gateup_q4k_nx_param_2,\n" " .param .u64 fc_moe_gateup_q4k_nx_param_3,\n" " .param .u64 fc_moe_gateup_q4k_nx_param_4,\n" " .param .u32 fc_moe_gateup_q4k_nx_param_5,\n" " .param .u32 fc_moe_gateup_q4k_nx_param_6,\n" " .param .u32 fc_moe_gateup_q4k_nx_param_7,\n" " .param .u32 fc_moe_gateup_q4k_nx_param_8\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .f32 %f<242>;\n" " .reg .b32 %r<171>;\n" " .reg .b64 %rd<73>;\n" "\n" "\n" " ld.param.u64 %rd25, [fc_moe_gateup_q4k_nx_param_0];\n" " ld.param.u64 %rd26, [fc_moe_gateup_q4k_nx_param_1];\n" " ld.param.u64 %rd27, [fc_moe_gateup_q4k_nx_param_2];\n" " ld.param.u64 %rd30, [fc_moe_gateup_q4k_nx_param_3];\n" " ld.param.u64 %rd28, [fc_moe_gateup_q4k_nx_param_4];\n" " ld.param.u32 %r32, [fc_moe_gateup_q4k_nx_param_5];\n" " ld.param.u32 %r28, [fc_moe_gateup_q4k_nx_param_6];\n" " ld.param.u32 %r29, [fc_moe_gateup_q4k_nx_param_7];\n" " ld.param.u32 %r30, [fc_moe_gateup_q4k_nx_param_8];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r33, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r34, %ctaid.x;\n" " shl.b32 %r35, %r34, 3;\n" " add.s32 %r36, %r35, %r33;\n" " mov.u32 %r37, %ctaid.z;\n" " mul.wide.s32 %rd1, %r29, %r37;\n" " mul.wide.s32 %rd31, %r32, %r37;\n" " mov.u32 %r38, %ctaid.y;\n" " cvt.s64.s32 %rd32, %r38;\n" " add.s64 %rd2, %rd31, %rd32;\n" " shl.b64 %rd33, %rd2, 2;\n" " add.s64 %rd29, %rd30, %rd33;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd29];\n" " // end inline asm\n" " cvt.s64.s32 %rd3, %r28;\n" " cvt.s64.s32 %rd4, %r36;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f238, 0f00000000;\n" " mov.f32 %f239, %f238;\n" " @%p1 bra $L__BB14_15;\n" "\n" " cvt.s64.s32 %rd34, %r30;\n" " cvt.s64.s32 %rd35, %r31;\n" " mul.lo.s64 %rd36, %rd35, %rd3;\n" " add.s64 %rd37, %rd36, %rd4;\n" " mul.lo.s64 %rd38, %rd37, %rd34;\n" " cvt.u64.u32 %rd5, %r2;\n" " mov.u32 %r41, -2;\n" " sub.s32 %r4, %r41, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd6, %r1;\n" " shl.b32 %r42, %r1, 2;\n" " mov.u32 %r43, sh_x;\n" " add.s32 %r6, %r43, %r42;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd7, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd8, %r8;\n" " add.s32 %r9, %r1, 768;\n" " shl.b64 %rd39, %rd1, 2;\n" " add.s64 %rd9, %rd27, %rd39;\n" " cvta.to.global.u64 %rd40, %rd26;\n" " add.s64 %rd72, %rd40, %rd38;\n" " cvta.to.global.u64 %rd41, %rd25;\n" " add.s64 %rd71, %rd41, %rd38;\n" " mov.f32 %f239, 0f00000000;\n" " mov.u32 %r164, 0;\n" " shl.b64 %rd58, %rd5, 2;\n" " mov.u32 %r165, %r164;\n" " mov.f32 %f238, %f239;\n" "\n" "$L__BB14_2:\n" " add.s32 %r44, %r164, %r5;\n" " max.s32 %r45, %r44, -2049;\n" " sub.s32 %r12, %r4, %r45;\n" " sub.s32 %r46, %r29, %r165;\n" " min.s32 %r13, %r46, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB14_10;\n" "\n" " shr.u32 %r47, %r12, 8;\n" " add.s32 %r48, %r47, 1;\n" " cvt.s64.s32 %rd14, %r165;\n" " add.s64 %rd15, %rd1, %rd14;\n" " and.b32 %r14, %r48, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r166, %r1;\n" " @%p3 bra $L__BB14_7;\n" "\n" " add.s64 %rd43, %rd15, %rd6;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd27, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r166, %r7;\n" " @%p4 bra $L__BB14_7;\n" "\n" " add.s64 %rd46, %rd15, %rd7;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd45, %rd27, %rd47;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd45];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r166, %r8;\n" " @%p5 bra $L__BB14_7;\n" "\n" " add.s64 %rd49, %rd15, %rd8;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd48, %rd27, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r166, %r9;\n" "\n" "$L__BB14_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB14_10;\n" "\n" " cvt.s64.s32 %rd51, %r166;\n" " add.s64 %rd52, %rd51, %rd14;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd68, %rd9, %rd53;\n" " shl.b32 %r49, %r166, 2;\n" " mov.u32 %r50, sh_x;\n" " add.s32 %r51, %r50, %r49;\n" " add.s32 %r167, %r51, 2048;\n" "\n" "$L__BB14_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd68];\n" " // end inline asm\n" " st.shared.f32 [%r167+-2048], %f20;\n" " add.s64 %rd55, %rd68, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd55];\n" " // end inline asm\n" " st.shared.f32 [%r167+-1024], %f21;\n" " add.s64 %rd56, %rd68, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd56];\n" " // end inline asm\n" " st.shared.f32 [%r167], %f22;\n" " add.s64 %rd57, %rd68, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd57];\n" " // end inline asm\n" " st.shared.f32 [%r167+1024], %f23;\n" " add.s64 %rd68, %rd68, 4096;\n" " add.s32 %r167, %r167, 4096;\n" " add.s32 %r166, %r166, 1024;\n" " setp.lt.s32 %p7, %r166, %r13;\n" " @%p7 bra $L__BB14_9;\n" "\n" "$L__BB14_10:\n" " cvt.u32.u64 %r52, %rd4;\n" " setp.ge.s32 %p8, %r52, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB14_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB14_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r169, 0;\n" " mov.u32 %r170, %r43;\n" "\n" "$L__BB14_13:\n" " add.s64 %rd59, %rd71, %rd58;\n" " add.s64 %rd60, %rd72, %rd58;\n" " ld.global.v2.u32 {%r55, %r56}, [%rd71+128];\n" " ld.global.v2.u32 {%r59, %r60}, [%rd72+128];\n" " ld.global.v2.u32 {%r63, %r64}, [%rd71+136];\n" " ld.global.v2.u32 {%r67, %r68}, [%rd72+136];\n" " ld.global.v2.f32 {%f24, %f25}, [%rd71+144];\n" " ld.global.v2.f32 {%f28, %f29}, [%rd72+144];\n" " ld.global.u32 %r71, [%rd59];\n" " shr.u32 %r72, %r71, 16;\n" " ld.global.u32 %r73, [%rd60];\n" " shr.u32 %r74, %r73, 16;\n" " and.b32 %r75, %r55, 255;\n" " cvt.rn.f32.u32 %f32, %r75;\n" " mul.rn.f32 %f33, %f24, %f32;\n" " and.b32 %r76, %r71, 15;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f34, %r77;\n" " mov.f32 %f35, 0fCB000000;\n" " mov.f32 %f36, 0f3F800000;\n" " fma.rn.f32 %f37, %f34, %f36, %f35;\n" " mul.rn.f32 %f38, %f33, %f37;\n" " and.b32 %r78, %r63, 255;\n" " cvt.rn.f32.u32 %f39, %r78;\n" " mul.rn.f32 %f40, %f25, %f39;\n" " sub.rn.f32 %f41, %f38, %f40;\n" " and.b32 %r79, %r59, 255;\n" " cvt.rn.f32.u32 %f42, %r79;\n" " mul.rn.f32 %f43, %f28, %f42;\n" " and.b32 %r80, %r73, 15;\n" " or.b32 %r81, %r80, 1258291200;\n" " mov.b32 %f44, %r81;\n" " fma.rn.f32 %f45, %f44, %f36, %f35;\n" " mul.rn.f32 %f46, %f43, %f45;\n" " and.b32 %r82, %r67, 255;\n" " cvt.rn.f32.u32 %f47, %r82;\n" " mul.rn.f32 %f48, %f29, %f47;\n" " sub.rn.f32 %f49, %f46, %f48;\n" " shl.b32 %r83, %r2, 2;\n" " add.s32 %r84, %r170, %r83;\n" " ld.shared.f32 %f50, [%r84];\n" " mov.f32 %f51, 0f00000000;\n" " fma.rn.f32 %f52, %f41, %f50, %f51;\n" " fma.rn.f32 %f53, %f49, %f50, %f51;\n" " bfe.u32 %r85, %r55, 8, 8;\n" " cvt.rn.f32.u32 %f54, %r85;\n" " mul.rn.f32 %f55, %f24, %f54;\n" " and.b32 %r86, %r71, 240;\n" " or.b32 %r87, %r86, 1258291200;\n" " mov.b32 %f56, %r87;\n" " mov.f32 %f57, 0fC9000000;\n" " mov.f32 %f58, 0f3D800000;\n" " fma.rn.f32 %f59, %f56, %f58, %f57;\n" " mul.rn.f32 %f60, %f55, %f59;\n" " bfe.u32 %r88, %r63, 8, 8;\n" " cvt.rn.f32.u32 %f61, %r88;\n" " mul.rn.f32 %f62, %f25, %f61;\n" " sub.rn.f32 %f63, %f60, %f62;\n" " bfe.u32 %r89, %r59, 8, 8;\n" " cvt.rn.f32.u32 %f64, %r89;\n" " mul.rn.f32 %f65, %f28, %f64;\n" " and.b32 %r90, %r73, 240;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f66, %r91;\n" " fma.rn.f32 %f67, %f66, %f58, %f57;\n" " mul.rn.f32 %f68, %f65, %f67;\n" " bfe.u32 %r92, %r67, 8, 8;\n" " cvt.rn.f32.u32 %f69, %r92;\n" " mul.rn.f32 %f70, %f29, %f69;\n" " sub.rn.f32 %f71, %f68, %f70;\n" " ld.shared.f32 %f72, [%r84+128];\n" " fma.rn.f32 %f73, %f63, %f72, %f52;\n" " fma.rn.f32 %f74, %f71, %f72, %f53;\n" " bfe.u32 %r93, %r55, 16, 8;\n" " cvt.rn.f32.u32 %f75, %r93;\n" " mul.rn.f32 %f76, %f24, %f75;\n" " and.b32 %r94, %r71, 3840;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f77, %r95;\n" " mov.f32 %f78, 0fC7000000;\n" " mov.f32 %f79, 0f3B800000;\n" " fma.rn.f32 %f80, %f77, %f79, %f78;\n" " mul.rn.f32 %f81, %f76, %f80;\n" " bfe.u32 %r96, %r63, 16, 8;\n" " cvt.rn.f32.u32 %f82, %r96;\n" " mul.rn.f32 %f83, %f25, %f82;\n" " sub.rn.f32 %f84, %f81, %f83;\n" " bfe.u32 %r97, %r59, 16, 8;\n" " cvt.rn.f32.u32 %f85, %r97;\n" " mul.rn.f32 %f86, %f28, %f85;\n" " and.b32 %r98, %r73, 3840;\n" " or.b32 %r99, %r98, 1258291200;\n" " mov.b32 %f87, %r99;\n" " fma.rn.f32 %f88, %f87, %f79, %f78;\n" " mul.rn.f32 %f89, %f86, %f88;\n" " bfe.u32 %r100, %r67, 16, 8;\n" " cvt.rn.f32.u32 %f90, %r100;\n" " mul.rn.f32 %f91, %f29, %f90;\n" " sub.rn.f32 %f92, %f89, %f91;\n" " ld.shared.f32 %f93, [%r84+256];\n" " fma.rn.f32 %f94, %f84, %f93, %f73;\n" " fma.rn.f32 %f95, %f92, %f93, %f74;\n" " shr.u32 %r101, %r55, 24;\n" " cvt.rn.f32.u32 %f96, %r101;\n" " mul.rn.f32 %f97, %f24, %f96;\n" " and.b32 %r102, %r71, 61440;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f98, %r103;\n" " mov.f32 %f99, 0fC5000000;\n" " mov.f32 %f100, 0f39800000;\n" " fma.rn.f32 %f101, %f98, %f100, %f99;\n" " mul.rn.f32 %f102, %f97, %f101;\n" " shr.u32 %r104, %r63, 24;\n" " cvt.rn.f32.u32 %f103, %r104;\n" " mul.rn.f32 %f104, %f25, %f103;\n" " sub.rn.f32 %f105, %f102, %f104;\n" " shr.u32 %r105, %r59, 24;\n" " cvt.rn.f32.u32 %f106, %r105;\n" " mul.rn.f32 %f107, %f28, %f106;\n" " and.b32 %r106, %r73, 61440;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f108, %r107;\n" " fma.rn.f32 %f109, %f108, %f100, %f99;\n" " mul.rn.f32 %f110, %f107, %f109;\n" " shr.u32 %r108, %r67, 24;\n" " cvt.rn.f32.u32 %f111, %r108;\n" " mul.rn.f32 %f112, %f29, %f111;\n" " sub.rn.f32 %f113, %f110, %f112;\n" " ld.shared.f32 %f114, [%r84+384];\n" " fma.rn.f32 %f115, %f105, %f114, %f94;\n" " fma.rn.f32 %f116, %f113, %f114, %f95;\n" " and.b32 %r109, %r56, 255;\n" " cvt.rn.f32.u32 %f117, %r109;\n" " mul.rn.f32 %f118, %f24, %f117;\n" " and.b32 %r110, %r71, 983040;\n" " or.b32 %r111, %r110, 1258291200;\n" " mov.b32 %f119, %r111;\n" " mov.f32 %f120, 0fC3000000;\n" " mov.f32 %f121, 0f37800000;\n" " fma.rn.f32 %f122, %f119, %f121, %f120;\n" " mul.rn.f32 %f123, %f118, %f122;\n" " and.b32 %r112, %r64, 255;\n" " cvt.rn.f32.u32 %f124, %r112;\n" " mul.rn.f32 %f125, %f25, %f124;\n" " sub.rn.f32 %f126, %f123, %f125;\n" " and.b32 %r113, %r60, 255;\n" " cvt.rn.f32.u32 %f127, %r113;\n" " mul.rn.f32 %f128, %f28, %f127;\n" " and.b32 %r114, %r73, 983040;\n" " or.b32 %r115, %r114, 1258291200;\n" " mov.b32 %f129, %r115;\n" " fma.rn.f32 %f130, %f129, %f121, %f120;\n" " mul.rn.f32 %f131, %f128, %f130;\n" " and.b32 %r116, %r68, 255;\n" " cvt.rn.f32.u32 %f132, %r116;\n" " mul.rn.f32 %f133, %f29, %f132;\n" " sub.rn.f32 %f134, %f131, %f133;\n" " ld.shared.f32 %f135, [%r84+512];\n" " fma.rn.f32 %f136, %f126, %f135, %f115;\n" " fma.rn.f32 %f137, %f134, %f135, %f116;\n" " bfe.u32 %r117, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f138, %r117;\n" " mul.rn.f32 %f139, %f24, %f138;\n" " and.b32 %r118, %r72, 240;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f140, %r119;\n" " fma.rn.f32 %f141, %f140, %f58, %f57;\n" " mul.rn.f32 %f142, %f139, %f141;\n" " bfe.u32 %r120, %r64, 8, 8;\n" " cvt.rn.f32.u32 %f143, %r120;\n" " mul.rn.f32 %f144, %f25, %f143;\n" " sub.rn.f32 %f145, %f142, %f144;\n" " bfe.u32 %r121, %r60, 8, 8;\n" " cvt.rn.f32.u32 %f146, %r121;\n" " mul.rn.f32 %f147, %f28, %f146;\n" " and.b32 %r122, %r74, 240;\n" " or.b32 %r123, %r122, 1258291200;\n" " mov.b32 %f148, %r123;\n" " fma.rn.f32 %f149, %f148, %f58, %f57;\n" " mul.rn.f32 %f150, %f147, %f149;\n" " bfe.u32 %r124, %r68, 8, 8;\n" " cvt.rn.f32.u32 %f151, %r124;\n" " mul.rn.f32 %f152, %f29, %f151;\n" " sub.rn.f32 %f153, %f150, %f152;\n" " ld.shared.f32 %f154, [%r84+640];\n" " fma.rn.f32 %f155, %f145, %f154, %f136;\n" " fma.rn.f32 %f156, %f153, %f154, %f137;\n" " bfe.u32 %r125, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f157, %r125;\n" " mul.rn.f32 %f158, %f24, %f157;\n" " and.b32 %r126, %r72, 3840;\n" " or.b32 %r127, %r126, 1258291200;\n" " mov.b32 %f159, %r127;\n" " fma.rn.f32 %f160, %f159, %f79, %f78;\n" " mul.rn.f32 %f161, %f158, %f160;\n" " bfe.u32 %r128, %r64, 16, 8;\n" " cvt.rn.f32.u32 %f162, %r128;\n" " mul.rn.f32 %f163, %f25, %f162;\n" " sub.rn.f32 %f164, %f161, %f163;\n" " bfe.u32 %r129, %r60, 16, 8;\n" " cvt.rn.f32.u32 %f165, %r129;\n" " mul.rn.f32 %f166, %f28, %f165;\n" " and.b32 %r130, %r74, 3840;\n" " or.b32 %r131, %r130, 1258291200;\n" " mov.b32 %f167, %r131;\n" " fma.rn.f32 %f168, %f167, %f79, %f78;\n" " mul.rn.f32 %f169, %f166, %f168;\n" " bfe.u32 %r132, %r68, 16, 8;\n" " cvt.rn.f32.u32 %f170, %r132;\n" " mul.rn.f32 %f171, %f29, %f170;\n" " sub.rn.f32 %f172, %f169, %f171;\n" " ld.shared.f32 %f173, [%r84+768];\n" " fma.rn.f32 %f174, %f164, %f173, %f155;\n" " fma.rn.f32 %f175, %f172, %f173, %f156;\n" " shr.u32 %r133, %r56, 24;\n" " cvt.rn.f32.u32 %f176, %r133;\n" " mul.rn.f32 %f177, %f24, %f176;\n" " and.b32 %r134, %r72, 61440;\n" " or.b32 %r135, %r134, 1258291200;\n" " mov.b32 %f178, %r135;\n" " fma.rn.f32 %f179, %f178, %f100, %f99;\n" " mul.rn.f32 %f180, %f177, %f179;\n" " shr.u32 %r136, %r64, 24;\n" " cvt.rn.f32.u32 %f181, %r136;\n" " mul.rn.f32 %f182, %f25, %f181;\n" " sub.rn.f32 %f183, %f180, %f182;\n" " shr.u32 %r137, %r60, 24;\n" " cvt.rn.f32.u32 %f184, %r137;\n" " mul.rn.f32 %f185, %f28, %f184;\n" " and.b32 %r138, %r74, 61440;\n" " or.b32 %r139, %r138, 1258291200;\n" " mov.b32 %f186, %r139;\n" " fma.rn.f32 %f187, %f186, %f100, %f99;\n" " mul.rn.f32 %f188, %f185, %f187;\n" " shr.u32 %r140, %r68, 24;\n" " cvt.rn.f32.u32 %f189, %r140;\n" " mul.rn.f32 %f190, %f29, %f189;\n" " sub.rn.f32 %f191, %f188, %f190;\n" " ld.shared.f32 %f192, [%r84+896];\n" " fma.rn.f32 %f193, %f183, %f192, %f174;\n" " fma.rn.f32 %f194, %f191, %f192, %f175;\n" " add.f32 %f238, %f238, %f193;\n" " add.f32 %f239, %f239, %f194;\n" " add.s64 %rd71, %rd71, 152;\n" " add.s64 %rd72, %rd72, 152;\n" " add.s32 %r170, %r170, 1024;\n" " add.s32 %r169, %r169, 1;\n" " setp.lt.s32 %p10, %r169, %r21;\n" " @%p10 bra $L__BB14_13;\n" "\n" "$L__BB14_14:\n" " add.s32 %r165, %r165, 2048;\n" " setp.lt.s32 %p11, %r165, %r29;\n" " add.s32 %r164, %r164, 2048;\n" " @%p11 bra $L__BB14_2;\n" "\n" "$L__BB14_15:\n" " cvt.u32.u64 %r161, %rd4;\n" " setp.ge.s32 %p12, %r161, %r28;\n" " mov.u32 %r151, 16;\n" " mov.u32 %r160, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f238, %r151, %r160;\n" " // end inline asm\n" " add.f32 %f198, %f238, %f195;\n" " mov.u32 %r153, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f197, %f198, %r153, %r160;\n" " // end inline asm\n" " add.f32 %f200, %f198, %f197;\n" " mov.u32 %r155, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f199, %f200, %r155, %r160;\n" " // end inline asm\n" " add.f32 %f202, %f200, %f199;\n" " mov.u32 %r157, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f201, %f202, %r157, %r160;\n" " // end inline asm\n" " add.f32 %f204, %f202, %f201;\n" " mov.u32 %r159, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f203, %f204, %r159, %r160;\n" " // end inline asm\n" " add.f32 %f11, %f204, %f203;\n" " // begin inline asm\n" " shfl.down.b32 %f205, %f239, %r151, %r160;\n" " // end inline asm\n" " add.f32 %f208, %f239, %f205;\n" " // begin inline asm\n" " shfl.down.b32 %f207, %f208, %r153, %r160;\n" " // end inline asm\n" " add.f32 %f210, %f208, %f207;\n" " // begin inline asm\n" " shfl.down.b32 %f209, %f210, %r155, %r160;\n" " // end inline asm\n" " add.f32 %f212, %f210, %f209;\n" " // begin inline asm\n" " shfl.down.b32 %f211, %f212, %r157, %r160;\n" " // end inline asm\n" " add.f32 %f214, %f212, %f211;\n" " // begin inline asm\n" " shfl.down.b32 %f213, %f214, %r159, %r160;\n" " // end inline asm\n" " add.f32 %f12, %f214, %f213;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB14_17;\n" "\n" " neg.f32 %f215, %f11;\n" " mov.f32 %f216, 0f3F000000;\n" " mov.f32 %f217, 0f3BBB989D;\n" " fma.rn.f32 %f218, %f215, %f217, %f216;\n" " mov.f32 %f219, 0f3FB8AA3B;\n" " mov.f32 %f220, 0f437C0000;\n" " cvt.sat.f32.f32 %f221, %f218;\n" " mov.f32 %f222, 0f4B400001;\n" " fma.rm.f32 %f223, %f221, %f220, %f222;\n" " add.f32 %f224, %f223, 0fCB40007F;\n" " neg.f32 %f225, %f224;\n" " fma.rn.f32 %f226, %f215, %f219, %f225;\n" " mov.f32 %f227, 0f32A57060;\n" " fma.rn.f32 %f228, %f215, %f227, %f226;\n" " mov.b32 %r162, %f223;\n" " shl.b32 %r163, %r162, 23;\n" " mov.b32 %f229, %r163;\n" " ex2.approx.ftz.f32 %f230, %f228;\n" " fma.rn.f32 %f231, %f230, %f229, 0f3F800000;\n" " div.rn.f32 %f232, %f11, %f231;\n" " mul.f32 %f233, %f12, %f232;\n" " mul.lo.s64 %rd61, %rd2, %rd3;\n" " add.s64 %rd62, %rd61, %rd4;\n" " cvta.to.global.u64 %rd63, %rd28;\n" " shl.b64 %rd64, %rd62, 2;\n" " add.s64 %rd65, %rd63, %rd64;\n" " st.global.f32 [%rd65], %f233;\n" "\n" "$L__BB14_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_f16_nx\n" ".visible .entry fc_moe_gateup_f16_nx(\n" " .param .u64 fc_moe_gateup_f16_nx_param_0,\n" " .param .u64 fc_moe_gateup_f16_nx_param_1,\n" " .param .u64 fc_moe_gateup_f16_nx_param_2,\n" " .param .u64 fc_moe_gateup_f16_nx_param_3,\n" " .param .u64 fc_moe_gateup_f16_nx_param_4,\n" " .param .u32 fc_moe_gateup_f16_nx_param_5,\n" " .param .u32 fc_moe_gateup_f16_nx_param_6,\n" " .param .u32 fc_moe_gateup_f16_nx_param_7,\n" " .param .u32 fc_moe_gateup_f16_nx_param_8\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .b16 %rs<17>;\n" " .reg .f32 %f<112>;\n" " .reg .b32 %r<87>;\n" " .reg .b64 %rd<73>;\n" "\n" "\n" " ld.param.u64 %rd25, [fc_moe_gateup_f16_nx_param_0];\n" " ld.param.u64 %rd26, [fc_moe_gateup_f16_nx_param_1];\n" " ld.param.u64 %rd27, [fc_moe_gateup_f16_nx_param_2];\n" " ld.param.u64 %rd30, [fc_moe_gateup_f16_nx_param_3];\n" " ld.param.u64 %rd28, [fc_moe_gateup_f16_nx_param_4];\n" " ld.param.u32 %r32, [fc_moe_gateup_f16_nx_param_5];\n" " ld.param.u32 %r28, [fc_moe_gateup_f16_nx_param_6];\n" " ld.param.u32 %r29, [fc_moe_gateup_f16_nx_param_7];\n" " ld.param.u32 %r30, [fc_moe_gateup_f16_nx_param_8];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r33, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r34, %ctaid.x;\n" " shl.b32 %r35, %r34, 3;\n" " add.s32 %r36, %r35, %r33;\n" " mov.u32 %r37, %ctaid.z;\n" " mul.wide.s32 %rd1, %r29, %r37;\n" " mul.wide.s32 %rd31, %r32, %r37;\n" " mov.u32 %r38, %ctaid.y;\n" " cvt.s64.s32 %rd32, %r38;\n" " add.s64 %rd2, %rd31, %rd32;\n" " shl.b64 %rd33, %rd2, 2;\n" " add.s64 %rd29, %rd30, %rd33;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd29];\n" " // end inline asm\n" " cvt.s64.s32 %rd3, %r28;\n" " cvt.s64.s32 %rd4, %r36;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f108, 0f00000000;\n" " mov.f32 %f109, %f108;\n" " @%p1 bra $L__BB15_15;\n" "\n" " cvt.s64.s32 %rd34, %r30;\n" " cvt.s64.s32 %rd35, %r31;\n" " mul.lo.s64 %rd36, %rd35, %rd3;\n" " add.s64 %rd37, %rd36, %rd4;\n" " mul.lo.s64 %rd38, %rd37, %rd34;\n" " mov.u32 %r41, -2;\n" " sub.s32 %r4, %r41, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd5, %r1;\n" " shl.b32 %r42, %r1, 2;\n" " mov.u32 %r43, sh_x;\n" " add.s32 %r6, %r43, %r42;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd6, %r7;\n" " cvt.u64.u32 %rd7, %r2;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd8, %r8;\n" " add.s32 %r9, %r1, 768;\n" " shl.b64 %rd39, %rd1, 2;\n" " add.s64 %rd9, %rd27, %rd39;\n" " cvta.to.global.u64 %rd40, %rd26;\n" " add.s64 %rd72, %rd40, %rd38;\n" " cvta.to.global.u64 %rd41, %rd25;\n" " add.s64 %rd71, %rd41, %rd38;\n" " mov.f32 %f109, 0f00000000;\n" " mov.u32 %r80, 0;\n" " shl.b64 %rd58, %rd7, 1;\n" " mov.u32 %r81, %r80;\n" " mov.f32 %f108, %f109;\n" "\n" "$L__BB15_2:\n" " add.s32 %r44, %r80, %r5;\n" " max.s32 %r45, %r44, -2049;\n" " sub.s32 %r12, %r4, %r45;\n" " sub.s32 %r46, %r29, %r81;\n" " min.s32 %r13, %r46, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB15_10;\n" "\n" " shr.u32 %r47, %r12, 8;\n" " add.s32 %r48, %r47, 1;\n" " cvt.s64.s32 %rd14, %r81;\n" " add.s64 %rd15, %rd1, %rd14;\n" " and.b32 %r14, %r48, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r82, %r1;\n" " @%p3 bra $L__BB15_7;\n" "\n" " add.s64 %rd43, %rd15, %rd5;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd27, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r82, %r7;\n" " @%p4 bra $L__BB15_7;\n" "\n" " add.s64 %rd46, %rd15, %rd6;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd45, %rd27, %rd47;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd45];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r82, %r8;\n" " @%p5 bra $L__BB15_7;\n" "\n" " add.s64 %rd49, %rd15, %rd8;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd48, %rd27, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r82, %r9;\n" "\n" "$L__BB15_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB15_10;\n" "\n" " cvt.s64.s32 %rd51, %r82;\n" " add.s64 %rd52, %rd51, %rd14;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd68, %rd9, %rd53;\n" " shl.b32 %r49, %r82, 2;\n" " mov.u32 %r50, sh_x;\n" " add.s32 %r51, %r50, %r49;\n" " add.s32 %r83, %r51, 2048;\n" "\n" "$L__BB15_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd68];\n" " // end inline asm\n" " st.shared.f32 [%r83+-2048], %f20;\n" " add.s64 %rd55, %rd68, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd55];\n" " // end inline asm\n" " st.shared.f32 [%r83+-1024], %f21;\n" " add.s64 %rd56, %rd68, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd56];\n" " // end inline asm\n" " st.shared.f32 [%r83], %f22;\n" " add.s64 %rd57, %rd68, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd57];\n" " // end inline asm\n" " st.shared.f32 [%r83+1024], %f23;\n" " add.s64 %rd68, %rd68, 4096;\n" " add.s32 %r83, %r83, 4096;\n" " add.s32 %r82, %r82, 1024;\n" " setp.lt.s32 %p7, %r82, %r13;\n" " @%p7 bra $L__BB15_9;\n" "\n" "$L__BB15_10:\n" " cvt.u32.u64 %r52, %rd4;\n" " setp.ge.s32 %p8, %r52, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB15_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB15_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r85, 0;\n" " mov.u32 %r86, %r43;\n" "\n" "$L__BB15_13:\n" " shl.b32 %r55, %r2, 2;\n" " add.s32 %r56, %r86, %r55;\n" " ld.shared.f32 %f40, [%r56];\n" " add.s64 %rd59, %rd71, %rd58;\n" " ld.global.u16 %rs1, [%rd59];\n" " // begin inline asm\n" " cvt.f32.f16 %f24, %rs1;\n" " // end inline asm\n" " mov.f32 %f41, 0f00000000;\n" " fma.rn.f32 %f42, %f24, %f40, %f41;\n" " add.s64 %rd60, %rd72, %rd58;\n" " ld.global.u16 %rs2, [%rd60];\n" " // begin inline asm\n" " cvt.f32.f16 %f25, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f43, %f25, %f40, %f41;\n" " ld.shared.f32 %f44, [%r56+128];\n" " ld.global.u16 %rs3, [%rd59+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f26, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f45, %f26, %f44, %f42;\n" " ld.global.u16 %rs4, [%rd60+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f27, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f46, %f27, %f44, %f43;\n" " ld.shared.f32 %f47, [%r56+256];\n" " ld.global.u16 %rs5, [%rd59+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f28, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f48, %f28, %f47, %f45;\n" " ld.global.u16 %rs6, [%rd60+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f29, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f49, %f29, %f47, %f46;\n" " ld.shared.f32 %f50, [%r56+384];\n" " ld.global.u16 %rs7, [%rd59+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f30, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f51, %f30, %f50, %f48;\n" " ld.global.u16 %rs8, [%rd60+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f31, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f52, %f31, %f50, %f49;\n" " ld.shared.f32 %f53, [%r56+512];\n" " ld.global.u16 %rs9, [%rd59+256];\n" " // begin inline asm\n" " cvt.f32.f16 %f32, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f54, %f32, %f53, %f51;\n" " ld.global.u16 %rs10, [%rd60+256];\n" " // begin inline asm\n" " cvt.f32.f16 %f33, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f55, %f33, %f53, %f52;\n" " ld.shared.f32 %f56, [%r56+640];\n" " ld.global.u16 %rs11, [%rd59+320];\n" " // begin inline asm\n" " cvt.f32.f16 %f34, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f57, %f34, %f56, %f54;\n" " ld.global.u16 %rs12, [%rd60+320];\n" " // begin inline asm\n" " cvt.f32.f16 %f35, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f58, %f35, %f56, %f55;\n" " ld.shared.f32 %f59, [%r56+768];\n" " ld.global.u16 %rs13, [%rd59+384];\n" " // begin inline asm\n" " cvt.f32.f16 %f36, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f60, %f36, %f59, %f57;\n" " ld.global.u16 %rs14, [%rd60+384];\n" " // begin inline asm\n" " cvt.f32.f16 %f37, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f61, %f37, %f59, %f58;\n" " ld.shared.f32 %f62, [%r56+896];\n" " ld.global.u16 %rs15, [%rd59+448];\n" " // begin inline asm\n" " cvt.f32.f16 %f38, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f63, %f38, %f62, %f60;\n" " ld.global.u16 %rs16, [%rd60+448];\n" " // begin inline asm\n" " cvt.f32.f16 %f39, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f64, %f39, %f62, %f61;\n" " add.f32 %f108, %f108, %f63;\n" " add.f32 %f109, %f109, %f64;\n" " add.s64 %rd71, %rd71, 512;\n" " add.s64 %rd72, %rd72, 512;\n" " add.s32 %r86, %r86, 1024;\n" " add.s32 %r85, %r85, 1;\n" " setp.lt.s32 %p10, %r85, %r21;\n" " @%p10 bra $L__BB15_13;\n" "\n" "$L__BB15_14:\n" " add.s32 %r81, %r81, 2048;\n" " setp.lt.s32 %p11, %r81, %r29;\n" " add.s32 %r80, %r80, 2048;\n" " @%p11 bra $L__BB15_2;\n" "\n" "$L__BB15_15:\n" " cvt.u32.u64 %r77, %rd4;\n" " setp.ge.s32 %p12, %r77, %r28;\n" " mov.u32 %r67, 16;\n" " mov.u32 %r76, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f65, %f108, %r67, %r76;\n" " // end inline asm\n" " add.f32 %f68, %f108, %f65;\n" " mov.u32 %r69, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f67, %f68, %r69, %r76;\n" " // end inline asm\n" " add.f32 %f70, %f68, %f67;\n" " mov.u32 %r71, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f69, %f70, %r71, %r76;\n" " // end inline asm\n" " add.f32 %f72, %f70, %f69;\n" " mov.u32 %r73, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f71, %f72, %r73, %r76;\n" " // end inline asm\n" " add.f32 %f74, %f72, %f71;\n" " mov.u32 %r75, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f73, %f74, %r75, %r76;\n" " // end inline asm\n" " add.f32 %f11, %f74, %f73;\n" " // begin inline asm\n" " shfl.down.b32 %f75, %f109, %r67, %r76;\n" " // end inline asm\n" " add.f32 %f78, %f109, %f75;\n" " // begin inline asm\n" " shfl.down.b32 %f77, %f78, %r69, %r76;\n" " // end inline asm\n" " add.f32 %f80, %f78, %f77;\n" " // begin inline asm\n" " shfl.down.b32 %f79, %f80, %r71, %r76;\n" " // end inline asm\n" " add.f32 %f82, %f80, %f79;\n" " // begin inline asm\n" " shfl.down.b32 %f81, %f82, %r73, %r76;\n" " // end inline asm\n" " add.f32 %f84, %f82, %f81;\n" " // begin inline asm\n" " shfl.down.b32 %f83, %f84, %r75, %r76;\n" " // end inline asm\n" " add.f32 %f12, %f84, %f83;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB15_17;\n" "\n" " neg.f32 %f85, %f11;\n" " mov.f32 %f86, 0f3F000000;\n" " mov.f32 %f87, 0f3BBB989D;\n" " fma.rn.f32 %f88, %f85, %f87, %f86;\n" " mov.f32 %f89, 0f3FB8AA3B;\n" " mov.f32 %f90, 0f437C0000;\n" " cvt.sat.f32.f32 %f91, %f88;\n" " mov.f32 %f92, 0f4B400001;\n" " fma.rm.f32 %f93, %f91, %f90, %f92;\n" " add.f32 %f94, %f93, 0fCB40007F;\n" " neg.f32 %f95, %f94;\n" " fma.rn.f32 %f96, %f85, %f89, %f95;\n" " mov.f32 %f97, 0f32A57060;\n" " fma.rn.f32 %f98, %f85, %f97, %f96;\n" " mov.b32 %r78, %f93;\n" " shl.b32 %r79, %r78, 23;\n" " mov.b32 %f99, %r79;\n" " ex2.approx.ftz.f32 %f100, %f98;\n" " fma.rn.f32 %f101, %f100, %f99, 0f3F800000;\n" " div.rn.f32 %f102, %f11, %f101;\n" " mul.f32 %f103, %f12, %f102;\n" " mul.lo.s64 %rd61, %rd2, %rd3;\n" " add.s64 %rd62, %rd61, %rd4;\n" " cvta.to.global.u64 %rd63, %rd28;\n" " shl.b64 %rd64, %rd62, 2;\n" " add.s64 %rd65, %rd63, %rd64;\n" " st.global.f32 [%rd65], %f103;\n" "\n" "$L__BB15_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_bf16_nx\n" ".visible .entry fc_moe_gateup_bf16_nx(\n" " .param .u64 fc_moe_gateup_bf16_nx_param_0,\n" " .param .u64 fc_moe_gateup_bf16_nx_param_1,\n" " .param .u64 fc_moe_gateup_bf16_nx_param_2,\n" " .param .u64 fc_moe_gateup_bf16_nx_param_3,\n" " .param .u64 fc_moe_gateup_bf16_nx_param_4,\n" " .param .u32 fc_moe_gateup_bf16_nx_param_5,\n" " .param .u32 fc_moe_gateup_bf16_nx_param_6,\n" " .param .u32 fc_moe_gateup_bf16_nx_param_7,\n" " .param .u32 fc_moe_gateup_bf16_nx_param_8\n" ")\n" "{\n" " .reg .pred %p<15>;\n" " .reg .f32 %f<112>;\n" " .reg .b32 %r<119>;\n" " .reg .b64 %rd<73>;\n" "\n" "\n" " ld.param.u64 %rd25, [fc_moe_gateup_bf16_nx_param_0];\n" " ld.param.u64 %rd26, [fc_moe_gateup_bf16_nx_param_1];\n" " ld.param.u64 %rd27, [fc_moe_gateup_bf16_nx_param_2];\n" " ld.param.u64 %rd30, [fc_moe_gateup_bf16_nx_param_3];\n" " ld.param.u64 %rd28, [fc_moe_gateup_bf16_nx_param_4];\n" " ld.param.u32 %r32, [fc_moe_gateup_bf16_nx_param_5];\n" " ld.param.u32 %r28, [fc_moe_gateup_bf16_nx_param_6];\n" " ld.param.u32 %r29, [fc_moe_gateup_bf16_nx_param_7];\n" " ld.param.u32 %r30, [fc_moe_gateup_bf16_nx_param_8];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r33, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r34, %ctaid.x;\n" " shl.b32 %r35, %r34, 3;\n" " add.s32 %r36, %r35, %r33;\n" " mov.u32 %r37, %ctaid.z;\n" " mul.wide.s32 %rd1, %r29, %r37;\n" " mul.wide.s32 %rd31, %r32, %r37;\n" " mov.u32 %r38, %ctaid.y;\n" " cvt.s64.s32 %rd32, %r38;\n" " add.s64 %rd2, %rd31, %rd32;\n" " shl.b64 %rd33, %rd2, 2;\n" " add.s64 %rd29, %rd30, %rd33;\n" " // begin inline asm\n" " ld.global.nc.s32 %r31, [%rd29];\n" " // end inline asm\n" " cvt.s64.s32 %rd3, %r28;\n" " cvt.s64.s32 %rd4, %r36;\n" " setp.lt.s32 %p1, %r29, 1;\n" " mov.f32 %f108, 0f00000000;\n" " mov.f32 %f109, %f108;\n" " @%p1 bra $L__BB16_15;\n" "\n" " cvt.s64.s32 %rd34, %r30;\n" " cvt.s64.s32 %rd35, %r31;\n" " mul.lo.s64 %rd36, %rd35, %rd3;\n" " add.s64 %rd37, %rd36, %rd4;\n" " mul.lo.s64 %rd38, %rd37, %rd34;\n" " mov.u32 %r41, -2;\n" " sub.s32 %r4, %r41, %r1;\n" " not.b32 %r5, %r29;\n" " cvt.s64.s32 %rd5, %r1;\n" " shl.b32 %r42, %r1, 2;\n" " mov.u32 %r43, sh_x;\n" " add.s32 %r6, %r43, %r42;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd6, %r7;\n" " cvt.u64.u32 %rd7, %r2;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd8, %r8;\n" " add.s32 %r9, %r1, 768;\n" " shl.b64 %rd39, %rd1, 2;\n" " add.s64 %rd9, %rd27, %rd39;\n" " cvta.to.global.u64 %rd40, %rd26;\n" " add.s64 %rd72, %rd40, %rd38;\n" " cvta.to.global.u64 %rd41, %rd25;\n" " add.s64 %rd71, %rd41, %rd38;\n" " mov.f32 %f109, 0f00000000;\n" " mov.u32 %r112, 0;\n" " shl.b64 %rd58, %rd7, 1;\n" " mov.u32 %r113, %r112;\n" " mov.f32 %f108, %f109;\n" "\n" "$L__BB16_2:\n" " add.s32 %r44, %r112, %r5;\n" " max.s32 %r45, %r44, -2049;\n" " sub.s32 %r12, %r4, %r45;\n" " sub.s32 %r46, %r29, %r113;\n" " min.s32 %r13, %r46, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB16_10;\n" "\n" " shr.u32 %r47, %r12, 8;\n" " add.s32 %r48, %r47, 1;\n" " cvt.s64.s32 %rd14, %r113;\n" " add.s64 %rd15, %rd1, %rd14;\n" " and.b32 %r14, %r48, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r114, %r1;\n" " @%p3 bra $L__BB16_7;\n" "\n" " add.s64 %rd43, %rd15, %rd5;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd42, %rd27, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f17, [%rd42];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f17;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r114, %r7;\n" " @%p4 bra $L__BB16_7;\n" "\n" " add.s64 %rd46, %rd15, %rd6;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd45, %rd27, %rd47;\n" " // begin inline asm\n" " ld.global.nc.f32 %f18, [%rd45];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f18;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r114, %r8;\n" " @%p5 bra $L__BB16_7;\n" "\n" " add.s64 %rd49, %rd15, %rd8;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd48, %rd27, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f19, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f19;\n" " mov.u32 %r114, %r9;\n" "\n" "$L__BB16_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB16_10;\n" "\n" " cvt.s64.s32 %rd51, %r114;\n" " add.s64 %rd52, %rd51, %rd14;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd68, %rd9, %rd53;\n" " shl.b32 %r49, %r114, 2;\n" " mov.u32 %r50, sh_x;\n" " add.s32 %r51, %r50, %r49;\n" " add.s32 %r115, %r51, 2048;\n" "\n" "$L__BB16_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f20, [%rd68];\n" " // end inline asm\n" " st.shared.f32 [%r115+-2048], %f20;\n" " add.s64 %rd55, %rd68, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f21, [%rd55];\n" " // end inline asm\n" " st.shared.f32 [%r115+-1024], %f21;\n" " add.s64 %rd56, %rd68, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f22, [%rd56];\n" " // end inline asm\n" " st.shared.f32 [%r115], %f22;\n" " add.s64 %rd57, %rd68, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd57];\n" " // end inline asm\n" " st.shared.f32 [%r115+1024], %f23;\n" " add.s64 %rd68, %rd68, 4096;\n" " add.s32 %r115, %r115, 4096;\n" " add.s32 %r114, %r114, 1024;\n" " setp.lt.s32 %p7, %r114, %r13;\n" " @%p7 bra $L__BB16_9;\n" "\n" "$L__BB16_10:\n" " cvt.u32.u64 %r52, %rd4;\n" " setp.ge.s32 %p8, %r52, %r28;\n" " bar.sync 0;\n" " @%p8 bra $L__BB16_14;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB16_14;\n" "\n" " shr.s32 %r21, %r13, 8;\n" " mov.u32 %r117, 0;\n" " mov.u32 %r118, %r43;\n" "\n" "$L__BB16_13:\n" " add.s64 %rd59, %rd71, %rd58;\n" " ld.global.u16 %r55, [%rd59];\n" " shl.b32 %r56, %r55, 16;\n" " mov.b32 %f24, %r56;\n" " shl.b32 %r57, %r2, 2;\n" " add.s32 %r58, %r118, %r57;\n" " ld.shared.f32 %f25, [%r58];\n" " mov.f32 %f26, 0f00000000;\n" " fma.rn.f32 %f27, %f24, %f25, %f26;\n" " add.s64 %rd60, %rd72, %rd58;\n" " ld.global.u16 %r59, [%rd60];\n" " shl.b32 %r60, %r59, 16;\n" " mov.b32 %f28, %r60;\n" " fma.rn.f32 %f29, %f28, %f25, %f26;\n" " ld.global.u16 %r61, [%rd59+64];\n" " shl.b32 %r62, %r61, 16;\n" " mov.b32 %f30, %r62;\n" " ld.shared.f32 %f31, [%r58+128];\n" " fma.rn.f32 %f32, %f30, %f31, %f27;\n" " ld.global.u16 %r63, [%rd60+64];\n" " shl.b32 %r64, %r63, 16;\n" " mov.b32 %f33, %r64;\n" " fma.rn.f32 %f34, %f33, %f31, %f29;\n" " ld.global.u16 %r65, [%rd59+128];\n" " shl.b32 %r66, %r65, 16;\n" " mov.b32 %f35, %r66;\n" " ld.shared.f32 %f36, [%r58+256];\n" " fma.rn.f32 %f37, %f35, %f36, %f32;\n" " ld.global.u16 %r67, [%rd60+128];\n" " shl.b32 %r68, %r67, 16;\n" " mov.b32 %f38, %r68;\n" " fma.rn.f32 %f39, %f38, %f36, %f34;\n" " ld.global.u16 %r69, [%rd59+192];\n" " shl.b32 %r70, %r69, 16;\n" " mov.b32 %f40, %r70;\n" " ld.shared.f32 %f41, [%r58+384];\n" " fma.rn.f32 %f42, %f40, %f41, %f37;\n" " ld.global.u16 %r71, [%rd60+192];\n" " shl.b32 %r72, %r71, 16;\n" " mov.b32 %f43, %r72;\n" " fma.rn.f32 %f44, %f43, %f41, %f39;\n" " ld.global.u16 %r73, [%rd59+256];\n" " shl.b32 %r74, %r73, 16;\n" " mov.b32 %f45, %r74;\n" " ld.shared.f32 %f46, [%r58+512];\n" " fma.rn.f32 %f47, %f45, %f46, %f42;\n" " ld.global.u16 %r75, [%rd60+256];\n" " shl.b32 %r76, %r75, 16;\n" " mov.b32 %f48, %r76;\n" " fma.rn.f32 %f49, %f48, %f46, %f44;\n" " ld.global.u16 %r77, [%rd59+320];\n" " shl.b32 %r78, %r77, 16;\n" " mov.b32 %f50, %r78;\n" " ld.shared.f32 %f51, [%r58+640];\n" " fma.rn.f32 %f52, %f50, %f51, %f47;\n" " ld.global.u16 %r79, [%rd60+320];\n" " shl.b32 %r80, %r79, 16;\n" " mov.b32 %f53, %r80;\n" " fma.rn.f32 %f54, %f53, %f51, %f49;\n" " ld.global.u16 %r81, [%rd59+384];\n" " shl.b32 %r82, %r81, 16;\n" " mov.b32 %f55, %r82;\n" " ld.shared.f32 %f56, [%r58+768];\n" " fma.rn.f32 %f57, %f55, %f56, %f52;\n" " ld.global.u16 %r83, [%rd60+384];\n" " shl.b32 %r84, %r83, 16;\n" " mov.b32 %f58, %r84;\n" " fma.rn.f32 %f59, %f58, %f56, %f54;\n" " ld.global.u16 %r85, [%rd59+448];\n" " shl.b32 %r86, %r85, 16;\n" " mov.b32 %f60, %r86;\n" " ld.shared.f32 %f61, [%r58+896];\n" " fma.rn.f32 %f62, %f60, %f61, %f57;\n" " ld.global.u16 %r87, [%rd60+448];\n" " shl.b32 %r88, %r87, 16;\n" " mov.b32 %f63, %r88;\n" " fma.rn.f32 %f64, %f63, %f61, %f59;\n" " add.f32 %f108, %f108, %f62;\n" " add.f32 %f109, %f109, %f64;\n" " add.s64 %rd71, %rd71, 512;\n" " add.s64 %rd72, %rd72, 512;\n" " add.s32 %r118, %r118, 1024;\n" " add.s32 %r117, %r117, 1;\n" " setp.lt.s32 %p10, %r117, %r21;\n" " @%p10 bra $L__BB16_13;\n" "\n" "$L__BB16_14:\n" " add.s32 %r113, %r113, 2048;\n" " setp.lt.s32 %p11, %r113, %r29;\n" " add.s32 %r112, %r112, 2048;\n" " @%p11 bra $L__BB16_2;\n" "\n" "$L__BB16_15:\n" " cvt.u32.u64 %r109, %rd4;\n" " setp.ge.s32 %p12, %r109, %r28;\n" " mov.u32 %r99, 16;\n" " mov.u32 %r108, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f65, %f108, %r99, %r108;\n" " // end inline asm\n" " add.f32 %f68, %f108, %f65;\n" " mov.u32 %r101, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f67, %f68, %r101, %r108;\n" " // end inline asm\n" " add.f32 %f70, %f68, %f67;\n" " mov.u32 %r103, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f69, %f70, %r103, %r108;\n" " // end inline asm\n" " add.f32 %f72, %f70, %f69;\n" " mov.u32 %r105, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f71, %f72, %r105, %r108;\n" " // end inline asm\n" " add.f32 %f74, %f72, %f71;\n" " mov.u32 %r107, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f73, %f74, %r107, %r108;\n" " // end inline asm\n" " add.f32 %f11, %f74, %f73;\n" " // begin inline asm\n" " shfl.down.b32 %f75, %f109, %r99, %r108;\n" " // end inline asm\n" " add.f32 %f78, %f109, %f75;\n" " // begin inline asm\n" " shfl.down.b32 %f77, %f78, %r101, %r108;\n" " // end inline asm\n" " add.f32 %f80, %f78, %f77;\n" " // begin inline asm\n" " shfl.down.b32 %f79, %f80, %r103, %r108;\n" " // end inline asm\n" " add.f32 %f82, %f80, %f79;\n" " // begin inline asm\n" " shfl.down.b32 %f81, %f82, %r105, %r108;\n" " // end inline asm\n" " add.f32 %f84, %f82, %f81;\n" " // begin inline asm\n" " shfl.down.b32 %f83, %f84, %r107, %r108;\n" " // end inline asm\n" " add.f32 %f12, %f84, %f83;\n" " setp.ne.s32 %p13, %r2, 0;\n" " or.pred %p14, %p13, %p12;\n" " @%p14 bra $L__BB16_17;\n" "\n" " neg.f32 %f85, %f11;\n" " mov.f32 %f86, 0f3F000000;\n" " mov.f32 %f87, 0f3BBB989D;\n" " fma.rn.f32 %f88, %f85, %f87, %f86;\n" " mov.f32 %f89, 0f3FB8AA3B;\n" " mov.f32 %f90, 0f437C0000;\n" " cvt.sat.f32.f32 %f91, %f88;\n" " mov.f32 %f92, 0f4B400001;\n" " fma.rm.f32 %f93, %f91, %f90, %f92;\n" " add.f32 %f94, %f93, 0fCB40007F;\n" " neg.f32 %f95, %f94;\n" " fma.rn.f32 %f96, %f85, %f89, %f95;\n" " mov.f32 %f97, 0f32A57060;\n" " fma.rn.f32 %f98, %f85, %f97, %f96;\n" " mov.b32 %r110, %f93;\n" " shl.b32 %r111, %r110, 23;\n" " mov.b32 %f99, %r111;\n" " ex2.approx.ftz.f32 %f100, %f98;\n" " fma.rn.f32 %f101, %f100, %f99, 0f3F800000;\n" " div.rn.f32 %f102, %f11, %f101;\n" " mul.f32 %f103, %f12, %f102;\n" " mul.lo.s64 %rd61, %rd2, %rd3;\n" " add.s64 %rd62, %rd61, %rd4;\n" " cvta.to.global.u64 %rd63, %rd28;\n" " shl.b64 %rd64, %rd62, 2;\n" " add.s64 %rd65, %rd63, %rd64;\n" " st.global.f32 [%rd65], %f103;\n" "\n" "$L__BB16_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_f16_vec_nx\n" ".visible .entry fc_moe_gateup_f16_vec_nx(\n" " .param .u64 fc_moe_gateup_f16_vec_nx_param_0,\n" " .param .u64 fc_moe_gateup_f16_vec_nx_param_1,\n" " .param .u64 fc_moe_gateup_f16_vec_nx_param_2,\n" " .param .u64 fc_moe_gateup_f16_vec_nx_param_3,\n" " .param .u64 fc_moe_gateup_f16_vec_nx_param_4,\n" " .param .u32 fc_moe_gateup_f16_vec_nx_param_5,\n" " .param .u32 fc_moe_gateup_f16_vec_nx_param_6,\n" " .param .u32 fc_moe_gateup_f16_vec_nx_param_7,\n" " .param .u32 fc_moe_gateup_f16_vec_nx_param_8\n" ")\n" "{\n" " .reg .pred %p<17>;\n" " .reg .b16 %rs<49>;\n" " .reg .f32 %f<230>;\n" " .reg .b32 %r<168>;\n" " .reg .b64 %rd<88>;\n" "\n" "\n" " ld.param.u64 %rd31, [fc_moe_gateup_f16_vec_nx_param_0];\n" " ld.param.u64 %rd32, [fc_moe_gateup_f16_vec_nx_param_1];\n" " ld.param.u64 %rd33, [fc_moe_gateup_f16_vec_nx_param_2];\n" " ld.param.u64 %rd36, [fc_moe_gateup_f16_vec_nx_param_3];\n" " ld.param.u64 %rd34, [fc_moe_gateup_f16_vec_nx_param_4];\n" " ld.param.u32 %r35, [fc_moe_gateup_f16_vec_nx_param_5];\n" " ld.param.u32 %r31, [fc_moe_gateup_f16_vec_nx_param_6];\n" " ld.param.u32 %r32, [fc_moe_gateup_f16_vec_nx_param_7];\n" " ld.param.u32 %r33, [fc_moe_gateup_f16_vec_nx_param_8];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r36, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r37, %ctaid.x;\n" " shl.b32 %r38, %r37, 3;\n" " add.s32 %r39, %r38, %r36;\n" " mov.u32 %r40, %ctaid.z;\n" " mul.wide.s32 %rd1, %r32, %r40;\n" " mul.wide.s32 %rd37, %r35, %r40;\n" " mov.u32 %r41, %ctaid.y;\n" " cvt.s64.s32 %rd38, %r41;\n" " add.s64 %rd2, %rd37, %rd38;\n" " shl.b64 %rd39, %rd2, 2;\n" " add.s64 %rd35, %rd36, %rd39;\n" " // begin inline asm\n" " ld.global.nc.s32 %r34, [%rd35];\n" " // end inline asm\n" " cvt.s64.s32 %rd3, %r31;\n" " cvt.s64.s32 %rd4, %r39;\n" " setp.lt.s32 %p1, %r32, 1;\n" " mov.f32 %f226, 0f00000000;\n" " mov.f32 %f227, %f226;\n" " @%p1 bra $L__BB17_18;\n" "\n" " cvt.s64.s32 %rd40, %r33;\n" " cvt.s64.s32 %rd41, %r34;\n" " mul.lo.s64 %rd42, %rd41, %rd3;\n" " add.s64 %rd43, %rd42, %rd4;\n" " mul.lo.s64 %rd44, %rd43, %rd40;\n" " cvt.u64.u32 %rd5, %r2;\n" " mov.u32 %r44, -2;\n" " sub.s32 %r4, %r44, %r1;\n" " cvt.s64.s32 %rd6, %r1;\n" " shl.b32 %r45, %r1, 2;\n" " mov.u32 %r46, sh_x4;\n" " add.s32 %r6, %r46, %r45;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd7, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd8, %r8;\n" " add.s32 %r9, %r1, 768;\n" " shl.b64 %rd45, %rd1, 2;\n" " add.s64 %rd9, %rd33, %rd45;\n" " cvta.to.global.u64 %rd46, %rd32;\n" " add.s64 %rd87, %rd46, %rd44;\n" " cvta.to.global.u64 %rd47, %rd31;\n" " add.s64 %rd86, %rd47, %rd44;\n" " mov.f32 %f227, 0f00000000;\n" " mov.u32 %r160, 0;\n" " mov.u32 %r161, %r160;\n" " mov.f32 %f226, %f227;\n" "\n" "$L__BB17_2:\n" " not.b32 %r159, %r32;\n" " add.s32 %r47, %r160, %r159;\n" " max.s32 %r48, %r47, -2049;\n" " sub.s32 %r12, %r4, %r48;\n" " sub.s32 %r49, %r32, %r161;\n" " min.s32 %r13, %r49, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB17_10;\n" "\n" " shr.u32 %r50, %r12, 8;\n" " add.s32 %r51, %r50, 1;\n" " cvt.s64.s32 %rd14, %r161;\n" " add.s64 %rd15, %rd1, %rd14;\n" " and.b32 %r14, %r51, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r162, %r1;\n" " @%p3 bra $L__BB17_7;\n" "\n" " add.s64 %rd49, %rd15, %rd6;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd48, %rd33, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f23;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r162, %r7;\n" " @%p4 bra $L__BB17_7;\n" "\n" " add.s64 %rd52, %rd15, %rd7;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd51, %rd33, %rd53;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd51];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f24;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r162, %r8;\n" " @%p5 bra $L__BB17_7;\n" "\n" " add.s64 %rd55, %rd15, %rd8;\n" " shl.b64 %rd56, %rd55, 2;\n" " add.s64 %rd54, %rd33, %rd56;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd54];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f25;\n" " mov.u32 %r162, %r9;\n" "\n" "$L__BB17_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB17_10;\n" "\n" " cvt.s64.s32 %rd57, %r162;\n" " add.s64 %rd58, %rd57, %rd14;\n" " shl.b64 %rd59, %rd58, 2;\n" " add.s64 %rd79, %rd9, %rd59;\n" " shl.b32 %r52, %r162, 2;\n" " mov.u32 %r53, sh_x4;\n" " add.s32 %r54, %r53, %r52;\n" " add.s32 %r163, %r54, 2048;\n" "\n" "$L__BB17_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd79];\n" " // end inline asm\n" " st.shared.f32 [%r163+-2048], %f26;\n" " add.s64 %rd61, %rd79, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd61];\n" " // end inline asm\n" " st.shared.f32 [%r163+-1024], %f27;\n" " add.s64 %rd62, %rd79, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd62];\n" " // end inline asm\n" " st.shared.f32 [%r163], %f28;\n" " add.s64 %rd63, %rd79, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd63];\n" " // end inline asm\n" " st.shared.f32 [%r163+1024], %f29;\n" " add.s64 %rd79, %rd79, 4096;\n" " add.s32 %r163, %r163, 4096;\n" " add.s32 %r162, %r162, 1024;\n" " setp.lt.s32 %p7, %r162, %r13;\n" " @%p7 bra $L__BB17_9;\n" "\n" "$L__BB17_10:\n" " cvt.u32.u64 %r55, %rd4;\n" " setp.ge.s32 %p8, %r55, %r31;\n" " bar.sync 0;\n" " @%p8 bra $L__BB17_17;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB17_17;\n" "\n" " shr.s32 %r57, %r13, 8;\n" " max.s32 %r21, %r57, 1;\n" " and.b32 %r22, %r21, 1;\n" " setp.lt.s32 %p10, %r13, 512;\n" " mov.u32 %r167, %r46;\n" " @%p10 bra $L__BB17_15;\n" "\n" " sub.s32 %r166, %r21, %r22;\n" " mov.u32 %r167, sh_x4;\n" "\n" "$L__BB17_14:\n" " shl.b64 %rd65, %rd5, 3;\n" " add.s64 %rd66, %rd86, %rd65;\n" " ld.global.v2.u32 {%r59, %r60}, [%rd66];\n" " add.s64 %rd67, %rd87, %rd65;\n" " ld.global.v2.u32 {%r63, %r64}, [%rd67];\n" " shl.b32 %r67, %r2, 4;\n" " add.s32 %r68, %r167, %r67;\n" " ld.shared.v4.f32 {%f63, %f64, %f65, %f66}, [%r68];\n" " cvt.u16.u32 %rs1, %r59;\n" " // begin inline asm\n" " cvt.f32.f16 %f31, %rs1;\n" " // end inline asm\n" " mov.f32 %f71, 0f00000000;\n" " fma.rn.f32 %f72, %f31, %f63, %f71;\n" " cvt.u16.u32 %rs2, %r63;\n" " // begin inline asm\n" " cvt.f32.f16 %f32, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f73, %f32, %f63, %f71;\n" " shr.u32 %r69, %r59, 16;\n" " cvt.u16.u32 %rs3, %r69;\n" " // begin inline asm\n" " cvt.f32.f16 %f33, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f74, %f33, %f64, %f72;\n" " shr.u32 %r70, %r63, 16;\n" " cvt.u16.u32 %rs4, %r70;\n" " // begin inline asm\n" " cvt.f32.f16 %f34, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f75, %f34, %f64, %f73;\n" " cvt.u16.u32 %rs5, %r60;\n" " // begin inline asm\n" " cvt.f32.f16 %f35, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f76, %f35, %f65, %f74;\n" " cvt.u16.u32 %rs6, %r64;\n" " // begin inline asm\n" " cvt.f32.f16 %f36, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f77, %f36, %f65, %f75;\n" " shr.u32 %r71, %r60, 16;\n" " cvt.u16.u32 %rs7, %r71;\n" " // begin inline asm\n" " cvt.f32.f16 %f37, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f78, %f37, %f66, %f76;\n" " shr.u32 %r72, %r64, 16;\n" " cvt.u16.u32 %rs8, %r72;\n" " // begin inline asm\n" " cvt.f32.f16 %f38, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f79, %f38, %f66, %f77;\n" " ld.global.v2.u32 {%r73, %r74}, [%rd66+256];\n" " ld.global.v2.u32 {%r77, %r78}, [%rd67+256];\n" " ld.shared.v4.f32 {%f80, %f81, %f82, %f83}, [%r68+512];\n" " cvt.u16.u32 %rs9, %r73;\n" " // begin inline asm\n" " cvt.f32.f16 %f39, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f88, %f39, %f80, %f78;\n" " cvt.u16.u32 %rs10, %r77;\n" " // begin inline asm\n" " cvt.f32.f16 %f40, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f89, %f40, %f80, %f79;\n" " shr.u32 %r81, %r73, 16;\n" " cvt.u16.u32 %rs11, %r81;\n" " // begin inline asm\n" " cvt.f32.f16 %f41, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f90, %f41, %f81, %f88;\n" " shr.u32 %r82, %r77, 16;\n" " cvt.u16.u32 %rs12, %r82;\n" " // begin inline asm\n" " cvt.f32.f16 %f42, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f91, %f42, %f81, %f89;\n" " cvt.u16.u32 %rs13, %r74;\n" " // begin inline asm\n" " cvt.f32.f16 %f43, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f92, %f43, %f82, %f90;\n" " cvt.u16.u32 %rs14, %r78;\n" " // begin inline asm\n" " cvt.f32.f16 %f44, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f93, %f44, %f82, %f91;\n" " shr.u32 %r83, %r74, 16;\n" " cvt.u16.u32 %rs15, %r83;\n" " // begin inline asm\n" " cvt.f32.f16 %f45, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f94, %f45, %f83, %f92;\n" " shr.u32 %r84, %r78, 16;\n" " cvt.u16.u32 %rs16, %r84;\n" " // begin inline asm\n" " cvt.f32.f16 %f46, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f95, %f46, %f83, %f93;\n" " add.f32 %f96, %f226, %f94;\n" " add.f32 %f97, %f227, %f95;\n" " ld.global.v2.u32 {%r85, %r86}, [%rd66+512];\n" " ld.global.v2.u32 {%r89, %r90}, [%rd67+512];\n" " ld.shared.v4.f32 {%f98, %f99, %f100, %f101}, [%r68+1024];\n" " cvt.u16.u32 %rs17, %r85;\n" " // begin inline asm\n" " cvt.f32.f16 %f47, %rs17;\n" " // end inline asm\n" " fma.rn.f32 %f106, %f47, %f98, %f71;\n" " cvt.u16.u32 %rs18, %r89;\n" " // begin inline asm\n" " cvt.f32.f16 %f48, %rs18;\n" " // end inline asm\n" " fma.rn.f32 %f107, %f48, %f98, %f71;\n" " shr.u32 %r93, %r85, 16;\n" " cvt.u16.u32 %rs19, %r93;\n" " // begin inline asm\n" " cvt.f32.f16 %f49, %rs19;\n" " // end inline asm\n" " fma.rn.f32 %f108, %f49, %f99, %f106;\n" " shr.u32 %r94, %r89, 16;\n" " cvt.u16.u32 %rs20, %r94;\n" " // begin inline asm\n" " cvt.f32.f16 %f50, %rs20;\n" " // end inline asm\n" " fma.rn.f32 %f109, %f50, %f99, %f107;\n" " cvt.u16.u32 %rs21, %r86;\n" " // begin inline asm\n" " cvt.f32.f16 %f51, %rs21;\n" " // end inline asm\n" " fma.rn.f32 %f110, %f51, %f100, %f108;\n" " cvt.u16.u32 %rs22, %r90;\n" " // begin inline asm\n" " cvt.f32.f16 %f52, %rs22;\n" " // end inline asm\n" " fma.rn.f32 %f111, %f52, %f100, %f109;\n" " shr.u32 %r95, %r86, 16;\n" " cvt.u16.u32 %rs23, %r95;\n" " // begin inline asm\n" " cvt.f32.f16 %f53, %rs23;\n" " // end inline asm\n" " fma.rn.f32 %f112, %f53, %f101, %f110;\n" " shr.u32 %r96, %r90, 16;\n" " cvt.u16.u32 %rs24, %r96;\n" " // begin inline asm\n" " cvt.f32.f16 %f54, %rs24;\n" " // end inline asm\n" " fma.rn.f32 %f113, %f54, %f101, %f111;\n" " ld.global.v2.u32 {%r97, %r98}, [%rd66+768];\n" " ld.global.v2.u32 {%r101, %r102}, [%rd67+768];\n" " ld.shared.v4.f32 {%f114, %f115, %f116, %f117}, [%r68+1536];\n" " cvt.u16.u32 %rs25, %r97;\n" " // begin inline asm\n" " cvt.f32.f16 %f55, %rs25;\n" " // end inline asm\n" " fma.rn.f32 %f122, %f55, %f114, %f112;\n" " cvt.u16.u32 %rs26, %r101;\n" " // begin inline asm\n" " cvt.f32.f16 %f56, %rs26;\n" " // end inline asm\n" " fma.rn.f32 %f123, %f56, %f114, %f113;\n" " shr.u32 %r105, %r97, 16;\n" " cvt.u16.u32 %rs27, %r105;\n" " // begin inline asm\n" " cvt.f32.f16 %f57, %rs27;\n" " // end inline asm\n" " fma.rn.f32 %f124, %f57, %f115, %f122;\n" " shr.u32 %r106, %r101, 16;\n" " cvt.u16.u32 %rs28, %r106;\n" " // begin inline asm\n" " cvt.f32.f16 %f58, %rs28;\n" " // end inline asm\n" " fma.rn.f32 %f125, %f58, %f115, %f123;\n" " cvt.u16.u32 %rs29, %r98;\n" " // begin inline asm\n" " cvt.f32.f16 %f59, %rs29;\n" " // end inline asm\n" " fma.rn.f32 %f126, %f59, %f116, %f124;\n" " cvt.u16.u32 %rs30, %r102;\n" " // begin inline asm\n" " cvt.f32.f16 %f60, %rs30;\n" " // end inline asm\n" " fma.rn.f32 %f127, %f60, %f116, %f125;\n" " shr.u32 %r107, %r98, 16;\n" " cvt.u16.u32 %rs31, %r107;\n" " // begin inline asm\n" " cvt.f32.f16 %f61, %rs31;\n" " // end inline asm\n" " fma.rn.f32 %f128, %f61, %f117, %f126;\n" " shr.u32 %r108, %r102, 16;\n" " cvt.u16.u32 %rs32, %r108;\n" " // begin inline asm\n" " cvt.f32.f16 %f62, %rs32;\n" " // end inline asm\n" " fma.rn.f32 %f129, %f62, %f117, %f127;\n" " add.f32 %f226, %f96, %f128;\n" " add.f32 %f227, %f97, %f129;\n" " add.s64 %rd86, %rd86, 1024;\n" " add.s64 %rd87, %rd87, 1024;\n" " add.s32 %r167, %r167, 2048;\n" " add.s32 %r166, %r166, -2;\n" " setp.ne.s32 %p11, %r166, 0;\n" " @%p11 bra $L__BB17_14;\n" "\n" "$L__BB17_15:\n" " setp.eq.s32 %p12, %r22, 0;\n" " @%p12 bra $L__BB17_17;\n" "\n" " shl.b64 %rd68, %rd5, 3;\n" " add.s64 %rd69, %rd86, %rd68;\n" " ld.global.v2.u32 {%r109, %r110}, [%rd69];\n" " add.s64 %rd70, %rd87, %rd68;\n" " ld.global.v2.u32 {%r113, %r114}, [%rd70];\n" " shl.b32 %r117, %r2, 4;\n" " add.s32 %r118, %r167, %r117;\n" " ld.shared.v4.f32 {%f146, %f147, %f148, %f149}, [%r118];\n" " cvt.u16.u32 %rs33, %r109;\n" " // begin inline asm\n" " cvt.f32.f16 %f130, %rs33;\n" " // end inline asm\n" " mov.f32 %f154, 0f00000000;\n" " fma.rn.f32 %f155, %f130, %f146, %f154;\n" " cvt.u16.u32 %rs34, %r113;\n" " // begin inline asm\n" " cvt.f32.f16 %f131, %rs34;\n" " // end inline asm\n" " fma.rn.f32 %f156, %f131, %f146, %f154;\n" " shr.u32 %r119, %r109, 16;\n" " cvt.u16.u32 %rs35, %r119;\n" " // begin inline asm\n" " cvt.f32.f16 %f132, %rs35;\n" " // end inline asm\n" " fma.rn.f32 %f157, %f132, %f147, %f155;\n" " shr.u32 %r120, %r113, 16;\n" " cvt.u16.u32 %rs36, %r120;\n" " // begin inline asm\n" " cvt.f32.f16 %f133, %rs36;\n" " // end inline asm\n" " fma.rn.f32 %f158, %f133, %f147, %f156;\n" " cvt.u16.u32 %rs37, %r110;\n" " // begin inline asm\n" " cvt.f32.f16 %f134, %rs37;\n" " // end inline asm\n" " fma.rn.f32 %f159, %f134, %f148, %f157;\n" " cvt.u16.u32 %rs38, %r114;\n" " // begin inline asm\n" " cvt.f32.f16 %f135, %rs38;\n" " // end inline asm\n" " fma.rn.f32 %f160, %f135, %f148, %f158;\n" " shr.u32 %r121, %r110, 16;\n" " cvt.u16.u32 %rs39, %r121;\n" " // begin inline asm\n" " cvt.f32.f16 %f136, %rs39;\n" " // end inline asm\n" " fma.rn.f32 %f161, %f136, %f149, %f159;\n" " shr.u32 %r122, %r114, 16;\n" " cvt.u16.u32 %rs40, %r122;\n" " // begin inline asm\n" " cvt.f32.f16 %f137, %rs40;\n" " // end inline asm\n" " fma.rn.f32 %f162, %f137, %f149, %f160;\n" " ld.global.v2.u32 {%r123, %r124}, [%rd69+256];\n" " ld.global.v2.u32 {%r127, %r128}, [%rd70+256];\n" " ld.shared.v4.f32 {%f163, %f164, %f165, %f166}, [%r118+512];\n" " cvt.u16.u32 %rs41, %r123;\n" " // begin inline asm\n" " cvt.f32.f16 %f138, %rs41;\n" " // end inline asm\n" " fma.rn.f32 %f171, %f138, %f163, %f161;\n" " cvt.u16.u32 %rs42, %r127;\n" " // begin inline asm\n" " cvt.f32.f16 %f139, %rs42;\n" " // end inline asm\n" " fma.rn.f32 %f172, %f139, %f163, %f162;\n" " shr.u32 %r131, %r123, 16;\n" " cvt.u16.u32 %rs43, %r131;\n" " // begin inline asm\n" " cvt.f32.f16 %f140, %rs43;\n" " // end inline asm\n" " fma.rn.f32 %f173, %f140, %f164, %f171;\n" " shr.u32 %r132, %r127, 16;\n" " cvt.u16.u32 %rs44, %r132;\n" " // begin inline asm\n" " cvt.f32.f16 %f141, %rs44;\n" " // end inline asm\n" " fma.rn.f32 %f174, %f141, %f164, %f172;\n" " cvt.u16.u32 %rs45, %r124;\n" " // begin inline asm\n" " cvt.f32.f16 %f142, %rs45;\n" " // end inline asm\n" " fma.rn.f32 %f175, %f142, %f165, %f173;\n" " cvt.u16.u32 %rs46, %r128;\n" " // begin inline asm\n" " cvt.f32.f16 %f143, %rs46;\n" " // end inline asm\n" " fma.rn.f32 %f176, %f143, %f165, %f174;\n" " shr.u32 %r133, %r124, 16;\n" " cvt.u16.u32 %rs47, %r133;\n" " // begin inline asm\n" " cvt.f32.f16 %f144, %rs47;\n" " // end inline asm\n" " fma.rn.f32 %f177, %f144, %f166, %f175;\n" " shr.u32 %r134, %r128, 16;\n" " cvt.u16.u32 %rs48, %r134;\n" " // begin inline asm\n" " cvt.f32.f16 %f145, %rs48;\n" " // end inline asm\n" " fma.rn.f32 %f178, %f145, %f166, %f176;\n" " add.f32 %f226, %f226, %f177;\n" " add.f32 %f227, %f227, %f178;\n" " add.s64 %rd87, %rd87, 512;\n" " add.s64 %rd86, %rd86, 512;\n" "\n" "$L__BB17_17:\n" " add.s32 %r161, %r161, 2048;\n" " setp.lt.s32 %p13, %r161, %r32;\n" " add.s32 %r160, %r160, 2048;\n" " @%p13 bra $L__BB17_2;\n" "\n" "$L__BB17_18:\n" " cvt.u32.u64 %r155, %rd4;\n" " setp.ge.s32 %p14, %r155, %r31;\n" " mov.u32 %r145, 16;\n" " mov.u32 %r154, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f179, %f226, %r145, %r154;\n" " // end inline asm\n" " add.f32 %f182, %f226, %f179;\n" " mov.u32 %r147, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f181, %f182, %r147, %r154;\n" " // end inline asm\n" " add.f32 %f184, %f182, %f181;\n" " mov.u32 %r149, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f183, %f184, %r149, %r154;\n" " // end inline asm\n" " add.f32 %f186, %f184, %f183;\n" " mov.u32 %r151, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f185, %f186, %r151, %r154;\n" " // end inline asm\n" " add.f32 %f188, %f186, %f185;\n" " mov.u32 %r153, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f187, %f188, %r153, %r154;\n" " // end inline asm\n" " add.f32 %f17, %f188, %f187;\n" " // begin inline asm\n" " shfl.down.b32 %f189, %f227, %r145, %r154;\n" " // end inline asm\n" " add.f32 %f192, %f227, %f189;\n" " // begin inline asm\n" " shfl.down.b32 %f191, %f192, %r147, %r154;\n" " // end inline asm\n" " add.f32 %f194, %f192, %f191;\n" " // begin inline asm\n" " shfl.down.b32 %f193, %f194, %r149, %r154;\n" " // end inline asm\n" " add.f32 %f196, %f194, %f193;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f196, %r151, %r154;\n" " // end inline asm\n" " add.f32 %f198, %f196, %f195;\n" " // begin inline asm\n" " shfl.down.b32 %f197, %f198, %r153, %r154;\n" " // end inline asm\n" " add.f32 %f18, %f198, %f197;\n" " setp.ne.s32 %p15, %r2, 0;\n" " or.pred %p16, %p15, %p14;\n" " @%p16 bra $L__BB17_20;\n" "\n" " ld.param.u32 %r158, [fc_moe_gateup_f16_vec_nx_param_6];\n" " cvt.s64.s32 %rd76, %r158;\n" " neg.f32 %f199, %f17;\n" " mov.f32 %f200, 0f3F000000;\n" " mov.f32 %f201, 0f3BBB989D;\n" " fma.rn.f32 %f202, %f199, %f201, %f200;\n" " mov.f32 %f203, 0f3FB8AA3B;\n" " mov.f32 %f204, 0f437C0000;\n" " cvt.sat.f32.f32 %f205, %f202;\n" " mov.f32 %f206, 0f4B400001;\n" " fma.rm.f32 %f207, %f205, %f204, %f206;\n" " add.f32 %f208, %f207, 0fCB40007F;\n" " neg.f32 %f209, %f208;\n" " fma.rn.f32 %f210, %f199, %f203, %f209;\n" " mov.f32 %f211, 0f32A57060;\n" " fma.rn.f32 %f212, %f199, %f211, %f210;\n" " mov.b32 %r156, %f207;\n" " shl.b32 %r157, %r156, 23;\n" " mov.b32 %f213, %r157;\n" " ex2.approx.ftz.f32 %f214, %f212;\n" " fma.rn.f32 %f215, %f214, %f213, 0f3F800000;\n" " div.rn.f32 %f216, %f17, %f215;\n" " mul.f32 %f217, %f18, %f216;\n" " mul.lo.s64 %rd71, %rd2, %rd76;\n" " add.s64 %rd72, %rd71, %rd4;\n" " cvta.to.global.u64 %rd73, %rd34;\n" " shl.b64 %rd74, %rd72, 2;\n" " add.s64 %rd75, %rd73, %rd74;\n" " st.global.f32 [%rd75], %f217;\n" "\n" "$L__BB17_20:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_bf16_vec_nx\n" ".visible .entry fc_moe_gateup_bf16_vec_nx(\n" " .param .u64 fc_moe_gateup_bf16_vec_nx_param_0,\n" " .param .u64 fc_moe_gateup_bf16_vec_nx_param_1,\n" " .param .u64 fc_moe_gateup_bf16_vec_nx_param_2,\n" " .param .u64 fc_moe_gateup_bf16_vec_nx_param_3,\n" " .param .u64 fc_moe_gateup_bf16_vec_nx_param_4,\n" " .param .u32 fc_moe_gateup_bf16_vec_nx_param_5,\n" " .param .u32 fc_moe_gateup_bf16_vec_nx_param_6,\n" " .param .u32 fc_moe_gateup_bf16_vec_nx_param_7,\n" " .param .u32 fc_moe_gateup_bf16_vec_nx_param_8\n" ")\n" "{\n" " .reg .pred %p<17>;\n" " .reg .f32 %f<230>;\n" " .reg .b32 %r<192>;\n" " .reg .b64 %rd<88>;\n" "\n" "\n" " ld.param.u64 %rd31, [fc_moe_gateup_bf16_vec_nx_param_0];\n" " ld.param.u64 %rd32, [fc_moe_gateup_bf16_vec_nx_param_1];\n" " ld.param.u64 %rd33, [fc_moe_gateup_bf16_vec_nx_param_2];\n" " ld.param.u64 %rd36, [fc_moe_gateup_bf16_vec_nx_param_3];\n" " ld.param.u64 %rd34, [fc_moe_gateup_bf16_vec_nx_param_4];\n" " ld.param.u32 %r35, [fc_moe_gateup_bf16_vec_nx_param_5];\n" " ld.param.u32 %r31, [fc_moe_gateup_bf16_vec_nx_param_6];\n" " ld.param.u32 %r32, [fc_moe_gateup_bf16_vec_nx_param_7];\n" " ld.param.u32 %r33, [fc_moe_gateup_bf16_vec_nx_param_8];\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r36, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r37, %ctaid.x;\n" " shl.b32 %r38, %r37, 3;\n" " add.s32 %r39, %r38, %r36;\n" " mov.u32 %r40, %ctaid.z;\n" " mul.wide.s32 %rd1, %r32, %r40;\n" " mul.wide.s32 %rd37, %r35, %r40;\n" " mov.u32 %r41, %ctaid.y;\n" " cvt.s64.s32 %rd38, %r41;\n" " add.s64 %rd2, %rd37, %rd38;\n" " shl.b64 %rd39, %rd2, 2;\n" " add.s64 %rd35, %rd36, %rd39;\n" " // begin inline asm\n" " ld.global.nc.s32 %r34, [%rd35];\n" " // end inline asm\n" " cvt.s64.s32 %rd3, %r31;\n" " cvt.s64.s32 %rd4, %r39;\n" " setp.lt.s32 %p1, %r32, 1;\n" " mov.f32 %f226, 0f00000000;\n" " mov.f32 %f227, %f226;\n" " @%p1 bra $L__BB18_18;\n" "\n" " cvt.s64.s32 %rd40, %r33;\n" " cvt.s64.s32 %rd41, %r34;\n" " mul.lo.s64 %rd42, %rd41, %rd3;\n" " add.s64 %rd43, %rd42, %rd4;\n" " mul.lo.s64 %rd44, %rd43, %rd40;\n" " cvt.u64.u32 %rd5, %r2;\n" " mov.u32 %r44, -2;\n" " sub.s32 %r4, %r44, %r1;\n" " cvt.s64.s32 %rd6, %r1;\n" " shl.b32 %r45, %r1, 2;\n" " mov.u32 %r46, sh_x4;\n" " add.s32 %r6, %r46, %r45;\n" " add.s32 %r7, %r1, 256;\n" " cvt.s64.s32 %rd7, %r7;\n" " add.s32 %r8, %r1, 512;\n" " cvt.s64.s32 %rd8, %r8;\n" " add.s32 %r9, %r1, 768;\n" " shl.b64 %rd45, %rd1, 2;\n" " add.s64 %rd9, %rd33, %rd45;\n" " cvta.to.global.u64 %rd46, %rd32;\n" " add.s64 %rd87, %rd46, %rd44;\n" " cvta.to.global.u64 %rd47, %rd31;\n" " add.s64 %rd86, %rd47, %rd44;\n" " mov.f32 %f227, 0f00000000;\n" " mov.u32 %r184, 0;\n" " mov.u32 %r185, %r184;\n" " mov.f32 %f226, %f227;\n" "\n" "$L__BB18_2:\n" " not.b32 %r183, %r32;\n" " add.s32 %r47, %r184, %r183;\n" " max.s32 %r48, %r47, -2049;\n" " sub.s32 %r12, %r4, %r48;\n" " sub.s32 %r49, %r32, %r185;\n" " min.s32 %r13, %r49, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p2, %r1, %r13;\n" " @%p2 bra $L__BB18_10;\n" "\n" " shr.u32 %r50, %r12, 8;\n" " add.s32 %r51, %r50, 1;\n" " cvt.s64.s32 %rd14, %r185;\n" " add.s64 %rd15, %rd1, %rd14;\n" " and.b32 %r14, %r51, 3;\n" " setp.eq.s32 %p3, %r14, 0;\n" " mov.u32 %r186, %r1;\n" " @%p3 bra $L__BB18_7;\n" "\n" " add.s64 %rd49, %rd15, %rd6;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd48, %rd33, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f23, [%rd48];\n" " // end inline asm\n" " st.shared.f32 [%r6], %f23;\n" " setp.eq.s32 %p4, %r14, 1;\n" " mov.u32 %r186, %r7;\n" " @%p4 bra $L__BB18_7;\n" "\n" " add.s64 %rd52, %rd15, %rd7;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd51, %rd33, %rd53;\n" " // begin inline asm\n" " ld.global.nc.f32 %f24, [%rd51];\n" " // end inline asm\n" " st.shared.f32 [%r6+1024], %f24;\n" " setp.eq.s32 %p5, %r14, 2;\n" " mov.u32 %r186, %r8;\n" " @%p5 bra $L__BB18_7;\n" "\n" " add.s64 %rd55, %rd15, %rd8;\n" " shl.b64 %rd56, %rd55, 2;\n" " add.s64 %rd54, %rd33, %rd56;\n" " // begin inline asm\n" " ld.global.nc.f32 %f25, [%rd54];\n" " // end inline asm\n" " st.shared.f32 [%r6+2048], %f25;\n" " mov.u32 %r186, %r9;\n" "\n" "$L__BB18_7:\n" " setp.lt.u32 %p6, %r12, 768;\n" " @%p6 bra $L__BB18_10;\n" "\n" " cvt.s64.s32 %rd57, %r186;\n" " add.s64 %rd58, %rd57, %rd14;\n" " shl.b64 %rd59, %rd58, 2;\n" " add.s64 %rd79, %rd9, %rd59;\n" " shl.b32 %r52, %r186, 2;\n" " mov.u32 %r53, sh_x4;\n" " add.s32 %r54, %r53, %r52;\n" " add.s32 %r187, %r54, 2048;\n" "\n" "$L__BB18_9:\n" " // begin inline asm\n" " ld.global.nc.f32 %f26, [%rd79];\n" " // end inline asm\n" " st.shared.f32 [%r187+-2048], %f26;\n" " add.s64 %rd61, %rd79, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f27, [%rd61];\n" " // end inline asm\n" " st.shared.f32 [%r187+-1024], %f27;\n" " add.s64 %rd62, %rd79, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f28, [%rd62];\n" " // end inline asm\n" " st.shared.f32 [%r187], %f28;\n" " add.s64 %rd63, %rd79, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f29, [%rd63];\n" " // end inline asm\n" " st.shared.f32 [%r187+1024], %f29;\n" " add.s64 %rd79, %rd79, 4096;\n" " add.s32 %r187, %r187, 4096;\n" " add.s32 %r186, %r186, 1024;\n" " setp.lt.s32 %p7, %r186, %r13;\n" " @%p7 bra $L__BB18_9;\n" "\n" "$L__BB18_10:\n" " cvt.u32.u64 %r55, %rd4;\n" " setp.ge.s32 %p8, %r55, %r31;\n" " bar.sync 0;\n" " @%p8 bra $L__BB18_17;\n" "\n" " setp.lt.s32 %p9, %r13, 256;\n" " @%p9 bra $L__BB18_17;\n" "\n" " shr.s32 %r57, %r13, 8;\n" " max.s32 %r21, %r57, 1;\n" " and.b32 %r22, %r21, 1;\n" " setp.lt.s32 %p10, %r13, 512;\n" " mov.u32 %r191, %r46;\n" " @%p10 bra $L__BB18_15;\n" "\n" " sub.s32 %r190, %r21, %r22;\n" " mov.u32 %r191, sh_x4;\n" "\n" "$L__BB18_14:\n" " shl.b64 %rd65, %rd5, 3;\n" " add.s64 %rd66, %rd86, %rd65;\n" " ld.global.v2.u32 {%r59, %r60}, [%rd66];\n" " add.s64 %rd67, %rd87, %rd65;\n" " ld.global.v2.u32 {%r63, %r64}, [%rd67];\n" " shl.b32 %r67, %r2, 4;\n" " add.s32 %r68, %r191, %r67;\n" " ld.shared.v4.f32 {%f31, %f32, %f33, %f34}, [%r68];\n" " shl.b32 %r69, %r59, 16;\n" " mov.b32 %f39, %r69;\n" " mov.f32 %f40, 0f00000000;\n" " fma.rn.f32 %f41, %f39, %f31, %f40;\n" " shl.b32 %r70, %r63, 16;\n" " mov.b32 %f42, %r70;\n" " fma.rn.f32 %f43, %f42, %f31, %f40;\n" " and.b32 %r71, %r59, -65536;\n" " mov.b32 %f44, %r71;\n" " fma.rn.f32 %f45, %f44, %f32, %f41;\n" " and.b32 %r72, %r63, -65536;\n" " mov.b32 %f46, %r72;\n" " fma.rn.f32 %f47, %f46, %f32, %f43;\n" " shl.b32 %r73, %r60, 16;\n" " mov.b32 %f48, %r73;\n" " fma.rn.f32 %f49, %f48, %f33, %f45;\n" " shl.b32 %r74, %r64, 16;\n" " mov.b32 %f50, %r74;\n" " fma.rn.f32 %f51, %f50, %f33, %f47;\n" " and.b32 %r75, %r60, -65536;\n" " mov.b32 %f52, %r75;\n" " fma.rn.f32 %f53, %f52, %f34, %f49;\n" " and.b32 %r76, %r64, -65536;\n" " mov.b32 %f54, %r76;\n" " fma.rn.f32 %f55, %f54, %f34, %f51;\n" " ld.global.v2.u32 {%r77, %r78}, [%rd66+256];\n" " ld.global.v2.u32 {%r81, %r82}, [%rd67+256];\n" " ld.shared.v4.f32 {%f56, %f57, %f58, %f59}, [%r68+512];\n" " shl.b32 %r85, %r77, 16;\n" " mov.b32 %f64, %r85;\n" " fma.rn.f32 %f65, %f64, %f56, %f53;\n" " shl.b32 %r86, %r81, 16;\n" " mov.b32 %f66, %r86;\n" " fma.rn.f32 %f67, %f66, %f56, %f55;\n" " and.b32 %r87, %r77, -65536;\n" " mov.b32 %f68, %r87;\n" " fma.rn.f32 %f69, %f68, %f57, %f65;\n" " and.b32 %r88, %r81, -65536;\n" " mov.b32 %f70, %r88;\n" " fma.rn.f32 %f71, %f70, %f57, %f67;\n" " shl.b32 %r89, %r78, 16;\n" " mov.b32 %f72, %r89;\n" " fma.rn.f32 %f73, %f72, %f58, %f69;\n" " shl.b32 %r90, %r82, 16;\n" " mov.b32 %f74, %r90;\n" " fma.rn.f32 %f75, %f74, %f58, %f71;\n" " and.b32 %r91, %r78, -65536;\n" " mov.b32 %f76, %r91;\n" " fma.rn.f32 %f77, %f76, %f59, %f73;\n" " and.b32 %r92, %r82, -65536;\n" " mov.b32 %f78, %r92;\n" " fma.rn.f32 %f79, %f78, %f59, %f75;\n" " add.f32 %f80, %f226, %f77;\n" " add.f32 %f81, %f227, %f79;\n" " ld.global.v2.u32 {%r93, %r94}, [%rd66+512];\n" " ld.global.v2.u32 {%r97, %r98}, [%rd67+512];\n" " ld.shared.v4.f32 {%f82, %f83, %f84, %f85}, [%r68+1024];\n" " shl.b32 %r101, %r93, 16;\n" " mov.b32 %f90, %r101;\n" " fma.rn.f32 %f91, %f90, %f82, %f40;\n" " shl.b32 %r102, %r97, 16;\n" " mov.b32 %f92, %r102;\n" " fma.rn.f32 %f93, %f92, %f82, %f40;\n" " and.b32 %r103, %r93, -65536;\n" " mov.b32 %f94, %r103;\n" " fma.rn.f32 %f95, %f94, %f83, %f91;\n" " and.b32 %r104, %r97, -65536;\n" " mov.b32 %f96, %r104;\n" " fma.rn.f32 %f97, %f96, %f83, %f93;\n" " shl.b32 %r105, %r94, 16;\n" " mov.b32 %f98, %r105;\n" " fma.rn.f32 %f99, %f98, %f84, %f95;\n" " shl.b32 %r106, %r98, 16;\n" " mov.b32 %f100, %r106;\n" " fma.rn.f32 %f101, %f100, %f84, %f97;\n" " and.b32 %r107, %r94, -65536;\n" " mov.b32 %f102, %r107;\n" " fma.rn.f32 %f103, %f102, %f85, %f99;\n" " and.b32 %r108, %r98, -65536;\n" " mov.b32 %f104, %r108;\n" " fma.rn.f32 %f105, %f104, %f85, %f101;\n" " ld.global.v2.u32 {%r109, %r110}, [%rd66+768];\n" " ld.global.v2.u32 {%r113, %r114}, [%rd67+768];\n" " ld.shared.v4.f32 {%f106, %f107, %f108, %f109}, [%r68+1536];\n" " shl.b32 %r117, %r109, 16;\n" " mov.b32 %f114, %r117;\n" " fma.rn.f32 %f115, %f114, %f106, %f103;\n" " shl.b32 %r118, %r113, 16;\n" " mov.b32 %f116, %r118;\n" " fma.rn.f32 %f117, %f116, %f106, %f105;\n" " and.b32 %r119, %r109, -65536;\n" " mov.b32 %f118, %r119;\n" " fma.rn.f32 %f119, %f118, %f107, %f115;\n" " and.b32 %r120, %r113, -65536;\n" " mov.b32 %f120, %r120;\n" " fma.rn.f32 %f121, %f120, %f107, %f117;\n" " shl.b32 %r121, %r110, 16;\n" " mov.b32 %f122, %r121;\n" " fma.rn.f32 %f123, %f122, %f108, %f119;\n" " shl.b32 %r122, %r114, 16;\n" " mov.b32 %f124, %r122;\n" " fma.rn.f32 %f125, %f124, %f108, %f121;\n" " and.b32 %r123, %r110, -65536;\n" " mov.b32 %f126, %r123;\n" " fma.rn.f32 %f127, %f126, %f109, %f123;\n" " and.b32 %r124, %r114, -65536;\n" " mov.b32 %f128, %r124;\n" " fma.rn.f32 %f129, %f128, %f109, %f125;\n" " add.f32 %f226, %f80, %f127;\n" " add.f32 %f227, %f81, %f129;\n" " add.s64 %rd86, %rd86, 1024;\n" " add.s64 %rd87, %rd87, 1024;\n" " add.s32 %r191, %r191, 2048;\n" " add.s32 %r190, %r190, -2;\n" " setp.ne.s32 %p11, %r190, 0;\n" " @%p11 bra $L__BB18_14;\n" "\n" "$L__BB18_15:\n" " setp.eq.s32 %p12, %r22, 0;\n" " @%p12 bra $L__BB18_17;\n" "\n" " shl.b64 %rd68, %rd5, 3;\n" " add.s64 %rd69, %rd86, %rd68;\n" " ld.global.v2.u32 {%r125, %r126}, [%rd69];\n" " add.s64 %rd70, %rd87, %rd68;\n" " ld.global.v2.u32 {%r129, %r130}, [%rd70];\n" " shl.b32 %r133, %r2, 4;\n" " add.s32 %r134, %r191, %r133;\n" " ld.shared.v4.f32 {%f130, %f131, %f132, %f133}, [%r134];\n" " shl.b32 %r135, %r125, 16;\n" " mov.b32 %f138, %r135;\n" " mov.f32 %f139, 0f00000000;\n" " fma.rn.f32 %f140, %f138, %f130, %f139;\n" " shl.b32 %r136, %r129, 16;\n" " mov.b32 %f141, %r136;\n" " fma.rn.f32 %f142, %f141, %f130, %f139;\n" " and.b32 %r137, %r125, -65536;\n" " mov.b32 %f143, %r137;\n" " fma.rn.f32 %f144, %f143, %f131, %f140;\n" " and.b32 %r138, %r129, -65536;\n" " mov.b32 %f145, %r138;\n" " fma.rn.f32 %f146, %f145, %f131, %f142;\n" " shl.b32 %r139, %r126, 16;\n" " mov.b32 %f147, %r139;\n" " fma.rn.f32 %f148, %f147, %f132, %f144;\n" " shl.b32 %r140, %r130, 16;\n" " mov.b32 %f149, %r140;\n" " fma.rn.f32 %f150, %f149, %f132, %f146;\n" " and.b32 %r141, %r126, -65536;\n" " mov.b32 %f151, %r141;\n" " fma.rn.f32 %f152, %f151, %f133, %f148;\n" " and.b32 %r142, %r130, -65536;\n" " mov.b32 %f153, %r142;\n" " fma.rn.f32 %f154, %f153, %f133, %f150;\n" " ld.global.v2.u32 {%r143, %r144}, [%rd69+256];\n" " ld.global.v2.u32 {%r147, %r148}, [%rd70+256];\n" " ld.shared.v4.f32 {%f155, %f156, %f157, %f158}, [%r134+512];\n" " shl.b32 %r151, %r143, 16;\n" " mov.b32 %f163, %r151;\n" " fma.rn.f32 %f164, %f163, %f155, %f152;\n" " shl.b32 %r152, %r147, 16;\n" " mov.b32 %f165, %r152;\n" " fma.rn.f32 %f166, %f165, %f155, %f154;\n" " and.b32 %r153, %r143, -65536;\n" " mov.b32 %f167, %r153;\n" " fma.rn.f32 %f168, %f167, %f156, %f164;\n" " and.b32 %r154, %r147, -65536;\n" " mov.b32 %f169, %r154;\n" " fma.rn.f32 %f170, %f169, %f156, %f166;\n" " shl.b32 %r155, %r144, 16;\n" " mov.b32 %f171, %r155;\n" " fma.rn.f32 %f172, %f171, %f157, %f168;\n" " shl.b32 %r156, %r148, 16;\n" " mov.b32 %f173, %r156;\n" " fma.rn.f32 %f174, %f173, %f157, %f170;\n" " and.b32 %r157, %r144, -65536;\n" " mov.b32 %f175, %r157;\n" " fma.rn.f32 %f176, %f175, %f158, %f172;\n" " and.b32 %r158, %r148, -65536;\n" " mov.b32 %f177, %r158;\n" " fma.rn.f32 %f178, %f177, %f158, %f174;\n" " add.f32 %f226, %f226, %f176;\n" " add.f32 %f227, %f227, %f178;\n" " add.s64 %rd87, %rd87, 512;\n" " add.s64 %rd86, %rd86, 512;\n" "\n" "$L__BB18_17:\n" " add.s32 %r185, %r185, 2048;\n" " setp.lt.s32 %p13, %r185, %r32;\n" " add.s32 %r184, %r184, 2048;\n" " @%p13 bra $L__BB18_2;\n" "\n" "$L__BB18_18:\n" " cvt.u32.u64 %r179, %rd4;\n" " setp.ge.s32 %p14, %r179, %r31;\n" " mov.u32 %r169, 16;\n" " mov.u32 %r178, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f179, %f226, %r169, %r178;\n" " // end inline asm\n" " add.f32 %f182, %f226, %f179;\n" " mov.u32 %r171, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f181, %f182, %r171, %r178;\n" " // end inline asm\n" " add.f32 %f184, %f182, %f181;\n" " mov.u32 %r173, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f183, %f184, %r173, %r178;\n" " // end inline asm\n" " add.f32 %f186, %f184, %f183;\n" " mov.u32 %r175, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f185, %f186, %r175, %r178;\n" " // end inline asm\n" " add.f32 %f188, %f186, %f185;\n" " mov.u32 %r177, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f187, %f188, %r177, %r178;\n" " // end inline asm\n" " add.f32 %f17, %f188, %f187;\n" " // begin inline asm\n" " shfl.down.b32 %f189, %f227, %r169, %r178;\n" " // end inline asm\n" " add.f32 %f192, %f227, %f189;\n" " // begin inline asm\n" " shfl.down.b32 %f191, %f192, %r171, %r178;\n" " // end inline asm\n" " add.f32 %f194, %f192, %f191;\n" " // begin inline asm\n" " shfl.down.b32 %f193, %f194, %r173, %r178;\n" " // end inline asm\n" " add.f32 %f196, %f194, %f193;\n" " // begin inline asm\n" " shfl.down.b32 %f195, %f196, %r175, %r178;\n" " // end inline asm\n" " add.f32 %f198, %f196, %f195;\n" " // begin inline asm\n" " shfl.down.b32 %f197, %f198, %r177, %r178;\n" " // end inline asm\n" " add.f32 %f18, %f198, %f197;\n" " setp.ne.s32 %p15, %r2, 0;\n" " or.pred %p16, %p15, %p14;\n" " @%p16 bra $L__BB18_20;\n" "\n" " ld.param.u32 %r182, [fc_moe_gateup_bf16_vec_nx_param_6];\n" " cvt.s64.s32 %rd76, %r182;\n" " neg.f32 %f199, %f17;\n" " mov.f32 %f200, 0f3F000000;\n" " mov.f32 %f201, 0f3BBB989D;\n" " fma.rn.f32 %f202, %f199, %f201, %f200;\n" " mov.f32 %f203, 0f3FB8AA3B;\n" " mov.f32 %f204, 0f437C0000;\n" " cvt.sat.f32.f32 %f205, %f202;\n" " mov.f32 %f206, 0f4B400001;\n" " fma.rm.f32 %f207, %f205, %f204, %f206;\n" " add.f32 %f208, %f207, 0fCB40007F;\n" " neg.f32 %f209, %f208;\n" " fma.rn.f32 %f210, %f199, %f203, %f209;\n" " mov.f32 %f211, 0f32A57060;\n" " fma.rn.f32 %f212, %f199, %f211, %f210;\n" " mov.b32 %r180, %f207;\n" " shl.b32 %r181, %r180, 23;\n" " mov.b32 %f213, %r181;\n" " ex2.approx.ftz.f32 %f214, %f212;\n" " fma.rn.f32 %f215, %f214, %f213, 0f3F800000;\n" " div.rn.f32 %f216, %f17, %f215;\n" " mul.f32 %f217, %f18, %f216;\n" " mul.lo.s64 %rd71, %rd2, %rd76;\n" " add.s64 %rd72, %rd71, %rd4;\n" " cvta.to.global.u64 %rd73, %rd34;\n" " shl.b64 %rd74, %rd72, 2;\n" " add.s64 %rd75, %rd73, %rd74;\n" " st.global.f32 [%rd75], %f217;\n" "\n" "$L__BB18_20:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_q3k_nx\n" ".visible .entry fc_moe_down_q3k_nx(\n" " .param .u64 fc_moe_down_q3k_nx_param_0,\n" " .param .u64 fc_moe_down_q3k_nx_param_1,\n" " .param .u64 fc_moe_down_q3k_nx_param_2,\n" " .param .u64 fc_moe_down_q3k_nx_param_3,\n" " .param .u64 fc_moe_down_q3k_nx_param_4,\n" " .param .u32 fc_moe_down_q3k_nx_param_5,\n" " .param .u32 fc_moe_down_q3k_nx_param_6,\n" " .param .u32 fc_moe_down_q3k_nx_param_7,\n" " .param .u32 fc_moe_down_q3k_nx_param_8,\n" " .param .u32 fc_moe_down_q3k_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<194>;\n" " .reg .b32 %r<154>;\n" " .reg .b64 %rd<68>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ18fc_moe_down_q3k_nxE4sred[128];\n" "\n" " ld.param.u64 %rd27, [fc_moe_down_q3k_nx_param_0];\n" " ld.param.u64 %rd28, [fc_moe_down_q3k_nx_param_1];\n" " ld.param.u64 %rd29, [fc_moe_down_q3k_nx_param_2];\n" " ld.param.u64 %rd30, [fc_moe_down_q3k_nx_param_3];\n" " ld.param.u64 %rd31, [fc_moe_down_q3k_nx_param_4];\n" " ld.param.u32 %r30, [fc_moe_down_q3k_nx_param_5];\n" " ld.param.u32 %r31, [fc_moe_down_q3k_nx_param_6];\n" " ld.param.u32 %r32, [fc_moe_down_q3k_nx_param_7];\n" " ld.param.u32 %r33, [fc_moe_down_q3k_nx_param_8];\n" " ld.param.u32 %r34, [fc_moe_down_q3k_nx_param_9];\n" " mov.u32 %r35, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r35;\n" " cvt.s64.s32 %rd2, %r31;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r36, %ntid.x;\n" " shr.u32 %r37, %r36, 5;\n" " div.u32 %r4, %r37, %r34;\n" " div.s32 %r38, %r2, %r34;\n" " mul.lo.s32 %r39, %r38, %r34;\n" " sub.s32 %r147, %r2, %r39;\n" " mov.u32 %r40, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r40;\n" " add.s32 %r41, %r6, %r38;\n" " shl.b32 %r7, %r41, 1;\n" " setp.ge.s32 %p1, %r7, %r31;\n" " setp.ge.s32 %p2, %r147, %r30;\n" " mov.f32 %f187, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f188, %f187;\n" " @%p3 bra $L__BB19_6;\n" "\n" " cvt.s64.s32 %rd3, %r33;\n" " cvt.s64.s32 %rd4, %r7;\n" " shr.u32 %r42, %r3, 4;\n" " cvt.u64.u32 %rd5, %r42;\n" " cvt.u64.u32 %rd6, %r3;\n" " mul.wide.s32 %rd8, %r3, 4;\n" " add.s64 %rd7, %rd28, %rd8;\n" " mul.wide.s32 %rd32, %r42, 8;\n" " or.b64 %rd9, %rd32, 128;\n" " shr.s32 %r8, %r32, 8;\n" " add.s32 %r9, %r7, 1;\n" " cvt.s64.s32 %rd10, %r32;\n" " cvt.s64.s32 %rd33, %r30;\n" " mul.lo.s64 %rd11, %rd33, %rd1;\n" " cvta.to.global.u64 %rd12, %rd27;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r9, %r31;\n" " setp.lt.s32 %p5, %r32, 256;\n" " shl.b64 %rd52, %rd6, 2;\n" " shl.b64 %rd54, %rd5, 3;\n" " mov.f32 %f188, %f25;\n" " mov.f32 %f187, %f25;\n" "\n" "$L__BB19_2:\n" " cvt.s64.s32 %rd35, %r147;\n" " add.s64 %rd13, %rd11, %rd35;\n" " shl.b64 %rd36, %rd13, 2;\n" " add.s64 %rd34, %rd29, %rd36;\n" " // begin inline asm\n" " ld.global.nc.s32 %r43, [%rd34];\n" " // end inline asm\n" " cvt.s64.s32 %rd37, %r43;\n" " mul.lo.s64 %rd38, %rd37, %rd2;\n" " add.s64 %rd39, %rd38, %rd4;\n" " mul.lo.s64 %rd40, %rd39, %rd3;\n" " add.s64 %rd67, %rd12, %rd40;\n" " add.s64 %rd41, %rd67, %rd3;\n" " selp.b64 %rd66, %rd41, %rd67, %p4;\n" " mov.f32 %f185, %f25;\n" " mov.f32 %f186, %f25;\n" " @%p5 bra $L__BB19_5;\n" "\n" " mul.lo.s64 %rd42, %rd13, %rd10;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd65, %rd7, %rd43;\n" " add.s64 %rd64, %rd66, %rd8;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r148, 0;\n" " mov.f32 %f186, %f29;\n" " mov.f32 %f185, %f29;\n" "\n" "$L__BB19_4:\n" " add.s64 %rd53, %rd67, %rd52;\n" " add.s64 %rd55, %rd67, %rd54;\n" " ld.global.v2.u32 {%r45, %r46}, [%rd55+128];\n" " add.s64 %rd56, %rd66, %rd9;\n" " ld.global.v2.u32 {%r49, %r50}, [%rd56];\n" " ld.global.u32 %r53, [%rd53];\n" " shr.u32 %r54, %r53, 16;\n" " ld.global.u32 %r55, [%rd64];\n" " shr.u32 %r56, %r55, 16;\n" " ld.global.f32 %f38, [%rd66+144];\n" " ld.global.f32 %f39, [%rd67+144];\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd65];\n" " // end inline asm\n" " and.b32 %r57, %r53, 15;\n" " or.b32 %r58, %r57, 1258291200;\n" " mov.b32 %f40, %r58;\n" " mov.f32 %f41, 0fCB000004;\n" " mov.f32 %f42, 0f3F800000;\n" " fma.rn.f32 %f43, %f40, %f42, %f41;\n" " and.b32 %r59, %r45, 255;\n" " cvt.rn.f32.u32 %f44, %r59;\n" " mov.f32 %f45, 0f42000000;\n" " sub.rn.f32 %f46, %f44, %f45;\n" " and.b32 %r60, %r55, 15;\n" " or.b32 %r61, %r60, 1258291200;\n" " mov.b32 %f47, %r61;\n" " fma.rn.f32 %f48, %f47, %f42, %f41;\n" " and.b32 %r62, %r49, 255;\n" " cvt.rn.f32.u32 %f49, %r62;\n" " sub.rn.f32 %f50, %f49, %f45;\n" " mul.rn.f32 %f51, %f46, %f43;\n" " fma.rn.f32 %f53, %f51, %f30, %f29;\n" " mul.rn.f32 %f54, %f50, %f48;\n" " fma.rn.f32 %f55, %f54, %f30, %f29;\n" " add.s64 %rd45, %rd65, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd45];\n" " // end inline asm\n" " and.b32 %r63, %r53, 240;\n" " or.b32 %r64, %r63, 1258291200;\n" " mov.b32 %f56, %r64;\n" " mov.f32 %f57, 0fC9000040;\n" " mov.f32 %f58, 0f3D800000;\n" " fma.rn.f32 %f59, %f56, %f58, %f57;\n" " bfe.u32 %r65, %r45, 8, 8;\n" " cvt.rn.f32.u32 %f60, %r65;\n" " sub.rn.f32 %f61, %f60, %f45;\n" " and.b32 %r66, %r55, 240;\n" " or.b32 %r67, %r66, 1258291200;\n" " mov.b32 %f62, %r67;\n" " fma.rn.f32 %f63, %f62, %f58, %f57;\n" " bfe.u32 %r68, %r49, 8, 8;\n" " cvt.rn.f32.u32 %f64, %r68;\n" " sub.rn.f32 %f65, %f64, %f45;\n" " mul.rn.f32 %f66, %f61, %f59;\n" " fma.rn.f32 %f67, %f66, %f31, %f53;\n" " mul.rn.f32 %f68, %f65, %f63;\n" " fma.rn.f32 %f69, %f68, %f31, %f55;\n" " add.s64 %rd46, %rd65, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd46];\n" " // end inline asm\n" " and.b32 %r69, %r53, 3840;\n" " or.b32 %r70, %r69, 1258291200;\n" " mov.b32 %f70, %r70;\n" " mov.f32 %f71, 0fC7000400;\n" " mov.f32 %f72, 0f3B800000;\n" " fma.rn.f32 %f73, %f70, %f72, %f71;\n" " bfe.u32 %r71, %r45, 16, 8;\n" " cvt.rn.f32.u32 %f74, %r71;\n" " sub.rn.f32 %f75, %f74, %f45;\n" " and.b32 %r72, %r55, 3840;\n" " or.b32 %r73, %r72, 1258291200;\n" " mov.b32 %f76, %r73;\n" " fma.rn.f32 %f77, %f76, %f72, %f71;\n" " bfe.u32 %r74, %r49, 16, 8;\n" " cvt.rn.f32.u32 %f78, %r74;\n" " sub.rn.f32 %f79, %f78, %f45;\n" " mul.rn.f32 %f80, %f75, %f73;\n" " fma.rn.f32 %f81, %f80, %f32, %f67;\n" " mul.rn.f32 %f82, %f79, %f77;\n" " fma.rn.f32 %f83, %f82, %f32, %f69;\n" " add.s64 %rd47, %rd65, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd47];\n" " // end inline asm\n" " and.b32 %r75, %r53, 61440;\n" " or.b32 %r76, %r75, 1258291200;\n" " mov.b32 %f84, %r76;\n" " mov.f32 %f85, 0fC5004000;\n" " mov.f32 %f86, 0f39800000;\n" " fma.rn.f32 %f87, %f84, %f86, %f85;\n" " shr.u32 %r77, %r45, 24;\n" " cvt.rn.f32.u32 %f88, %r77;\n" " sub.rn.f32 %f89, %f88, %f45;\n" " and.b32 %r78, %r55, 61440;\n" " or.b32 %r79, %r78, 1258291200;\n" " mov.b32 %f90, %r79;\n" " fma.rn.f32 %f91, %f90, %f86, %f85;\n" " shr.u32 %r80, %r49, 24;\n" " cvt.rn.f32.u32 %f92, %r80;\n" " sub.rn.f32 %f93, %f92, %f45;\n" " mul.rn.f32 %f94, %f89, %f87;\n" " fma.rn.f32 %f95, %f94, %f33, %f81;\n" " mul.rn.f32 %f96, %f93, %f91;\n" " fma.rn.f32 %f97, %f96, %f33, %f83;\n" " add.s64 %rd48, %rd65, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd48];\n" " // end inline asm\n" " and.b32 %r81, %r53, 983040;\n" " or.b32 %r82, %r81, 1258291200;\n" " mov.b32 %f98, %r82;\n" " mov.f32 %f99, 0fC3040000;\n" " mov.f32 %f100, 0f37800000;\n" " fma.rn.f32 %f101, %f98, %f100, %f99;\n" " and.b32 %r83, %r46, 255;\n" " cvt.rn.f32.u32 %f102, %r83;\n" " sub.rn.f32 %f103, %f102, %f45;\n" " and.b32 %r84, %r55, 983040;\n" " or.b32 %r85, %r84, 1258291200;\n" " mov.b32 %f104, %r85;\n" " fma.rn.f32 %f105, %f104, %f100, %f99;\n" " and.b32 %r86, %r50, 255;\n" " cvt.rn.f32.u32 %f106, %r86;\n" " sub.rn.f32 %f107, %f106, %f45;\n" " mul.rn.f32 %f108, %f103, %f101;\n" " fma.rn.f32 %f109, %f108, %f34, %f95;\n" " mul.rn.f32 %f110, %f107, %f105;\n" " fma.rn.f32 %f111, %f110, %f34, %f97;\n" " add.s64 %rd49, %rd65, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd49];\n" " // end inline asm\n" " and.b32 %r87, %r54, 240;\n" " or.b32 %r88, %r87, 1258291200;\n" " mov.b32 %f112, %r88;\n" " fma.rn.f32 %f113, %f112, %f58, %f57;\n" " bfe.u32 %r89, %r46, 8, 8;\n" " cvt.rn.f32.u32 %f114, %r89;\n" " sub.rn.f32 %f115, %f114, %f45;\n" " and.b32 %r90, %r56, 240;\n" " or.b32 %r91, %r90, 1258291200;\n" " mov.b32 %f116, %r91;\n" " fma.rn.f32 %f117, %f116, %f58, %f57;\n" " bfe.u32 %r92, %r50, 8, 8;\n" " cvt.rn.f32.u32 %f118, %r92;\n" " sub.rn.f32 %f119, %f118, %f45;\n" " mul.rn.f32 %f120, %f115, %f113;\n" " fma.rn.f32 %f121, %f120, %f35, %f109;\n" " mul.rn.f32 %f122, %f119, %f117;\n" " fma.rn.f32 %f123, %f122, %f35, %f111;\n" " add.s64 %rd50, %rd65, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd50];\n" " // end inline asm\n" " and.b32 %r93, %r54, 3840;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f124, %r94;\n" " fma.rn.f32 %f125, %f124, %f72, %f71;\n" " bfe.u32 %r95, %r46, 16, 8;\n" " cvt.rn.f32.u32 %f126, %r95;\n" " sub.rn.f32 %f127, %f126, %f45;\n" " and.b32 %r96, %r56, 3840;\n" " or.b32 %r97, %r96, 1258291200;\n" " mov.b32 %f128, %r97;\n" " fma.rn.f32 %f129, %f128, %f72, %f71;\n" " bfe.u32 %r98, %r50, 16, 8;\n" " cvt.rn.f32.u32 %f130, %r98;\n" " sub.rn.f32 %f131, %f130, %f45;\n" " mul.rn.f32 %f132, %f127, %f125;\n" " fma.rn.f32 %f133, %f132, %f36, %f121;\n" " mul.rn.f32 %f134, %f131, %f129;\n" " fma.rn.f32 %f135, %f134, %f36, %f123;\n" " add.s64 %rd51, %rd65, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd51];\n" " // end inline asm\n" " and.b32 %r99, %r54, 61440;\n" " or.b32 %r100, %r99, 1258291200;\n" " mov.b32 %f136, %r100;\n" " fma.rn.f32 %f137, %f136, %f86, %f85;\n" " shr.u32 %r101, %r46, 24;\n" " cvt.rn.f32.u32 %f138, %r101;\n" " sub.rn.f32 %f139, %f138, %f45;\n" " and.b32 %r102, %r56, 61440;\n" " or.b32 %r103, %r102, 1258291200;\n" " mov.b32 %f140, %r103;\n" " fma.rn.f32 %f141, %f140, %f86, %f85;\n" " shr.u32 %r104, %r50, 24;\n" " cvt.rn.f32.u32 %f142, %r104;\n" " sub.rn.f32 %f143, %f142, %f45;\n" " mul.rn.f32 %f144, %f139, %f137;\n" " fma.rn.f32 %f145, %f144, %f37, %f133;\n" " mul.rn.f32 %f146, %f143, %f141;\n" " fma.rn.f32 %f147, %f146, %f37, %f135;\n" " fma.rn.f32 %f185, %f39, %f145, %f185;\n" " fma.rn.f32 %f186, %f38, %f147, %f186;\n" " add.s64 %rd67, %rd67, 152;\n" " add.s64 %rd66, %rd66, 152;\n" " add.s64 %rd65, %rd65, 1024;\n" " add.s64 %rd64, %rd64, 152;\n" " add.s32 %r148, %r148, 1;\n" " setp.lt.s32 %p6, %r148, %r8;\n" " @%p6 bra $L__BB19_4;\n" "\n" "$L__BB19_5:\n" " add.s64 %rd57, %rd30, %rd36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f148, [%rd57];\n" " // end inline asm\n" " fma.rn.f32 %f187, %f148, %f185, %f187;\n" " fma.rn.f32 %f188, %f148, %f186, %f188;\n" " add.s32 %r147, %r147, %r34;\n" " setp.lt.s32 %p7, %r147, %r30;\n" " @%p7 bra $L__BB19_2;\n" "\n" "$L__BB19_6:\n" " mov.u32 %r115, 16;\n" " mov.u32 %r124, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f149, %f187, %r115, %r124;\n" " // end inline asm\n" " add.f32 %f152, %f187, %f149;\n" " mov.u32 %r117, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f151, %f152, %r117, %r124;\n" " // end inline asm\n" " add.f32 %f154, %f152, %f151;\n" " mov.u32 %r119, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f153, %f154, %r119, %r124;\n" " // end inline asm\n" " add.f32 %f156, %f154, %f153;\n" " mov.u32 %r121, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f155, %f156, %r121, %r124;\n" " // end inline asm\n" " add.f32 %f158, %f156, %f155;\n" " mov.u32 %r123, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f157, %f158, %r123, %r124;\n" " // end inline asm\n" " add.f32 %f13, %f158, %f157;\n" " // begin inline asm\n" " shfl.down.b32 %f159, %f188, %r115, %r124;\n" " // end inline asm\n" " add.f32 %f162, %f188, %f159;\n" " // begin inline asm\n" " shfl.down.b32 %f161, %f162, %r117, %r124;\n" " // end inline asm\n" " add.f32 %f164, %f162, %f161;\n" " // begin inline asm\n" " shfl.down.b32 %f163, %f164, %r119, %r124;\n" " // end inline asm\n" " add.f32 %f166, %f164, %f163;\n" " // begin inline asm\n" " shfl.down.b32 %f165, %f166, %r121, %r124;\n" " // end inline asm\n" " add.f32 %f168, %f166, %f165;\n" " // begin inline asm\n" " shfl.down.b32 %f167, %f168, %r123, %r124;\n" " // end inline asm\n" " add.f32 %f14, %f168, %f167;\n" " setp.ne.s32 %p8, %r3, 0;\n" " mul.lo.s64 %rd26, %rd2, %rd1;\n" " @%p8 bra $L__BB19_8;\n" "\n" " mov.u32 %r146, %tid.x;\n" " shr.u32 %r145, %r146, 5;\n" " shl.b32 %r125, %r145, 3;\n" " mov.u32 %r126, _ZZ18fc_moe_down_q3k_nxE4sred;\n" " add.s32 %r127, %r126, %r125;\n" " st.shared.f32 [%r127], %f13;\n" " st.shared.f32 [%r127+4], %f14;\n" "\n" "$L__BB19_8:\n" " bar.sync 0;\n" " shl.b32 %r128, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r128;\n" " @%p9 bra $L__BB19_18;\n" "\n" " ld.param.u32 %r144, [fc_moe_down_q3k_nx_param_6];\n" " shr.u32 %r14, %r1, 1;\n" " add.s32 %r129, %r6, %r14;\n" " and.b32 %r15, %r1, 1;\n" " bfi.b32 %r16, %r129, %r15, 1, 31;\n" " setp.ge.s32 %p10, %r16, %r144;\n" " @%p10 bra $L__BB19_18;\n" "\n" " setp.lt.s32 %p11, %r34, 1;\n" " mov.f32 %f193, 0f00000000;\n" " @%p11 bra $L__BB19_17;\n" "\n" " mul.lo.s32 %r17, %r14, %r34;\n" " and.b32 %r153, %r34, 3;\n" " add.s32 %r131, %r34, -1;\n" " setp.lt.u32 %p12, %r131, 3;\n" " mov.f32 %f193, 0f00000000;\n" " mov.u32 %r151, 0;\n" " @%p12 bra $L__BB19_14;\n" "\n" " sub.s32 %r150, %r34, %r153;\n" " mov.f32 %f193, 0f00000000;\n" " mov.u32 %r151, 0;\n" "\n" "$L__BB19_13:\n" " add.s32 %r133, %r151, %r17;\n" " shl.b32 %r134, %r133, 3;\n" " mov.u32 %r135, _ZZ18fc_moe_down_q3k_nxE4sred;\n" " add.s32 %r136, %r135, %r134;\n" " shl.b32 %r137, %r15, 2;\n" " add.s32 %r138, %r136, %r137;\n" " ld.shared.f32 %f173, [%r138];\n" " add.f32 %f174, %f193, %f173;\n" " ld.shared.f32 %f175, [%r138+8];\n" " add.f32 %f176, %f174, %f175;\n" " ld.shared.f32 %f177, [%r138+16];\n" " add.f32 %f178, %f176, %f177;\n" " ld.shared.f32 %f179, [%r138+24];\n" " add.f32 %f193, %f178, %f179;\n" " add.s32 %r151, %r151, 4;\n" " add.s32 %r150, %r150, -4;\n" " setp.ne.s32 %p13, %r150, 0;\n" " @%p13 bra $L__BB19_13;\n" "\n" "$L__BB19_14:\n" " setp.eq.s32 %p14, %r153, 0;\n" " @%p14 bra $L__BB19_17;\n" "\n" " add.s32 %r139, %r151, %r17;\n" " shl.b32 %r140, %r139, 3;\n" " mov.u32 %r141, _ZZ18fc_moe_down_q3k_nxE4sred;\n" " add.s32 %r142, %r141, %r140;\n" " shl.b32 %r143, %r15, 2;\n" " add.s32 %r152, %r142, %r143;\n" "\n" "$L__BB19_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f180, [%r152];\n" " add.f32 %f193, %f193, %f180;\n" " add.s32 %r152, %r152, 8;\n" " add.s32 %r153, %r153, -1;\n" " setp.ne.s32 %p15, %r153, 0;\n" " @%p15 bra $L__BB19_16;\n" "\n" "$L__BB19_17:\n" " cvt.s64.s32 %rd59, %r16;\n" " add.s64 %rd60, %rd26, %rd59;\n" " cvta.to.global.u64 %rd61, %rd31;\n" " shl.b64 %rd62, %rd60, 2;\n" " add.s64 %rd63, %rd61, %rd62;\n" " st.global.f32 [%rd63], %f193;\n" "\n" "$L__BB19_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_q4k_nx\n" ".visible .entry fc_moe_down_q4k_nx(\n" " .param .u64 fc_moe_down_q4k_nx_param_0,\n" " .param .u64 fc_moe_down_q4k_nx_param_1,\n" " .param .u64 fc_moe_down_q4k_nx_param_2,\n" " .param .u64 fc_moe_down_q4k_nx_param_3,\n" " .param .u64 fc_moe_down_q4k_nx_param_4,\n" " .param .u32 fc_moe_down_q4k_nx_param_5,\n" " .param .u32 fc_moe_down_q4k_nx_param_6,\n" " .param .u32 fc_moe_down_q4k_nx_param_7,\n" " .param .u32 fc_moe_down_q4k_nx_param_8,\n" " .param .u32 fc_moe_down_q4k_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<247>;\n" " .reg .b32 %r<174>;\n" " .reg .b64 %rd<62>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ18fc_moe_down_q4k_nxE4sred[128];\n" "\n" " ld.param.u64 %rd25, [fc_moe_down_q4k_nx_param_0];\n" " ld.param.u64 %rd26, [fc_moe_down_q4k_nx_param_1];\n" " ld.param.u64 %rd27, [fc_moe_down_q4k_nx_param_2];\n" " ld.param.u64 %rd28, [fc_moe_down_q4k_nx_param_3];\n" " ld.param.u64 %rd29, [fc_moe_down_q4k_nx_param_4];\n" " ld.param.u32 %r30, [fc_moe_down_q4k_nx_param_5];\n" " ld.param.u32 %r31, [fc_moe_down_q4k_nx_param_6];\n" " ld.param.u32 %r32, [fc_moe_down_q4k_nx_param_7];\n" " ld.param.u32 %r33, [fc_moe_down_q4k_nx_param_8];\n" " ld.param.u32 %r34, [fc_moe_down_q4k_nx_param_9];\n" " mov.u32 %r35, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r35;\n" " cvt.s64.s32 %rd2, %r31;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r36, %ntid.x;\n" " shr.u32 %r37, %r36, 5;\n" " div.u32 %r4, %r37, %r34;\n" " div.s32 %r38, %r2, %r34;\n" " mul.lo.s32 %r39, %r38, %r34;\n" " sub.s32 %r167, %r2, %r39;\n" " mov.u32 %r40, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r40;\n" " add.s32 %r41, %r6, %r38;\n" " shl.b32 %r7, %r41, 1;\n" " setp.ge.s32 %p1, %r7, %r31;\n" " setp.ge.s32 %p2, %r167, %r30;\n" " mov.f32 %f240, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f241, %f240;\n" " @%p3 bra $L__BB20_6;\n" "\n" " cvt.s64.s32 %rd3, %r33;\n" " cvt.s64.s32 %rd4, %r7;\n" " cvt.u64.u32 %rd5, %r3;\n" " mul.wide.s32 %rd7, %r3, 4;\n" " add.s64 %rd6, %rd26, %rd7;\n" " shr.s32 %r8, %r32, 8;\n" " add.s32 %r9, %r7, 1;\n" " cvt.s64.s32 %rd8, %r32;\n" " cvt.s64.s32 %rd30, %r30;\n" " mul.lo.s64 %rd9, %rd30, %rd1;\n" " cvta.to.global.u64 %rd10, %rd25;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r9, %r31;\n" " setp.lt.s32 %p5, %r32, 256;\n" " shl.b64 %rd49, %rd5, 2;\n" " mov.f32 %f241, %f25;\n" " mov.f32 %f240, %f25;\n" "\n" "$L__BB20_2:\n" " cvt.s64.s32 %rd32, %r167;\n" " add.s64 %rd11, %rd9, %rd32;\n" " shl.b64 %rd33, %rd11, 2;\n" " add.s64 %rd31, %rd27, %rd33;\n" " // begin inline asm\n" " ld.global.nc.s32 %r42, [%rd31];\n" " // end inline asm\n" " cvt.s64.s32 %rd34, %r42;\n" " mul.lo.s64 %rd35, %rd34, %rd2;\n" " add.s64 %rd36, %rd35, %rd4;\n" " mul.lo.s64 %rd37, %rd36, %rd3;\n" " add.s64 %rd61, %rd10, %rd37;\n" " add.s64 %rd38, %rd61, %rd3;\n" " selp.b64 %rd60, %rd38, %rd61, %p4;\n" " mov.f32 %f238, %f25;\n" " mov.f32 %f239, %f25;\n" " @%p5 bra $L__BB20_5;\n" "\n" " mul.lo.s64 %rd39, %rd11, %rd8;\n" " shl.b64 %rd40, %rd39, 2;\n" " add.s64 %rd59, %rd6, %rd40;\n" " add.s64 %rd58, %rd60, %rd7;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r168, 0;\n" " mov.f32 %f239, %f29;\n" " mov.f32 %f238, %f29;\n" "\n" "$L__BB20_4:\n" " add.s64 %rd50, %rd61, %rd49;\n" " ld.global.v2.u32 {%r44, %r45}, [%rd61+128];\n" " ld.global.v2.u32 {%r48, %r49}, [%rd60+128];\n" " ld.global.v2.u32 {%r52, %r53}, [%rd61+136];\n" " ld.global.v2.u32 {%r56, %r57}, [%rd60+136];\n" " ld.global.v2.f32 {%f38, %f39}, [%rd61+144];\n" " ld.global.v2.f32 {%f42, %f43}, [%rd60+144];\n" " ld.global.u32 %r60, [%rd50];\n" " shr.u32 %r61, %r60, 16;\n" " ld.global.u32 %r62, [%rd58];\n" " shr.u32 %r63, %r62, 16;\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd59];\n" " // end inline asm\n" " and.b32 %r64, %r44, 255;\n" " cvt.rn.f32.u32 %f46, %r64;\n" " mul.rn.f32 %f47, %f38, %f46;\n" " and.b32 %r65, %r60, 15;\n" " or.b32 %r66, %r65, 1258291200;\n" " mov.b32 %f48, %r66;\n" " mov.f32 %f49, 0fCB000000;\n" " mov.f32 %f50, 0f3F800000;\n" " fma.rn.f32 %f51, %f48, %f50, %f49;\n" " mul.rn.f32 %f52, %f47, %f51;\n" " and.b32 %r67, %r52, 255;\n" " cvt.rn.f32.u32 %f53, %r67;\n" " mul.rn.f32 %f54, %f39, %f53;\n" " sub.rn.f32 %f55, %f52, %f54;\n" " and.b32 %r68, %r48, 255;\n" " cvt.rn.f32.u32 %f56, %r68;\n" " mul.rn.f32 %f57, %f42, %f56;\n" " and.b32 %r69, %r62, 15;\n" " or.b32 %r70, %r69, 1258291200;\n" " mov.b32 %f58, %r70;\n" " fma.rn.f32 %f59, %f58, %f50, %f49;\n" " mul.rn.f32 %f60, %f57, %f59;\n" " and.b32 %r71, %r56, 255;\n" " cvt.rn.f32.u32 %f61, %r71;\n" " mul.rn.f32 %f62, %f43, %f61;\n" " sub.rn.f32 %f63, %f60, %f62;\n" " fma.rn.f32 %f65, %f55, %f30, %f29;\n" " fma.rn.f32 %f66, %f63, %f30, %f29;\n" " add.s64 %rd42, %rd59, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd42];\n" " // end inline asm\n" " bfe.u32 %r72, %r44, 8, 8;\n" " cvt.rn.f32.u32 %f67, %r72;\n" " mul.rn.f32 %f68, %f38, %f67;\n" " and.b32 %r73, %r60, 240;\n" " or.b32 %r74, %r73, 1258291200;\n" " mov.b32 %f69, %r74;\n" " mov.f32 %f70, 0fC9000000;\n" " mov.f32 %f71, 0f3D800000;\n" " fma.rn.f32 %f72, %f69, %f71, %f70;\n" " mul.rn.f32 %f73, %f68, %f72;\n" " bfe.u32 %r75, %r52, 8, 8;\n" " cvt.rn.f32.u32 %f74, %r75;\n" " mul.rn.f32 %f75, %f39, %f74;\n" " sub.rn.f32 %f76, %f73, %f75;\n" " bfe.u32 %r76, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f77, %r76;\n" " mul.rn.f32 %f78, %f42, %f77;\n" " and.b32 %r77, %r62, 240;\n" " or.b32 %r78, %r77, 1258291200;\n" " mov.b32 %f79, %r78;\n" " fma.rn.f32 %f80, %f79, %f71, %f70;\n" " mul.rn.f32 %f81, %f78, %f80;\n" " bfe.u32 %r79, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f82, %r79;\n" " mul.rn.f32 %f83, %f43, %f82;\n" " sub.rn.f32 %f84, %f81, %f83;\n" " fma.rn.f32 %f85, %f76, %f31, %f65;\n" " fma.rn.f32 %f86, %f84, %f31, %f66;\n" " add.s64 %rd43, %rd59, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd43];\n" " // end inline asm\n" " bfe.u32 %r80, %r44, 16, 8;\n" " cvt.rn.f32.u32 %f87, %r80;\n" " mul.rn.f32 %f88, %f38, %f87;\n" " and.b32 %r81, %r60, 3840;\n" " or.b32 %r82, %r81, 1258291200;\n" " mov.b32 %f89, %r82;\n" " mov.f32 %f90, 0fC7000000;\n" " mov.f32 %f91, 0f3B800000;\n" " fma.rn.f32 %f92, %f89, %f91, %f90;\n" " mul.rn.f32 %f93, %f88, %f92;\n" " bfe.u32 %r83, %r52, 16, 8;\n" " cvt.rn.f32.u32 %f94, %r83;\n" " mul.rn.f32 %f95, %f39, %f94;\n" " sub.rn.f32 %f96, %f93, %f95;\n" " bfe.u32 %r84, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f97, %r84;\n" " mul.rn.f32 %f98, %f42, %f97;\n" " and.b32 %r85, %r62, 3840;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f99, %r86;\n" " fma.rn.f32 %f100, %f99, %f91, %f90;\n" " mul.rn.f32 %f101, %f98, %f100;\n" " bfe.u32 %r87, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f102, %r87;\n" " mul.rn.f32 %f103, %f43, %f102;\n" " sub.rn.f32 %f104, %f101, %f103;\n" " fma.rn.f32 %f105, %f96, %f32, %f85;\n" " fma.rn.f32 %f106, %f104, %f32, %f86;\n" " add.s64 %rd44, %rd59, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd44];\n" " // end inline asm\n" " shr.u32 %r88, %r44, 24;\n" " cvt.rn.f32.u32 %f107, %r88;\n" " mul.rn.f32 %f108, %f38, %f107;\n" " and.b32 %r89, %r60, 61440;\n" " or.b32 %r90, %r89, 1258291200;\n" " mov.b32 %f109, %r90;\n" " mov.f32 %f110, 0fC5000000;\n" " mov.f32 %f111, 0f39800000;\n" " fma.rn.f32 %f112, %f109, %f111, %f110;\n" " mul.rn.f32 %f113, %f108, %f112;\n" " shr.u32 %r91, %r52, 24;\n" " cvt.rn.f32.u32 %f114, %r91;\n" " mul.rn.f32 %f115, %f39, %f114;\n" " sub.rn.f32 %f116, %f113, %f115;\n" " shr.u32 %r92, %r48, 24;\n" " cvt.rn.f32.u32 %f117, %r92;\n" " mul.rn.f32 %f118, %f42, %f117;\n" " and.b32 %r93, %r62, 61440;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f119, %r94;\n" " fma.rn.f32 %f120, %f119, %f111, %f110;\n" " mul.rn.f32 %f121, %f118, %f120;\n" " shr.u32 %r95, %r56, 24;\n" " cvt.rn.f32.u32 %f122, %r95;\n" " mul.rn.f32 %f123, %f43, %f122;\n" " sub.rn.f32 %f124, %f121, %f123;\n" " fma.rn.f32 %f125, %f116, %f33, %f105;\n" " fma.rn.f32 %f126, %f124, %f33, %f106;\n" " add.s64 %rd45, %rd59, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd45];\n" " // end inline asm\n" " and.b32 %r96, %r45, 255;\n" " cvt.rn.f32.u32 %f127, %r96;\n" " mul.rn.f32 %f128, %f38, %f127;\n" " and.b32 %r97, %r60, 983040;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f129, %r98;\n" " mov.f32 %f130, 0fC3000000;\n" " mov.f32 %f131, 0f37800000;\n" " fma.rn.f32 %f132, %f129, %f131, %f130;\n" " mul.rn.f32 %f133, %f128, %f132;\n" " and.b32 %r99, %r53, 255;\n" " cvt.rn.f32.u32 %f134, %r99;\n" " mul.rn.f32 %f135, %f39, %f134;\n" " sub.rn.f32 %f136, %f133, %f135;\n" " and.b32 %r100, %r49, 255;\n" " cvt.rn.f32.u32 %f137, %r100;\n" " mul.rn.f32 %f138, %f42, %f137;\n" " and.b32 %r101, %r62, 983040;\n" " or.b32 %r102, %r101, 1258291200;\n" " mov.b32 %f139, %r102;\n" " fma.rn.f32 %f140, %f139, %f131, %f130;\n" " mul.rn.f32 %f141, %f138, %f140;\n" " and.b32 %r103, %r57, 255;\n" " cvt.rn.f32.u32 %f142, %r103;\n" " mul.rn.f32 %f143, %f43, %f142;\n" " sub.rn.f32 %f144, %f141, %f143;\n" " fma.rn.f32 %f145, %f136, %f34, %f125;\n" " fma.rn.f32 %f146, %f144, %f34, %f126;\n" " add.s64 %rd46, %rd59, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd46];\n" " // end inline asm\n" " bfe.u32 %r104, %r45, 8, 8;\n" " cvt.rn.f32.u32 %f147, %r104;\n" " mul.rn.f32 %f148, %f38, %f147;\n" " and.b32 %r105, %r61, 240;\n" " or.b32 %r106, %r105, 1258291200;\n" " mov.b32 %f149, %r106;\n" " fma.rn.f32 %f150, %f149, %f71, %f70;\n" " mul.rn.f32 %f151, %f148, %f150;\n" " bfe.u32 %r107, %r53, 8, 8;\n" " cvt.rn.f32.u32 %f152, %r107;\n" " mul.rn.f32 %f153, %f39, %f152;\n" " sub.rn.f32 %f154, %f151, %f153;\n" " bfe.u32 %r108, %r49, 8, 8;\n" " cvt.rn.f32.u32 %f155, %r108;\n" " mul.rn.f32 %f156, %f42, %f155;\n" " and.b32 %r109, %r63, 240;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f157, %r110;\n" " fma.rn.f32 %f158, %f157, %f71, %f70;\n" " mul.rn.f32 %f159, %f156, %f158;\n" " bfe.u32 %r111, %r57, 8, 8;\n" " cvt.rn.f32.u32 %f160, %r111;\n" " mul.rn.f32 %f161, %f43, %f160;\n" " sub.rn.f32 %f162, %f159, %f161;\n" " fma.rn.f32 %f163, %f154, %f35, %f145;\n" " fma.rn.f32 %f164, %f162, %f35, %f146;\n" " add.s64 %rd47, %rd59, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd47];\n" " // end inline asm\n" " bfe.u32 %r112, %r45, 16, 8;\n" " cvt.rn.f32.u32 %f165, %r112;\n" " mul.rn.f32 %f166, %f38, %f165;\n" " and.b32 %r113, %r61, 3840;\n" " or.b32 %r114, %r113, 1258291200;\n" " mov.b32 %f167, %r114;\n" " fma.rn.f32 %f168, %f167, %f91, %f90;\n" " mul.rn.f32 %f169, %f166, %f168;\n" " bfe.u32 %r115, %r53, 16, 8;\n" " cvt.rn.f32.u32 %f170, %r115;\n" " mul.rn.f32 %f171, %f39, %f170;\n" " sub.rn.f32 %f172, %f169, %f171;\n" " bfe.u32 %r116, %r49, 16, 8;\n" " cvt.rn.f32.u32 %f173, %r116;\n" " mul.rn.f32 %f174, %f42, %f173;\n" " and.b32 %r117, %r63, 3840;\n" " or.b32 %r118, %r117, 1258291200;\n" " mov.b32 %f175, %r118;\n" " fma.rn.f32 %f176, %f175, %f91, %f90;\n" " mul.rn.f32 %f177, %f174, %f176;\n" " bfe.u32 %r119, %r57, 16, 8;\n" " cvt.rn.f32.u32 %f178, %r119;\n" " mul.rn.f32 %f179, %f43, %f178;\n" " sub.rn.f32 %f180, %f177, %f179;\n" " fma.rn.f32 %f181, %f172, %f36, %f163;\n" " fma.rn.f32 %f182, %f180, %f36, %f164;\n" " add.s64 %rd48, %rd59, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd48];\n" " // end inline asm\n" " shr.u32 %r120, %r45, 24;\n" " cvt.rn.f32.u32 %f183, %r120;\n" " mul.rn.f32 %f184, %f38, %f183;\n" " and.b32 %r121, %r61, 61440;\n" " or.b32 %r122, %r121, 1258291200;\n" " mov.b32 %f185, %r122;\n" " fma.rn.f32 %f186, %f185, %f111, %f110;\n" " mul.rn.f32 %f187, %f184, %f186;\n" " shr.u32 %r123, %r53, 24;\n" " cvt.rn.f32.u32 %f188, %r123;\n" " mul.rn.f32 %f189, %f39, %f188;\n" " sub.rn.f32 %f190, %f187, %f189;\n" " shr.u32 %r124, %r49, 24;\n" " cvt.rn.f32.u32 %f191, %r124;\n" " mul.rn.f32 %f192, %f42, %f191;\n" " and.b32 %r125, %r63, 61440;\n" " or.b32 %r126, %r125, 1258291200;\n" " mov.b32 %f193, %r126;\n" " fma.rn.f32 %f194, %f193, %f111, %f110;\n" " mul.rn.f32 %f195, %f192, %f194;\n" " shr.u32 %r127, %r57, 24;\n" " cvt.rn.f32.u32 %f196, %r127;\n" " mul.rn.f32 %f197, %f43, %f196;\n" " sub.rn.f32 %f198, %f195, %f197;\n" " fma.rn.f32 %f199, %f190, %f37, %f181;\n" " fma.rn.f32 %f200, %f198, %f37, %f182;\n" " add.f32 %f238, %f238, %f199;\n" " add.f32 %f239, %f239, %f200;\n" " add.s64 %rd61, %rd61, 152;\n" " add.s64 %rd60, %rd60, 152;\n" " add.s64 %rd59, %rd59, 1024;\n" " add.s64 %rd58, %rd58, 152;\n" " add.s32 %r168, %r168, 1;\n" " setp.lt.s32 %p6, %r168, %r8;\n" " @%p6 bra $L__BB20_4;\n" "\n" "$L__BB20_5:\n" " add.s64 %rd51, %rd28, %rd33;\n" " // begin inline asm\n" " ld.global.nc.f32 %f201, [%rd51];\n" " // end inline asm\n" " fma.rn.f32 %f240, %f201, %f238, %f240;\n" " fma.rn.f32 %f241, %f201, %f239, %f241;\n" " add.s32 %r167, %r167, %r34;\n" " setp.lt.s32 %p7, %r167, %r30;\n" " @%p7 bra $L__BB20_2;\n" "\n" "$L__BB20_6:\n" " mov.u32 %r138, 16;\n" " mov.u32 %r147, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f202, %f240, %r138, %r147;\n" " // end inline asm\n" " add.f32 %f205, %f240, %f202;\n" " mov.u32 %r140, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f204, %f205, %r140, %r147;\n" " // end inline asm\n" " add.f32 %f207, %f205, %f204;\n" " mov.u32 %r142, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f206, %f207, %r142, %r147;\n" " // end inline asm\n" " add.f32 %f209, %f207, %f206;\n" " mov.u32 %r144, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f208, %f209, %r144, %r147;\n" " // end inline asm\n" " add.f32 %f211, %f209, %f208;\n" " mov.u32 %r146, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f210, %f211, %r146, %r147;\n" " // end inline asm\n" " add.f32 %f13, %f211, %f210;\n" " // begin inline asm\n" " shfl.down.b32 %f212, %f241, %r138, %r147;\n" " // end inline asm\n" " add.f32 %f215, %f241, %f212;\n" " // begin inline asm\n" " shfl.down.b32 %f214, %f215, %r140, %r147;\n" " // end inline asm\n" " add.f32 %f217, %f215, %f214;\n" " // begin inline asm\n" " shfl.down.b32 %f216, %f217, %r142, %r147;\n" " // end inline asm\n" " add.f32 %f219, %f217, %f216;\n" " // begin inline asm\n" " shfl.down.b32 %f218, %f219, %r144, %r147;\n" " // end inline asm\n" " add.f32 %f221, %f219, %f218;\n" " // begin inline asm\n" " shfl.down.b32 %f220, %f221, %r146, %r147;\n" " // end inline asm\n" " add.f32 %f14, %f221, %f220;\n" " setp.ne.s32 %p8, %r3, 0;\n" " mul.lo.s64 %rd24, %rd2, %rd1;\n" " @%p8 bra $L__BB20_8;\n" "\n" " shl.b32 %r148, %r2, 3;\n" " mov.u32 %r149, _ZZ18fc_moe_down_q4k_nxE4sred;\n" " add.s32 %r150, %r149, %r148;\n" " st.shared.f32 [%r150], %f13;\n" " st.shared.f32 [%r150+4], %f14;\n" "\n" "$L__BB20_8:\n" " bar.sync 0;\n" " shl.b32 %r151, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r151;\n" " @%p9 bra $L__BB20_18;\n" "\n" " shr.u32 %r14, %r1, 1;\n" " add.s32 %r152, %r6, %r14;\n" " and.b32 %r15, %r1, 1;\n" " bfi.b32 %r16, %r152, %r15, 1, 31;\n" " setp.ge.s32 %p10, %r16, %r31;\n" " @%p10 bra $L__BB20_18;\n" "\n" " setp.lt.s32 %p11, %r34, 1;\n" " mov.f32 %f246, 0f00000000;\n" " @%p11 bra $L__BB20_17;\n" "\n" " mul.lo.s32 %r17, %r14, %r34;\n" " and.b32 %r173, %r34, 3;\n" " add.s32 %r154, %r34, -1;\n" " setp.lt.u32 %p12, %r154, 3;\n" " mov.f32 %f246, 0f00000000;\n" " mov.u32 %r171, 0;\n" " @%p12 bra $L__BB20_14;\n" "\n" " sub.s32 %r170, %r34, %r173;\n" " mov.f32 %f246, 0f00000000;\n" " mov.u32 %r171, 0;\n" "\n" "$L__BB20_13:\n" " add.s32 %r156, %r171, %r17;\n" " shl.b32 %r157, %r156, 3;\n" " mov.u32 %r158, _ZZ18fc_moe_down_q4k_nxE4sred;\n" " add.s32 %r159, %r158, %r157;\n" " shl.b32 %r160, %r15, 2;\n" " add.s32 %r161, %r159, %r160;\n" " ld.shared.f32 %f226, [%r161];\n" " add.f32 %f227, %f246, %f226;\n" " ld.shared.f32 %f228, [%r161+8];\n" " add.f32 %f229, %f227, %f228;\n" " ld.shared.f32 %f230, [%r161+16];\n" " add.f32 %f231, %f229, %f230;\n" " ld.shared.f32 %f232, [%r161+24];\n" " add.f32 %f246, %f231, %f232;\n" " add.s32 %r171, %r171, 4;\n" " add.s32 %r170, %r170, -4;\n" " setp.ne.s32 %p13, %r170, 0;\n" " @%p13 bra $L__BB20_13;\n" "\n" "$L__BB20_14:\n" " setp.eq.s32 %p14, %r173, 0;\n" " @%p14 bra $L__BB20_17;\n" "\n" " add.s32 %r162, %r171, %r17;\n" " shl.b32 %r163, %r162, 3;\n" " mov.u32 %r164, _ZZ18fc_moe_down_q4k_nxE4sred;\n" " add.s32 %r165, %r164, %r163;\n" " shl.b32 %r166, %r15, 2;\n" " add.s32 %r172, %r165, %r166;\n" "\n" "$L__BB20_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f233, [%r172];\n" " add.f32 %f246, %f246, %f233;\n" " add.s32 %r172, %r172, 8;\n" " add.s32 %r173, %r173, -1;\n" " setp.ne.s32 %p15, %r173, 0;\n" " @%p15 bra $L__BB20_16;\n" "\n" "$L__BB20_17:\n" " cvt.s64.s32 %rd53, %r16;\n" " add.s64 %rd54, %rd24, %rd53;\n" " cvta.to.global.u64 %rd55, %rd29;\n" " shl.b64 %rd56, %rd54, 2;\n" " add.s64 %rd57, %rd55, %rd56;\n" " st.global.f32 [%rd57], %f246;\n" "\n" "$L__BB20_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_q6k_nx\n" ".visible .entry fc_moe_down_q6k_nx(\n" " .param .u64 fc_moe_down_q6k_nx_param_0,\n" " .param .u64 fc_moe_down_q6k_nx_param_1,\n" " .param .u64 fc_moe_down_q6k_nx_param_2,\n" " .param .u64 fc_moe_down_q6k_nx_param_3,\n" " .param .u64 fc_moe_down_q6k_nx_param_4,\n" " .param .u32 fc_moe_down_q6k_nx_param_5,\n" " .param .u32 fc_moe_down_q6k_nx_param_6,\n" " .param .u32 fc_moe_down_q6k_nx_param_7,\n" " .param .u32 fc_moe_down_q6k_nx_param_8,\n" " .param .u32 fc_moe_down_q6k_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<255>;\n" " .reg .b32 %r<190>;\n" " .reg .b64 %rd<77>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ18fc_moe_down_q6k_nxE4sred[128];\n" "\n" " ld.param.u64 %rd31, [fc_moe_down_q6k_nx_param_0];\n" " ld.param.u64 %rd32, [fc_moe_down_q6k_nx_param_1];\n" " ld.param.u64 %rd33, [fc_moe_down_q6k_nx_param_2];\n" " ld.param.u64 %rd34, [fc_moe_down_q6k_nx_param_3];\n" " ld.param.u32 %r30, [fc_moe_down_q6k_nx_param_5];\n" " ld.param.u32 %r31, [fc_moe_down_q6k_nx_param_6];\n" " ld.param.u32 %r32, [fc_moe_down_q6k_nx_param_7];\n" " ld.param.u32 %r33, [fc_moe_down_q6k_nx_param_8];\n" " ld.param.u32 %r34, [fc_moe_down_q6k_nx_param_9];\n" " mov.u32 %r35, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r35;\n" " cvt.s64.s32 %rd2, %r31;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r36, %ntid.x;\n" " shr.u32 %r37, %r36, 5;\n" " div.u32 %r4, %r37, %r34;\n" " div.s32 %r38, %r2, %r34;\n" " mul.lo.s32 %r39, %r38, %r34;\n" " sub.s32 %r183, %r2, %r39;\n" " mov.u32 %r40, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r40;\n" " add.s32 %r41, %r6, %r38;\n" " shl.b32 %r7, %r41, 1;\n" " setp.ge.s32 %p1, %r7, %r31;\n" " setp.ge.s32 %p2, %r183, %r30;\n" " mov.f32 %f248, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f249, %f248;\n" " @%p3 bra $L__BB21_6;\n" "\n" " cvt.s64.s32 %rd3, %r33;\n" " cvt.s64.s32 %rd4, %r7;\n" " shr.u32 %r42, %r3, 4;\n" " cvt.u64.u32 %rd5, %r42;\n" " cvt.u64.u32 %rd6, %r3;\n" " mul.wide.s32 %rd8, %r3, 4;\n" " add.s64 %rd7, %rd32, %rd8;\n" " mul.wide.s32 %rd36, %r3, 2;\n" " or.b64 %rd9, %rd36, 128;\n" " mul.wide.s32 %rd37, %r42, 8;\n" " or.b64 %rd10, %rd37, 192;\n" " shr.s32 %r8, %r32, 8;\n" " add.s32 %r9, %r7, 1;\n" " cvt.s64.s32 %rd11, %r32;\n" " cvt.s64.s32 %rd38, %r30;\n" " mul.lo.s64 %rd12, %rd38, %rd1;\n" " cvta.to.global.u64 %rd13, %rd31;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r9, %r31;\n" " setp.lt.s32 %p5, %r32, 256;\n" " shl.b64 %rd61, %rd5, 3;\n" " mov.f32 %f249, %f25;\n" " mov.f32 %f248, %f25;\n" "\n" "$L__BB21_2:\n" " cvt.s64.s32 %rd40, %r183;\n" " add.s64 %rd14, %rd12, %rd40;\n" " shl.b64 %rd41, %rd14, 2;\n" " add.s64 %rd39, %rd33, %rd41;\n" " // begin inline asm\n" " ld.global.nc.s32 %r43, [%rd39];\n" " // end inline asm\n" " cvt.s64.s32 %rd42, %r43;\n" " mul.lo.s64 %rd43, %rd42, %rd2;\n" " add.s64 %rd44, %rd43, %rd4;\n" " mul.lo.s64 %rd45, %rd44, %rd3;\n" " add.s64 %rd76, %rd13, %rd45;\n" " add.s64 %rd46, %rd76, %rd3;\n" " selp.b64 %rd75, %rd46, %rd76, %p4;\n" " mov.f32 %f246, %f25;\n" " mov.f32 %f247, %f25;\n" " @%p5 bra $L__BB21_5;\n" "\n" " mul.lo.s64 %rd47, %rd14, %rd11;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd74, %rd7, %rd48;\n" " add.s64 %rd73, %rd75, %rd8;\n" " add.s64 %rd72, %rd75, %rd9;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r184, 0;\n" " mov.f32 %f247, %f29;\n" " mov.f32 %f246, %f29;\n" "\n" "$L__BB21_4:\n" " shl.b64 %rd57, %rd6, 2;\n" " add.s64 %rd58, %rd76, %rd57;\n" " shl.b64 %rd59, %rd6, 1;\n" " add.s64 %rd60, %rd76, %rd59;\n" " ld.global.u16 %r45, [%rd60+128];\n" " ld.global.u16 %r46, [%rd72];\n" " ld.global.f32 %f38, [%rd76+208];\n" " ld.global.f32 %f39, [%rd75+208];\n" " add.s64 %rd62, %rd76, %rd61;\n" " ld.global.v2.u32 {%r47, %r48}, [%rd62+192];\n" " add.s64 %rd63, %rd75, %rd10;\n" " ld.global.v2.u32 {%r51, %r52}, [%rd63];\n" " ld.global.u32 %r55, [%rd58];\n" " shr.u32 %r56, %r55, 16;\n" " ld.global.u32 %r57, [%rd73];\n" " shr.u32 %r58, %r57, 16;\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd74];\n" " // end inline asm\n" " and.b32 %r59, %r55, 15;\n" " or.b32 %r60, %r59, 1258291200;\n" " mov.b32 %f40, %r60;\n" " mov.f32 %f41, 0fCB000020;\n" " mov.f32 %f42, 0f3F800000;\n" " fma.rn.f32 %f43, %f40, %f42, %f41;\n" " and.b32 %r61, %r45, 3;\n" " or.b32 %r62, %r61, 1258291200;\n" " mov.b32 %f44, %r62;\n" " mov.f32 %f45, 0fCD000000;\n" " mov.f32 %f46, 0f41800000;\n" " fma.rn.f32 %f47, %f44, %f46, %f45;\n" " add.rn.f32 %f48, %f43, %f47;\n" " and.b32 %r63, %r57, 15;\n" " or.b32 %r64, %r63, 1258291200;\n" " mov.b32 %f49, %r64;\n" " fma.rn.f32 %f50, %f49, %f42, %f41;\n" " and.b32 %r65, %r46, 3;\n" " or.b32 %r66, %r65, 1258291200;\n" " mov.b32 %f51, %r66;\n" " fma.rn.f32 %f52, %f51, %f46, %f45;\n" " add.rn.f32 %f53, %f50, %f52;\n" " and.b32 %r67, %r47, 255;\n" " cvt.rn.f32.u32 %f54, %r67;\n" " mov.f32 %f55, 0f43000000;\n" " sub.rn.f32 %f56, %f54, %f55;\n" " and.b32 %r68, %r51, 255;\n" " cvt.rn.f32.u32 %f57, %r68;\n" " sub.rn.f32 %f58, %f57, %f55;\n" " mul.rn.f32 %f59, %f56, %f48;\n" " fma.rn.f32 %f61, %f59, %f30, %f29;\n" " mul.rn.f32 %f62, %f58, %f53;\n" " fma.rn.f32 %f63, %f62, %f30, %f29;\n" " add.s64 %rd50, %rd74, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd50];\n" " // end inline asm\n" " and.b32 %r69, %r55, 3840;\n" " or.b32 %r70, %r69, 1258291200;\n" " mov.b32 %f64, %r70;\n" " mov.f32 %f65, 0fC7002000;\n" " mov.f32 %f66, 0f3B800000;\n" " fma.rn.f32 %f67, %f64, %f66, %f65;\n" " and.b32 %r71, %r45, 12;\n" " or.b32 %r72, %r71, 1258291200;\n" " mov.b32 %f68, %r72;\n" " mov.f32 %f69, 0fCC000000;\n" " mov.f32 %f70, 0f40800000;\n" " fma.rn.f32 %f71, %f68, %f70, %f69;\n" " add.rn.f32 %f72, %f67, %f71;\n" " and.b32 %r73, %r57, 3840;\n" " or.b32 %r74, %r73, 1258291200;\n" " mov.b32 %f73, %r74;\n" " fma.rn.f32 %f74, %f73, %f66, %f65;\n" " and.b32 %r75, %r46, 12;\n" " or.b32 %r76, %r75, 1258291200;\n" " mov.b32 %f75, %r76;\n" " fma.rn.f32 %f76, %f75, %f70, %f69;\n" " add.rn.f32 %f77, %f74, %f76;\n" " bfe.u32 %r77, %r47, 8, 8;\n" " cvt.rn.f32.u32 %f78, %r77;\n" " sub.rn.f32 %f79, %f78, %f55;\n" " bfe.u32 %r78, %r51, 8, 8;\n" " cvt.rn.f32.u32 %f80, %r78;\n" " sub.rn.f32 %f81, %f80, %f55;\n" " mul.rn.f32 %f82, %f79, %f72;\n" " fma.rn.f32 %f83, %f82, %f31, %f61;\n" " mul.rn.f32 %f84, %f81, %f77;\n" " fma.rn.f32 %f85, %f84, %f31, %f63;\n" " add.s64 %rd51, %rd74, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd51];\n" " // end inline asm\n" " and.b32 %r79, %r55, 240;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f86, %r80;\n" " mov.f32 %f87, 0fC9000200;\n" " mov.f32 %f88, 0f3D800000;\n" " fma.rn.f32 %f89, %f86, %f88, %f87;\n" " and.b32 %r81, %r45, 48;\n" " or.b32 %r82, %r81, 1258291200;\n" " mov.b32 %f90, %r82;\n" " mov.f32 %f91, 0fCB000000;\n" " fma.rn.f32 %f92, %f90, %f42, %f91;\n" " add.rn.f32 %f93, %f89, %f92;\n" " and.b32 %r83, %r57, 240;\n" " or.b32 %r84, %r83, 1258291200;\n" " mov.b32 %f94, %r84;\n" " fma.rn.f32 %f95, %f94, %f88, %f87;\n" " and.b32 %r85, %r46, 48;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f96, %r86;\n" " fma.rn.f32 %f97, %f96, %f42, %f91;\n" " add.rn.f32 %f98, %f95, %f97;\n" " bfe.u32 %r87, %r47, 16, 8;\n" " cvt.rn.f32.u32 %f99, %r87;\n" " sub.rn.f32 %f100, %f99, %f55;\n" " bfe.u32 %r88, %r51, 16, 8;\n" " cvt.rn.f32.u32 %f101, %r88;\n" " sub.rn.f32 %f102, %f101, %f55;\n" " mul.rn.f32 %f103, %f100, %f93;\n" " fma.rn.f32 %f104, %f103, %f32, %f83;\n" " mul.rn.f32 %f105, %f102, %f98;\n" " fma.rn.f32 %f106, %f105, %f32, %f85;\n" " add.s64 %rd52, %rd74, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd52];\n" " // end inline asm\n" " and.b32 %r89, %r55, 61440;\n" " or.b32 %r90, %r89, 1258291200;\n" " mov.b32 %f107, %r90;\n" " mov.f32 %f108, 0fC5020000;\n" " mov.f32 %f109, 0f39800000;\n" " fma.rn.f32 %f110, %f107, %f109, %f108;\n" " and.b32 %r91, %r45, 192;\n" " or.b32 %r92, %r91, 1258291200;\n" " mov.b32 %f111, %r92;\n" " mov.f32 %f112, 0fCA000000;\n" " mov.f32 %f113, 0f3E800000;\n" " fma.rn.f32 %f114, %f111, %f113, %f112;\n" " add.rn.f32 %f115, %f110, %f114;\n" " and.b32 %r93, %r57, 61440;\n" " or.b32 %r94, %r93, 1258291200;\n" " mov.b32 %f116, %r94;\n" " fma.rn.f32 %f117, %f116, %f109, %f108;\n" " and.b32 %r95, %r46, 192;\n" " or.b32 %r96, %r95, 1258291200;\n" " mov.b32 %f118, %r96;\n" " fma.rn.f32 %f119, %f118, %f113, %f112;\n" " add.rn.f32 %f120, %f117, %f119;\n" " shr.u32 %r97, %r47, 24;\n" " cvt.rn.f32.u32 %f121, %r97;\n" " sub.rn.f32 %f122, %f121, %f55;\n" " shr.u32 %r98, %r51, 24;\n" " cvt.rn.f32.u32 %f123, %r98;\n" " sub.rn.f32 %f124, %f123, %f55;\n" " mul.rn.f32 %f125, %f122, %f115;\n" " fma.rn.f32 %f126, %f125, %f33, %f104;\n" " mul.rn.f32 %f127, %f124, %f120;\n" " fma.rn.f32 %f128, %f127, %f33, %f106;\n" " add.s64 %rd53, %rd74, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd53];\n" " // end inline asm\n" " and.b32 %r99, %r55, 983040;\n" " or.b32 %r100, %r99, 1258291200;\n" " mov.b32 %f129, %r100;\n" " mov.f32 %f130, 0fC3200000;\n" " mov.f32 %f131, 0f37800000;\n" " fma.rn.f32 %f132, %f129, %f131, %f130;\n" " and.b32 %r101, %r45, 768;\n" " or.b32 %r102, %r101, 1258291200;\n" " mov.b32 %f133, %r102;\n" " mov.f32 %f134, 0fC9000000;\n" " fma.rn.f32 %f135, %f133, %f88, %f134;\n" " add.rn.f32 %f136, %f132, %f135;\n" " and.b32 %r103, %r57, 983040;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f137, %r104;\n" " fma.rn.f32 %f138, %f137, %f131, %f130;\n" " and.b32 %r105, %r46, 768;\n" " or.b32 %r106, %r105, 1258291200;\n" " mov.b32 %f139, %r106;\n" " fma.rn.f32 %f140, %f139, %f88, %f134;\n" " add.rn.f32 %f141, %f138, %f140;\n" " and.b32 %r107, %r48, 255;\n" " cvt.rn.f32.u32 %f142, %r107;\n" " sub.rn.f32 %f143, %f142, %f55;\n" " and.b32 %r108, %r52, 255;\n" " cvt.rn.f32.u32 %f144, %r108;\n" " sub.rn.f32 %f145, %f144, %f55;\n" " mul.rn.f32 %f146, %f143, %f136;\n" " fma.rn.f32 %f147, %f146, %f34, %f126;\n" " mul.rn.f32 %f148, %f145, %f141;\n" " fma.rn.f32 %f149, %f148, %f34, %f128;\n" " add.s64 %rd54, %rd74, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd54];\n" " // end inline asm\n" " and.b32 %r109, %r56, 3840;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f150, %r110;\n" " fma.rn.f32 %f151, %f150, %f66, %f65;\n" " and.b32 %r111, %r45, 3072;\n" " or.b32 %r112, %r111, 1258291200;\n" " mov.b32 %f152, %r112;\n" " mov.f32 %f153, 0fC8000000;\n" " mov.f32 %f154, 0f3C800000;\n" " fma.rn.f32 %f155, %f152, %f154, %f153;\n" " add.rn.f32 %f156, %f151, %f155;\n" " and.b32 %r113, %r58, 3840;\n" " or.b32 %r114, %r113, 1258291200;\n" " mov.b32 %f157, %r114;\n" " fma.rn.f32 %f158, %f157, %f66, %f65;\n" " and.b32 %r115, %r46, 3072;\n" " or.b32 %r116, %r115, 1258291200;\n" " mov.b32 %f159, %r116;\n" " fma.rn.f32 %f160, %f159, %f154, %f153;\n" " add.rn.f32 %f161, %f158, %f160;\n" " bfe.u32 %r117, %r48, 8, 8;\n" " cvt.rn.f32.u32 %f162, %r117;\n" " sub.rn.f32 %f163, %f162, %f55;\n" " bfe.u32 %r118, %r52, 8, 8;\n" " cvt.rn.f32.u32 %f164, %r118;\n" " sub.rn.f32 %f165, %f164, %f55;\n" " mul.rn.f32 %f166, %f163, %f156;\n" " fma.rn.f32 %f167, %f166, %f35, %f147;\n" " mul.rn.f32 %f168, %f165, %f161;\n" " fma.rn.f32 %f169, %f168, %f35, %f149;\n" " add.s64 %rd55, %rd74, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd55];\n" " // end inline asm\n" " and.b32 %r119, %r56, 240;\n" " or.b32 %r120, %r119, 1258291200;\n" " mov.b32 %f170, %r120;\n" " fma.rn.f32 %f171, %f170, %f88, %f87;\n" " and.b32 %r121, %r45, 12288;\n" " or.b32 %r122, %r121, 1258291200;\n" " mov.b32 %f172, %r122;\n" " mov.f32 %f173, 0fC7000000;\n" " fma.rn.f32 %f174, %f172, %f66, %f173;\n" " add.rn.f32 %f175, %f171, %f174;\n" " and.b32 %r123, %r58, 240;\n" " or.b32 %r124, %r123, 1258291200;\n" " mov.b32 %f176, %r124;\n" " fma.rn.f32 %f177, %f176, %f88, %f87;\n" " and.b32 %r125, %r46, 12288;\n" " or.b32 %r126, %r125, 1258291200;\n" " mov.b32 %f178, %r126;\n" " fma.rn.f32 %f179, %f178, %f66, %f173;\n" " add.rn.f32 %f180, %f177, %f179;\n" " bfe.u32 %r127, %r48, 16, 8;\n" " cvt.rn.f32.u32 %f181, %r127;\n" " sub.rn.f32 %f182, %f181, %f55;\n" " bfe.u32 %r128, %r52, 16, 8;\n" " cvt.rn.f32.u32 %f183, %r128;\n" " sub.rn.f32 %f184, %f183, %f55;\n" " mul.rn.f32 %f185, %f182, %f175;\n" " fma.rn.f32 %f186, %f185, %f36, %f167;\n" " mul.rn.f32 %f187, %f184, %f180;\n" " fma.rn.f32 %f188, %f187, %f36, %f169;\n" " add.s64 %rd56, %rd74, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd56];\n" " // end inline asm\n" " and.b32 %r129, %r56, 61440;\n" " or.b32 %r130, %r129, 1258291200;\n" " mov.b32 %f189, %r130;\n" " fma.rn.f32 %f190, %f189, %f109, %f108;\n" " and.b32 %r131, %r45, 49152;\n" " or.b32 %r132, %r131, 1258291200;\n" " mov.b32 %f191, %r132;\n" " mov.f32 %f192, 0fC6000000;\n" " mov.f32 %f193, 0f3A800000;\n" " fma.rn.f32 %f194, %f191, %f193, %f192;\n" " add.rn.f32 %f195, %f190, %f194;\n" " and.b32 %r133, %r58, 61440;\n" " or.b32 %r134, %r133, 1258291200;\n" " mov.b32 %f196, %r134;\n" " fma.rn.f32 %f197, %f196, %f109, %f108;\n" " and.b32 %r135, %r46, 49152;\n" " or.b32 %r136, %r135, 1258291200;\n" " mov.b32 %f198, %r136;\n" " fma.rn.f32 %f199, %f198, %f193, %f192;\n" " add.rn.f32 %f200, %f197, %f199;\n" " shr.u32 %r137, %r48, 24;\n" " cvt.rn.f32.u32 %f201, %r137;\n" " sub.rn.f32 %f202, %f201, %f55;\n" " shr.u32 %r138, %r52, 24;\n" " cvt.rn.f32.u32 %f203, %r138;\n" " sub.rn.f32 %f204, %f203, %f55;\n" " mul.rn.f32 %f205, %f202, %f195;\n" " fma.rn.f32 %f206, %f205, %f37, %f186;\n" " mul.rn.f32 %f207, %f204, %f200;\n" " fma.rn.f32 %f208, %f207, %f37, %f188;\n" " fma.rn.f32 %f246, %f38, %f206, %f246;\n" " fma.rn.f32 %f247, %f39, %f208, %f247;\n" " add.s64 %rd76, %rd76, 216;\n" " add.s64 %rd75, %rd75, 216;\n" " add.s64 %rd74, %rd74, 1024;\n" " add.s64 %rd73, %rd73, 216;\n" " add.s64 %rd72, %rd72, 216;\n" " add.s32 %r184, %r184, 1;\n" " setp.lt.s32 %p6, %r184, %r8;\n" " @%p6 bra $L__BB21_4;\n" "\n" "$L__BB21_5:\n" " add.s64 %rd64, %rd34, %rd41;\n" " // begin inline asm\n" " ld.global.nc.f32 %f209, [%rd64];\n" " // end inline asm\n" " fma.rn.f32 %f248, %f209, %f246, %f248;\n" " fma.rn.f32 %f249, %f209, %f247, %f249;\n" " add.s32 %r183, %r183, %r34;\n" " setp.lt.s32 %p7, %r183, %r30;\n" " @%p7 bra $L__BB21_2;\n" "\n" "$L__BB21_6:\n" " mov.u32 %r149, 16;\n" " mov.u32 %r158, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f210, %f248, %r149, %r158;\n" " // end inline asm\n" " add.f32 %f213, %f248, %f210;\n" " mov.u32 %r151, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f212, %f213, %r151, %r158;\n" " // end inline asm\n" " add.f32 %f215, %f213, %f212;\n" " mov.u32 %r153, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f214, %f215, %r153, %r158;\n" " // end inline asm\n" " add.f32 %f217, %f215, %f214;\n" " mov.u32 %r155, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f216, %f217, %r155, %r158;\n" " // end inline asm\n" " add.f32 %f219, %f217, %f216;\n" " mov.u32 %r157, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f218, %f219, %r157, %r158;\n" " // end inline asm\n" " add.f32 %f13, %f219, %f218;\n" " // begin inline asm\n" " shfl.down.b32 %f220, %f249, %r149, %r158;\n" " // end inline asm\n" " add.f32 %f223, %f249, %f220;\n" " // begin inline asm\n" " shfl.down.b32 %f222, %f223, %r151, %r158;\n" " // end inline asm\n" " add.f32 %f225, %f223, %f222;\n" " // begin inline asm\n" " shfl.down.b32 %f224, %f225, %r153, %r158;\n" " // end inline asm\n" " add.f32 %f227, %f225, %f224;\n" " // begin inline asm\n" " shfl.down.b32 %f226, %f227, %r155, %r158;\n" " // end inline asm\n" " add.f32 %f229, %f227, %f226;\n" " // begin inline asm\n" " shfl.down.b32 %f228, %f229, %r157, %r158;\n" " // end inline asm\n" " add.f32 %f14, %f229, %f228;\n" " setp.ne.s32 %p8, %r3, 0;\n" " mul.lo.s64 %rd30, %rd2, %rd1;\n" " @%p8 bra $L__BB21_8;\n" "\n" " mov.u32 %r182, %tid.x;\n" " shr.u32 %r181, %r182, 5;\n" " shl.b32 %r159, %r181, 3;\n" " mov.u32 %r160, _ZZ18fc_moe_down_q6k_nxE4sred;\n" " add.s32 %r161, %r160, %r159;\n" " st.shared.f32 [%r161], %f13;\n" " st.shared.f32 [%r161+4], %f14;\n" "\n" "$L__BB21_8:\n" " mov.u32 %r178, %tid.x;\n" " bar.sync 0;\n" " shl.b32 %r162, %r4, 1;\n" " setp.ge.s32 %p9, %r178, %r162;\n" " @%p9 bra $L__BB21_18;\n" "\n" " mov.u32 %r180, %tid.x;\n" " ld.param.u32 %r179, [fc_moe_down_q6k_nx_param_6];\n" " shr.u32 %r14, %r180, 1;\n" " add.s32 %r163, %r6, %r14;\n" " and.b32 %r15, %r180, 1;\n" " bfi.b32 %r16, %r163, %r15, 1, 31;\n" " setp.ge.s32 %p10, %r16, %r179;\n" " @%p10 bra $L__BB21_18;\n" "\n" " setp.lt.s32 %p11, %r34, 1;\n" " mov.f32 %f254, 0f00000000;\n" " @%p11 bra $L__BB21_17;\n" "\n" " mul.lo.s32 %r17, %r14, %r34;\n" " and.b32 %r189, %r34, 3;\n" " add.s32 %r165, %r34, -1;\n" " setp.lt.u32 %p12, %r165, 3;\n" " mov.f32 %f254, 0f00000000;\n" " mov.u32 %r187, 0;\n" " @%p12 bra $L__BB21_14;\n" "\n" " sub.s32 %r186, %r34, %r189;\n" " mov.f32 %f254, 0f00000000;\n" " mov.u32 %r187, 0;\n" "\n" "$L__BB21_13:\n" " add.s32 %r167, %r187, %r17;\n" " shl.b32 %r168, %r167, 3;\n" " mov.u32 %r169, _ZZ18fc_moe_down_q6k_nxE4sred;\n" " add.s32 %r170, %r169, %r168;\n" " shl.b32 %r171, %r15, 2;\n" " add.s32 %r172, %r170, %r171;\n" " ld.shared.f32 %f234, [%r172];\n" " add.f32 %f235, %f254, %f234;\n" " ld.shared.f32 %f236, [%r172+8];\n" " add.f32 %f237, %f235, %f236;\n" " ld.shared.f32 %f238, [%r172+16];\n" " add.f32 %f239, %f237, %f238;\n" " ld.shared.f32 %f240, [%r172+24];\n" " add.f32 %f254, %f239, %f240;\n" " add.s32 %r187, %r187, 4;\n" " add.s32 %r186, %r186, -4;\n" " setp.ne.s32 %p13, %r186, 0;\n" " @%p13 bra $L__BB21_13;\n" "\n" "$L__BB21_14:\n" " setp.eq.s32 %p14, %r189, 0;\n" " @%p14 bra $L__BB21_17;\n" "\n" " add.s32 %r173, %r187, %r17;\n" " shl.b32 %r174, %r173, 3;\n" " mov.u32 %r175, _ZZ18fc_moe_down_q6k_nxE4sred;\n" " add.s32 %r176, %r175, %r174;\n" " shl.b32 %r177, %r15, 2;\n" " add.s32 %r188, %r176, %r177;\n" "\n" "$L__BB21_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f241, [%r188];\n" " add.f32 %f254, %f254, %f241;\n" " add.s32 %r188, %r188, 8;\n" " add.s32 %r189, %r189, -1;\n" " setp.ne.s32 %p15, %r189, 0;\n" " @%p15 bra $L__BB21_16;\n" "\n" "$L__BB21_17:\n" " ld.param.u64 %rd71, [fc_moe_down_q6k_nx_param_4];\n" " cvt.s64.s32 %rd66, %r16;\n" " add.s64 %rd67, %rd30, %rd66;\n" " cvta.to.global.u64 %rd68, %rd71;\n" " shl.b64 %rd69, %rd67, 2;\n" " add.s64 %rd70, %rd68, %rd69;\n" " st.global.f32 [%rd70], %f254;\n" "\n" "$L__BB21_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_f16_nx\n" ".visible .entry fc_moe_down_f16_nx(\n" " .param .u64 fc_moe_down_f16_nx_param_0,\n" " .param .u64 fc_moe_down_f16_nx_param_1,\n" " .param .u64 fc_moe_down_f16_nx_param_2,\n" " .param .u64 fc_moe_down_f16_nx_param_3,\n" " .param .u64 fc_moe_down_f16_nx_param_4,\n" " .param .u32 fc_moe_down_f16_nx_param_5,\n" " .param .u32 fc_moe_down_f16_nx_param_6,\n" " .param .u32 fc_moe_down_f16_nx_param_7,\n" " .param .u32 fc_moe_down_f16_nx_param_8,\n" " .param .u32 fc_moe_down_f16_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .b16 %rs<17>;\n" " .reg .f32 %f<117>;\n" " .reg .b32 %r<91>;\n" " .reg .b64 %rd<61>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ18fc_moe_down_f16_nxE4sred[128];\n" "\n" " ld.param.u64 %rd22, [fc_moe_down_f16_nx_param_0];\n" " ld.param.u64 %rd23, [fc_moe_down_f16_nx_param_1];\n" " ld.param.u64 %rd24, [fc_moe_down_f16_nx_param_2];\n" " ld.param.u64 %rd25, [fc_moe_down_f16_nx_param_3];\n" " ld.param.u64 %rd26, [fc_moe_down_f16_nx_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_f16_nx_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_f16_nx_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_f16_nx_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_f16_nx_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_f16_nx_param_9];\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " cvt.s64.s32 %rd2, %r32;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r37, %ntid.x;\n" " shr.u32 %r38, %r37, 5;\n" " div.u32 %r4, %r38, %r35;\n" " div.s32 %r39, %r2, %r35;\n" " mul.lo.s32 %r40, %r39, %r35;\n" " sub.s32 %r84, %r2, %r40;\n" " mov.u32 %r41, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r41;\n" " add.s32 %r42, %r6, %r39;\n" " shl.b32 %r7, %r42, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r84, %r31;\n" " mov.f32 %f110, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f111, %f110;\n" " @%p3 bra $L__BB22_6;\n" "\n" " cvt.s64.s32 %rd3, %r34;\n" " cvt.s64.s32 %rd4, %r7;\n" " cvt.u64.u32 %rd5, %r3;\n" " mul.wide.s32 %rd27, %r3, 4;\n" " add.s64 %rd6, %rd23, %rd27;\n" " mul.wide.s32 %rd28, %r3, 2;\n" " or.b64 %rd7, %rd28, 256;\n" " shr.s32 %r8, %r33, 8;\n" " add.s32 %r9, %r7, 1;\n" " cvt.s64.s32 %rd8, %r33;\n" " cvt.s64.s32 %rd29, %r31;\n" " mul.lo.s64 %rd9, %rd29, %rd1;\n" " cvta.to.global.u64 %rd10, %rd22;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r33, 256;\n" " setp.lt.s32 %p5, %r9, %r32;\n" " shl.b64 %rd49, %rd5, 1;\n" " mov.f32 %f111, %f25;\n" " mov.f32 %f110, %f25;\n" "\n" "$L__BB22_2:\n" " cvt.s64.s32 %rd31, %r84;\n" " add.s64 %rd11, %rd9, %rd31;\n" " shl.b64 %rd32, %rd11, 2;\n" " add.s64 %rd30, %rd24, %rd32;\n" " // begin inline asm\n" " ld.global.nc.s32 %r43, [%rd30];\n" " // end inline asm\n" " mov.f32 %f108, %f25;\n" " mov.f32 %f109, %f25;\n" " @%p4 bra $L__BB22_5;\n" "\n" " cvt.s64.s32 %rd33, %r43;\n" " mul.lo.s64 %rd34, %rd33, %rd2;\n" " add.s64 %rd35, %rd34, %rd4;\n" " mul.lo.s64 %rd36, %rd35, %rd3;\n" " add.s64 %rd60, %rd10, %rd36;\n" " mul.lo.s64 %rd37, %rd11, %rd8;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd59, %rd6, %rd38;\n" " add.s64 %rd39, %rd60, %rd3;\n" " selp.b64 %rd40, %rd39, %rd60, %p5;\n" " add.s64 %rd58, %rd40, %rd7;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r85, 0;\n" " mov.f32 %f109, %f29;\n" " mov.f32 %f108, %f29;\n" "\n" "$L__BB22_4:\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd59];\n" " // end inline asm\n" " add.s64 %rd50, %rd60, %rd49;\n" " ld.global.u16 %rs1, [%rd50];\n" " // begin inline asm\n" " cvt.f32.f16 %f31, %rs1;\n" " // end inline asm\n" " fma.rn.f32 %f55, %f31, %f30, %f29;\n" " ld.global.u16 %rs2, [%rd58+-256];\n" " // begin inline asm\n" " cvt.f32.f16 %f32, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f56, %f32, %f30, %f29;\n" " add.s64 %rd42, %rd59, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd42];\n" " // end inline asm\n" " ld.global.u16 %rs3, [%rd50+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f34, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f57, %f34, %f33, %f55;\n" " ld.global.u16 %rs4, [%rd58+-192];\n" " // begin inline asm\n" " cvt.f32.f16 %f35, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f58, %f35, %f33, %f56;\n" " add.s64 %rd43, %rd59, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd43];\n" " // end inline asm\n" " ld.global.u16 %rs5, [%rd50+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f37, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f59, %f37, %f36, %f57;\n" " ld.global.u16 %rs6, [%rd58+-128];\n" " // begin inline asm\n" " cvt.f32.f16 %f38, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f60, %f38, %f36, %f58;\n" " add.s64 %rd44, %rd59, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f39, [%rd44];\n" " // end inline asm\n" " ld.global.u16 %rs7, [%rd50+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f40, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f61, %f40, %f39, %f59;\n" " ld.global.u16 %rs8, [%rd58+-64];\n" " // begin inline asm\n" " cvt.f32.f16 %f41, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f62, %f41, %f39, %f60;\n" " add.s64 %rd45, %rd59, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f42, [%rd45];\n" " // end inline asm\n" " ld.global.u16 %rs9, [%rd50+256];\n" " // begin inline asm\n" " cvt.f32.f16 %f43, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f63, %f43, %f42, %f61;\n" " ld.global.u16 %rs10, [%rd58];\n" " // begin inline asm\n" " cvt.f32.f16 %f44, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f64, %f44, %f42, %f62;\n" " add.s64 %rd46, %rd59, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f45, [%rd46];\n" " // end inline asm\n" " ld.global.u16 %rs11, [%rd50+320];\n" " // begin inline asm\n" " cvt.f32.f16 %f46, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f65, %f46, %f45, %f63;\n" " ld.global.u16 %rs12, [%rd58+64];\n" " // begin inline asm\n" " cvt.f32.f16 %f47, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f66, %f47, %f45, %f64;\n" " add.s64 %rd47, %rd59, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f48, [%rd47];\n" " // end inline asm\n" " ld.global.u16 %rs13, [%rd50+384];\n" " // begin inline asm\n" " cvt.f32.f16 %f49, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f67, %f49, %f48, %f65;\n" " ld.global.u16 %rs14, [%rd58+128];\n" " // begin inline asm\n" " cvt.f32.f16 %f50, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f68, %f50, %f48, %f66;\n" " add.s64 %rd48, %rd59, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f51, [%rd48];\n" " // end inline asm\n" " ld.global.u16 %rs15, [%rd50+448];\n" " // begin inline asm\n" " cvt.f32.f16 %f52, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f69, %f52, %f51, %f67;\n" " ld.global.u16 %rs16, [%rd58+192];\n" " // begin inline asm\n" " cvt.f32.f16 %f53, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f70, %f53, %f51, %f68;\n" " add.f32 %f108, %f108, %f69;\n" " add.f32 %f109, %f109, %f70;\n" " add.s64 %rd60, %rd60, 512;\n" " add.s64 %rd59, %rd59, 1024;\n" " add.s64 %rd58, %rd58, 512;\n" " add.s32 %r85, %r85, 1;\n" " setp.lt.s32 %p6, %r85, %r8;\n" " @%p6 bra $L__BB22_4;\n" "\n" "$L__BB22_5:\n" " add.s64 %rd51, %rd25, %rd32;\n" " // begin inline asm\n" " ld.global.nc.f32 %f71, [%rd51];\n" " // end inline asm\n" " fma.rn.f32 %f110, %f71, %f108, %f110;\n" " fma.rn.f32 %f111, %f71, %f109, %f111;\n" " add.s32 %r84, %r84, %r35;\n" " setp.lt.s32 %p7, %r84, %r31;\n" " @%p7 bra $L__BB22_2;\n" "\n" "$L__BB22_6:\n" " mov.u32 %r55, 16;\n" " mov.u32 %r64, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f72, %f110, %r55, %r64;\n" " // end inline asm\n" " add.f32 %f75, %f110, %f72;\n" " mov.u32 %r57, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f74, %f75, %r57, %r64;\n" " // end inline asm\n" " add.f32 %f77, %f75, %f74;\n" " mov.u32 %r59, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f76, %f77, %r59, %r64;\n" " // end inline asm\n" " add.f32 %f79, %f77, %f76;\n" " mov.u32 %r61, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f78, %f79, %r61, %r64;\n" " // end inline asm\n" " add.f32 %f81, %f79, %f78;\n" " mov.u32 %r63, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f80, %f81, %r63, %r64;\n" " // end inline asm\n" " add.f32 %f13, %f81, %f80;\n" " // begin inline asm\n" " shfl.down.b32 %f82, %f111, %r55, %r64;\n" " // end inline asm\n" " add.f32 %f85, %f111, %f82;\n" " // begin inline asm\n" " shfl.down.b32 %f84, %f85, %r57, %r64;\n" " // end inline asm\n" " add.f32 %f87, %f85, %f84;\n" " // begin inline asm\n" " shfl.down.b32 %f86, %f87, %r59, %r64;\n" " // end inline asm\n" " add.f32 %f89, %f87, %f86;\n" " // begin inline asm\n" " shfl.down.b32 %f88, %f89, %r61, %r64;\n" " // end inline asm\n" " add.f32 %f91, %f89, %f88;\n" " // begin inline asm\n" " shfl.down.b32 %f90, %f91, %r63, %r64;\n" " // end inline asm\n" " add.f32 %f14, %f91, %f90;\n" " setp.ne.s32 %p8, %r3, 0;\n" " mul.lo.s64 %rd21, %rd2, %rd1;\n" " @%p8 bra $L__BB22_8;\n" "\n" " shl.b32 %r65, %r2, 3;\n" " mov.u32 %r66, _ZZ18fc_moe_down_f16_nxE4sred;\n" " add.s32 %r67, %r66, %r65;\n" " st.shared.f32 [%r67], %f13;\n" " st.shared.f32 [%r67+4], %f14;\n" "\n" "$L__BB22_8:\n" " bar.sync 0;\n" " shl.b32 %r68, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r68;\n" " @%p9 bra $L__BB22_18;\n" "\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r69, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r69, %r16, 1, 31;\n" " setp.ge.s32 %p10, %r17, %r32;\n" " @%p10 bra $L__BB22_18;\n" "\n" " setp.lt.s32 %p11, %r35, 1;\n" " mov.f32 %f116, 0f00000000;\n" " @%p11 bra $L__BB22_17;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r90, %r35, 3;\n" " add.s32 %r71, %r35, -1;\n" " setp.lt.u32 %p12, %r71, 3;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r88, 0;\n" " @%p12 bra $L__BB22_14;\n" "\n" " sub.s32 %r87, %r35, %r90;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r88, 0;\n" "\n" "$L__BB22_13:\n" " add.s32 %r73, %r88, %r18;\n" " shl.b32 %r74, %r73, 3;\n" " mov.u32 %r75, _ZZ18fc_moe_down_f16_nxE4sred;\n" " add.s32 %r76, %r75, %r74;\n" " shl.b32 %r77, %r16, 2;\n" " add.s32 %r78, %r76, %r77;\n" " ld.shared.f32 %f96, [%r78];\n" " add.f32 %f97, %f116, %f96;\n" " ld.shared.f32 %f98, [%r78+8];\n" " add.f32 %f99, %f97, %f98;\n" " ld.shared.f32 %f100, [%r78+16];\n" " add.f32 %f101, %f99, %f100;\n" " ld.shared.f32 %f102, [%r78+24];\n" " add.f32 %f116, %f101, %f102;\n" " add.s32 %r88, %r88, 4;\n" " add.s32 %r87, %r87, -4;\n" " setp.ne.s32 %p13, %r87, 0;\n" " @%p13 bra $L__BB22_13;\n" "\n" "$L__BB22_14:\n" " setp.eq.s32 %p14, %r90, 0;\n" " @%p14 bra $L__BB22_17;\n" "\n" " add.s32 %r79, %r88, %r18;\n" " shl.b32 %r80, %r79, 3;\n" " mov.u32 %r81, _ZZ18fc_moe_down_f16_nxE4sred;\n" " add.s32 %r82, %r81, %r80;\n" " shl.b32 %r83, %r16, 2;\n" " add.s32 %r89, %r82, %r83;\n" "\n" "$L__BB22_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f103, [%r89];\n" " add.f32 %f116, %f116, %f103;\n" " add.s32 %r89, %r89, 8;\n" " add.s32 %r90, %r90, -1;\n" " setp.ne.s32 %p15, %r90, 0;\n" " @%p15 bra $L__BB22_16;\n" "\n" "$L__BB22_17:\n" " cvt.s64.s32 %rd53, %r17;\n" " add.s64 %rd54, %rd21, %rd53;\n" " cvta.to.global.u64 %rd55, %rd26;\n" " shl.b64 %rd56, %rd54, 2;\n" " add.s64 %rd57, %rd55, %rd56;\n" " st.global.f32 [%rd57], %f116;\n" "\n" "$L__BB22_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_bf16_nx\n" ".visible .entry fc_moe_down_bf16_nx(\n" " .param .u64 fc_moe_down_bf16_nx_param_0,\n" " .param .u64 fc_moe_down_bf16_nx_param_1,\n" " .param .u64 fc_moe_down_bf16_nx_param_2,\n" " .param .u64 fc_moe_down_bf16_nx_param_3,\n" " .param .u64 fc_moe_down_bf16_nx_param_4,\n" " .param .u32 fc_moe_down_bf16_nx_param_5,\n" " .param .u32 fc_moe_down_bf16_nx_param_6,\n" " .param .u32 fc_moe_down_bf16_nx_param_7,\n" " .param .u32 fc_moe_down_bf16_nx_param_8,\n" " .param .u32 fc_moe_down_bf16_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<16>;\n" " .reg .f32 %f<117>;\n" " .reg .b32 %r<123>;\n" " .reg .b64 %rd<61>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ19fc_moe_down_bf16_nxE4sred[128];\n" "\n" " ld.param.u64 %rd22, [fc_moe_down_bf16_nx_param_0];\n" " ld.param.u64 %rd23, [fc_moe_down_bf16_nx_param_1];\n" " ld.param.u64 %rd24, [fc_moe_down_bf16_nx_param_2];\n" " ld.param.u64 %rd25, [fc_moe_down_bf16_nx_param_3];\n" " ld.param.u64 %rd26, [fc_moe_down_bf16_nx_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_bf16_nx_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_bf16_nx_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_bf16_nx_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_bf16_nx_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_bf16_nx_param_9];\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " cvt.s64.s32 %rd2, %r32;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r37, %ntid.x;\n" " shr.u32 %r38, %r37, 5;\n" " div.u32 %r4, %r38, %r35;\n" " div.s32 %r39, %r2, %r35;\n" " mul.lo.s32 %r40, %r39, %r35;\n" " sub.s32 %r116, %r2, %r40;\n" " mov.u32 %r41, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r41;\n" " add.s32 %r42, %r6, %r39;\n" " shl.b32 %r7, %r42, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r116, %r31;\n" " mov.f32 %f110, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f111, %f110;\n" " @%p3 bra $L__BB23_6;\n" "\n" " cvt.s64.s32 %rd3, %r34;\n" " cvt.s64.s32 %rd4, %r7;\n" " cvt.u64.u32 %rd5, %r3;\n" " mul.wide.s32 %rd27, %r3, 4;\n" " add.s64 %rd6, %rd23, %rd27;\n" " mul.wide.s32 %rd28, %r3, 2;\n" " or.b64 %rd7, %rd28, 256;\n" " shr.s32 %r8, %r33, 8;\n" " add.s32 %r9, %r7, 1;\n" " cvt.s64.s32 %rd8, %r33;\n" " cvt.s64.s32 %rd29, %r31;\n" " mul.lo.s64 %rd9, %rd29, %rd1;\n" " cvta.to.global.u64 %rd10, %rd22;\n" " mov.f32 %f25, 0f00000000;\n" " setp.lt.s32 %p4, %r33, 256;\n" " setp.lt.s32 %p5, %r9, %r32;\n" " shl.b64 %rd49, %rd5, 1;\n" " mov.f32 %f111, %f25;\n" " mov.f32 %f110, %f25;\n" "\n" "$L__BB23_2:\n" " cvt.s64.s32 %rd31, %r116;\n" " add.s64 %rd11, %rd9, %rd31;\n" " shl.b64 %rd32, %rd11, 2;\n" " add.s64 %rd30, %rd24, %rd32;\n" " // begin inline asm\n" " ld.global.nc.s32 %r43, [%rd30];\n" " // end inline asm\n" " mov.f32 %f108, %f25;\n" " mov.f32 %f109, %f25;\n" " @%p4 bra $L__BB23_5;\n" "\n" " cvt.s64.s32 %rd33, %r43;\n" " mul.lo.s64 %rd34, %rd33, %rd2;\n" " add.s64 %rd35, %rd34, %rd4;\n" " mul.lo.s64 %rd36, %rd35, %rd3;\n" " add.s64 %rd60, %rd10, %rd36;\n" " mul.lo.s64 %rd37, %rd11, %rd8;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd59, %rd6, %rd38;\n" " add.s64 %rd39, %rd60, %rd3;\n" " selp.b64 %rd40, %rd39, %rd60, %p5;\n" " add.s64 %rd58, %rd40, %rd7;\n" " mov.f32 %f29, 0f00000000;\n" " mov.u32 %r117, 0;\n" " mov.f32 %f109, %f29;\n" " mov.f32 %f108, %f29;\n" "\n" "$L__BB23_4:\n" " // begin inline asm\n" " ld.global.nc.f32 %f30, [%rd59];\n" " // end inline asm\n" " add.s64 %rd50, %rd60, %rd49;\n" " ld.global.u16 %r45, [%rd50];\n" " shl.b32 %r46, %r45, 16;\n" " mov.b32 %f38, %r46;\n" " fma.rn.f32 %f40, %f38, %f30, %f29;\n" " ld.global.u16 %r47, [%rd58+-256];\n" " shl.b32 %r48, %r47, 16;\n" " mov.b32 %f41, %r48;\n" " fma.rn.f32 %f42, %f41, %f30, %f29;\n" " add.s64 %rd42, %rd59, 128;\n" " // begin inline asm\n" " ld.global.nc.f32 %f31, [%rd42];\n" " // end inline asm\n" " ld.global.u16 %r49, [%rd50+64];\n" " shl.b32 %r50, %r49, 16;\n" " mov.b32 %f43, %r50;\n" " fma.rn.f32 %f44, %f43, %f31, %f40;\n" " ld.global.u16 %r51, [%rd58+-192];\n" " shl.b32 %r52, %r51, 16;\n" " mov.b32 %f45, %r52;\n" " fma.rn.f32 %f46, %f45, %f31, %f42;\n" " add.s64 %rd43, %rd59, 256;\n" " // begin inline asm\n" " ld.global.nc.f32 %f32, [%rd43];\n" " // end inline asm\n" " ld.global.u16 %r53, [%rd50+128];\n" " shl.b32 %r54, %r53, 16;\n" " mov.b32 %f47, %r54;\n" " fma.rn.f32 %f48, %f47, %f32, %f44;\n" " ld.global.u16 %r55, [%rd58+-128];\n" " shl.b32 %r56, %r55, 16;\n" " mov.b32 %f49, %r56;\n" " fma.rn.f32 %f50, %f49, %f32, %f46;\n" " add.s64 %rd44, %rd59, 384;\n" " // begin inline asm\n" " ld.global.nc.f32 %f33, [%rd44];\n" " // end inline asm\n" " ld.global.u16 %r57, [%rd50+192];\n" " shl.b32 %r58, %r57, 16;\n" " mov.b32 %f51, %r58;\n" " fma.rn.f32 %f52, %f51, %f33, %f48;\n" " ld.global.u16 %r59, [%rd58+-64];\n" " shl.b32 %r60, %r59, 16;\n" " mov.b32 %f53, %r60;\n" " fma.rn.f32 %f54, %f53, %f33, %f50;\n" " add.s64 %rd45, %rd59, 512;\n" " // begin inline asm\n" " ld.global.nc.f32 %f34, [%rd45];\n" " // end inline asm\n" " ld.global.u16 %r61, [%rd50+256];\n" " shl.b32 %r62, %r61, 16;\n" " mov.b32 %f55, %r62;\n" " fma.rn.f32 %f56, %f55, %f34, %f52;\n" " ld.global.u16 %r63, [%rd58];\n" " shl.b32 %r64, %r63, 16;\n" " mov.b32 %f57, %r64;\n" " fma.rn.f32 %f58, %f57, %f34, %f54;\n" " add.s64 %rd46, %rd59, 640;\n" " // begin inline asm\n" " ld.global.nc.f32 %f35, [%rd46];\n" " // end inline asm\n" " ld.global.u16 %r65, [%rd50+320];\n" " shl.b32 %r66, %r65, 16;\n" " mov.b32 %f59, %r66;\n" " fma.rn.f32 %f60, %f59, %f35, %f56;\n" " ld.global.u16 %r67, [%rd58+64];\n" " shl.b32 %r68, %r67, 16;\n" " mov.b32 %f61, %r68;\n" " fma.rn.f32 %f62, %f61, %f35, %f58;\n" " add.s64 %rd47, %rd59, 768;\n" " // begin inline asm\n" " ld.global.nc.f32 %f36, [%rd47];\n" " // end inline asm\n" " ld.global.u16 %r69, [%rd50+384];\n" " shl.b32 %r70, %r69, 16;\n" " mov.b32 %f63, %r70;\n" " fma.rn.f32 %f64, %f63, %f36, %f60;\n" " ld.global.u16 %r71, [%rd58+128];\n" " shl.b32 %r72, %r71, 16;\n" " mov.b32 %f65, %r72;\n" " fma.rn.f32 %f66, %f65, %f36, %f62;\n" " add.s64 %rd48, %rd59, 896;\n" " // begin inline asm\n" " ld.global.nc.f32 %f37, [%rd48];\n" " // end inline asm\n" " ld.global.u16 %r73, [%rd50+448];\n" " shl.b32 %r74, %r73, 16;\n" " mov.b32 %f67, %r74;\n" " fma.rn.f32 %f68, %f67, %f37, %f64;\n" " ld.global.u16 %r75, [%rd58+192];\n" " shl.b32 %r76, %r75, 16;\n" " mov.b32 %f69, %r76;\n" " fma.rn.f32 %f70, %f69, %f37, %f66;\n" " add.f32 %f108, %f108, %f68;\n" " add.f32 %f109, %f109, %f70;\n" " add.s64 %rd60, %rd60, 512;\n" " add.s64 %rd59, %rd59, 1024;\n" " add.s64 %rd58, %rd58, 512;\n" " add.s32 %r117, %r117, 1;\n" " setp.lt.s32 %p6, %r117, %r8;\n" " @%p6 bra $L__BB23_4;\n" "\n" "$L__BB23_5:\n" " add.s64 %rd51, %rd25, %rd32;\n" " // begin inline asm\n" " ld.global.nc.f32 %f71, [%rd51];\n" " // end inline asm\n" " fma.rn.f32 %f110, %f71, %f108, %f110;\n" " fma.rn.f32 %f111, %f71, %f109, %f111;\n" " add.s32 %r116, %r116, %r35;\n" " setp.lt.s32 %p7, %r116, %r31;\n" " @%p7 bra $L__BB23_2;\n" "\n" "$L__BB23_6:\n" " mov.u32 %r87, 16;\n" " mov.u32 %r96, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f72, %f110, %r87, %r96;\n" " // end inline asm\n" " add.f32 %f75, %f110, %f72;\n" " mov.u32 %r89, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f74, %f75, %r89, %r96;\n" " // end inline asm\n" " add.f32 %f77, %f75, %f74;\n" " mov.u32 %r91, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f76, %f77, %r91, %r96;\n" " // end inline asm\n" " add.f32 %f79, %f77, %f76;\n" " mov.u32 %r93, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f78, %f79, %r93, %r96;\n" " // end inline asm\n" " add.f32 %f81, %f79, %f78;\n" " mov.u32 %r95, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f80, %f81, %r95, %r96;\n" " // end inline asm\n" " add.f32 %f13, %f81, %f80;\n" " // begin inline asm\n" " shfl.down.b32 %f82, %f111, %r87, %r96;\n" " // end inline asm\n" " add.f32 %f85, %f111, %f82;\n" " // begin inline asm\n" " shfl.down.b32 %f84, %f85, %r89, %r96;\n" " // end inline asm\n" " add.f32 %f87, %f85, %f84;\n" " // begin inline asm\n" " shfl.down.b32 %f86, %f87, %r91, %r96;\n" " // end inline asm\n" " add.f32 %f89, %f87, %f86;\n" " // begin inline asm\n" " shfl.down.b32 %f88, %f89, %r93, %r96;\n" " // end inline asm\n" " add.f32 %f91, %f89, %f88;\n" " // begin inline asm\n" " shfl.down.b32 %f90, %f91, %r95, %r96;\n" " // end inline asm\n" " add.f32 %f14, %f91, %f90;\n" " setp.ne.s32 %p8, %r3, 0;\n" " mul.lo.s64 %rd21, %rd2, %rd1;\n" " @%p8 bra $L__BB23_8;\n" "\n" " shl.b32 %r97, %r2, 3;\n" " mov.u32 %r98, _ZZ19fc_moe_down_bf16_nxE4sred;\n" " add.s32 %r99, %r98, %r97;\n" " st.shared.f32 [%r99], %f13;\n" " st.shared.f32 [%r99+4], %f14;\n" "\n" "$L__BB23_8:\n" " bar.sync 0;\n" " shl.b32 %r100, %r4, 1;\n" " setp.ge.s32 %p9, %r1, %r100;\n" " @%p9 bra $L__BB23_18;\n" "\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r101, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r101, %r16, 1, 31;\n" " setp.ge.s32 %p10, %r17, %r32;\n" " @%p10 bra $L__BB23_18;\n" "\n" " setp.lt.s32 %p11, %r35, 1;\n" " mov.f32 %f116, 0f00000000;\n" " @%p11 bra $L__BB23_17;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r122, %r35, 3;\n" " add.s32 %r103, %r35, -1;\n" " setp.lt.u32 %p12, %r103, 3;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r120, 0;\n" " @%p12 bra $L__BB23_14;\n" "\n" " sub.s32 %r119, %r35, %r122;\n" " mov.f32 %f116, 0f00000000;\n" " mov.u32 %r120, 0;\n" "\n" "$L__BB23_13:\n" " add.s32 %r105, %r120, %r18;\n" " shl.b32 %r106, %r105, 3;\n" " mov.u32 %r107, _ZZ19fc_moe_down_bf16_nxE4sred;\n" " add.s32 %r108, %r107, %r106;\n" " shl.b32 %r109, %r16, 2;\n" " add.s32 %r110, %r108, %r109;\n" " ld.shared.f32 %f96, [%r110];\n" " add.f32 %f97, %f116, %f96;\n" " ld.shared.f32 %f98, [%r110+8];\n" " add.f32 %f99, %f97, %f98;\n" " ld.shared.f32 %f100, [%r110+16];\n" " add.f32 %f101, %f99, %f100;\n" " ld.shared.f32 %f102, [%r110+24];\n" " add.f32 %f116, %f101, %f102;\n" " add.s32 %r120, %r120, 4;\n" " add.s32 %r119, %r119, -4;\n" " setp.ne.s32 %p13, %r119, 0;\n" " @%p13 bra $L__BB23_13;\n" "\n" "$L__BB23_14:\n" " setp.eq.s32 %p14, %r122, 0;\n" " @%p14 bra $L__BB23_17;\n" "\n" " add.s32 %r111, %r120, %r18;\n" " shl.b32 %r112, %r111, 3;\n" " mov.u32 %r113, _ZZ19fc_moe_down_bf16_nxE4sred;\n" " add.s32 %r114, %r113, %r112;\n" " shl.b32 %r115, %r16, 2;\n" " add.s32 %r121, %r114, %r115;\n" "\n" "$L__BB23_16:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f103, [%r121];\n" " add.f32 %f116, %f116, %f103;\n" " add.s32 %r121, %r121, 8;\n" " add.s32 %r122, %r122, -1;\n" " setp.ne.s32 %p15, %r122, 0;\n" " @%p15 bra $L__BB23_16;\n" "\n" "$L__BB23_17:\n" " cvt.s64.s32 %rd53, %r17;\n" " add.s64 %rd54, %rd21, %rd53;\n" " cvta.to.global.u64 %rd55, %rd26;\n" " shl.b64 %rd56, %rd54, 2;\n" " add.s64 %rd57, %rd55, %rd56;\n" " st.global.f32 [%rd57], %f116;\n" "\n" "$L__BB23_18:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_f16_vec_nx\n" ".visible .entry fc_moe_down_f16_vec_nx(\n" " .param .u64 fc_moe_down_f16_vec_nx_param_0,\n" " .param .u64 fc_moe_down_f16_vec_nx_param_1,\n" " .param .u64 fc_moe_down_f16_vec_nx_param_2,\n" " .param .u64 fc_moe_down_f16_vec_nx_param_3,\n" " .param .u64 fc_moe_down_f16_vec_nx_param_4,\n" " .param .u32 fc_moe_down_f16_vec_nx_param_5,\n" " .param .u32 fc_moe_down_f16_vec_nx_param_6,\n" " .param .u32 fc_moe_down_f16_vec_nx_param_7,\n" " .param .u32 fc_moe_down_f16_vec_nx_param_8,\n" " .param .u32 fc_moe_down_f16_vec_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<18>;\n" " .reg .b16 %rs<49>;\n" " .reg .f32 %f<213>;\n" " .reg .b32 %r<165>;\n" " .reg .b64 %rd<66>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ22fc_moe_down_f16_vec_nxE4sred[128];\n" "\n" " ld.param.u64 %rd23, [fc_moe_down_f16_vec_nx_param_0];\n" " ld.param.u64 %rd24, [fc_moe_down_f16_vec_nx_param_1];\n" " ld.param.u64 %rd25, [fc_moe_down_f16_vec_nx_param_2];\n" " ld.param.u64 %rd26, [fc_moe_down_f16_vec_nx_param_3];\n" " ld.param.u64 %rd27, [fc_moe_down_f16_vec_nx_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_f16_vec_nx_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_f16_vec_nx_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_f16_vec_nx_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_f16_vec_nx_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_f16_vec_nx_param_9];\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " cvt.s64.s32 %rd2, %r32;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r37, %ntid.x;\n" " shr.u32 %r38, %r37, 5;\n" " div.u32 %r4, %r38, %r35;\n" " div.s32 %r39, %r2, %r35;\n" " mul.lo.s32 %r40, %r39, %r35;\n" " sub.s32 %r158, %r2, %r40;\n" " mov.u32 %r41, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r41;\n" " add.s32 %r42, %r6, %r39;\n" " shl.b32 %r7, %r42, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r158, %r31;\n" " mov.f32 %f206, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f207, %f206;\n" " @%p3 bra $L__BB24_9;\n" "\n" " cvt.s64.s32 %rd3, %r34;\n" " shr.s32 %r43, %r33, 8;\n" " cvt.s64.s32 %rd4, %r7;\n" " cvt.u64.u32 %rd5, %r3;\n" " max.s32 %r44, %r43, 1;\n" " and.b32 %r8, %r44, 1;\n" " sub.s32 %r9, %r44, %r8;\n" " add.s32 %r10, %r7, 1;\n" " cvt.s64.s32 %rd6, %r33;\n" " cvt.s64.s32 %rd28, %r31;\n" " mul.lo.s64 %rd7, %rd28, %rd1;\n" " cvta.to.global.u64 %rd8, %rd23;\n" " mov.f32 %f31, 0f00000000;\n" " setp.lt.s32 %p4, %r10, %r32;\n" " setp.lt.s32 %p5, %r33, 256;\n" " setp.lt.s32 %p6, %r33, 512;\n" " setp.eq.s32 %p8, %r8, 0;\n" " mov.f32 %f207, %f31;\n" " mov.f32 %f206, %f31;\n" "\n" "$L__BB24_2:\n" " cvt.s64.s32 %rd30, %r158;\n" " add.s64 %rd9, %rd7, %rd30;\n" " shl.b64 %rd31, %rd9, 2;\n" " add.s64 %rd29, %rd25, %rd31;\n" " // begin inline asm\n" " ld.global.nc.s32 %r45, [%rd29];\n" " // end inline asm\n" " cvt.s64.s32 %rd32, %r45;\n" " mul.lo.s64 %rd33, %rd32, %rd2;\n" " add.s64 %rd34, %rd33, %rd4;\n" " mul.lo.s64 %rd35, %rd34, %rd3;\n" " add.s64 %rd65, %rd8, %rd35;\n" " add.s64 %rd36, %rd65, %rd3;\n" " selp.b64 %rd64, %rd36, %rd65, %p4;\n" " mov.f32 %f200, %f31;\n" " mov.f32 %f201, %f31;\n" " @%p5 bra $L__BB24_8;\n" "\n" " mul.lo.s64 %rd37, %rd9, %rd6;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd63, %rd24, %rd38;\n" " mov.f32 %f201, 0f00000000;\n" " mov.f32 %f200, %f201;\n" " @%p6 bra $L__BB24_6;\n" "\n" " mov.f32 %f38, 0f00000000;\n" " mov.f32 %f201, %f38;\n" " mov.f32 %f200, %f38;\n" " mov.u32 %r159, %r9;\n" "\n" "$L__BB24_5:\n" " shl.b64 %rd43, %rd5, 3;\n" " add.s64 %rd44, %rd65, %rd43;\n" " ld.global.v2.u32 {%r46, %r47}, [%rd44];\n" " add.s64 %rd45, %rd64, %rd43;\n" " ld.global.v2.u32 {%r50, %r51}, [%rd45];\n" " shl.b64 %rd46, %rd5, 4;\n" " add.s64 %rd39, %rd63, %rd46;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f39,%f40,%f41,%f42}, [%rd39];\n" " // end inline asm\n" " cvt.u16.u32 %rs1, %r46;\n" " // begin inline asm\n" " cvt.f32.f16 %f43, %rs1;\n" " // end inline asm\n" " fma.rn.f32 %f88, %f43, %f39, %f38;\n" " cvt.u16.u32 %rs2, %r50;\n" " // begin inline asm\n" " cvt.f32.f16 %f44, %rs2;\n" " // end inline asm\n" " fma.rn.f32 %f89, %f44, %f39, %f38;\n" " shr.u32 %r54, %r46, 16;\n" " cvt.u16.u32 %rs3, %r54;\n" " // begin inline asm\n" " cvt.f32.f16 %f45, %rs3;\n" " // end inline asm\n" " fma.rn.f32 %f90, %f45, %f40, %f88;\n" " shr.u32 %r55, %r50, 16;\n" " cvt.u16.u32 %rs4, %r55;\n" " // begin inline asm\n" " cvt.f32.f16 %f46, %rs4;\n" " // end inline asm\n" " fma.rn.f32 %f91, %f46, %f40, %f89;\n" " cvt.u16.u32 %rs5, %r47;\n" " // begin inline asm\n" " cvt.f32.f16 %f47, %rs5;\n" " // end inline asm\n" " fma.rn.f32 %f92, %f47, %f41, %f90;\n" " cvt.u16.u32 %rs6, %r51;\n" " // begin inline asm\n" " cvt.f32.f16 %f48, %rs6;\n" " // end inline asm\n" " fma.rn.f32 %f93, %f48, %f41, %f91;\n" " shr.u32 %r56, %r47, 16;\n" " cvt.u16.u32 %rs7, %r56;\n" " // begin inline asm\n" " cvt.f32.f16 %f49, %rs7;\n" " // end inline asm\n" " fma.rn.f32 %f94, %f49, %f42, %f92;\n" " shr.u32 %r57, %r51, 16;\n" " cvt.u16.u32 %rs8, %r57;\n" " // begin inline asm\n" " cvt.f32.f16 %f50, %rs8;\n" " // end inline asm\n" " fma.rn.f32 %f95, %f50, %f42, %f93;\n" " ld.global.v2.u32 {%r58, %r59}, [%rd44+256];\n" " ld.global.v2.u32 {%r62, %r63}, [%rd45+256];\n" " add.s64 %rd40, %rd39, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f51,%f52,%f53,%f54}, [%rd40];\n" " // end inline asm\n" " cvt.u16.u32 %rs9, %r58;\n" " // begin inline asm\n" " cvt.f32.f16 %f55, %rs9;\n" " // end inline asm\n" " fma.rn.f32 %f96, %f55, %f51, %f94;\n" " cvt.u16.u32 %rs10, %r62;\n" " // begin inline asm\n" " cvt.f32.f16 %f56, %rs10;\n" " // end inline asm\n" " fma.rn.f32 %f97, %f56, %f51, %f95;\n" " shr.u32 %r66, %r58, 16;\n" " cvt.u16.u32 %rs11, %r66;\n" " // begin inline asm\n" " cvt.f32.f16 %f57, %rs11;\n" " // end inline asm\n" " fma.rn.f32 %f98, %f57, %f52, %f96;\n" " shr.u32 %r67, %r62, 16;\n" " cvt.u16.u32 %rs12, %r67;\n" " // begin inline asm\n" " cvt.f32.f16 %f58, %rs12;\n" " // end inline asm\n" " fma.rn.f32 %f99, %f58, %f52, %f97;\n" " cvt.u16.u32 %rs13, %r59;\n" " // begin inline asm\n" " cvt.f32.f16 %f59, %rs13;\n" " // end inline asm\n" " fma.rn.f32 %f100, %f59, %f53, %f98;\n" " cvt.u16.u32 %rs14, %r63;\n" " // begin inline asm\n" " cvt.f32.f16 %f60, %rs14;\n" " // end inline asm\n" " fma.rn.f32 %f101, %f60, %f53, %f99;\n" " shr.u32 %r68, %r59, 16;\n" " cvt.u16.u32 %rs15, %r68;\n" " // begin inline asm\n" " cvt.f32.f16 %f61, %rs15;\n" " // end inline asm\n" " fma.rn.f32 %f102, %f61, %f54, %f100;\n" " shr.u32 %r69, %r63, 16;\n" " cvt.u16.u32 %rs16, %r69;\n" " // begin inline asm\n" " cvt.f32.f16 %f62, %rs16;\n" " // end inline asm\n" " fma.rn.f32 %f103, %f62, %f54, %f101;\n" " add.f32 %f104, %f200, %f102;\n" " add.f32 %f105, %f201, %f103;\n" " ld.global.v2.u32 {%r70, %r71}, [%rd44+512];\n" " ld.global.v2.u32 {%r74, %r75}, [%rd45+512];\n" " add.s64 %rd41, %rd39, 1024;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f63,%f64,%f65,%f66}, [%rd41];\n" " // end inline asm\n" " cvt.u16.u32 %rs17, %r70;\n" " // begin inline asm\n" " cvt.f32.f16 %f67, %rs17;\n" " // end inline asm\n" " fma.rn.f32 %f106, %f67, %f63, %f38;\n" " cvt.u16.u32 %rs18, %r74;\n" " // begin inline asm\n" " cvt.f32.f16 %f68, %rs18;\n" " // end inline asm\n" " fma.rn.f32 %f107, %f68, %f63, %f38;\n" " shr.u32 %r78, %r70, 16;\n" " cvt.u16.u32 %rs19, %r78;\n" " // begin inline asm\n" " cvt.f32.f16 %f69, %rs19;\n" " // end inline asm\n" " fma.rn.f32 %f108, %f69, %f64, %f106;\n" " shr.u32 %r79, %r74, 16;\n" " cvt.u16.u32 %rs20, %r79;\n" " // begin inline asm\n" " cvt.f32.f16 %f70, %rs20;\n" " // end inline asm\n" " fma.rn.f32 %f109, %f70, %f64, %f107;\n" " cvt.u16.u32 %rs21, %r71;\n" " // begin inline asm\n" " cvt.f32.f16 %f71, %rs21;\n" " // end inline asm\n" " fma.rn.f32 %f110, %f71, %f65, %f108;\n" " cvt.u16.u32 %rs22, %r75;\n" " // begin inline asm\n" " cvt.f32.f16 %f72, %rs22;\n" " // end inline asm\n" " fma.rn.f32 %f111, %f72, %f65, %f109;\n" " shr.u32 %r80, %r71, 16;\n" " cvt.u16.u32 %rs23, %r80;\n" " // begin inline asm\n" " cvt.f32.f16 %f73, %rs23;\n" " // end inline asm\n" " fma.rn.f32 %f112, %f73, %f66, %f110;\n" " shr.u32 %r81, %r75, 16;\n" " cvt.u16.u32 %rs24, %r81;\n" " // begin inline asm\n" " cvt.f32.f16 %f74, %rs24;\n" " // end inline asm\n" " fma.rn.f32 %f113, %f74, %f66, %f111;\n" " ld.global.v2.u32 {%r82, %r83}, [%rd44+768];\n" " ld.global.v2.u32 {%r86, %r87}, [%rd45+768];\n" " add.s64 %rd42, %rd39, 1536;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f75,%f76,%f77,%f78}, [%rd42];\n" " // end inline asm\n" " cvt.u16.u32 %rs25, %r82;\n" " // begin inline asm\n" " cvt.f32.f16 %f79, %rs25;\n" " // end inline asm\n" " fma.rn.f32 %f114, %f79, %f75, %f112;\n" " cvt.u16.u32 %rs26, %r86;\n" " // begin inline asm\n" " cvt.f32.f16 %f80, %rs26;\n" " // end inline asm\n" " fma.rn.f32 %f115, %f80, %f75, %f113;\n" " shr.u32 %r90, %r82, 16;\n" " cvt.u16.u32 %rs27, %r90;\n" " // begin inline asm\n" " cvt.f32.f16 %f81, %rs27;\n" " // end inline asm\n" " fma.rn.f32 %f116, %f81, %f76, %f114;\n" " shr.u32 %r91, %r86, 16;\n" " cvt.u16.u32 %rs28, %r91;\n" " // begin inline asm\n" " cvt.f32.f16 %f82, %rs28;\n" " // end inline asm\n" " fma.rn.f32 %f117, %f82, %f76, %f115;\n" " cvt.u16.u32 %rs29, %r83;\n" " // begin inline asm\n" " cvt.f32.f16 %f83, %rs29;\n" " // end inline asm\n" " fma.rn.f32 %f118, %f83, %f77, %f116;\n" " cvt.u16.u32 %rs30, %r87;\n" " // begin inline asm\n" " cvt.f32.f16 %f84, %rs30;\n" " // end inline asm\n" " fma.rn.f32 %f119, %f84, %f77, %f117;\n" " shr.u32 %r92, %r83, 16;\n" " cvt.u16.u32 %rs31, %r92;\n" " // begin inline asm\n" " cvt.f32.f16 %f85, %rs31;\n" " // end inline asm\n" " fma.rn.f32 %f120, %f85, %f78, %f118;\n" " shr.u32 %r93, %r87, 16;\n" " cvt.u16.u32 %rs32, %r93;\n" " // begin inline asm\n" " cvt.f32.f16 %f86, %rs32;\n" " // end inline asm\n" " fma.rn.f32 %f121, %f86, %f78, %f119;\n" " add.f32 %f200, %f104, %f120;\n" " add.f32 %f201, %f105, %f121;\n" " add.s64 %rd65, %rd65, 1024;\n" " add.s64 %rd64, %rd64, 1024;\n" " add.s64 %rd63, %rd63, 2048;\n" " add.s32 %r159, %r159, -2;\n" " setp.ne.s32 %p7, %r159, 0;\n" " @%p7 bra $L__BB24_5;\n" "\n" "$L__BB24_6:\n" " @%p8 bra $L__BB24_8;\n" "\n" " shl.b64 %rd49, %rd5, 3;\n" " add.s64 %rd50, %rd65, %rd49;\n" " ld.global.v2.u32 {%r94, %r95}, [%rd50];\n" " add.s64 %rd51, %rd64, %rd49;\n" " ld.global.v2.u32 {%r98, %r99}, [%rd51];\n" " shl.b64 %rd52, %rd5, 4;\n" " add.s64 %rd47, %rd63, %rd52;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f122,%f123,%f124,%f125}, [%rd47];\n" " // end inline asm\n" " cvt.u16.u32 %rs33, %r94;\n" " // begin inline asm\n" " cvt.f32.f16 %f126, %rs33;\n" " // end inline asm\n" " mov.f32 %f146, 0f00000000;\n" " fma.rn.f32 %f147, %f126, %f122, %f146;\n" " cvt.u16.u32 %rs34, %r98;\n" " // begin inline asm\n" " cvt.f32.f16 %f127, %rs34;\n" " // end inline asm\n" " fma.rn.f32 %f148, %f127, %f122, %f146;\n" " shr.u32 %r102, %r94, 16;\n" " cvt.u16.u32 %rs35, %r102;\n" " // begin inline asm\n" " cvt.f32.f16 %f128, %rs35;\n" " // end inline asm\n" " fma.rn.f32 %f149, %f128, %f123, %f147;\n" " shr.u32 %r103, %r98, 16;\n" " cvt.u16.u32 %rs36, %r103;\n" " // begin inline asm\n" " cvt.f32.f16 %f129, %rs36;\n" " // end inline asm\n" " fma.rn.f32 %f150, %f129, %f123, %f148;\n" " cvt.u16.u32 %rs37, %r95;\n" " // begin inline asm\n" " cvt.f32.f16 %f130, %rs37;\n" " // end inline asm\n" " fma.rn.f32 %f151, %f130, %f124, %f149;\n" " cvt.u16.u32 %rs38, %r99;\n" " // begin inline asm\n" " cvt.f32.f16 %f131, %rs38;\n" " // end inline asm\n" " fma.rn.f32 %f152, %f131, %f124, %f150;\n" " shr.u32 %r104, %r95, 16;\n" " cvt.u16.u32 %rs39, %r104;\n" " // begin inline asm\n" " cvt.f32.f16 %f132, %rs39;\n" " // end inline asm\n" " fma.rn.f32 %f153, %f132, %f125, %f151;\n" " shr.u32 %r105, %r99, 16;\n" " cvt.u16.u32 %rs40, %r105;\n" " // begin inline asm\n" " cvt.f32.f16 %f133, %rs40;\n" " // end inline asm\n" " fma.rn.f32 %f154, %f133, %f125, %f152;\n" " ld.global.v2.u32 {%r106, %r107}, [%rd50+256];\n" " ld.global.v2.u32 {%r110, %r111}, [%rd51+256];\n" " add.s64 %rd48, %rd47, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f134,%f135,%f136,%f137}, [%rd48];\n" " // end inline asm\n" " cvt.u16.u32 %rs41, %r106;\n" " // begin inline asm\n" " cvt.f32.f16 %f138, %rs41;\n" " // end inline asm\n" " fma.rn.f32 %f155, %f138, %f134, %f153;\n" " cvt.u16.u32 %rs42, %r110;\n" " // begin inline asm\n" " cvt.f32.f16 %f139, %rs42;\n" " // end inline asm\n" " fma.rn.f32 %f156, %f139, %f134, %f154;\n" " shr.u32 %r114, %r106, 16;\n" " cvt.u16.u32 %rs43, %r114;\n" " // begin inline asm\n" " cvt.f32.f16 %f140, %rs43;\n" " // end inline asm\n" " fma.rn.f32 %f157, %f140, %f135, %f155;\n" " shr.u32 %r115, %r110, 16;\n" " cvt.u16.u32 %rs44, %r115;\n" " // begin inline asm\n" " cvt.f32.f16 %f141, %rs44;\n" " // end inline asm\n" " fma.rn.f32 %f158, %f141, %f135, %f156;\n" " cvt.u16.u32 %rs45, %r107;\n" " // begin inline asm\n" " cvt.f32.f16 %f142, %rs45;\n" " // end inline asm\n" " fma.rn.f32 %f159, %f142, %f136, %f157;\n" " cvt.u16.u32 %rs46, %r111;\n" " // begin inline asm\n" " cvt.f32.f16 %f143, %rs46;\n" " // end inline asm\n" " fma.rn.f32 %f160, %f143, %f136, %f158;\n" " shr.u32 %r116, %r107, 16;\n" " cvt.u16.u32 %rs47, %r116;\n" " // begin inline asm\n" " cvt.f32.f16 %f144, %rs47;\n" " // end inline asm\n" " fma.rn.f32 %f161, %f144, %f137, %f159;\n" " shr.u32 %r117, %r111, 16;\n" " cvt.u16.u32 %rs48, %r117;\n" " // begin inline asm\n" " cvt.f32.f16 %f145, %rs48;\n" " // end inline asm\n" " fma.rn.f32 %f162, %f145, %f137, %f160;\n" " add.f32 %f200, %f200, %f161;\n" " add.f32 %f201, %f201, %f162;\n" "\n" "$L__BB24_8:\n" " add.s64 %rd53, %rd26, %rd31;\n" " // begin inline asm\n" " ld.global.nc.f32 %f163, [%rd53];\n" " // end inline asm\n" " fma.rn.f32 %f206, %f163, %f200, %f206;\n" " fma.rn.f32 %f207, %f163, %f201, %f207;\n" " add.s32 %r158, %r158, %r35;\n" " setp.lt.s32 %p9, %r158, %r31;\n" " @%p9 bra $L__BB24_2;\n" "\n" "$L__BB24_9:\n" " mov.u32 %r128, 16;\n" " mov.u32 %r137, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f164, %f206, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f167, %f206, %f164;\n" " mov.u32 %r130, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f166, %f167, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f169, %f167, %f166;\n" " mov.u32 %r132, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f168, %f169, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f171, %f169, %f168;\n" " mov.u32 %r134, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f170, %f171, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f173, %f171, %f170;\n" " mov.u32 %r136, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f172, %f173, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f19, %f173, %f172;\n" " // begin inline asm\n" " shfl.down.b32 %f174, %f207, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f177, %f207, %f174;\n" " // begin inline asm\n" " shfl.down.b32 %f176, %f177, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f179, %f177, %f176;\n" " // begin inline asm\n" " shfl.down.b32 %f178, %f179, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f181, %f179, %f178;\n" " // begin inline asm\n" " shfl.down.b32 %f180, %f181, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f183, %f181, %f180;\n" " // begin inline asm\n" " shfl.down.b32 %f182, %f183, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f20, %f183, %f182;\n" " setp.ne.s32 %p10, %r3, 0;\n" " mul.lo.s64 %rd22, %rd2, %rd1;\n" " @%p10 bra $L__BB24_11;\n" "\n" " shl.b32 %r138, %r2, 3;\n" " mov.u32 %r139, _ZZ22fc_moe_down_f16_vec_nxE4sred;\n" " add.s32 %r140, %r139, %r138;\n" " st.shared.f32 [%r140], %f19;\n" " st.shared.f32 [%r140+4], %f20;\n" "\n" "$L__BB24_11:\n" " bar.sync 0;\n" " shl.b32 %r141, %r4, 1;\n" " setp.ge.s32 %p11, %r1, %r141;\n" " @%p11 bra $L__BB24_21;\n" "\n" " ld.param.u32 %r157, [fc_moe_down_f16_vec_nx_param_6];\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r142, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r142, %r16, 1, 31;\n" " setp.ge.s32 %p12, %r17, %r157;\n" " @%p12 bra $L__BB24_21;\n" "\n" " setp.lt.s32 %p13, %r35, 1;\n" " mov.f32 %f212, 0f00000000;\n" " @%p13 bra $L__BB24_20;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r164, %r35, 3;\n" " add.s32 %r144, %r35, -1;\n" " setp.lt.u32 %p14, %r144, 3;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r162, 0;\n" " @%p14 bra $L__BB24_17;\n" "\n" " sub.s32 %r161, %r35, %r164;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r162, 0;\n" "\n" "$L__BB24_16:\n" " add.s32 %r146, %r162, %r18;\n" " shl.b32 %r147, %r146, 3;\n" " mov.u32 %r148, _ZZ22fc_moe_down_f16_vec_nxE4sred;\n" " add.s32 %r149, %r148, %r147;\n" " shl.b32 %r150, %r16, 2;\n" " add.s32 %r151, %r149, %r150;\n" " ld.shared.f32 %f188, [%r151];\n" " add.f32 %f189, %f212, %f188;\n" " ld.shared.f32 %f190, [%r151+8];\n" " add.f32 %f191, %f189, %f190;\n" " ld.shared.f32 %f192, [%r151+16];\n" " add.f32 %f193, %f191, %f192;\n" " ld.shared.f32 %f194, [%r151+24];\n" " add.f32 %f212, %f193, %f194;\n" " add.s32 %r162, %r162, 4;\n" " add.s32 %r161, %r161, -4;\n" " setp.ne.s32 %p15, %r161, 0;\n" " @%p15 bra $L__BB24_16;\n" "\n" "$L__BB24_17:\n" " setp.eq.s32 %p16, %r164, 0;\n" " @%p16 bra $L__BB24_20;\n" "\n" " add.s32 %r152, %r162, %r18;\n" " shl.b32 %r153, %r152, 3;\n" " mov.u32 %r154, _ZZ22fc_moe_down_f16_vec_nxE4sred;\n" " add.s32 %r155, %r154, %r153;\n" " shl.b32 %r156, %r16, 2;\n" " add.s32 %r163, %r155, %r156;\n" "\n" "$L__BB24_19:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f195, [%r163];\n" " add.f32 %f212, %f212, %f195;\n" " add.s32 %r163, %r163, 8;\n" " add.s32 %r164, %r164, -1;\n" " setp.ne.s32 %p17, %r164, 0;\n" " @%p17 bra $L__BB24_19;\n" "\n" "$L__BB24_20:\n" " cvt.s64.s32 %rd55, %r17;\n" " add.s64 %rd56, %rd22, %rd55;\n" " cvta.to.global.u64 %rd57, %rd27;\n" " shl.b64 %rd58, %rd56, 2;\n" " add.s64 %rd59, %rd57, %rd58;\n" " st.global.f32 [%rd59], %f212;\n" "\n" "$L__BB24_21:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_down_bf16_vec_nx\n" ".visible .entry fc_moe_down_bf16_vec_nx(\n" " .param .u64 fc_moe_down_bf16_vec_nx_param_0,\n" " .param .u64 fc_moe_down_bf16_vec_nx_param_1,\n" " .param .u64 fc_moe_down_bf16_vec_nx_param_2,\n" " .param .u64 fc_moe_down_bf16_vec_nx_param_3,\n" " .param .u64 fc_moe_down_bf16_vec_nx_param_4,\n" " .param .u32 fc_moe_down_bf16_vec_nx_param_5,\n" " .param .u32 fc_moe_down_bf16_vec_nx_param_6,\n" " .param .u32 fc_moe_down_bf16_vec_nx_param_7,\n" " .param .u32 fc_moe_down_bf16_vec_nx_param_8,\n" " .param .u32 fc_moe_down_bf16_vec_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<18>;\n" " .reg .f32 %f<213>;\n" " .reg .b32 %r<189>;\n" " .reg .b64 %rd<66>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ23fc_moe_down_bf16_vec_nxE4sred[128];\n" "\n" " ld.param.u64 %rd23, [fc_moe_down_bf16_vec_nx_param_0];\n" " ld.param.u64 %rd24, [fc_moe_down_bf16_vec_nx_param_1];\n" " ld.param.u64 %rd25, [fc_moe_down_bf16_vec_nx_param_2];\n" " ld.param.u64 %rd26, [fc_moe_down_bf16_vec_nx_param_3];\n" " ld.param.u64 %rd27, [fc_moe_down_bf16_vec_nx_param_4];\n" " ld.param.u32 %r31, [fc_moe_down_bf16_vec_nx_param_5];\n" " ld.param.u32 %r32, [fc_moe_down_bf16_vec_nx_param_6];\n" " ld.param.u32 %r33, [fc_moe_down_bf16_vec_nx_param_7];\n" " ld.param.u32 %r34, [fc_moe_down_bf16_vec_nx_param_8];\n" " ld.param.u32 %r35, [fc_moe_down_bf16_vec_nx_param_9];\n" " mov.u32 %r36, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r36;\n" " cvt.s64.s32 %rd2, %r32;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r2, %r1, 5;\n" " and.b32 %r3, %r1, 31;\n" " mov.u32 %r37, %ntid.x;\n" " shr.u32 %r38, %r37, 5;\n" " div.u32 %r4, %r38, %r35;\n" " div.s32 %r39, %r2, %r35;\n" " mul.lo.s32 %r40, %r39, %r35;\n" " sub.s32 %r182, %r2, %r40;\n" " mov.u32 %r41, %ctaid.x;\n" " mul.lo.s32 %r6, %r4, %r41;\n" " add.s32 %r42, %r6, %r39;\n" " shl.b32 %r7, %r42, 1;\n" " setp.ge.s32 %p1, %r7, %r32;\n" " setp.ge.s32 %p2, %r182, %r31;\n" " mov.f32 %f206, 0f00000000;\n" " or.pred %p3, %p1, %p2;\n" " mov.f32 %f207, %f206;\n" " @%p3 bra $L__BB25_9;\n" "\n" " cvt.s64.s32 %rd3, %r34;\n" " shr.s32 %r43, %r33, 8;\n" " cvt.s64.s32 %rd4, %r7;\n" " cvt.u64.u32 %rd5, %r3;\n" " max.s32 %r44, %r43, 1;\n" " and.b32 %r8, %r44, 1;\n" " sub.s32 %r9, %r44, %r8;\n" " add.s32 %r10, %r7, 1;\n" " cvt.s64.s32 %rd6, %r33;\n" " cvt.s64.s32 %rd28, %r31;\n" " mul.lo.s64 %rd7, %rd28, %rd1;\n" " cvta.to.global.u64 %rd8, %rd23;\n" " mov.f32 %f31, 0f00000000;\n" " setp.lt.s32 %p4, %r10, %r32;\n" " setp.lt.s32 %p5, %r33, 256;\n" " setp.lt.s32 %p6, %r33, 512;\n" " setp.eq.s32 %p8, %r8, 0;\n" " mov.f32 %f207, %f31;\n" " mov.f32 %f206, %f31;\n" "\n" "$L__BB25_2:\n" " cvt.s64.s32 %rd30, %r182;\n" " add.s64 %rd9, %rd7, %rd30;\n" " shl.b64 %rd31, %rd9, 2;\n" " add.s64 %rd29, %rd25, %rd31;\n" " // begin inline asm\n" " ld.global.nc.s32 %r45, [%rd29];\n" " // end inline asm\n" " cvt.s64.s32 %rd32, %r45;\n" " mul.lo.s64 %rd33, %rd32, %rd2;\n" " add.s64 %rd34, %rd33, %rd4;\n" " mul.lo.s64 %rd35, %rd34, %rd3;\n" " add.s64 %rd65, %rd8, %rd35;\n" " add.s64 %rd36, %rd65, %rd3;\n" " selp.b64 %rd64, %rd36, %rd65, %p4;\n" " mov.f32 %f200, %f31;\n" " mov.f32 %f201, %f31;\n" " @%p5 bra $L__BB25_8;\n" "\n" " mul.lo.s64 %rd37, %rd9, %rd6;\n" " shl.b64 %rd38, %rd37, 2;\n" " add.s64 %rd63, %rd24, %rd38;\n" " mov.f32 %f201, 0f00000000;\n" " mov.f32 %f200, %f201;\n" " @%p6 bra $L__BB25_6;\n" "\n" " mov.f32 %f38, 0f00000000;\n" " mov.f32 %f201, %f38;\n" " mov.f32 %f200, %f38;\n" " mov.u32 %r183, %r9;\n" "\n" "$L__BB25_5:\n" " shl.b64 %rd43, %rd5, 3;\n" " add.s64 %rd44, %rd65, %rd43;\n" " ld.global.v2.u32 {%r46, %r47}, [%rd44];\n" " add.s64 %rd45, %rd64, %rd43;\n" " ld.global.v2.u32 {%r50, %r51}, [%rd45];\n" " shl.b64 %rd46, %rd5, 4;\n" " add.s64 %rd39, %rd63, %rd46;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f39,%f40,%f41,%f42}, [%rd39];\n" " // end inline asm\n" " shl.b32 %r54, %r46, 16;\n" " mov.b32 %f55, %r54;\n" " fma.rn.f32 %f57, %f55, %f39, %f38;\n" " shl.b32 %r55, %r50, 16;\n" " mov.b32 %f58, %r55;\n" " fma.rn.f32 %f59, %f58, %f39, %f38;\n" " and.b32 %r56, %r46, -65536;\n" " mov.b32 %f60, %r56;\n" " fma.rn.f32 %f61, %f60, %f40, %f57;\n" " and.b32 %r57, %r50, -65536;\n" " mov.b32 %f62, %r57;\n" " fma.rn.f32 %f63, %f62, %f40, %f59;\n" " shl.b32 %r58, %r47, 16;\n" " mov.b32 %f64, %r58;\n" " fma.rn.f32 %f65, %f64, %f41, %f61;\n" " shl.b32 %r59, %r51, 16;\n" " mov.b32 %f66, %r59;\n" " fma.rn.f32 %f67, %f66, %f41, %f63;\n" " and.b32 %r60, %r47, -65536;\n" " mov.b32 %f68, %r60;\n" " fma.rn.f32 %f69, %f68, %f42, %f65;\n" " and.b32 %r61, %r51, -65536;\n" " mov.b32 %f70, %r61;\n" " fma.rn.f32 %f71, %f70, %f42, %f67;\n" " ld.global.v2.u32 {%r62, %r63}, [%rd44+256];\n" " ld.global.v2.u32 {%r66, %r67}, [%rd45+256];\n" " add.s64 %rd40, %rd39, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f43,%f44,%f45,%f46}, [%rd40];\n" " // end inline asm\n" " shl.b32 %r70, %r62, 16;\n" " mov.b32 %f72, %r70;\n" " fma.rn.f32 %f73, %f72, %f43, %f69;\n" " shl.b32 %r71, %r66, 16;\n" " mov.b32 %f74, %r71;\n" " fma.rn.f32 %f75, %f74, %f43, %f71;\n" " and.b32 %r72, %r62, -65536;\n" " mov.b32 %f76, %r72;\n" " fma.rn.f32 %f77, %f76, %f44, %f73;\n" " and.b32 %r73, %r66, -65536;\n" " mov.b32 %f78, %r73;\n" " fma.rn.f32 %f79, %f78, %f44, %f75;\n" " shl.b32 %r74, %r63, 16;\n" " mov.b32 %f80, %r74;\n" " fma.rn.f32 %f81, %f80, %f45, %f77;\n" " shl.b32 %r75, %r67, 16;\n" " mov.b32 %f82, %r75;\n" " fma.rn.f32 %f83, %f82, %f45, %f79;\n" " and.b32 %r76, %r63, -65536;\n" " mov.b32 %f84, %r76;\n" " fma.rn.f32 %f85, %f84, %f46, %f81;\n" " and.b32 %r77, %r67, -65536;\n" " mov.b32 %f86, %r77;\n" " fma.rn.f32 %f87, %f86, %f46, %f83;\n" " add.f32 %f88, %f200, %f85;\n" " add.f32 %f89, %f201, %f87;\n" " ld.global.v2.u32 {%r78, %r79}, [%rd44+512];\n" " ld.global.v2.u32 {%r82, %r83}, [%rd45+512];\n" " add.s64 %rd41, %rd39, 1024;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f47,%f48,%f49,%f50}, [%rd41];\n" " // end inline asm\n" " shl.b32 %r86, %r78, 16;\n" " mov.b32 %f90, %r86;\n" " fma.rn.f32 %f91, %f90, %f47, %f38;\n" " shl.b32 %r87, %r82, 16;\n" " mov.b32 %f92, %r87;\n" " fma.rn.f32 %f93, %f92, %f47, %f38;\n" " and.b32 %r88, %r78, -65536;\n" " mov.b32 %f94, %r88;\n" " fma.rn.f32 %f95, %f94, %f48, %f91;\n" " and.b32 %r89, %r82, -65536;\n" " mov.b32 %f96, %r89;\n" " fma.rn.f32 %f97, %f96, %f48, %f93;\n" " shl.b32 %r90, %r79, 16;\n" " mov.b32 %f98, %r90;\n" " fma.rn.f32 %f99, %f98, %f49, %f95;\n" " shl.b32 %r91, %r83, 16;\n" " mov.b32 %f100, %r91;\n" " fma.rn.f32 %f101, %f100, %f49, %f97;\n" " and.b32 %r92, %r79, -65536;\n" " mov.b32 %f102, %r92;\n" " fma.rn.f32 %f103, %f102, %f50, %f99;\n" " and.b32 %r93, %r83, -65536;\n" " mov.b32 %f104, %r93;\n" " fma.rn.f32 %f105, %f104, %f50, %f101;\n" " ld.global.v2.u32 {%r94, %r95}, [%rd44+768];\n" " ld.global.v2.u32 {%r98, %r99}, [%rd45+768];\n" " add.s64 %rd42, %rd39, 1536;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f51,%f52,%f53,%f54}, [%rd42];\n" " // end inline asm\n" " shl.b32 %r102, %r94, 16;\n" " mov.b32 %f106, %r102;\n" " fma.rn.f32 %f107, %f106, %f51, %f103;\n" " shl.b32 %r103, %r98, 16;\n" " mov.b32 %f108, %r103;\n" " fma.rn.f32 %f109, %f108, %f51, %f105;\n" " and.b32 %r104, %r94, -65536;\n" " mov.b32 %f110, %r104;\n" " fma.rn.f32 %f111, %f110, %f52, %f107;\n" " and.b32 %r105, %r98, -65536;\n" " mov.b32 %f112, %r105;\n" " fma.rn.f32 %f113, %f112, %f52, %f109;\n" " shl.b32 %r106, %r95, 16;\n" " mov.b32 %f114, %r106;\n" " fma.rn.f32 %f115, %f114, %f53, %f111;\n" " shl.b32 %r107, %r99, 16;\n" " mov.b32 %f116, %r107;\n" " fma.rn.f32 %f117, %f116, %f53, %f113;\n" " and.b32 %r108, %r95, -65536;\n" " mov.b32 %f118, %r108;\n" " fma.rn.f32 %f119, %f118, %f54, %f115;\n" " and.b32 %r109, %r99, -65536;\n" " mov.b32 %f120, %r109;\n" " fma.rn.f32 %f121, %f120, %f54, %f117;\n" " add.f32 %f200, %f88, %f119;\n" " add.f32 %f201, %f89, %f121;\n" " add.s64 %rd65, %rd65, 1024;\n" " add.s64 %rd64, %rd64, 1024;\n" " add.s64 %rd63, %rd63, 2048;\n" " add.s32 %r183, %r183, -2;\n" " setp.ne.s32 %p7, %r183, 0;\n" " @%p7 bra $L__BB25_5;\n" "\n" "$L__BB25_6:\n" " @%p8 bra $L__BB25_8;\n" "\n" " shl.b64 %rd49, %rd5, 3;\n" " add.s64 %rd50, %rd65, %rd49;\n" " ld.global.v2.u32 {%r110, %r111}, [%rd50];\n" " add.s64 %rd51, %rd64, %rd49;\n" " ld.global.v2.u32 {%r114, %r115}, [%rd51];\n" " shl.b64 %rd52, %rd5, 4;\n" " add.s64 %rd47, %rd63, %rd52;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f122,%f123,%f124,%f125}, [%rd47];\n" " // end inline asm\n" " shl.b32 %r118, %r110, 16;\n" " mov.b32 %f130, %r118;\n" " mov.f32 %f131, 0f00000000;\n" " fma.rn.f32 %f132, %f130, %f122, %f131;\n" " shl.b32 %r119, %r114, 16;\n" " mov.b32 %f133, %r119;\n" " fma.rn.f32 %f134, %f133, %f122, %f131;\n" " and.b32 %r120, %r110, -65536;\n" " mov.b32 %f135, %r120;\n" " fma.rn.f32 %f136, %f135, %f123, %f132;\n" " and.b32 %r121, %r114, -65536;\n" " mov.b32 %f137, %r121;\n" " fma.rn.f32 %f138, %f137, %f123, %f134;\n" " shl.b32 %r122, %r111, 16;\n" " mov.b32 %f139, %r122;\n" " fma.rn.f32 %f140, %f139, %f124, %f136;\n" " shl.b32 %r123, %r115, 16;\n" " mov.b32 %f141, %r123;\n" " fma.rn.f32 %f142, %f141, %f124, %f138;\n" " and.b32 %r124, %r111, -65536;\n" " mov.b32 %f143, %r124;\n" " fma.rn.f32 %f144, %f143, %f125, %f140;\n" " and.b32 %r125, %r115, -65536;\n" " mov.b32 %f145, %r125;\n" " fma.rn.f32 %f146, %f145, %f125, %f142;\n" " ld.global.v2.u32 {%r126, %r127}, [%rd50+256];\n" " ld.global.v2.u32 {%r130, %r131}, [%rd51+256];\n" " add.s64 %rd48, %rd47, 512;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f126,%f127,%f128,%f129}, [%rd48];\n" " // end inline asm\n" " shl.b32 %r134, %r126, 16;\n" " mov.b32 %f147, %r134;\n" " fma.rn.f32 %f148, %f147, %f126, %f144;\n" " shl.b32 %r135, %r130, 16;\n" " mov.b32 %f149, %r135;\n" " fma.rn.f32 %f150, %f149, %f126, %f146;\n" " and.b32 %r136, %r126, -65536;\n" " mov.b32 %f151, %r136;\n" " fma.rn.f32 %f152, %f151, %f127, %f148;\n" " and.b32 %r137, %r130, -65536;\n" " mov.b32 %f153, %r137;\n" " fma.rn.f32 %f154, %f153, %f127, %f150;\n" " shl.b32 %r138, %r127, 16;\n" " mov.b32 %f155, %r138;\n" " fma.rn.f32 %f156, %f155, %f128, %f152;\n" " shl.b32 %r139, %r131, 16;\n" " mov.b32 %f157, %r139;\n" " fma.rn.f32 %f158, %f157, %f128, %f154;\n" " and.b32 %r140, %r127, -65536;\n" " mov.b32 %f159, %r140;\n" " fma.rn.f32 %f160, %f159, %f129, %f156;\n" " and.b32 %r141, %r131, -65536;\n" " mov.b32 %f161, %r141;\n" " fma.rn.f32 %f162, %f161, %f129, %f158;\n" " add.f32 %f200, %f200, %f160;\n" " add.f32 %f201, %f201, %f162;\n" "\n" "$L__BB25_8:\n" " add.s64 %rd53, %rd26, %rd31;\n" " // begin inline asm\n" " ld.global.nc.f32 %f163, [%rd53];\n" " // end inline asm\n" " fma.rn.f32 %f206, %f163, %f200, %f206;\n" " fma.rn.f32 %f207, %f163, %f201, %f207;\n" " add.s32 %r182, %r182, %r35;\n" " setp.lt.s32 %p9, %r182, %r31;\n" " @%p9 bra $L__BB25_2;\n" "\n" "$L__BB25_9:\n" " mov.u32 %r152, 16;\n" " mov.u32 %r161, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f164, %f206, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f167, %f206, %f164;\n" " mov.u32 %r154, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f166, %f167, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f169, %f167, %f166;\n" " mov.u32 %r156, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f168, %f169, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f171, %f169, %f168;\n" " mov.u32 %r158, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f170, %f171, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f173, %f171, %f170;\n" " mov.u32 %r160, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f172, %f173, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f19, %f173, %f172;\n" " // begin inline asm\n" " shfl.down.b32 %f174, %f207, %r152, %r161;\n" " // end inline asm\n" " add.f32 %f177, %f207, %f174;\n" " // begin inline asm\n" " shfl.down.b32 %f176, %f177, %r154, %r161;\n" " // end inline asm\n" " add.f32 %f179, %f177, %f176;\n" " // begin inline asm\n" " shfl.down.b32 %f178, %f179, %r156, %r161;\n" " // end inline asm\n" " add.f32 %f181, %f179, %f178;\n" " // begin inline asm\n" " shfl.down.b32 %f180, %f181, %r158, %r161;\n" " // end inline asm\n" " add.f32 %f183, %f181, %f180;\n" " // begin inline asm\n" " shfl.down.b32 %f182, %f183, %r160, %r161;\n" " // end inline asm\n" " add.f32 %f20, %f183, %f182;\n" " setp.ne.s32 %p10, %r3, 0;\n" " mul.lo.s64 %rd22, %rd2, %rd1;\n" " @%p10 bra $L__BB25_11;\n" "\n" " shl.b32 %r162, %r2, 3;\n" " mov.u32 %r163, _ZZ23fc_moe_down_bf16_vec_nxE4sred;\n" " add.s32 %r164, %r163, %r162;\n" " st.shared.f32 [%r164], %f19;\n" " st.shared.f32 [%r164+4], %f20;\n" "\n" "$L__BB25_11:\n" " bar.sync 0;\n" " shl.b32 %r165, %r4, 1;\n" " setp.ge.s32 %p11, %r1, %r165;\n" " @%p11 bra $L__BB25_21;\n" "\n" " ld.param.u32 %r181, [fc_moe_down_bf16_vec_nx_param_6];\n" " shr.u32 %r15, %r1, 1;\n" " add.s32 %r166, %r6, %r15;\n" " and.b32 %r16, %r1, 1;\n" " bfi.b32 %r17, %r166, %r16, 1, 31;\n" " setp.ge.s32 %p12, %r17, %r181;\n" " @%p12 bra $L__BB25_21;\n" "\n" " setp.lt.s32 %p13, %r35, 1;\n" " mov.f32 %f212, 0f00000000;\n" " @%p13 bra $L__BB25_20;\n" "\n" " mul.lo.s32 %r18, %r15, %r35;\n" " and.b32 %r188, %r35, 3;\n" " add.s32 %r168, %r35, -1;\n" " setp.lt.u32 %p14, %r168, 3;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r186, 0;\n" " @%p14 bra $L__BB25_17;\n" "\n" " sub.s32 %r185, %r35, %r188;\n" " mov.f32 %f212, 0f00000000;\n" " mov.u32 %r186, 0;\n" "\n" "$L__BB25_16:\n" " add.s32 %r170, %r186, %r18;\n" " shl.b32 %r171, %r170, 3;\n" " mov.u32 %r172, _ZZ23fc_moe_down_bf16_vec_nxE4sred;\n" " add.s32 %r173, %r172, %r171;\n" " shl.b32 %r174, %r16, 2;\n" " add.s32 %r175, %r173, %r174;\n" " ld.shared.f32 %f188, [%r175];\n" " add.f32 %f189, %f212, %f188;\n" " ld.shared.f32 %f190, [%r175+8];\n" " add.f32 %f191, %f189, %f190;\n" " ld.shared.f32 %f192, [%r175+16];\n" " add.f32 %f193, %f191, %f192;\n" " ld.shared.f32 %f194, [%r175+24];\n" " add.f32 %f212, %f193, %f194;\n" " add.s32 %r186, %r186, 4;\n" " add.s32 %r185, %r185, -4;\n" " setp.ne.s32 %p15, %r185, 0;\n" " @%p15 bra $L__BB25_16;\n" "\n" "$L__BB25_17:\n" " setp.eq.s32 %p16, %r188, 0;\n" " @%p16 bra $L__BB25_20;\n" "\n" " add.s32 %r176, %r186, %r18;\n" " shl.b32 %r177, %r176, 3;\n" " mov.u32 %r178, _ZZ23fc_moe_down_bf16_vec_nxE4sred;\n" " add.s32 %r179, %r178, %r177;\n" " shl.b32 %r180, %r16, 2;\n" " add.s32 %r187, %r179, %r180;\n" "\n" "$L__BB25_19:\n" " .pragma \"nounroll\";\n" " ld.shared.f32 %f195, [%r187];\n" " add.f32 %f212, %f212, %f195;\n" " add.s32 %r187, %r187, 8;\n" " add.s32 %r188, %r188, -1;\n" " setp.ne.s32 %p17, %r188, 0;\n" " @%p17 bra $L__BB25_19;\n" "\n" "$L__BB25_20:\n" " cvt.s64.s32 %rd55, %r17;\n" " add.s64 %rd56, %rd22, %rd55;\n" " cvta.to.global.u64 %rd57, %rd27;\n" " shl.b64 %rd58, %rd56, 2;\n" " add.s64 %rd59, %rd57, %rd58;\n" " st.global.f32 [%rd59], %f212;\n" "\n" "$L__BB25_21:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_silu_mul\n" ".visible .entry fc_moe_silu_mul(\n" " .param .u64 fc_moe_silu_mul_param_0,\n" " .param .u64 fc_moe_silu_mul_param_1,\n" " .param .u64 fc_moe_silu_mul_param_2,\n" " .param .u32 fc_moe_silu_mul_param_3\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<22>;\n" " .reg .b32 %r<8>;\n" " .reg .b64 %rd<11>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_moe_silu_mul_param_0];\n" " ld.param.u64 %rd2, [fc_moe_silu_mul_param_1];\n" " ld.param.u64 %rd3, [fc_moe_silu_mul_param_2];\n" " ld.param.u32 %r2, [fc_moe_silu_mul_param_3];\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r1, %r3, %r4, %r5;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB26_2;\n" "\n" " cvta.to.global.u64 %rd4, %rd1;\n" " mul.wide.s32 %rd5, %r1, 4;\n" " add.s64 %rd6, %rd4, %rd5;\n" " ld.global.f32 %f1, [%rd6];\n" " neg.f32 %f2, %f1;\n" " mov.f32 %f3, 0f3F000000;\n" " mov.f32 %f4, 0f3BBB989D;\n" " fma.rn.f32 %f5, %f2, %f4, %f3;\n" " mov.f32 %f6, 0f3FB8AA3B;\n" " mov.f32 %f7, 0f437C0000;\n" " cvt.sat.f32.f32 %f8, %f5;\n" " mov.f32 %f9, 0f4B400001;\n" " fma.rm.f32 %f10, %f8, %f7, %f9;\n" " add.f32 %f11, %f10, 0fCB40007F;\n" " neg.f32 %f12, %f11;\n" " fma.rn.f32 %f13, %f2, %f6, %f12;\n" " mov.f32 %f14, 0f32A57060;\n" " fma.rn.f32 %f15, %f2, %f14, %f13;\n" " mov.b32 %r6, %f10;\n" " shl.b32 %r7, %r6, 23;\n" " mov.b32 %f16, %r7;\n" " ex2.approx.ftz.f32 %f17, %f15;\n" " fma.rn.f32 %f18, %f17, %f16, 0f3F800000;\n" " div.rn.f32 %f19, %f1, %f18;\n" " cvta.to.global.u64 %rd7, %rd2;\n" " add.s64 %rd8, %rd7, %rd5;\n" " ld.global.f32 %f20, [%rd8];\n" " mul.f32 %f21, %f20, %f19;\n" " cvta.to.global.u64 %rd9, %rd3;\n" " add.s64 %rd10, %rd9, %rd5;\n" " st.global.f32 [%rd10], %f21;\n" "\n" "$L__BB26_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_shexp_dot_gate_nx\n" ".visible .entry fc_moe_shexp_dot_gate_nx(\n" " .param .u64 fc_moe_shexp_dot_gate_nx_param_0,\n" " .param .u64 fc_moe_shexp_dot_gate_nx_param_1,\n" " .param .u64 fc_moe_shexp_dot_gate_nx_param_2,\n" " .param .u32 fc_moe_shexp_dot_gate_nx_param_3\n" ")\n" "{\n" " .reg .pred %p<14>;\n" " .reg .f32 %f<62>;\n" " .reg .b32 %r<56>;\n" " .reg .b64 %rd<50>;\n" "\n" "\n" " ld.param.u64 %rd18, [fc_moe_shexp_dot_gate_nx_param_0];\n" " ld.param.u64 %rd19, [fc_moe_shexp_dot_gate_nx_param_1];\n" " ld.param.u64 %rd20, [fc_moe_shexp_dot_gate_nx_param_2];\n" " ld.param.u32 %r25, [fc_moe_shexp_dot_gate_nx_param_3];\n" " cvta.to.global.u64 %rd1, %rd19;\n" " cvta.to.global.u64 %rd2, %rd18;\n" " cvta.to.global.u64 %rd3, %rd20;\n" " mov.u32 %r26, %ctaid.x;\n" " mul.wide.s32 %rd4, %r25, %r26;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r54, %tid.x;\n" " setp.ge.s32 %p1, %r54, %r25;\n" " mov.f32 %f61, 0f00000000;\n" " @%p1 bra $L__BB27_7;\n" "\n" " not.b32 %r27, %r54;\n" " add.s32 %r28, %r27, %r25;\n" " div.u32 %r3, %r28, %r1;\n" " add.s32 %r29, %r3, 1;\n" " and.b32 %r48, %r29, 3;\n" " setp.eq.s32 %p2, %r48, 0;\n" " mov.f32 %f61, 0f00000000;\n" " mov.u32 %r49, %r54;\n" " @%p2 bra $L__BB27_4;\n" "\n" " cvt.s64.s32 %rd21, %r54;\n" " add.s64 %rd22, %rd4, %rd21;\n" " shl.b64 %rd23, %rd22, 2;\n" " add.s64 %rd48, %rd1, %rd23;\n" " mul.wide.s32 %rd6, %r1, 4;\n" " mul.wide.s32 %rd24, %r54, 4;\n" " add.s64 %rd47, %rd2, %rd24;\n" " mov.f32 %f61, 0f00000000;\n" " mov.u32 %r49, %r54;\n" "\n" "$L__BB27_3:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f13, [%rd48];\n" " ld.global.f32 %f14, [%rd47];\n" " fma.rn.f32 %f61, %f14, %f13, %f61;\n" " add.s32 %r49, %r49, %r1;\n" " add.s64 %rd48, %rd48, %rd6;\n" " add.s64 %rd47, %rd47, %rd6;\n" " add.s32 %r48, %r48, -1;\n" " setp.ne.s32 %p3, %r48, 0;\n" " @%p3 bra $L__BB27_3;\n" "\n" "$L__BB27_4:\n" " setp.lt.u32 %p4, %r3, 3;\n" " @%p4 bra $L__BB27_7;\n" "\n" " mul.wide.s32 %rd12, %r1, 4;\n" "\n" "$L__BB27_6:\n" " cvt.s64.s32 %rd25, %r49;\n" " mul.wide.s32 %rd26, %r49, 4;\n" " add.s64 %rd27, %rd2, %rd26;\n" " add.s64 %rd28, %rd4, %rd25;\n" " shl.b64 %rd29, %rd28, 2;\n" " add.s64 %rd30, %rd1, %rd29;\n" " ld.global.f32 %f15, [%rd30];\n" " ld.global.f32 %f16, [%rd27];\n" " fma.rn.f32 %f17, %f16, %f15, %f61;\n" " add.s64 %rd31, %rd27, %rd12;\n" " add.s64 %rd32, %rd30, %rd12;\n" " ld.global.f32 %f18, [%rd32];\n" " ld.global.f32 %f19, [%rd31];\n" " fma.rn.f32 %f20, %f19, %f18, %f17;\n" " add.s32 %r30, %r49, %r1;\n" " add.s32 %r31, %r30, %r1;\n" " add.s64 %rd33, %rd31, %rd12;\n" " add.s64 %rd34, %rd32, %rd12;\n" " ld.global.f32 %f21, [%rd34];\n" " ld.global.f32 %f22, [%rd33];\n" " fma.rn.f32 %f23, %f22, %f21, %f20;\n" " add.s32 %r32, %r31, %r1;\n" " add.s64 %rd35, %rd33, %rd12;\n" " add.s64 %rd36, %rd34, %rd12;\n" " ld.global.f32 %f24, [%rd36];\n" " ld.global.f32 %f25, [%rd35];\n" " fma.rn.f32 %f61, %f25, %f24, %f23;\n" " add.s32 %r49, %r32, %r1;\n" " setp.lt.s32 %p5, %r49, %r25;\n" " @%p5 bra $L__BB27_6;\n" "\n" "$L__BB27_7:\n" " shl.b32 %r33, %r54, 2;\n" " mov.u32 %r34, sdata;\n" " add.s32 %r12, %r34, %r33;\n" " st.shared.f32 [%r12], %f61;\n" " bar.sync 0;\n" " setp.lt.s32 %p6, %r1, 2;\n" " @%p6 bra $L__BB27_12;\n" "\n" " shr.u32 %r35, %r1, 31;\n" " add.s32 %r36, %r1, %r35;\n" " shr.s32 %r51, %r36, 1;\n" "\n" "$L__BB27_9:\n" " setp.ge.s32 %p7, %r54, %r51;\n" " @%p7 bra $L__BB27_11;\n" "\n" " shl.b32 %r37, %r51, 2;\n" " add.s32 %r38, %r12, %r37;\n" " ld.shared.f32 %f26, [%r12];\n" " ld.shared.f32 %f27, [%r38];\n" " add.f32 %f28, %f27, %f26;\n" " st.shared.f32 [%r12], %f28;\n" "\n" "$L__BB27_11:\n" " bar.sync 0;\n" " shr.s32 %r15, %r51, 1;\n" " setp.gt.s32 %p8, %r51, 1;\n" " mov.u32 %r51, %r15;\n" " @%p8 bra $L__BB27_9;\n" "\n" "$L__BB27_12:\n" " ld.shared.f32 %f29, [sdata];\n" " neg.f32 %f30, %f29;\n" " mov.f32 %f31, 0f3F000000;\n" " mov.f32 %f32, 0f3BBB989D;\n" " fma.rn.f32 %f33, %f30, %f32, %f31;\n" " mov.f32 %f34, 0f3FB8AA3B;\n" " mov.f32 %f35, 0f437C0000;\n" " cvt.sat.f32.f32 %f36, %f33;\n" " mov.f32 %f37, 0f4B400001;\n" " fma.rm.f32 %f38, %f36, %f35, %f37;\n" " add.f32 %f39, %f38, 0fCB40007F;\n" " neg.f32 %f40, %f39;\n" " fma.rn.f32 %f41, %f30, %f34, %f40;\n" " mov.f32 %f42, 0f32A57060;\n" " fma.rn.f32 %f43, %f30, %f42, %f41;\n" " mov.b32 %r39, %f38;\n" " shl.b32 %r40, %r39, 23;\n" " mov.b32 %f44, %r40;\n" " ex2.approx.ftz.f32 %f45, %f43;\n" " fma.rn.f32 %f46, %f45, %f44, 0f3F800000;\n" " rcp.rn.f32 %f8, %f46;\n" " @%p1 bra $L__BB27_19;\n" "\n" " not.b32 %r41, %r54;\n" " add.s32 %r42, %r41, %r25;\n" " div.u32 %r16, %r42, %r1;\n" " add.s32 %r43, %r16, 1;\n" " and.b32 %r53, %r43, 3;\n" " setp.eq.s32 %p10, %r53, 0;\n" " @%p10 bra $L__BB27_16;\n" "\n" " cvt.s64.s32 %rd37, %r54;\n" " add.s64 %rd38, %rd4, %rd37;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd49, %rd3, %rd39;\n" " mul.wide.s32 %rd14, %r1, 4;\n" "\n" "$L__BB27_15:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f47, [%rd49];\n" " mul.f32 %f48, %f8, %f47;\n" " st.global.f32 [%rd49], %f48;\n" " add.s32 %r54, %r54, %r1;\n" " add.s64 %rd49, %rd49, %rd14;\n" " add.s32 %r53, %r53, -1;\n" " setp.ne.s32 %p11, %r53, 0;\n" " @%p11 bra $L__BB27_15;\n" "\n" "$L__BB27_16:\n" " setp.lt.u32 %p12, %r16, 3;\n" " @%p12 bra $L__BB27_19;\n" "\n" " mul.wide.s32 %rd17, %r1, 4;\n" "\n" "$L__BB27_18:\n" " cvt.s64.s32 %rd40, %r54;\n" " add.s64 %rd41, %rd4, %rd40;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd43, %rd3, %rd42;\n" " ld.global.f32 %f49, [%rd43];\n" " mul.f32 %f50, %f8, %f49;\n" " st.global.f32 [%rd43], %f50;\n" " add.s64 %rd44, %rd43, %rd17;\n" " ld.global.f32 %f51, [%rd44];\n" " mul.f32 %f52, %f8, %f51;\n" " st.global.f32 [%rd44], %f52;\n" " add.s32 %r44, %r54, %r1;\n" " add.s32 %r45, %r44, %r1;\n" " add.s64 %rd45, %rd44, %rd17;\n" " ld.global.f32 %f53, [%rd45];\n" " mul.f32 %f54, %f8, %f53;\n" " st.global.f32 [%rd45], %f54;\n" " add.s32 %r46, %r45, %r1;\n" " add.s64 %rd46, %rd45, %rd17;\n" " ld.global.f32 %f55, [%rd46];\n" " mul.f32 %f56, %f8, %f55;\n" " st.global.f32 [%rd46], %f56;\n" " add.s32 %r54, %r46, %r1;\n" " setp.lt.s32 %p13, %r54, %r25;\n" " @%p13 bra $L__BB27_18;\n" "\n" "$L__BB27_19:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_shexp_dot_gate\n" ".visible .entry fc_moe_shexp_dot_gate(\n" " .param .u64 fc_moe_shexp_dot_gate_param_0,\n" " .param .u64 fc_moe_shexp_dot_gate_param_1,\n" " .param .u64 fc_moe_shexp_dot_gate_param_2,\n" " .param .u32 fc_moe_shexp_dot_gate_param_3\n" ")\n" "{\n" " .reg .pred %p<14>;\n" " .reg .f32 %f<62>;\n" " .reg .b32 %r<57>;\n" " .reg .b64 %rd<39>;\n" "\n" "\n" " ld.param.u64 %rd15, [fc_moe_shexp_dot_gate_param_0];\n" " ld.param.u64 %rd16, [fc_moe_shexp_dot_gate_param_1];\n" " ld.param.u64 %rd17, [fc_moe_shexp_dot_gate_param_2];\n" " ld.param.u32 %r27, [fc_moe_shexp_dot_gate_param_3];\n" " cvta.to.global.u64 %rd1, %rd16;\n" " cvta.to.global.u64 %rd2, %rd15;\n" " cvta.to.global.u64 %rd3, %rd17;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r55, %tid.x;\n" " setp.ge.s32 %p1, %r55, %r27;\n" " mov.f32 %f61, 0f00000000;\n" " @%p1 bra $L__BB28_7;\n" "\n" " not.b32 %r28, %r55;\n" " add.s32 %r29, %r28, %r27;\n" " div.u32 %r3, %r29, %r1;\n" " add.s32 %r30, %r3, 1;\n" " and.b32 %r49, %r30, 3;\n" " setp.eq.s32 %p2, %r49, 0;\n" " mov.f32 %f61, 0f00000000;\n" " mov.u32 %r50, %r55;\n" " @%p2 bra $L__BB28_4;\n" "\n" " mul.wide.s32 %rd18, %r55, 4;\n" " add.s64 %rd37, %rd1, %rd18;\n" " mul.wide.s32 %rd5, %r1, 4;\n" " add.s64 %rd36, %rd2, %rd18;\n" " mov.f32 %f61, 0f00000000;\n" " mov.u32 %r50, %r55;\n" "\n" "$L__BB28_3:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f13, [%rd37];\n" " ld.global.f32 %f14, [%rd36];\n" " fma.rn.f32 %f61, %f14, %f13, %f61;\n" " add.s32 %r50, %r50, %r1;\n" " add.s64 %rd37, %rd37, %rd5;\n" " add.s64 %rd36, %rd36, %rd5;\n" " add.s32 %r49, %r49, -1;\n" " setp.ne.s32 %p3, %r49, 0;\n" " @%p3 bra $L__BB28_3;\n" "\n" "$L__BB28_4:\n" " setp.lt.u32 %p4, %r3, 3;\n" " @%p4 bra $L__BB28_7;\n" "\n" " shl.b32 %r10, %r1, 2;\n" " cvt.s64.s32 %rd22, %r10;\n" "\n" "$L__BB28_6:\n" " mul.wide.s32 %rd19, %r50, 4;\n" " add.s64 %rd20, %rd2, %rd19;\n" " add.s64 %rd21, %rd1, %rd19;\n" " ld.global.f32 %f15, [%rd21];\n" " ld.global.f32 %f16, [%rd20];\n" " fma.rn.f32 %f17, %f16, %f15, %f61;\n" " add.s64 %rd23, %rd20, %rd22;\n" " add.s64 %rd24, %rd21, %rd22;\n" " ld.global.f32 %f18, [%rd24];\n" " ld.global.f32 %f19, [%rd23];\n" " fma.rn.f32 %f20, %f19, %f18, %f17;\n" " add.s32 %r31, %r50, %r1;\n" " add.s32 %r32, %r31, %r1;\n" " add.s64 %rd25, %rd23, %rd22;\n" " add.s64 %rd26, %rd24, %rd22;\n" " ld.global.f32 %f21, [%rd26];\n" " ld.global.f32 %f22, [%rd25];\n" " fma.rn.f32 %f23, %f22, %f21, %f20;\n" " add.s32 %r33, %r32, %r1;\n" " add.s64 %rd27, %rd25, %rd22;\n" " add.s64 %rd28, %rd26, %rd22;\n" " ld.global.f32 %f24, [%rd28];\n" " ld.global.f32 %f25, [%rd27];\n" " fma.rn.f32 %f61, %f25, %f24, %f23;\n" " add.s32 %r50, %r33, %r1;\n" " setp.lt.s32 %p5, %r50, %r27;\n" " @%p5 bra $L__BB28_6;\n" "\n" "$L__BB28_7:\n" " shl.b32 %r34, %r55, 2;\n" " mov.u32 %r35, sdata;\n" " add.s32 %r13, %r35, %r34;\n" " st.shared.f32 [%r13], %f61;\n" " bar.sync 0;\n" " setp.lt.s32 %p6, %r1, 2;\n" " @%p6 bra $L__BB28_12;\n" "\n" " shr.u32 %r36, %r1, 31;\n" " add.s32 %r37, %r1, %r36;\n" " shr.s32 %r52, %r37, 1;\n" "\n" "$L__BB28_9:\n" " setp.ge.s32 %p7, %r55, %r52;\n" " @%p7 bra $L__BB28_11;\n" "\n" " shl.b32 %r38, %r52, 2;\n" " add.s32 %r39, %r13, %r38;\n" " ld.shared.f32 %f26, [%r13];\n" " ld.shared.f32 %f27, [%r39];\n" " add.f32 %f28, %f27, %f26;\n" " st.shared.f32 [%r13], %f28;\n" "\n" "$L__BB28_11:\n" " bar.sync 0;\n" " shr.s32 %r16, %r52, 1;\n" " setp.gt.s32 %p8, %r52, 1;\n" " mov.u32 %r52, %r16;\n" " @%p8 bra $L__BB28_9;\n" "\n" "$L__BB28_12:\n" " ld.shared.f32 %f29, [sdata];\n" " neg.f32 %f30, %f29;\n" " mov.f32 %f31, 0f3F000000;\n" " mov.f32 %f32, 0f3BBB989D;\n" " fma.rn.f32 %f33, %f30, %f32, %f31;\n" " mov.f32 %f34, 0f3FB8AA3B;\n" " mov.f32 %f35, 0f437C0000;\n" " cvt.sat.f32.f32 %f36, %f33;\n" " mov.f32 %f37, 0f4B400001;\n" " fma.rm.f32 %f38, %f36, %f35, %f37;\n" " add.f32 %f39, %f38, 0fCB40007F;\n" " neg.f32 %f40, %f39;\n" " fma.rn.f32 %f41, %f30, %f34, %f40;\n" " mov.f32 %f42, 0f32A57060;\n" " fma.rn.f32 %f43, %f30, %f42, %f41;\n" " mov.b32 %r40, %f38;\n" " shl.b32 %r41, %r40, 23;\n" " mov.b32 %f44, %r41;\n" " ex2.approx.ftz.f32 %f45, %f43;\n" " fma.rn.f32 %f46, %f45, %f44, 0f3F800000;\n" " rcp.rn.f32 %f8, %f46;\n" " @%p1 bra $L__BB28_19;\n" "\n" " not.b32 %r42, %r55;\n" " add.s32 %r43, %r42, %r27;\n" " div.u32 %r17, %r43, %r1;\n" " add.s32 %r44, %r17, 1;\n" " and.b32 %r54, %r44, 3;\n" " setp.eq.s32 %p10, %r54, 0;\n" " @%p10 bra $L__BB28_16;\n" "\n" " mul.wide.s32 %rd29, %r55, 4;\n" " add.s64 %rd38, %rd3, %rd29;\n" " mul.wide.s32 %rd12, %r1, 4;\n" "\n" "$L__BB28_15:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f47, [%rd38];\n" " mul.f32 %f48, %f8, %f47;\n" " st.global.f32 [%rd38], %f48;\n" " add.s32 %r55, %r55, %r1;\n" " add.s64 %rd38, %rd38, %rd12;\n" " add.s32 %r54, %r54, -1;\n" " setp.ne.s32 %p11, %r54, 0;\n" " @%p11 bra $L__BB28_15;\n" "\n" "$L__BB28_16:\n" " setp.lt.u32 %p12, %r17, 3;\n" " @%p12 bra $L__BB28_19;\n" "\n" " shl.b32 %r24, %r1, 2;\n" " cvt.s64.s32 %rd32, %r24;\n" "\n" "$L__BB28_18:\n" " mul.wide.s32 %rd30, %r55, 4;\n" " add.s64 %rd31, %rd3, %rd30;\n" " ld.global.f32 %f49, [%rd31];\n" " mul.f32 %f50, %f8, %f49;\n" " st.global.f32 [%rd31], %f50;\n" " add.s64 %rd33, %rd31, %rd32;\n" " ld.global.f32 %f51, [%rd33];\n" " mul.f32 %f52, %f8, %f51;\n" " st.global.f32 [%rd33], %f52;\n" " add.s32 %r45, %r55, %r1;\n" " add.s32 %r46, %r45, %r1;\n" " add.s64 %rd34, %rd33, %rd32;\n" " ld.global.f32 %f53, [%rd34];\n" " mul.f32 %f54, %f8, %f53;\n" " st.global.f32 [%rd34], %f54;\n" " add.s32 %r47, %r46, %r1;\n" " add.s64 %rd35, %rd34, %rd32;\n" " ld.global.f32 %f55, [%rd35];\n" " mul.f32 %f56, %f8, %f55;\n" " st.global.f32 [%rd35], %f56;\n" " add.s32 %r55, %r47, %r1;\n" " setp.lt.s32 %p13, %r55, %r27;\n" " @%p13 bra $L__BB28_18;\n" "\n" "$L__BB28_19:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_dedup_build\n" ".visible .entry fc_moe_dedup_build(\n" " .param .u64 fc_moe_dedup_build_param_0,\n" " .param .u64 fc_moe_dedup_build_param_1,\n" " .param .u64 fc_moe_dedup_build_param_2,\n" " .param .u32 fc_moe_dedup_build_param_3,\n" " .param .u32 fc_moe_dedup_build_param_4,\n" " .param .u32 fc_moe_dedup_build_param_5,\n" " .param .u32 fc_moe_dedup_build_param_6\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .b32 %r<34>;\n" " .reg .b64 %rd<30>;\n" "\n" "\n" " ld.param.u64 %rd11, [fc_moe_dedup_build_param_0];\n" " ld.param.u64 %rd13, [fc_moe_dedup_build_param_1];\n" " ld.param.u64 %rd12, [fc_moe_dedup_build_param_2];\n" " ld.param.u32 %r17, [fc_moe_dedup_build_param_3];\n" " ld.param.u32 %r18, [fc_moe_dedup_build_param_4];\n" " ld.param.u32 %r19, [fc_moe_dedup_build_param_5];\n" " ld.param.u32 %r20, [fc_moe_dedup_build_param_6];\n" " cvta.to.global.u64 %rd1, %rd13;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r33, %tid.x;\n" " setp.ge.s32 %p1, %r33, %r17;\n" " @%p1 bra $L__BB29_7;\n" "\n" " not.b32 %r21, %r33;\n" " add.s32 %r22, %r21, %r17;\n" " div.u32 %r3, %r22, %r1;\n" " add.s32 %r23, %r3, 1;\n" " and.b32 %r30, %r23, 3;\n" " setp.eq.s32 %p2, %r30, 0;\n" " mov.u32 %r31, %r33;\n" " @%p2 bra $L__BB29_4;\n" "\n" " mul.wide.s32 %rd14, %r33, 4;\n" " add.s64 %rd29, %rd1, %rd14;\n" " mul.wide.s32 %rd3, %r1, 4;\n" " mov.u32 %r31, %r33;\n" "\n" "$L__BB29_3:\n" " .pragma \"nounroll\";\n" " mov.u32 %r24, 0;\n" " st.global.u32 [%rd29], %r24;\n" " add.s32 %r31, %r31, %r1;\n" " add.s64 %rd29, %rd29, %rd3;\n" " add.s32 %r30, %r30, -1;\n" " setp.ne.s32 %p3, %r30, 0;\n" " @%p3 bra $L__BB29_3;\n" "\n" "$L__BB29_4:\n" " setp.lt.u32 %p4, %r3, 3;\n" " @%p4 bra $L__BB29_7;\n" "\n" " mul.wide.s32 %rd6, %r1, 4;\n" " mov.u32 %r25, 0;\n" "\n" "$L__BB29_6:\n" " mul.wide.s32 %rd15, %r31, 4;\n" " add.s64 %rd16, %rd1, %rd15;\n" " st.global.u32 [%rd16], %r25;\n" " add.s64 %rd17, %rd16, %rd6;\n" " st.global.u32 [%rd17], %r25;\n" " add.s32 %r26, %r31, %r1;\n" " add.s32 %r27, %r26, %r1;\n" " add.s64 %rd18, %rd17, %rd6;\n" " st.global.u32 [%rd18], %r25;\n" " add.s32 %r28, %r27, %r1;\n" " add.s64 %rd19, %rd18, %rd6;\n" " st.global.u32 [%rd19], %r25;\n" " add.s32 %r31, %r28, %r1;\n" " setp.lt.s32 %p5, %r31, %r17;\n" " @%p5 bra $L__BB29_6;\n" "\n" "$L__BB29_7:\n" " bar.sync 0;\n" " mul.lo.s32 %r12, %r19, %r18;\n" " setp.ge.s32 %p6, %r33, %r12;\n" " @%p6 bra $L__BB29_13;\n" "\n" " cvt.s64.s32 %rd7, %r20;\n" " cvta.to.global.u64 %rd8, %rd12;\n" " cvta.to.global.u64 %rd9, %rd11;\n" "\n" "$L__BB29_9:\n" " mul.wide.s32 %rd20, %r33, 4;\n" " add.s64 %rd21, %rd9, %rd20;\n" " ld.global.u32 %r14, [%rd21];\n" " cvt.s64.s32 %rd10, %r14;\n" " setp.lt.s32 %p7, %r14, 0;\n" " setp.ge.s32 %p8, %r14, %r17;\n" " or.pred %p9, %p7, %p8;\n" " @%p9 bra $L__BB29_12;\n" "\n" " mul.wide.s32 %rd22, %r14, 4;\n" " add.s64 %rd23, %rd1, %rd22;\n" " atom.global.add.u32 %r15, [%rd23], 1;\n" " setp.ge.s32 %p10, %r15, %r20;\n" " @%p10 bra $L__BB29_12;\n" "\n" " mul.lo.s64 %rd24, %rd10, %rd7;\n" " cvt.s64.s32 %rd25, %r15;\n" " add.s64 %rd26, %rd24, %rd25;\n" " shl.b64 %rd27, %rd26, 2;\n" " add.s64 %rd28, %rd8, %rd27;\n" " st.global.u32 [%rd28], %r33;\n" "\n" "$L__BB29_12:\n" " add.s32 %r33, %r33, %r1;\n" " setp.lt.s32 %p11, %r33, %r12;\n" " @%p11 bra $L__BB29_9;\n" "\n" "$L__BB29_13:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_q3k_dedup\n" ".visible .entry fc_moe_gateup_q3k_dedup(\n" " .param .u64 fc_moe_gateup_q3k_dedup_param_0,\n" " .param .u64 fc_moe_gateup_q3k_dedup_param_1,\n" " .param .u64 fc_moe_gateup_q3k_dedup_param_2,\n" " .param .u64 fc_moe_gateup_q3k_dedup_param_3,\n" " .param .u64 fc_moe_gateup_q3k_dedup_param_4,\n" " .param .u64 fc_moe_gateup_q3k_dedup_param_5,\n" " .param .u32 fc_moe_gateup_q3k_dedup_param_6,\n" " .param .u32 fc_moe_gateup_q3k_dedup_param_7,\n" " .param .u32 fc_moe_gateup_q3k_dedup_param_8,\n" " .param .u32 fc_moe_gateup_q3k_dedup_param_9,\n" " .param .u32 fc_moe_gateup_q3k_dedup_param_10\n" ")\n" "{\n" " .reg .pred %p<56>;\n" " .reg .f32 %f<782>;\n" " .reg .b32 %r<307>;\n" " .reg .b64 %rd<262>;\n" "\n" "\n" " ld.param.u64 %rd99, [fc_moe_gateup_q3k_dedup_param_0];\n" " ld.param.u64 %rd100, [fc_moe_gateup_q3k_dedup_param_1];\n" " ld.param.u64 %rd261, [fc_moe_gateup_q3k_dedup_param_2];\n" " ld.param.u64 %rd101, [fc_moe_gateup_q3k_dedup_param_3];\n" " ld.param.u64 %rd98, [fc_moe_gateup_q3k_dedup_param_4];\n" " ld.param.u64 %rd102, [fc_moe_gateup_q3k_dedup_param_5];\n" " ld.param.u32 %r26, [fc_moe_gateup_q3k_dedup_param_6];\n" " ld.param.u32 %r27, [fc_moe_gateup_q3k_dedup_param_7];\n" " ld.param.u32 %r28, [fc_moe_gateup_q3k_dedup_param_8];\n" " ld.param.u32 %r29, [fc_moe_gateup_q3k_dedup_param_9];\n" " ld.param.u32 %r30, [fc_moe_gateup_q3k_dedup_param_10];\n" " cvta.to.global.u64 %rd1, %rd100;\n" " cvta.to.global.u64 %rd2, %rd99;\n" " cvta.to.global.u64 %rd3, %rd102;\n" " mov.u32 %r31, %tid.x;\n" " shr.u32 %r32, %r31, 5;\n" " and.b32 %r1, %r31, 31;\n" " mov.u32 %r33, %ctaid.x;\n" " shl.b32 %r34, %r33, 3;\n" " add.s32 %r2, %r34, %r32;\n" " mov.u32 %r35, %ctaid.y;\n" " cvt.s64.s32 %rd4, %r35;\n" " cvta.to.global.u64 %rd103, %rd101;\n" " mul.wide.s32 %rd104, %r35, 4;\n" " add.s64 %rd105, %rd103, %rd104;\n" " mov.u32 %r36, %ctaid.z;\n" " shl.b32 %r3, %r36, 3;\n" " ld.global.u32 %r4, [%rd105];\n" " setp.le.s32 %p2, %r4, %r3;\n" " @%p2 bra $L__BB30_69;\n" "\n" " sub.s32 %r5, %r4, %r3;\n" " min.s32 %r6, %r5, 8;\n" " cvt.s64.s32 %rd106, %r30;\n" " mul.lo.s64 %rd107, %rd106, %rd4;\n" " cvt.s64.s32 %rd108, %r3;\n" " add.s64 %rd109, %rd107, %rd108;\n" " setp.lt.s32 %p3, %r5, 1;\n" " cvta.to.global.u64 %rd110, %rd98;\n" " shl.b64 %rd111, %rd109, 2;\n" " add.s64 %rd5, %rd110, %rd111;\n" " mov.u32 %r298, -1;\n" " @%p3 bra $L__BB30_3;\n" "\n" " ld.global.u32 %r298, [%rd5];\n" "\n" "$L__BB30_3:\n" " setp.lt.s32 %p4, %r298, 0;\n" " mov.u64 %rd252, 0;\n" " @%p4 bra $L__BB30_5;\n" "\n" " div.s32 %r38, %r298, %r26;\n" " cvt.s64.s32 %rd252, %r38;\n" "\n" "$L__BB30_5:\n" " setp.lt.s32 %p5, %r6, 2;\n" " mov.u32 %r299, -1;\n" " @%p5 bra $L__BB30_7;\n" "\n" " ld.global.u32 %r299, [%rd5+4];\n" "\n" "$L__BB30_7:\n" " setp.lt.s32 %p6, %r299, 0;\n" " mov.u64 %rd253, 0;\n" " @%p6 bra $L__BB30_9;\n" "\n" " div.s32 %r40, %r299, %r26;\n" " cvt.s64.s32 %rd253, %r40;\n" "\n" "$L__BB30_9:\n" " setp.lt.s32 %p7, %r6, 3;\n" " mov.u32 %r300, -1;\n" " @%p7 bra $L__BB30_11;\n" "\n" " ld.global.u32 %r300, [%rd5+8];\n" "\n" "$L__BB30_11:\n" " setp.lt.s32 %p8, %r300, 0;\n" " mov.u64 %rd254, 0;\n" " @%p8 bra $L__BB30_13;\n" "\n" " div.s32 %r42, %r300, %r26;\n" " cvt.s64.s32 %rd254, %r42;\n" "\n" "$L__BB30_13:\n" " setp.lt.s32 %p9, %r6, 4;\n" " mov.u32 %r301, -1;\n" " @%p9 bra $L__BB30_15;\n" "\n" " ld.global.u32 %r301, [%rd5+12];\n" "\n" "$L__BB30_15:\n" " setp.lt.s32 %p10, %r301, 0;\n" " mov.u64 %rd255, 0;\n" " @%p10 bra $L__BB30_17;\n" "\n" " div.s32 %r44, %r301, %r26;\n" " cvt.s64.s32 %rd255, %r44;\n" "\n" "$L__BB30_17:\n" " setp.lt.s32 %p11, %r6, 5;\n" " mov.u32 %r302, -1;\n" " @%p11 bra $L__BB30_19;\n" "\n" " ld.global.u32 %r302, [%rd5+16];\n" "\n" "$L__BB30_19:\n" " setp.lt.s32 %p12, %r302, 0;\n" " mov.u64 %rd256, 0;\n" " @%p12 bra $L__BB30_21;\n" "\n" " div.s32 %r46, %r302, %r26;\n" " cvt.s64.s32 %rd256, %r46;\n" "\n" "$L__BB30_21:\n" " setp.lt.s32 %p13, %r6, 6;\n" " mov.u32 %r303, -1;\n" " @%p13 bra $L__BB30_23;\n" "\n" " ld.global.u32 %r303, [%rd5+20];\n" "\n" "$L__BB30_23:\n" " setp.lt.s32 %p14, %r303, 0;\n" " mov.u64 %rd257, 0;\n" " @%p14 bra $L__BB30_25;\n" "\n" " div.s32 %r48, %r303, %r26;\n" " cvt.s64.s32 %rd257, %r48;\n" "\n" "$L__BB30_25:\n" " setp.lt.s32 %p15, %r6, 7;\n" " mov.u32 %r304, -1;\n" " @%p15 bra $L__BB30_27;\n" "\n" " ld.global.u32 %r304, [%rd5+24];\n" "\n" "$L__BB30_27:\n" " setp.lt.s32 %p16, %r304, 0;\n" " mov.u64 %rd258, 0;\n" " @%p16 bra $L__BB30_29;\n" "\n" " div.s32 %r50, %r304, %r26;\n" " cvt.s64.s32 %rd258, %r50;\n" "\n" "$L__BB30_29:\n" " setp.lt.s32 %p17, %r6, 8;\n" " mov.u32 %r305, -1;\n" " @%p17 bra $L__BB30_31;\n" "\n" " ld.global.u32 %r305, [%rd5+28];\n" "\n" "$L__BB30_31:\n" " setp.lt.s32 %p18, %r305, 0;\n" " mov.u64 %rd259, 0;\n" " @%p18 bra $L__BB30_33;\n" "\n" " div.s32 %r52, %r305, %r26;\n" " cvt.s64.s32 %rd259, %r52;\n" "\n" "$L__BB30_33:\n" " cvt.s64.s32 %rd22, %r27;\n" " cvt.s64.s32 %rd23, %r2;\n" " setp.ge.s32 %p19, %r2, %r27;\n" " mov.f32 %f766, 0f00000000;\n" " mov.f32 %f767, %f766;\n" " mov.f32 %f768, %f766;\n" " mov.f32 %f769, %f766;\n" " mov.f32 %f770, %f766;\n" " mov.f32 %f771, %f766;\n" " mov.f32 %f772, %f766;\n" " mov.f32 %f773, %f766;\n" " mov.f32 %f774, %f766;\n" " mov.f32 %f775, %f766;\n" " mov.f32 %f776, %f766;\n" " mov.f32 %f777, %f766;\n" " mov.f32 %f778, %f766;\n" " mov.f32 %f779, %f766;\n" " mov.f32 %f780, %f766;\n" " mov.f32 %f781, %f766;\n" " @%p19 bra $L__BB30_53;\n" "\n" " setp.lt.s32 %p20, %r28, 256;\n" " mov.f32 %f767, %f766;\n" " mov.f32 %f768, %f766;\n" " mov.f32 %f769, %f766;\n" " mov.f32 %f770, %f766;\n" " mov.f32 %f771, %f766;\n" " mov.f32 %f772, %f766;\n" " mov.f32 %f773, %f766;\n" " mov.f32 %f774, %f766;\n" " mov.f32 %f775, %f766;\n" " mov.f32 %f776, %f766;\n" " mov.f32 %f777, %f766;\n" " mov.f32 %f778, %f766;\n" " mov.f32 %f779, %f766;\n" " mov.f32 %f780, %f766;\n" " mov.f32 %f781, %f766;\n" " @%p20 bra $L__BB30_53;\n" "\n" " shr.u32 %r54, %r1, 4;\n" " cvt.s64.s32 %rd120, %r28;\n" " mul.lo.s64 %rd121, %rd259, %rd120;\n" " shl.b64 %rd122, %rd121, 2;\n" " mul.wide.u32 %rd123, %r1, 4;\n" " add.s64 %rd24, %rd122, %rd123;\n" " add.s64 %rd25, %rd24, 896;\n" " add.s64 %rd26, %rd24, 768;\n" " add.s64 %rd27, %rd24, 640;\n" " add.s64 %rd28, %rd24, 512;\n" " add.s64 %rd29, %rd24, 384;\n" " add.s64 %rd30, %rd24, 256;\n" " add.s64 %rd31, %rd24, 128;\n" " mul.lo.s64 %rd124, %rd258, %rd120;\n" " shl.b64 %rd125, %rd124, 2;\n" " add.s64 %rd32, %rd125, %rd123;\n" " add.s64 %rd33, %rd32, 896;\n" " add.s64 %rd34, %rd32, 768;\n" " add.s64 %rd35, %rd32, 640;\n" " add.s64 %rd36, %rd32, 512;\n" " add.s64 %rd37, %rd32, 384;\n" " add.s64 %rd38, %rd32, 256;\n" " add.s64 %rd39, %rd32, 128;\n" " mul.lo.s64 %rd126, %rd257, %rd120;\n" " shl.b64 %rd127, %rd126, 2;\n" " add.s64 %rd40, %rd127, %rd123;\n" " add.s64 %rd41, %rd40, 896;\n" " add.s64 %rd42, %rd40, 768;\n" " add.s64 %rd43, %rd40, 640;\n" " add.s64 %rd44, %rd40, 512;\n" " add.s64 %rd45, %rd40, 384;\n" " add.s64 %rd46, %rd40, 256;\n" " add.s64 %rd47, %rd40, 128;\n" " mul.lo.s64 %rd128, %rd256, %rd120;\n" " shl.b64 %rd129, %rd128, 2;\n" " add.s64 %rd48, %rd129, %rd123;\n" " add.s64 %rd49, %rd48, 896;\n" " add.s64 %rd50, %rd48, 768;\n" " add.s64 %rd51, %rd48, 640;\n" " add.s64 %rd52, %rd48, 512;\n" " add.s64 %rd53, %rd48, 384;\n" " add.s64 %rd54, %rd48, 256;\n" " add.s64 %rd55, %rd48, 128;\n" " mul.lo.s64 %rd130, %rd255, %rd120;\n" " shl.b64 %rd131, %rd130, 2;\n" " add.s64 %rd56, %rd131, %rd123;\n" " add.s64 %rd57, %rd56, 896;\n" " add.s64 %rd58, %rd56, 768;\n" " add.s64 %rd59, %rd56, 640;\n" " add.s64 %rd60, %rd56, 512;\n" " add.s64 %rd61, %rd56, 384;\n" " add.s64 %rd62, %rd56, 256;\n" " add.s64 %rd63, %rd56, 128;\n" " mul.lo.s64 %rd132, %rd254, %rd120;\n" " shl.b64 %rd133, %rd132, 2;\n" " add.s64 %rd64, %rd133, %rd123;\n" " add.s64 %rd65, %rd64, 896;\n" " add.s64 %rd66, %rd64, 768;\n" " add.s64 %rd67, %rd64, 640;\n" " add.s64 %rd68, %rd64, 512;\n" " add.s64 %rd69, %rd64, 384;\n" " add.s64 %rd70, %rd64, 256;\n" " add.s64 %rd71, %rd64, 128;\n" " mul.lo.s64 %rd134, %rd253, %rd120;\n" " shl.b64 %rd135, %rd134, 2;\n" " add.s64 %rd72, %rd135, %rd123;\n" " add.s64 %rd73, %rd72, 896;\n" " add.s64 %rd74, %rd72, 768;\n" " add.s64 %rd75, %rd72, 640;\n" " add.s64 %rd76, %rd72, 512;\n" " add.s64 %rd77, %rd72, 384;\n" " add.s64 %rd78, %rd72, 256;\n" " add.s64 %rd79, %rd72, 128;\n" " mul.lo.s64 %rd136, %rd252, %rd120;\n" " shl.b64 %rd137, %rd136, 2;\n" " add.s64 %rd80, %rd137, %rd123;\n" " add.s64 %rd81, %rd80, 896;\n" " add.s64 %rd82, %rd80, 768;\n" " add.s64 %rd83, %rd80, 640;\n" " add.s64 %rd84, %rd80, 512;\n" " add.s64 %rd85, %rd80, 384;\n" " add.s64 %rd86, %rd80, 256;\n" " add.s64 %rd87, %rd80, 128;\n" " add.s64 %rd88, %rd1, %rd123;\n" " mul.wide.u32 %rd138, %r54, 8;\n" " add.s64 %rd89, %rd1, %rd138;\n" " add.s64 %rd90, %rd2, %rd123;\n" " add.s64 %rd91, %rd2, %rd138;\n" " mul.lo.s64 %rd139, %rd22, %rd4;\n" " add.s64 %rd140, %rd139, %rd23;\n" " cvt.s64.s32 %rd141, %r29;\n" " mul.lo.s64 %rd260, %rd140, %rd141;\n" " shr.s32 %r23, %r28, 8;\n" " mov.u32 %r306, 0;\n" " mov.f32 %f766, 0f00000000;\n" " mov.f32 %f148, 0fCB000004;\n" " mov.f32 %f149, 0f3F800000;\n" " mov.f32 %f767, %f766;\n" " mov.f32 %f768, %f766;\n" " mov.f32 %f769, %f766;\n" " mov.f32 %f770, %f766;\n" " mov.f32 %f771, %f766;\n" " mov.f32 %f772, %f766;\n" " mov.f32 %f773, %f766;\n" " mov.f32 %f774, %f766;\n" " mov.f32 %f775, %f766;\n" " mov.f32 %f776, %f766;\n" " mov.f32 %f777, %f766;\n" " mov.f32 %f778, %f766;\n" " mov.f32 %f779, %f766;\n" " mov.f32 %f780, %f766;\n" " mov.f32 %f781, %f766;\n" "\n" "$L__BB30_36:\n" " add.s64 %rd142, %rd90, %rd260;\n" " add.s64 %rd143, %rd2, %rd260;\n" " ld.global.f32 %f17, [%rd143+144];\n" " add.s64 %rd144, %rd91, %rd260;\n" " ld.global.v2.u32 {%r55, %r56}, [%rd144+128];\n" " ld.global.u32 %r59, [%rd142];\n" " shr.u32 %r60, %r59, 16;\n" " and.b32 %r61, %r59, 15;\n" " or.b32 %r62, %r61, 1258291200;\n" " mov.b32 %f147, %r62;\n" " fma.rn.f32 %f150, %f147, %f149, %f148;\n" " and.b32 %r63, %r55, 255;\n" " cvt.rn.f32.u32 %f151, %r63;\n" " mov.f32 %f152, 0f42000000;\n" " sub.rn.f32 %f153, %f151, %f152;\n" " mul.rn.f32 %f18, %f153, %f150;\n" " and.b32 %r64, %r59, 240;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f154, %r65;\n" " mov.f32 %f155, 0fC9000040;\n" " mov.f32 %f156, 0f3D800000;\n" " fma.rn.f32 %f157, %f154, %f156, %f155;\n" " bfe.u32 %r66, %r55, 8, 8;\n" " cvt.rn.f32.u32 %f158, %r66;\n" " sub.rn.f32 %f159, %f158, %f152;\n" " mul.rn.f32 %f19, %f159, %f157;\n" " and.b32 %r67, %r59, 3840;\n" " or.b32 %r68, %r67, 1258291200;\n" " mov.b32 %f160, %r68;\n" " mov.f32 %f161, 0fC7000400;\n" " mov.f32 %f162, 0f3B800000;\n" " fma.rn.f32 %f163, %f160, %f162, %f161;\n" " bfe.u32 %r69, %r55, 16, 8;\n" " cvt.rn.f32.u32 %f164, %r69;\n" " sub.rn.f32 %f165, %f164, %f152;\n" " mul.rn.f32 %f20, %f165, %f163;\n" " and.b32 %r70, %r59, 61440;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f166, %r71;\n" " mov.f32 %f167, 0fC5004000;\n" " mov.f32 %f168, 0f39800000;\n" " fma.rn.f32 %f169, %f166, %f168, %f167;\n" " shr.u32 %r72, %r55, 24;\n" " cvt.rn.f32.u32 %f170, %r72;\n" " sub.rn.f32 %f171, %f170, %f152;\n" " mul.rn.f32 %f21, %f171, %f169;\n" " and.b32 %r73, %r59, 983040;\n" " or.b32 %r74, %r73, 1258291200;\n" " mov.b32 %f172, %r74;\n" " mov.f32 %f173, 0fC3040000;\n" " mov.f32 %f174, 0f37800000;\n" " fma.rn.f32 %f175, %f172, %f174, %f173;\n" " and.b32 %r75, %r56, 255;\n" " cvt.rn.f32.u32 %f176, %r75;\n" " sub.rn.f32 %f177, %f176, %f152;\n" " mul.rn.f32 %f22, %f177, %f175;\n" " and.b32 %r76, %r60, 240;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f178, %r77;\n" " fma.rn.f32 %f179, %f178, %f156, %f155;\n" " bfe.u32 %r78, %r56, 8, 8;\n" " cvt.rn.f32.u32 %f180, %r78;\n" " sub.rn.f32 %f181, %f180, %f152;\n" " mul.rn.f32 %f23, %f181, %f179;\n" " and.b32 %r79, %r60, 3840;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f182, %r80;\n" " fma.rn.f32 %f183, %f182, %f162, %f161;\n" " bfe.u32 %r81, %r56, 16, 8;\n" " cvt.rn.f32.u32 %f184, %r81;\n" " sub.rn.f32 %f185, %f184, %f152;\n" " mul.rn.f32 %f24, %f185, %f183;\n" " and.b32 %r82, %r60, 61440;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f186, %r83;\n" " fma.rn.f32 %f187, %f186, %f168, %f167;\n" " shr.u32 %r84, %r56, 24;\n" " cvt.rn.f32.u32 %f188, %r84;\n" " sub.rn.f32 %f189, %f188, %f152;\n" " mul.rn.f32 %f25, %f189, %f187;\n" " add.s64 %rd145, %rd88, %rd260;\n" " add.s64 %rd146, %rd1, %rd260;\n" " ld.global.f32 %f26, [%rd146+144];\n" " add.s64 %rd147, %rd89, %rd260;\n" " ld.global.v2.u32 {%r85, %r86}, [%rd147+128];\n" " ld.global.u32 %r89, [%rd145];\n" " shr.u32 %r90, %r89, 16;\n" " and.b32 %r91, %r89, 15;\n" " or.b32 %r92, %r91, 1258291200;\n" " mov.b32 %f190, %r92;\n" " fma.rn.f32 %f191, %f190, %f149, %f148;\n" " and.b32 %r93, %r85, 255;\n" " cvt.rn.f32.u32 %f192, %r93;\n" " sub.rn.f32 %f193, %f192, %f152;\n" " mul.rn.f32 %f27, %f193, %f191;\n" " and.b32 %r94, %r89, 240;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f194, %r95;\n" " fma.rn.f32 %f195, %f194, %f156, %f155;\n" " bfe.u32 %r96, %r85, 8, 8;\n" " cvt.rn.f32.u32 %f196, %r96;\n" " sub.rn.f32 %f197, %f196, %f152;\n" " mul.rn.f32 %f28, %f197, %f195;\n" " and.b32 %r97, %r89, 3840;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f198, %r98;\n" " fma.rn.f32 %f199, %f198, %f162, %f161;\n" " bfe.u32 %r99, %r85, 16, 8;\n" " cvt.rn.f32.u32 %f200, %r99;\n" " sub.rn.f32 %f201, %f200, %f152;\n" " mul.rn.f32 %f29, %f201, %f199;\n" " and.b32 %r100, %r89, 61440;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f202, %r101;\n" " fma.rn.f32 %f203, %f202, %f168, %f167;\n" " shr.u32 %r102, %r85, 24;\n" " cvt.rn.f32.u32 %f204, %r102;\n" " sub.rn.f32 %f205, %f204, %f152;\n" " mul.rn.f32 %f30, %f205, %f203;\n" " and.b32 %r103, %r89, 983040;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f206, %r104;\n" " fma.rn.f32 %f207, %f206, %f174, %f173;\n" " and.b32 %r105, %r86, 255;\n" " cvt.rn.f32.u32 %f208, %r105;\n" " sub.rn.f32 %f209, %f208, %f152;\n" " mul.rn.f32 %f31, %f209, %f207;\n" " and.b32 %r106, %r90, 240;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f210, %r107;\n" " fma.rn.f32 %f211, %f210, %f156, %f155;\n" " bfe.u32 %r108, %r86, 8, 8;\n" " cvt.rn.f32.u32 %f212, %r108;\n" " sub.rn.f32 %f213, %f212, %f152;\n" " mul.rn.f32 %f32, %f213, %f211;\n" " and.b32 %r109, %r90, 3840;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f214, %r110;\n" " fma.rn.f32 %f215, %f214, %f162, %f161;\n" " bfe.u32 %r111, %r86, 16, 8;\n" " cvt.rn.f32.u32 %f216, %r111;\n" " sub.rn.f32 %f217, %f216, %f152;\n" " mul.rn.f32 %f33, %f217, %f215;\n" " and.b32 %r112, %r90, 61440;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f218, %r113;\n" " fma.rn.f32 %f219, %f218, %f168, %f167;\n" " shr.u32 %r114, %r86, 24;\n" " cvt.rn.f32.u32 %f220, %r114;\n" " sub.rn.f32 %f221, %f220, %f152;\n" " mul.rn.f32 %f34, %f221, %f219;\n" " @%p3 bra $L__BB30_38;\n" "\n" " add.s64 %rd148, %rd261, %rd80;\n" " // begin inline asm\n" " ld.global.nc.f32 %f222, [%rd148];\n" " // end inline asm\n" " mov.f32 %f230, 0f00000000;\n" " fma.rn.f32 %f231, %f18, %f222, %f230;\n" " fma.rn.f32 %f232, %f27, %f222, %f230;\n" " add.s64 %rd149, %rd261, %rd87;\n" " // begin inline asm\n" " ld.global.nc.f32 %f223, [%rd149];\n" " // end inline asm\n" " fma.rn.f32 %f233, %f19, %f223, %f231;\n" " fma.rn.f32 %f234, %f28, %f223, %f232;\n" " add.s64 %rd150, %rd261, %rd86;\n" " // begin inline asm\n" " ld.global.nc.f32 %f224, [%rd150];\n" " // end inline asm\n" " fma.rn.f32 %f235, %f20, %f224, %f233;\n" " fma.rn.f32 %f236, %f29, %f224, %f234;\n" " add.s64 %rd151, %rd261, %rd85;\n" " // begin inline asm\n" " ld.global.nc.f32 %f225, [%rd151];\n" " // end inline asm\n" " fma.rn.f32 %f237, %f21, %f225, %f235;\n" " fma.rn.f32 %f238, %f30, %f225, %f236;\n" " add.s64 %rd152, %rd261, %rd84;\n" " // begin inline asm\n" " ld.global.nc.f32 %f226, [%rd152];\n" " // end inline asm\n" " fma.rn.f32 %f239, %f22, %f226, %f237;\n" " fma.rn.f32 %f240, %f31, %f226, %f238;\n" " add.s64 %rd153, %rd261, %rd83;\n" " // begin inline asm\n" " ld.global.nc.f32 %f227, [%rd153];\n" " // end inline asm\n" " fma.rn.f32 %f241, %f23, %f227, %f239;\n" " fma.rn.f32 %f242, %f32, %f227, %f240;\n" " add.s64 %rd154, %rd261, %rd82;\n" " // begin inline asm\n" " ld.global.nc.f32 %f228, [%rd154];\n" " // end inline asm\n" " fma.rn.f32 %f243, %f24, %f228, %f241;\n" " fma.rn.f32 %f244, %f33, %f228, %f242;\n" " add.s64 %rd155, %rd261, %rd81;\n" " // begin inline asm\n" " ld.global.nc.f32 %f229, [%rd155];\n" " // end inline asm\n" " fma.rn.f32 %f245, %f25, %f229, %f243;\n" " fma.rn.f32 %f246, %f34, %f229, %f244;\n" " fma.rn.f32 %f774, %f17, %f245, %f774;\n" " fma.rn.f32 %f766, %f26, %f246, %f766;\n" "\n" "$L__BB30_38:\n" " @%p5 bra $L__BB30_40;\n" "\n" " add.s64 %rd156, %rd261, %rd72;\n" " // begin inline asm\n" " ld.global.nc.f32 %f247, [%rd156];\n" " // end inline asm\n" " mov.f32 %f255, 0f00000000;\n" " fma.rn.f32 %f256, %f18, %f247, %f255;\n" " fma.rn.f32 %f257, %f27, %f247, %f255;\n" " add.s64 %rd157, %rd261, %rd79;\n" " // begin inline asm\n" " ld.global.nc.f32 %f248, [%rd157];\n" " // end inline asm\n" " fma.rn.f32 %f258, %f19, %f248, %f256;\n" " fma.rn.f32 %f259, %f28, %f248, %f257;\n" " add.s64 %rd158, %rd261, %rd78;\n" " // begin inline asm\n" " ld.global.nc.f32 %f249, [%rd158];\n" " // end inline asm\n" " fma.rn.f32 %f260, %f20, %f249, %f258;\n" " fma.rn.f32 %f261, %f29, %f249, %f259;\n" " add.s64 %rd159, %rd261, %rd77;\n" " // begin inline asm\n" " ld.global.nc.f32 %f250, [%rd159];\n" " // end inline asm\n" " fma.rn.f32 %f262, %f21, %f250, %f260;\n" " fma.rn.f32 %f263, %f30, %f250, %f261;\n" " add.s64 %rd160, %rd261, %rd76;\n" " // begin inline asm\n" " ld.global.nc.f32 %f251, [%rd160];\n" " // end inline asm\n" " fma.rn.f32 %f264, %f22, %f251, %f262;\n" " fma.rn.f32 %f265, %f31, %f251, %f263;\n" " add.s64 %rd161, %rd261, %rd75;\n" " // begin inline asm\n" " ld.global.nc.f32 %f252, [%rd161];\n" " // end inline asm\n" " fma.rn.f32 %f266, %f23, %f252, %f264;\n" " fma.rn.f32 %f267, %f32, %f252, %f265;\n" " add.s64 %rd162, %rd261, %rd74;\n" " // begin inline asm\n" " ld.global.nc.f32 %f253, [%rd162];\n" " // end inline asm\n" " fma.rn.f32 %f268, %f24, %f253, %f266;\n" " fma.rn.f32 %f269, %f33, %f253, %f267;\n" " add.s64 %rd163, %rd261, %rd73;\n" " // begin inline asm\n" " ld.global.nc.f32 %f254, [%rd163];\n" " // end inline asm\n" " fma.rn.f32 %f270, %f25, %f254, %f268;\n" " fma.rn.f32 %f271, %f34, %f254, %f269;\n" " fma.rn.f32 %f775, %f17, %f270, %f775;\n" " fma.rn.f32 %f767, %f26, %f271, %f767;\n" "\n" "$L__BB30_40:\n" " @%p7 bra $L__BB30_42;\n" "\n" " add.s64 %rd164, %rd261, %rd64;\n" " // begin inline asm\n" " ld.global.nc.f32 %f272, [%rd164];\n" " // end inline asm\n" " mov.f32 %f280, 0f00000000;\n" " fma.rn.f32 %f281, %f18, %f272, %f280;\n" " fma.rn.f32 %f282, %f27, %f272, %f280;\n" " add.s64 %rd165, %rd261, %rd71;\n" " // begin inline asm\n" " ld.global.nc.f32 %f273, [%rd165];\n" " // end inline asm\n" " fma.rn.f32 %f283, %f19, %f273, %f281;\n" " fma.rn.f32 %f284, %f28, %f273, %f282;\n" " add.s64 %rd166, %rd261, %rd70;\n" " // begin inline asm\n" " ld.global.nc.f32 %f274, [%rd166];\n" " // end inline asm\n" " fma.rn.f32 %f285, %f20, %f274, %f283;\n" " fma.rn.f32 %f286, %f29, %f274, %f284;\n" " add.s64 %rd167, %rd261, %rd69;\n" " // begin inline asm\n" " ld.global.nc.f32 %f275, [%rd167];\n" " // end inline asm\n" " fma.rn.f32 %f287, %f21, %f275, %f285;\n" " fma.rn.f32 %f288, %f30, %f275, %f286;\n" " add.s64 %rd168, %rd261, %rd68;\n" " // begin inline asm\n" " ld.global.nc.f32 %f276, [%rd168];\n" " // end inline asm\n" " fma.rn.f32 %f289, %f22, %f276, %f287;\n" " fma.rn.f32 %f290, %f31, %f276, %f288;\n" " add.s64 %rd169, %rd261, %rd67;\n" " // begin inline asm\n" " ld.global.nc.f32 %f277, [%rd169];\n" " // end inline asm\n" " fma.rn.f32 %f291, %f23, %f277, %f289;\n" " fma.rn.f32 %f292, %f32, %f277, %f290;\n" " add.s64 %rd170, %rd261, %rd66;\n" " // begin inline asm\n" " ld.global.nc.f32 %f278, [%rd170];\n" " // end inline asm\n" " fma.rn.f32 %f293, %f24, %f278, %f291;\n" " fma.rn.f32 %f294, %f33, %f278, %f292;\n" " add.s64 %rd171, %rd261, %rd65;\n" " // begin inline asm\n" " ld.global.nc.f32 %f279, [%rd171];\n" " // end inline asm\n" " fma.rn.f32 %f295, %f25, %f279, %f293;\n" " fma.rn.f32 %f296, %f34, %f279, %f294;\n" " fma.rn.f32 %f776, %f17, %f295, %f776;\n" " fma.rn.f32 %f768, %f26, %f296, %f768;\n" "\n" "$L__BB30_42:\n" " @%p9 bra $L__BB30_44;\n" "\n" " add.s64 %rd172, %rd261, %rd56;\n" " // begin inline asm\n" " ld.global.nc.f32 %f297, [%rd172];\n" " // end inline asm\n" " mov.f32 %f305, 0f00000000;\n" " fma.rn.f32 %f306, %f18, %f297, %f305;\n" " fma.rn.f32 %f307, %f27, %f297, %f305;\n" " add.s64 %rd173, %rd261, %rd63;\n" " // begin inline asm\n" " ld.global.nc.f32 %f298, [%rd173];\n" " // end inline asm\n" " fma.rn.f32 %f308, %f19, %f298, %f306;\n" " fma.rn.f32 %f309, %f28, %f298, %f307;\n" " add.s64 %rd174, %rd261, %rd62;\n" " // begin inline asm\n" " ld.global.nc.f32 %f299, [%rd174];\n" " // end inline asm\n" " fma.rn.f32 %f310, %f20, %f299, %f308;\n" " fma.rn.f32 %f311, %f29, %f299, %f309;\n" " add.s64 %rd175, %rd261, %rd61;\n" " // begin inline asm\n" " ld.global.nc.f32 %f300, [%rd175];\n" " // end inline asm\n" " fma.rn.f32 %f312, %f21, %f300, %f310;\n" " fma.rn.f32 %f313, %f30, %f300, %f311;\n" " add.s64 %rd176, %rd261, %rd60;\n" " // begin inline asm\n" " ld.global.nc.f32 %f301, [%rd176];\n" " // end inline asm\n" " fma.rn.f32 %f314, %f22, %f301, %f312;\n" " fma.rn.f32 %f315, %f31, %f301, %f313;\n" " add.s64 %rd177, %rd261, %rd59;\n" " // begin inline asm\n" " ld.global.nc.f32 %f302, [%rd177];\n" " // end inline asm\n" " fma.rn.f32 %f316, %f23, %f302, %f314;\n" " fma.rn.f32 %f317, %f32, %f302, %f315;\n" " add.s64 %rd178, %rd261, %rd58;\n" " // begin inline asm\n" " ld.global.nc.f32 %f303, [%rd178];\n" " // end inline asm\n" " fma.rn.f32 %f318, %f24, %f303, %f316;\n" " fma.rn.f32 %f319, %f33, %f303, %f317;\n" " add.s64 %rd179, %rd261, %rd57;\n" " // begin inline asm\n" " ld.global.nc.f32 %f304, [%rd179];\n" " // end inline asm\n" " fma.rn.f32 %f320, %f25, %f304, %f318;\n" " fma.rn.f32 %f321, %f34, %f304, %f319;\n" " fma.rn.f32 %f777, %f17, %f320, %f777;\n" " fma.rn.f32 %f769, %f26, %f321, %f769;\n" "\n" "$L__BB30_44:\n" " @%p11 bra $L__BB30_46;\n" "\n" " add.s64 %rd180, %rd261, %rd48;\n" " // begin inline asm\n" " ld.global.nc.f32 %f322, [%rd180];\n" " // end inline asm\n" " mov.f32 %f330, 0f00000000;\n" " fma.rn.f32 %f331, %f18, %f322, %f330;\n" " fma.rn.f32 %f332, %f27, %f322, %f330;\n" " add.s64 %rd181, %rd261, %rd55;\n" " // begin inline asm\n" " ld.global.nc.f32 %f323, [%rd181];\n" " // end inline asm\n" " fma.rn.f32 %f333, %f19, %f323, %f331;\n" " fma.rn.f32 %f334, %f28, %f323, %f332;\n" " add.s64 %rd182, %rd261, %rd54;\n" " // begin inline asm\n" " ld.global.nc.f32 %f324, [%rd182];\n" " // end inline asm\n" " fma.rn.f32 %f335, %f20, %f324, %f333;\n" " fma.rn.f32 %f336, %f29, %f324, %f334;\n" " add.s64 %rd183, %rd261, %rd53;\n" " // begin inline asm\n" " ld.global.nc.f32 %f325, [%rd183];\n" " // end inline asm\n" " fma.rn.f32 %f337, %f21, %f325, %f335;\n" " fma.rn.f32 %f338, %f30, %f325, %f336;\n" " add.s64 %rd184, %rd261, %rd52;\n" " // begin inline asm\n" " ld.global.nc.f32 %f326, [%rd184];\n" " // end inline asm\n" " fma.rn.f32 %f339, %f22, %f326, %f337;\n" " fma.rn.f32 %f340, %f31, %f326, %f338;\n" " add.s64 %rd185, %rd261, %rd51;\n" " // begin inline asm\n" " ld.global.nc.f32 %f327, [%rd185];\n" " // end inline asm\n" " fma.rn.f32 %f341, %f23, %f327, %f339;\n" " fma.rn.f32 %f342, %f32, %f327, %f340;\n" " add.s64 %rd186, %rd261, %rd50;\n" " // begin inline asm\n" " ld.global.nc.f32 %f328, [%rd186];\n" " // end inline asm\n" " fma.rn.f32 %f343, %f24, %f328, %f341;\n" " fma.rn.f32 %f344, %f33, %f328, %f342;\n" " add.s64 %rd187, %rd261, %rd49;\n" " // begin inline asm\n" " ld.global.nc.f32 %f329, [%rd187];\n" " // end inline asm\n" " fma.rn.f32 %f345, %f25, %f329, %f343;\n" " fma.rn.f32 %f346, %f34, %f329, %f344;\n" " fma.rn.f32 %f778, %f17, %f345, %f778;\n" " fma.rn.f32 %f770, %f26, %f346, %f770;\n" "\n" "$L__BB30_46:\n" " @%p13 bra $L__BB30_48;\n" "\n" " add.s64 %rd188, %rd261, %rd40;\n" " // begin inline asm\n" " ld.global.nc.f32 %f347, [%rd188];\n" " // end inline asm\n" " mov.f32 %f355, 0f00000000;\n" " fma.rn.f32 %f356, %f18, %f347, %f355;\n" " fma.rn.f32 %f357, %f27, %f347, %f355;\n" " add.s64 %rd189, %rd261, %rd47;\n" " // begin inline asm\n" " ld.global.nc.f32 %f348, [%rd189];\n" " // end inline asm\n" " fma.rn.f32 %f358, %f19, %f348, %f356;\n" " fma.rn.f32 %f359, %f28, %f348, %f357;\n" " add.s64 %rd190, %rd261, %rd46;\n" " // begin inline asm\n" " ld.global.nc.f32 %f349, [%rd190];\n" " // end inline asm\n" " fma.rn.f32 %f360, %f20, %f349, %f358;\n" " fma.rn.f32 %f361, %f29, %f349, %f359;\n" " add.s64 %rd191, %rd261, %rd45;\n" " // begin inline asm\n" " ld.global.nc.f32 %f350, [%rd191];\n" " // end inline asm\n" " fma.rn.f32 %f362, %f21, %f350, %f360;\n" " fma.rn.f32 %f363, %f30, %f350, %f361;\n" " add.s64 %rd192, %rd261, %rd44;\n" " // begin inline asm\n" " ld.global.nc.f32 %f351, [%rd192];\n" " // end inline asm\n" " fma.rn.f32 %f364, %f22, %f351, %f362;\n" " fma.rn.f32 %f365, %f31, %f351, %f363;\n" " add.s64 %rd193, %rd261, %rd43;\n" " // begin inline asm\n" " ld.global.nc.f32 %f352, [%rd193];\n" " // end inline asm\n" " fma.rn.f32 %f366, %f23, %f352, %f364;\n" " fma.rn.f32 %f367, %f32, %f352, %f365;\n" " add.s64 %rd194, %rd261, %rd42;\n" " // begin inline asm\n" " ld.global.nc.f32 %f353, [%rd194];\n" " // end inline asm\n" " fma.rn.f32 %f368, %f24, %f353, %f366;\n" " fma.rn.f32 %f369, %f33, %f353, %f367;\n" " add.s64 %rd195, %rd261, %rd41;\n" " // begin inline asm\n" " ld.global.nc.f32 %f354, [%rd195];\n" " // end inline asm\n" " fma.rn.f32 %f370, %f25, %f354, %f368;\n" " fma.rn.f32 %f371, %f34, %f354, %f369;\n" " fma.rn.f32 %f779, %f17, %f370, %f779;\n" " fma.rn.f32 %f771, %f26, %f371, %f771;\n" "\n" "$L__BB30_48:\n" " @%p15 bra $L__BB30_50;\n" "\n" " add.s64 %rd196, %rd261, %rd32;\n" " // begin inline asm\n" " ld.global.nc.f32 %f372, [%rd196];\n" " // end inline asm\n" " mov.f32 %f380, 0f00000000;\n" " fma.rn.f32 %f381, %f18, %f372, %f380;\n" " fma.rn.f32 %f382, %f27, %f372, %f380;\n" " add.s64 %rd197, %rd261, %rd39;\n" " // begin inline asm\n" " ld.global.nc.f32 %f373, [%rd197];\n" " // end inline asm\n" " fma.rn.f32 %f383, %f19, %f373, %f381;\n" " fma.rn.f32 %f384, %f28, %f373, %f382;\n" " add.s64 %rd198, %rd261, %rd38;\n" " // begin inline asm\n" " ld.global.nc.f32 %f374, [%rd198];\n" " // end inline asm\n" " fma.rn.f32 %f385, %f20, %f374, %f383;\n" " fma.rn.f32 %f386, %f29, %f374, %f384;\n" " add.s64 %rd199, %rd261, %rd37;\n" " // begin inline asm\n" " ld.global.nc.f32 %f375, [%rd199];\n" " // end inline asm\n" " fma.rn.f32 %f387, %f21, %f375, %f385;\n" " fma.rn.f32 %f388, %f30, %f375, %f386;\n" " add.s64 %rd200, %rd261, %rd36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f376, [%rd200];\n" " // end inline asm\n" " fma.rn.f32 %f389, %f22, %f376, %f387;\n" " fma.rn.f32 %f390, %f31, %f376, %f388;\n" " add.s64 %rd201, %rd261, %rd35;\n" " // begin inline asm\n" " ld.global.nc.f32 %f377, [%rd201];\n" " // end inline asm\n" " fma.rn.f32 %f391, %f23, %f377, %f389;\n" " fma.rn.f32 %f392, %f32, %f377, %f390;\n" " add.s64 %rd202, %rd261, %rd34;\n" " // begin inline asm\n" " ld.global.nc.f32 %f378, [%rd202];\n" " // end inline asm\n" " fma.rn.f32 %f393, %f24, %f378, %f391;\n" " fma.rn.f32 %f394, %f33, %f378, %f392;\n" " add.s64 %rd203, %rd261, %rd33;\n" " // begin inline asm\n" " ld.global.nc.f32 %f379, [%rd203];\n" " // end inline asm\n" " fma.rn.f32 %f395, %f25, %f379, %f393;\n" " fma.rn.f32 %f396, %f34, %f379, %f394;\n" " fma.rn.f32 %f780, %f17, %f395, %f780;\n" " fma.rn.f32 %f772, %f26, %f396, %f772;\n" "\n" "$L__BB30_50:\n" " @%p17 bra $L__BB30_52;\n" "\n" " add.s64 %rd204, %rd261, %rd24;\n" " // begin inline asm\n" " ld.global.nc.f32 %f397, [%rd204];\n" " // end inline asm\n" " mov.f32 %f405, 0f00000000;\n" " fma.rn.f32 %f406, %f18, %f397, %f405;\n" " fma.rn.f32 %f407, %f27, %f397, %f405;\n" " add.s64 %rd205, %rd261, %rd31;\n" " // begin inline asm\n" " ld.global.nc.f32 %f398, [%rd205];\n" " // end inline asm\n" " fma.rn.f32 %f408, %f19, %f398, %f406;\n" " fma.rn.f32 %f409, %f28, %f398, %f407;\n" " add.s64 %rd206, %rd261, %rd30;\n" " // begin inline asm\n" " ld.global.nc.f32 %f399, [%rd206];\n" " // end inline asm\n" " fma.rn.f32 %f410, %f20, %f399, %f408;\n" " fma.rn.f32 %f411, %f29, %f399, %f409;\n" " add.s64 %rd207, %rd261, %rd29;\n" " // begin inline asm\n" " ld.global.nc.f32 %f400, [%rd207];\n" " // end inline asm\n" " fma.rn.f32 %f412, %f21, %f400, %f410;\n" " fma.rn.f32 %f413, %f30, %f400, %f411;\n" " add.s64 %rd208, %rd261, %rd28;\n" " // begin inline asm\n" " ld.global.nc.f32 %f401, [%rd208];\n" " // end inline asm\n" " fma.rn.f32 %f414, %f22, %f401, %f412;\n" " fma.rn.f32 %f415, %f31, %f401, %f413;\n" " add.s64 %rd209, %rd261, %rd27;\n" " // begin inline asm\n" " ld.global.nc.f32 %f402, [%rd209];\n" " // end inline asm\n" " fma.rn.f32 %f416, %f23, %f402, %f414;\n" " fma.rn.f32 %f417, %f32, %f402, %f415;\n" " add.s64 %rd210, %rd261, %rd26;\n" " // begin inline asm\n" " ld.global.nc.f32 %f403, [%rd210];\n" " // end inline asm\n" " fma.rn.f32 %f418, %f24, %f403, %f416;\n" " fma.rn.f32 %f419, %f33, %f403, %f417;\n" " add.s64 %rd211, %rd261, %rd25;\n" " // begin inline asm\n" " ld.global.nc.f32 %f404, [%rd211];\n" " // end inline asm\n" " fma.rn.f32 %f420, %f25, %f404, %f418;\n" " fma.rn.f32 %f421, %f34, %f404, %f419;\n" " fma.rn.f32 %f781, %f17, %f420, %f781;\n" " fma.rn.f32 %f773, %f26, %f421, %f773;\n" "\n" "$L__BB30_52:\n" " add.s64 %rd261, %rd261, 1024;\n" " add.s64 %rd260, %rd260, 152;\n" " add.s32 %r306, %r306, 1;\n" " setp.lt.s32 %p29, %r306, %r23;\n" " @%p29 bra $L__BB30_36;\n" "\n" "$L__BB30_53:\n" " mov.u32 %r297, %tid.x;\n" " and.b32 %r296, %r297, 31;\n" " mov.u32 %r295, %ctaid.x;\n" " shr.u32 %r294, %r297, 5;\n" " shl.b32 %r293, %r295, 3;\n" " add.s32 %r292, %r293, %r294;\n" " ld.param.u32 %r291, [fc_moe_gateup_q3k_dedup_param_7];\n" " setp.eq.s32 %p30, %r296, 0;\n" " setp.lt.s32 %p31, %r292, %r291;\n" " and.pred %p1, %p30, %p31;\n" " not.pred %p32, %p1;\n" " mov.u32 %r125, 16;\n" " mov.u32 %r134, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f422, %f774, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f425, %f774, %f422;\n" " mov.u32 %r127, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f424, %f425, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f427, %f425, %f424;\n" " mov.u32 %r129, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f426, %f427, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f429, %f427, %f426;\n" " mov.u32 %r131, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f428, %f429, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f431, %f429, %f428;\n" " mov.u32 %r133, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f430, %f431, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f83, %f431, %f430;\n" " // begin inline asm\n" " shfl.down.b32 %f432, %f766, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f435, %f766, %f432;\n" " // begin inline asm\n" " shfl.down.b32 %f434, %f435, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f437, %f435, %f434;\n" " // begin inline asm\n" " shfl.down.b32 %f436, %f437, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f439, %f437, %f436;\n" " // begin inline asm\n" " shfl.down.b32 %f438, %f439, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f441, %f439, %f438;\n" " // begin inline asm\n" " shfl.down.b32 %f440, %f441, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f84, %f441, %f440;\n" " or.pred %p34, %p32, %p3;\n" " @%p34 bra $L__BB30_55;\n" "\n" " neg.f32 %f442, %f83;\n" " mov.f32 %f443, 0f3F000000;\n" " mov.f32 %f444, 0f3BBB989D;\n" " fma.rn.f32 %f445, %f442, %f444, %f443;\n" " mov.f32 %f446, 0f3FB8AA3B;\n" " mov.f32 %f447, 0f437C0000;\n" " cvt.sat.f32.f32 %f448, %f445;\n" " mov.f32 %f449, 0f4B400001;\n" " fma.rm.f32 %f450, %f448, %f447, %f449;\n" " add.f32 %f451, %f450, 0fCB40007F;\n" " neg.f32 %f452, %f451;\n" " fma.rn.f32 %f453, %f442, %f446, %f452;\n" " mov.f32 %f454, 0f32A57060;\n" " fma.rn.f32 %f455, %f442, %f454, %f453;\n" " mov.b32 %r135, %f450;\n" " shl.b32 %r136, %r135, 23;\n" " mov.b32 %f456, %r136;\n" " ex2.approx.ftz.f32 %f457, %f455;\n" " fma.rn.f32 %f458, %f457, %f456, 0f3F800000;\n" " div.rn.f32 %f459, %f83, %f458;\n" " mul.f32 %f460, %f84, %f459;\n" " cvt.s64.s32 %rd212, %r298;\n" " mul.lo.s64 %rd213, %rd212, %rd22;\n" " add.s64 %rd214, %rd213, %rd23;\n" " shl.b64 %rd215, %rd214, 2;\n" " add.s64 %rd216, %rd3, %rd215;\n" " st.global.f32 [%rd216], %f460;\n" "\n" "$L__BB30_55:\n" " // begin inline asm\n" " shfl.down.b32 %f461, %f775, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f464, %f775, %f461;\n" " // begin inline asm\n" " shfl.down.b32 %f463, %f464, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f466, %f464, %f463;\n" " // begin inline asm\n" " shfl.down.b32 %f465, %f466, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f468, %f466, %f465;\n" " // begin inline asm\n" " shfl.down.b32 %f467, %f468, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f470, %f468, %f467;\n" " // begin inline asm\n" " shfl.down.b32 %f469, %f470, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f85, %f470, %f469;\n" " // begin inline asm\n" " shfl.down.b32 %f471, %f767, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f474, %f767, %f471;\n" " // begin inline asm\n" " shfl.down.b32 %f473, %f474, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f476, %f474, %f473;\n" " // begin inline asm\n" " shfl.down.b32 %f475, %f476, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f478, %f476, %f475;\n" " // begin inline asm\n" " shfl.down.b32 %f477, %f478, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f480, %f478, %f477;\n" " // begin inline asm\n" " shfl.down.b32 %f479, %f480, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f86, %f480, %f479;\n" " or.pred %p37, %p32, %p5;\n" " @%p37 bra $L__BB30_57;\n" "\n" " neg.f32 %f481, %f85;\n" " mov.f32 %f482, 0f3F000000;\n" " mov.f32 %f483, 0f3BBB989D;\n" " fma.rn.f32 %f484, %f481, %f483, %f482;\n" " mov.f32 %f485, 0f3FB8AA3B;\n" " mov.f32 %f486, 0f437C0000;\n" " cvt.sat.f32.f32 %f487, %f484;\n" " mov.f32 %f488, 0f4B400001;\n" " fma.rm.f32 %f489, %f487, %f486, %f488;\n" " add.f32 %f490, %f489, 0fCB40007F;\n" " neg.f32 %f491, %f490;\n" " fma.rn.f32 %f492, %f481, %f485, %f491;\n" " mov.f32 %f493, 0f32A57060;\n" " fma.rn.f32 %f494, %f481, %f493, %f492;\n" " mov.b32 %r157, %f489;\n" " shl.b32 %r158, %r157, 23;\n" " mov.b32 %f495, %r158;\n" " ex2.approx.ftz.f32 %f496, %f494;\n" " fma.rn.f32 %f497, %f496, %f495, 0f3F800000;\n" " div.rn.f32 %f498, %f85, %f497;\n" " mul.f32 %f499, %f86, %f498;\n" " cvt.s64.s32 %rd217, %r299;\n" " mul.lo.s64 %rd218, %rd217, %rd22;\n" " add.s64 %rd219, %rd218, %rd23;\n" " shl.b64 %rd220, %rd219, 2;\n" " add.s64 %rd221, %rd3, %rd220;\n" " st.global.f32 [%rd221], %f499;\n" "\n" "$L__BB30_57:\n" " // begin inline asm\n" " shfl.down.b32 %f500, %f776, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f503, %f776, %f500;\n" " // begin inline asm\n" " shfl.down.b32 %f502, %f503, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f505, %f503, %f502;\n" " // begin inline asm\n" " shfl.down.b32 %f504, %f505, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f507, %f505, %f504;\n" " // begin inline asm\n" " shfl.down.b32 %f506, %f507, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f509, %f507, %f506;\n" " // begin inline asm\n" " shfl.down.b32 %f508, %f509, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f87, %f509, %f508;\n" " // begin inline asm\n" " shfl.down.b32 %f510, %f768, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f513, %f768, %f510;\n" " // begin inline asm\n" " shfl.down.b32 %f512, %f513, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f515, %f513, %f512;\n" " // begin inline asm\n" " shfl.down.b32 %f514, %f515, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f517, %f515, %f514;\n" " // begin inline asm\n" " shfl.down.b32 %f516, %f517, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f519, %f517, %f516;\n" " // begin inline asm\n" " shfl.down.b32 %f518, %f519, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f88, %f519, %f518;\n" " or.pred %p40, %p32, %p7;\n" " @%p40 bra $L__BB30_59;\n" "\n" " neg.f32 %f520, %f87;\n" " mov.f32 %f521, 0f3F000000;\n" " mov.f32 %f522, 0f3BBB989D;\n" " fma.rn.f32 %f523, %f520, %f522, %f521;\n" " mov.f32 %f524, 0f3FB8AA3B;\n" " mov.f32 %f525, 0f437C0000;\n" " cvt.sat.f32.f32 %f526, %f523;\n" " mov.f32 %f527, 0f4B400001;\n" " fma.rm.f32 %f528, %f526, %f525, %f527;\n" " add.f32 %f529, %f528, 0fCB40007F;\n" " neg.f32 %f530, %f529;\n" " fma.rn.f32 %f531, %f520, %f524, %f530;\n" " mov.f32 %f532, 0f32A57060;\n" " fma.rn.f32 %f533, %f520, %f532, %f531;\n" " mov.b32 %r179, %f528;\n" " shl.b32 %r180, %r179, 23;\n" " mov.b32 %f534, %r180;\n" " ex2.approx.ftz.f32 %f535, %f533;\n" " fma.rn.f32 %f536, %f535, %f534, 0f3F800000;\n" " div.rn.f32 %f537, %f87, %f536;\n" " mul.f32 %f538, %f88, %f537;\n" " cvt.s64.s32 %rd222, %r300;\n" " mul.lo.s64 %rd223, %rd222, %rd22;\n" " add.s64 %rd224, %rd223, %rd23;\n" " shl.b64 %rd225, %rd224, 2;\n" " add.s64 %rd226, %rd3, %rd225;\n" " st.global.f32 [%rd226], %f538;\n" "\n" "$L__BB30_59:\n" " // begin inline asm\n" " shfl.down.b32 %f539, %f777, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f542, %f777, %f539;\n" " // begin inline asm\n" " shfl.down.b32 %f541, %f542, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f544, %f542, %f541;\n" " // begin inline asm\n" " shfl.down.b32 %f543, %f544, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f546, %f544, %f543;\n" " // begin inline asm\n" " shfl.down.b32 %f545, %f546, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f548, %f546, %f545;\n" " // begin inline asm\n" " shfl.down.b32 %f547, %f548, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f89, %f548, %f547;\n" " // begin inline asm\n" " shfl.down.b32 %f549, %f769, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f552, %f769, %f549;\n" " // begin inline asm\n" " shfl.down.b32 %f551, %f552, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f554, %f552, %f551;\n" " // begin inline asm\n" " shfl.down.b32 %f553, %f554, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f556, %f554, %f553;\n" " // begin inline asm\n" " shfl.down.b32 %f555, %f556, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f558, %f556, %f555;\n" " // begin inline asm\n" " shfl.down.b32 %f557, %f558, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f90, %f558, %f557;\n" " or.pred %p43, %p32, %p9;\n" " @%p43 bra $L__BB30_61;\n" "\n" " neg.f32 %f559, %f89;\n" " mov.f32 %f560, 0f3F000000;\n" " mov.f32 %f561, 0f3BBB989D;\n" " fma.rn.f32 %f562, %f559, %f561, %f560;\n" " mov.f32 %f563, 0f3FB8AA3B;\n" " mov.f32 %f564, 0f437C0000;\n" " cvt.sat.f32.f32 %f565, %f562;\n" " mov.f32 %f566, 0f4B400001;\n" " fma.rm.f32 %f567, %f565, %f564, %f566;\n" " add.f32 %f568, %f567, 0fCB40007F;\n" " neg.f32 %f569, %f568;\n" " fma.rn.f32 %f570, %f559, %f563, %f569;\n" " mov.f32 %f571, 0f32A57060;\n" " fma.rn.f32 %f572, %f559, %f571, %f570;\n" " mov.b32 %r201, %f567;\n" " shl.b32 %r202, %r201, 23;\n" " mov.b32 %f573, %r202;\n" " ex2.approx.ftz.f32 %f574, %f572;\n" " fma.rn.f32 %f575, %f574, %f573, 0f3F800000;\n" " div.rn.f32 %f576, %f89, %f575;\n" " mul.f32 %f577, %f90, %f576;\n" " cvt.s64.s32 %rd227, %r301;\n" " mul.lo.s64 %rd228, %rd227, %rd22;\n" " add.s64 %rd229, %rd228, %rd23;\n" " shl.b64 %rd230, %rd229, 2;\n" " add.s64 %rd231, %rd3, %rd230;\n" " st.global.f32 [%rd231], %f577;\n" "\n" "$L__BB30_61:\n" " // begin inline asm\n" " shfl.down.b32 %f578, %f778, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f581, %f778, %f578;\n" " // begin inline asm\n" " shfl.down.b32 %f580, %f581, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f583, %f581, %f580;\n" " // begin inline asm\n" " shfl.down.b32 %f582, %f583, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f585, %f583, %f582;\n" " // begin inline asm\n" " shfl.down.b32 %f584, %f585, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f587, %f585, %f584;\n" " // begin inline asm\n" " shfl.down.b32 %f586, %f587, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f91, %f587, %f586;\n" " // begin inline asm\n" " shfl.down.b32 %f588, %f770, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f591, %f770, %f588;\n" " // begin inline asm\n" " shfl.down.b32 %f590, %f591, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f593, %f591, %f590;\n" " // begin inline asm\n" " shfl.down.b32 %f592, %f593, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f595, %f593, %f592;\n" " // begin inline asm\n" " shfl.down.b32 %f594, %f595, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f597, %f595, %f594;\n" " // begin inline asm\n" " shfl.down.b32 %f596, %f597, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f92, %f597, %f596;\n" " or.pred %p46, %p32, %p11;\n" " @%p46 bra $L__BB30_63;\n" "\n" " neg.f32 %f598, %f91;\n" " mov.f32 %f599, 0f3F000000;\n" " mov.f32 %f600, 0f3BBB989D;\n" " fma.rn.f32 %f601, %f598, %f600, %f599;\n" " mov.f32 %f602, 0f3FB8AA3B;\n" " mov.f32 %f603, 0f437C0000;\n" " cvt.sat.f32.f32 %f604, %f601;\n" " mov.f32 %f605, 0f4B400001;\n" " fma.rm.f32 %f606, %f604, %f603, %f605;\n" " add.f32 %f607, %f606, 0fCB40007F;\n" " neg.f32 %f608, %f607;\n" " fma.rn.f32 %f609, %f598, %f602, %f608;\n" " mov.f32 %f610, 0f32A57060;\n" " fma.rn.f32 %f611, %f598, %f610, %f609;\n" " mov.b32 %r223, %f606;\n" " shl.b32 %r224, %r223, 23;\n" " mov.b32 %f612, %r224;\n" " ex2.approx.ftz.f32 %f613, %f611;\n" " fma.rn.f32 %f614, %f613, %f612, 0f3F800000;\n" " div.rn.f32 %f615, %f91, %f614;\n" " mul.f32 %f616, %f92, %f615;\n" " cvt.s64.s32 %rd232, %r302;\n" " mul.lo.s64 %rd233, %rd232, %rd22;\n" " add.s64 %rd234, %rd233, %rd23;\n" " shl.b64 %rd235, %rd234, 2;\n" " add.s64 %rd236, %rd3, %rd235;\n" " st.global.f32 [%rd236], %f616;\n" "\n" "$L__BB30_63:\n" " // begin inline asm\n" " shfl.down.b32 %f617, %f779, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f620, %f779, %f617;\n" " // begin inline asm\n" " shfl.down.b32 %f619, %f620, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f622, %f620, %f619;\n" " // begin inline asm\n" " shfl.down.b32 %f621, %f622, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f624, %f622, %f621;\n" " // begin inline asm\n" " shfl.down.b32 %f623, %f624, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f626, %f624, %f623;\n" " // begin inline asm\n" " shfl.down.b32 %f625, %f626, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f93, %f626, %f625;\n" " // begin inline asm\n" " shfl.down.b32 %f627, %f771, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f630, %f771, %f627;\n" " // begin inline asm\n" " shfl.down.b32 %f629, %f630, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f632, %f630, %f629;\n" " // begin inline asm\n" " shfl.down.b32 %f631, %f632, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f634, %f632, %f631;\n" " // begin inline asm\n" " shfl.down.b32 %f633, %f634, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f636, %f634, %f633;\n" " // begin inline asm\n" " shfl.down.b32 %f635, %f636, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f94, %f636, %f635;\n" " or.pred %p49, %p32, %p13;\n" " @%p49 bra $L__BB30_65;\n" "\n" " neg.f32 %f637, %f93;\n" " mov.f32 %f638, 0f3F000000;\n" " mov.f32 %f639, 0f3BBB989D;\n" " fma.rn.f32 %f640, %f637, %f639, %f638;\n" " mov.f32 %f641, 0f3FB8AA3B;\n" " mov.f32 %f642, 0f437C0000;\n" " cvt.sat.f32.f32 %f643, %f640;\n" " mov.f32 %f644, 0f4B400001;\n" " fma.rm.f32 %f645, %f643, %f642, %f644;\n" " add.f32 %f646, %f645, 0fCB40007F;\n" " neg.f32 %f647, %f646;\n" " fma.rn.f32 %f648, %f637, %f641, %f647;\n" " mov.f32 %f649, 0f32A57060;\n" " fma.rn.f32 %f650, %f637, %f649, %f648;\n" " mov.b32 %r245, %f645;\n" " shl.b32 %r246, %r245, 23;\n" " mov.b32 %f651, %r246;\n" " ex2.approx.ftz.f32 %f652, %f650;\n" " fma.rn.f32 %f653, %f652, %f651, 0f3F800000;\n" " div.rn.f32 %f654, %f93, %f653;\n" " mul.f32 %f655, %f94, %f654;\n" " cvt.s64.s32 %rd237, %r303;\n" " mul.lo.s64 %rd238, %rd237, %rd22;\n" " add.s64 %rd239, %rd238, %rd23;\n" " shl.b64 %rd240, %rd239, 2;\n" " add.s64 %rd241, %rd3, %rd240;\n" " st.global.f32 [%rd241], %f655;\n" "\n" "$L__BB30_65:\n" " // begin inline asm\n" " shfl.down.b32 %f656, %f780, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f659, %f780, %f656;\n" " // begin inline asm\n" " shfl.down.b32 %f658, %f659, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f661, %f659, %f658;\n" " // begin inline asm\n" " shfl.down.b32 %f660, %f661, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f663, %f661, %f660;\n" " // begin inline asm\n" " shfl.down.b32 %f662, %f663, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f665, %f663, %f662;\n" " // begin inline asm\n" " shfl.down.b32 %f664, %f665, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f95, %f665, %f664;\n" " // begin inline asm\n" " shfl.down.b32 %f666, %f772, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f669, %f772, %f666;\n" " // begin inline asm\n" " shfl.down.b32 %f668, %f669, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f671, %f669, %f668;\n" " // begin inline asm\n" " shfl.down.b32 %f670, %f671, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f673, %f671, %f670;\n" " // begin inline asm\n" " shfl.down.b32 %f672, %f673, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f675, %f673, %f672;\n" " // begin inline asm\n" " shfl.down.b32 %f674, %f675, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f96, %f675, %f674;\n" " or.pred %p52, %p32, %p15;\n" " @%p52 bra $L__BB30_67;\n" "\n" " neg.f32 %f676, %f95;\n" " mov.f32 %f677, 0f3F000000;\n" " mov.f32 %f678, 0f3BBB989D;\n" " fma.rn.f32 %f679, %f676, %f678, %f677;\n" " mov.f32 %f680, 0f3FB8AA3B;\n" " mov.f32 %f681, 0f437C0000;\n" " cvt.sat.f32.f32 %f682, %f679;\n" " mov.f32 %f683, 0f4B400001;\n" " fma.rm.f32 %f684, %f682, %f681, %f683;\n" " add.f32 %f685, %f684, 0fCB40007F;\n" " neg.f32 %f686, %f685;\n" " fma.rn.f32 %f687, %f676, %f680, %f686;\n" " mov.f32 %f688, 0f32A57060;\n" " fma.rn.f32 %f689, %f676, %f688, %f687;\n" " mov.b32 %r267, %f684;\n" " shl.b32 %r268, %r267, 23;\n" " mov.b32 %f690, %r268;\n" " ex2.approx.ftz.f32 %f691, %f689;\n" " fma.rn.f32 %f692, %f691, %f690, 0f3F800000;\n" " div.rn.f32 %f693, %f95, %f692;\n" " mul.f32 %f694, %f96, %f693;\n" " cvt.s64.s32 %rd242, %r304;\n" " mul.lo.s64 %rd243, %rd242, %rd22;\n" " add.s64 %rd244, %rd243, %rd23;\n" " shl.b64 %rd245, %rd244, 2;\n" " add.s64 %rd246, %rd3, %rd245;\n" " st.global.f32 [%rd246], %f694;\n" "\n" "$L__BB30_67:\n" " // begin inline asm\n" " shfl.down.b32 %f695, %f781, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f698, %f781, %f695;\n" " // begin inline asm\n" " shfl.down.b32 %f697, %f698, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f700, %f698, %f697;\n" " // begin inline asm\n" " shfl.down.b32 %f699, %f700, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f702, %f700, %f699;\n" " // begin inline asm\n" " shfl.down.b32 %f701, %f702, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f704, %f702, %f701;\n" " // begin inline asm\n" " shfl.down.b32 %f703, %f704, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f97, %f704, %f703;\n" " // begin inline asm\n" " shfl.down.b32 %f705, %f773, %r125, %r134;\n" " // end inline asm\n" " add.f32 %f708, %f773, %f705;\n" " // begin inline asm\n" " shfl.down.b32 %f707, %f708, %r127, %r134;\n" " // end inline asm\n" " add.f32 %f710, %f708, %f707;\n" " // begin inline asm\n" " shfl.down.b32 %f709, %f710, %r129, %r134;\n" " // end inline asm\n" " add.f32 %f712, %f710, %f709;\n" " // begin inline asm\n" " shfl.down.b32 %f711, %f712, %r131, %r134;\n" " // end inline asm\n" " add.f32 %f714, %f712, %f711;\n" " // begin inline asm\n" " shfl.down.b32 %f713, %f714, %r133, %r134;\n" " // end inline asm\n" " add.f32 %f98, %f714, %f713;\n" " or.pred %p55, %p32, %p17;\n" " @%p55 bra $L__BB30_69;\n" "\n" " neg.f32 %f715, %f97;\n" " mov.f32 %f716, 0f3F000000;\n" " mov.f32 %f717, 0f3BBB989D;\n" " fma.rn.f32 %f718, %f715, %f717, %f716;\n" " mov.f32 %f719, 0f3FB8AA3B;\n" " mov.f32 %f720, 0f437C0000;\n" " cvt.sat.f32.f32 %f721, %f718;\n" " mov.f32 %f722, 0f4B400001;\n" " fma.rm.f32 %f723, %f721, %f720, %f722;\n" " add.f32 %f724, %f723, 0fCB40007F;\n" " neg.f32 %f725, %f724;\n" " fma.rn.f32 %f726, %f715, %f719, %f725;\n" " mov.f32 %f727, 0f32A57060;\n" " fma.rn.f32 %f728, %f715, %f727, %f726;\n" " mov.b32 %r289, %f723;\n" " shl.b32 %r290, %r289, 23;\n" " mov.b32 %f729, %r290;\n" " ex2.approx.ftz.f32 %f730, %f728;\n" " fma.rn.f32 %f731, %f730, %f729, 0f3F800000;\n" " div.rn.f32 %f732, %f97, %f731;\n" " mul.f32 %f733, %f98, %f732;\n" " cvt.s64.s32 %rd247, %r305;\n" " mul.lo.s64 %rd248, %rd247, %rd22;\n" " add.s64 %rd249, %rd248, %rd23;\n" " shl.b64 %rd250, %rd249, 2;\n" " add.s64 %rd251, %rd3, %rd250;\n" " st.global.f32 [%rd251], %f733;\n" "\n" "$L__BB30_69:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_q3k_dedup_shx\n" ".visible .entry fc_moe_gateup_q3k_dedup_shx(\n" " .param .u64 fc_moe_gateup_q3k_dedup_shx_param_0,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shx_param_1,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shx_param_2,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shx_param_3,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shx_param_4,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shx_param_5,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shx_param_6,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shx_param_7,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shx_param_8,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shx_param_9,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shx_param_10\n" ")\n" "{\n" " .reg .pred %p<45>;\n" " .reg .f32 %f<806>;\n" " .reg .b32 %r<307>;\n" " .reg .b64 %rd<104>;\n" " // demoted variable\n" " .shared .align 16 .b8 _ZZ27fc_moe_gateup_q3k_dedup_shxE4sh_x[8192];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj[32];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ27fc_moe_gateup_q3k_dedup_shxE6sh_tok[32];\n" "\n" " ld.param.u64 %rd18, [fc_moe_gateup_q3k_dedup_shx_param_0];\n" " ld.param.u64 %rd19, [fc_moe_gateup_q3k_dedup_shx_param_1];\n" " ld.param.u64 %rd20, [fc_moe_gateup_q3k_dedup_shx_param_2];\n" " ld.param.u64 %rd22, [fc_moe_gateup_q3k_dedup_shx_param_3];\n" " ld.param.u64 %rd21, [fc_moe_gateup_q3k_dedup_shx_param_4];\n" " ld.param.u64 %rd23, [fc_moe_gateup_q3k_dedup_shx_param_5];\n" " ld.param.u32 %r22, [fc_moe_gateup_q3k_dedup_shx_param_6];\n" " ld.param.u32 %r23, [fc_moe_gateup_q3k_dedup_shx_param_7];\n" " ld.param.u32 %r24, [fc_moe_gateup_q3k_dedup_shx_param_8];\n" " ld.param.u32 %r25, [fc_moe_gateup_q3k_dedup_shx_param_9];\n" " ld.param.u32 %r26, [fc_moe_gateup_q3k_dedup_shx_param_10];\n" " cvta.to.global.u64 %rd1, %rd23;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r27, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r28, %ctaid.x;\n" " shl.b32 %r29, %r28, 3;\n" " add.s32 %r3, %r29, %r27;\n" " mov.u32 %r30, %ctaid.y;\n" " cvt.s64.s32 %rd2, %r30;\n" " cvta.to.global.u64 %rd24, %rd22;\n" " mul.wide.s32 %rd25, %r30, 4;\n" " add.s64 %rd26, %rd24, %rd25;\n" " mov.u32 %r31, %ctaid.z;\n" " shl.b32 %r4, %r31, 3;\n" " ld.global.u32 %r5, [%rd26];\n" " setp.le.s32 %p2, %r5, %r4;\n" " @%p2 bra $L__BB31_48;\n" "\n" " sub.s32 %r6, %r5, %r4;\n" " min.s32 %r7, %r6, 8;\n" " setp.gt.s32 %p3, %r1, 7;\n" " @%p3 bra $L__BB31_7;\n" "\n" " setp.ge.s32 %p4, %r1, %r7;\n" " mov.u32 %r304, -1;\n" " @%p4 bra $L__BB31_4;\n" "\n" " cvt.s64.s32 %rd27, %r26;\n" " mul.lo.s64 %rd28, %rd27, %rd2;\n" " cvt.s64.s32 %rd29, %r1;\n" " cvt.s64.s32 %rd30, %r4;\n" " add.s64 %rd31, %rd30, %rd29;\n" " add.s64 %rd32, %rd31, %rd28;\n" " cvta.to.global.u64 %rd33, %rd21;\n" " shl.b64 %rd34, %rd32, 2;\n" " add.s64 %rd35, %rd33, %rd34;\n" " ld.global.u32 %r304, [%rd35];\n" "\n" "$L__BB31_4:\n" " shl.b32 %r34, %r1, 2;\n" " mov.u32 %r35, _ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj;\n" " add.s32 %r36, %r35, %r34;\n" " mov.u32 %r305, 0;\n" " st.shared.u32 [%r36], %r304;\n" " setp.lt.s32 %p5, %r304, 0;\n" " @%p5 bra $L__BB31_6;\n" "\n" " div.s32 %r305, %r304, %r22;\n" "\n" "$L__BB31_6:\n" " mov.u32 %r38, _ZZ27fc_moe_gateup_q3k_dedup_shxE6sh_tok;\n" " add.s32 %r39, %r38, %r34;\n" " st.shared.u32 [%r39], %r305;\n" "\n" "$L__BB31_7:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd3, %r23;\n" " cvt.s64.s32 %rd4, %r3;\n" " setp.lt.s32 %p6, %r24, 256;\n" " mov.f32 %f774, 0f00000000;\n" " mov.f32 %f775, %f774;\n" " mov.f32 %f776, %f774;\n" " mov.f32 %f777, %f774;\n" " mov.f32 %f778, %f774;\n" " mov.f32 %f779, %f774;\n" " mov.f32 %f780, %f774;\n" " mov.f32 %f781, %f774;\n" " mov.f32 %f782, %f774;\n" " mov.f32 %f783, %f774;\n" " mov.f32 %f784, %f774;\n" " mov.f32 %f785, %f774;\n" " mov.f32 %f786, %f774;\n" " mov.f32 %f787, %f774;\n" " mov.f32 %f788, %f774;\n" " mov.f32 %f789, %f774;\n" " @%p6 bra $L__BB31_32;\n" "\n" " cvt.s64.s32 %rd36, %r25;\n" " mul.lo.s64 %rd37, %rd3, %rd2;\n" " add.s64 %rd38, %rd37, %rd4;\n" " mul.lo.s64 %rd39, %rd38, %rd36;\n" " cvt.s64.s32 %rd5, %r24;\n" " cvt.u64.u32 %rd6, %r2;\n" " shr.u32 %r41, %r2, 4;\n" " cvt.u64.u32 %rd7, %r41;\n" " shl.b32 %r42, %r1, 2;\n" " and.b32 %r43, %r42, 252;\n" " shr.s32 %r12, %r1, 6;\n" " shl.b32 %r44, %r12, 2;\n" " mov.u32 %r45, _ZZ27fc_moe_gateup_q3k_dedup_shxE6sh_tok;\n" " add.s32 %r13, %r45, %r44;\n" " cvt.u64.u32 %rd8, %r43;\n" " shl.b32 %r46, %r12, 8;\n" " or.b32 %r47, %r46, %r43;\n" " shl.b32 %r48, %r47, 2;\n" " mov.u32 %r49, _ZZ27fc_moe_gateup_q3k_dedup_shxE4sh_x;\n" " add.s32 %r14, %r49, %r48;\n" " add.s32 %r50, %r1, 256;\n" " shr.s32 %r15, %r50, 6;\n" " shl.b32 %r51, %r50, 2;\n" " and.b32 %r52, %r51, 252;\n" " shl.b32 %r53, %r15, 2;\n" " add.s32 %r16, %r45, %r53;\n" " shl.b32 %r54, %r15, 8;\n" " or.b32 %r55, %r54, %r52;\n" " shl.b32 %r56, %r55, 2;\n" " add.s32 %r17, %r49, %r56;\n" " shl.b32 %r57, %r2, 2;\n" " add.s32 %r18, %r49, %r57;\n" " shr.s32 %r19, %r24, 8;\n" " cvta.to.global.u64 %rd40, %rd19;\n" " add.s64 %rd103, %rd40, %rd39;\n" " cvta.to.global.u64 %rd41, %rd18;\n" " add.s64 %rd102, %rd41, %rd39;\n" " mov.u32 %r306, 0;\n" " mov.f32 %f774, 0f00000000;\n" " setp.ge.s32 %p7, %r12, %r7;\n" " setp.ge.s32 %p8, %r15, %r7;\n" " setp.ge.s32 %p9, %r3, %r23;\n" " setp.lt.s32 %p10, %r6, 1;\n" " shl.b64 %rd54, %rd6, 2;\n" " shl.b64 %rd56, %rd7, 3;\n" " mov.f32 %f775, %f774;\n" " mov.f32 %f776, %f774;\n" " mov.f32 %f777, %f774;\n" " mov.f32 %f778, %f774;\n" " mov.f32 %f779, %f774;\n" " mov.f32 %f780, %f774;\n" " mov.f32 %f781, %f774;\n" " mov.f32 %f782, %f774;\n" " mov.f32 %f783, %f774;\n" " mov.f32 %f784, %f774;\n" " mov.f32 %f785, %f774;\n" " mov.f32 %f786, %f774;\n" " mov.f32 %f787, %f774;\n" " mov.f32 %f788, %f774;\n" " mov.f32 %f789, %f774;\n" "\n" "$L__BB31_9:\n" " bar.sync 0;\n" " mul.wide.u32 %rd13, %r306, 256;\n" " @%p7 bra $L__BB31_11;\n" "\n" " ld.shared.s32 %rd43, [%r13];\n" " mul.lo.s64 %rd44, %rd43, %rd5;\n" " add.s64 %rd45, %rd13, %rd8;\n" " add.s64 %rd46, %rd45, %rd44;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd42, %rd20, %rd47;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f147,%f148,%f149,%f150}, [%rd42];\n" " // end inline asm\n" " st.shared.v4.f32 [%r14], {%f147, %f148, %f149, %f150};\n" "\n" "$L__BB31_11:\n" " @%p8 bra $L__BB31_13;\n" "\n" " ld.shared.s32 %rd49, [%r16];\n" " mul.lo.s64 %rd50, %rd49, %rd5;\n" " add.s64 %rd51, %rd13, %rd8;\n" " add.s64 %rd52, %rd51, %rd50;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd48, %rd20, %rd53;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f151,%f152,%f153,%f154}, [%rd48];\n" " // end inline asm\n" " st.shared.v4.f32 [%r17], {%f151, %f152, %f153, %f154};\n" "\n" "$L__BB31_13:\n" " bar.sync 0;\n" " @%p9 bra $L__BB31_31;\n" "\n" " add.s64 %rd55, %rd102, %rd54;\n" " ld.global.f32 %f17, [%rd102+144];\n" " add.s64 %rd57, %rd102, %rd56;\n" " ld.global.v2.u32 {%r58, %r59}, [%rd57+128];\n" " ld.global.u32 %r62, [%rd55];\n" " shr.u32 %r63, %r62, 16;\n" " and.b32 %r64, %r62, 15;\n" " or.b32 %r65, %r64, 1258291200;\n" " mov.b32 %f155, %r65;\n" " mov.f32 %f156, 0fCB000004;\n" " mov.f32 %f157, 0f3F800000;\n" " fma.rn.f32 %f158, %f155, %f157, %f156;\n" " and.b32 %r66, %r58, 255;\n" " cvt.rn.f32.u32 %f159, %r66;\n" " mov.f32 %f160, 0f42000000;\n" " sub.rn.f32 %f161, %f159, %f160;\n" " mul.rn.f32 %f18, %f161, %f158;\n" " and.b32 %r67, %r62, 240;\n" " or.b32 %r68, %r67, 1258291200;\n" " mov.b32 %f162, %r68;\n" " mov.f32 %f163, 0fC9000040;\n" " mov.f32 %f164, 0f3D800000;\n" " fma.rn.f32 %f165, %f162, %f164, %f163;\n" " bfe.u32 %r69, %r58, 8, 8;\n" " cvt.rn.f32.u32 %f166, %r69;\n" " sub.rn.f32 %f167, %f166, %f160;\n" " mul.rn.f32 %f19, %f167, %f165;\n" " and.b32 %r70, %r62, 3840;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f168, %r71;\n" " mov.f32 %f169, 0fC7000400;\n" " mov.f32 %f170, 0f3B800000;\n" " fma.rn.f32 %f171, %f168, %f170, %f169;\n" " bfe.u32 %r72, %r58, 16, 8;\n" " cvt.rn.f32.u32 %f172, %r72;\n" " sub.rn.f32 %f173, %f172, %f160;\n" " mul.rn.f32 %f20, %f173, %f171;\n" " and.b32 %r73, %r62, 61440;\n" " or.b32 %r74, %r73, 1258291200;\n" " mov.b32 %f174, %r74;\n" " mov.f32 %f175, 0fC5004000;\n" " mov.f32 %f176, 0f39800000;\n" " fma.rn.f32 %f177, %f174, %f176, %f175;\n" " shr.u32 %r75, %r58, 24;\n" " cvt.rn.f32.u32 %f178, %r75;\n" " sub.rn.f32 %f179, %f178, %f160;\n" " mul.rn.f32 %f21, %f179, %f177;\n" " and.b32 %r76, %r62, 983040;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f180, %r77;\n" " mov.f32 %f181, 0fC3040000;\n" " mov.f32 %f182, 0f37800000;\n" " fma.rn.f32 %f183, %f180, %f182, %f181;\n" " and.b32 %r78, %r59, 255;\n" " cvt.rn.f32.u32 %f184, %r78;\n" " sub.rn.f32 %f185, %f184, %f160;\n" " mul.rn.f32 %f22, %f185, %f183;\n" " and.b32 %r79, %r63, 240;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f186, %r80;\n" " fma.rn.f32 %f187, %f186, %f164, %f163;\n" " bfe.u32 %r81, %r59, 8, 8;\n" " cvt.rn.f32.u32 %f188, %r81;\n" " sub.rn.f32 %f189, %f188, %f160;\n" " mul.rn.f32 %f23, %f189, %f187;\n" " and.b32 %r82, %r63, 3840;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f190, %r83;\n" " fma.rn.f32 %f191, %f190, %f170, %f169;\n" " bfe.u32 %r84, %r59, 16, 8;\n" " cvt.rn.f32.u32 %f192, %r84;\n" " sub.rn.f32 %f193, %f192, %f160;\n" " mul.rn.f32 %f24, %f193, %f191;\n" " and.b32 %r85, %r63, 61440;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f194, %r86;\n" " fma.rn.f32 %f195, %f194, %f176, %f175;\n" " shr.u32 %r87, %r59, 24;\n" " cvt.rn.f32.u32 %f196, %r87;\n" " sub.rn.f32 %f197, %f196, %f160;\n" " mul.rn.f32 %f25, %f197, %f195;\n" " add.s64 %rd58, %rd103, %rd54;\n" " ld.global.f32 %f26, [%rd103+144];\n" " add.s64 %rd59, %rd103, %rd56;\n" " ld.global.v2.u32 {%r88, %r89}, [%rd59+128];\n" " ld.global.u32 %r92, [%rd58];\n" " shr.u32 %r93, %r92, 16;\n" " and.b32 %r94, %r92, 15;\n" " or.b32 %r95, %r94, 1258291200;\n" " mov.b32 %f198, %r95;\n" " fma.rn.f32 %f199, %f198, %f157, %f156;\n" " and.b32 %r96, %r88, 255;\n" " cvt.rn.f32.u32 %f200, %r96;\n" " sub.rn.f32 %f201, %f200, %f160;\n" " mul.rn.f32 %f27, %f201, %f199;\n" " and.b32 %r97, %r92, 240;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f202, %r98;\n" " fma.rn.f32 %f203, %f202, %f164, %f163;\n" " bfe.u32 %r99, %r88, 8, 8;\n" " cvt.rn.f32.u32 %f204, %r99;\n" " sub.rn.f32 %f205, %f204, %f160;\n" " mul.rn.f32 %f28, %f205, %f203;\n" " and.b32 %r100, %r92, 3840;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f206, %r101;\n" " fma.rn.f32 %f207, %f206, %f170, %f169;\n" " bfe.u32 %r102, %r88, 16, 8;\n" " cvt.rn.f32.u32 %f208, %r102;\n" " sub.rn.f32 %f209, %f208, %f160;\n" " mul.rn.f32 %f29, %f209, %f207;\n" " and.b32 %r103, %r92, 61440;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f210, %r104;\n" " fma.rn.f32 %f211, %f210, %f176, %f175;\n" " shr.u32 %r105, %r88, 24;\n" " cvt.rn.f32.u32 %f212, %r105;\n" " sub.rn.f32 %f213, %f212, %f160;\n" " mul.rn.f32 %f30, %f213, %f211;\n" " and.b32 %r106, %r92, 983040;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f214, %r107;\n" " fma.rn.f32 %f215, %f214, %f182, %f181;\n" " and.b32 %r108, %r89, 255;\n" " cvt.rn.f32.u32 %f216, %r108;\n" " sub.rn.f32 %f217, %f216, %f160;\n" " mul.rn.f32 %f31, %f217, %f215;\n" " and.b32 %r109, %r93, 240;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f218, %r110;\n" " fma.rn.f32 %f219, %f218, %f164, %f163;\n" " bfe.u32 %r111, %r89, 8, 8;\n" " cvt.rn.f32.u32 %f220, %r111;\n" " sub.rn.f32 %f221, %f220, %f160;\n" " mul.rn.f32 %f32, %f221, %f219;\n" " and.b32 %r112, %r93, 3840;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f222, %r113;\n" " fma.rn.f32 %f223, %f222, %f170, %f169;\n" " bfe.u32 %r114, %r89, 16, 8;\n" " cvt.rn.f32.u32 %f224, %r114;\n" " sub.rn.f32 %f225, %f224, %f160;\n" " mul.rn.f32 %f33, %f225, %f223;\n" " and.b32 %r115, %r93, 61440;\n" " or.b32 %r116, %r115, 1258291200;\n" " mov.b32 %f226, %r116;\n" " fma.rn.f32 %f227, %f226, %f176, %f175;\n" " shr.u32 %r117, %r89, 24;\n" " cvt.rn.f32.u32 %f228, %r117;\n" " sub.rn.f32 %f229, %f228, %f160;\n" " mul.rn.f32 %f34, %f229, %f227;\n" " @%p10 bra $L__BB31_16;\n" "\n" " ld.shared.f32 %f230, [%r18];\n" " mov.f32 %f231, 0f00000000;\n" " fma.rn.f32 %f232, %f18, %f230, %f231;\n" " fma.rn.f32 %f233, %f27, %f230, %f231;\n" " ld.shared.f32 %f234, [%r18+128];\n" " fma.rn.f32 %f235, %f19, %f234, %f232;\n" " fma.rn.f32 %f236, %f28, %f234, %f233;\n" " ld.shared.f32 %f237, [%r18+256];\n" " fma.rn.f32 %f238, %f20, %f237, %f235;\n" " fma.rn.f32 %f239, %f29, %f237, %f236;\n" " ld.shared.f32 %f240, [%r18+384];\n" " fma.rn.f32 %f241, %f21, %f240, %f238;\n" " fma.rn.f32 %f242, %f30, %f240, %f239;\n" " ld.shared.f32 %f243, [%r18+512];\n" " fma.rn.f32 %f244, %f22, %f243, %f241;\n" " fma.rn.f32 %f245, %f31, %f243, %f242;\n" " ld.shared.f32 %f246, [%r18+640];\n" " fma.rn.f32 %f247, %f23, %f246, %f244;\n" " fma.rn.f32 %f248, %f32, %f246, %f245;\n" " ld.shared.f32 %f249, [%r18+768];\n" " fma.rn.f32 %f250, %f24, %f249, %f247;\n" " fma.rn.f32 %f251, %f33, %f249, %f248;\n" " ld.shared.f32 %f252, [%r18+896];\n" " fma.rn.f32 %f253, %f25, %f252, %f250;\n" " fma.rn.f32 %f254, %f34, %f252, %f251;\n" " fma.rn.f32 %f782, %f17, %f253, %f782;\n" " fma.rn.f32 %f774, %f26, %f254, %f774;\n" "\n" "$L__BB31_16:\n" " setp.lt.s32 %p11, %r7, 2;\n" " @%p11 bra $L__BB31_18;\n" "\n" " ld.shared.f32 %f255, [%r18+1024];\n" " mov.f32 %f256, 0f00000000;\n" " fma.rn.f32 %f257, %f18, %f255, %f256;\n" " fma.rn.f32 %f258, %f27, %f255, %f256;\n" " ld.shared.f32 %f259, [%r18+1152];\n" " fma.rn.f32 %f260, %f19, %f259, %f257;\n" " fma.rn.f32 %f261, %f28, %f259, %f258;\n" " ld.shared.f32 %f262, [%r18+1280];\n" " fma.rn.f32 %f263, %f20, %f262, %f260;\n" " fma.rn.f32 %f264, %f29, %f262, %f261;\n" " ld.shared.f32 %f265, [%r18+1408];\n" " fma.rn.f32 %f266, %f21, %f265, %f263;\n" " fma.rn.f32 %f267, %f30, %f265, %f264;\n" " ld.shared.f32 %f268, [%r18+1536];\n" " fma.rn.f32 %f269, %f22, %f268, %f266;\n" " fma.rn.f32 %f270, %f31, %f268, %f267;\n" " ld.shared.f32 %f271, [%r18+1664];\n" " fma.rn.f32 %f272, %f23, %f271, %f269;\n" " fma.rn.f32 %f273, %f32, %f271, %f270;\n" " ld.shared.f32 %f274, [%r18+1792];\n" " fma.rn.f32 %f275, %f24, %f274, %f272;\n" " fma.rn.f32 %f276, %f33, %f274, %f273;\n" " ld.shared.f32 %f277, [%r18+1920];\n" " fma.rn.f32 %f278, %f25, %f277, %f275;\n" " fma.rn.f32 %f279, %f34, %f277, %f276;\n" " fma.rn.f32 %f783, %f17, %f278, %f783;\n" " fma.rn.f32 %f775, %f26, %f279, %f775;\n" "\n" "$L__BB31_18:\n" " setp.lt.s32 %p12, %r7, 3;\n" " @%p12 bra $L__BB31_20;\n" "\n" " ld.shared.f32 %f280, [%r18+2048];\n" " mov.f32 %f281, 0f00000000;\n" " fma.rn.f32 %f282, %f18, %f280, %f281;\n" " fma.rn.f32 %f283, %f27, %f280, %f281;\n" " ld.shared.f32 %f284, [%r18+2176];\n" " fma.rn.f32 %f285, %f19, %f284, %f282;\n" " fma.rn.f32 %f286, %f28, %f284, %f283;\n" " ld.shared.f32 %f287, [%r18+2304];\n" " fma.rn.f32 %f288, %f20, %f287, %f285;\n" " fma.rn.f32 %f289, %f29, %f287, %f286;\n" " ld.shared.f32 %f290, [%r18+2432];\n" " fma.rn.f32 %f291, %f21, %f290, %f288;\n" " fma.rn.f32 %f292, %f30, %f290, %f289;\n" " ld.shared.f32 %f293, [%r18+2560];\n" " fma.rn.f32 %f294, %f22, %f293, %f291;\n" " fma.rn.f32 %f295, %f31, %f293, %f292;\n" " ld.shared.f32 %f296, [%r18+2688];\n" " fma.rn.f32 %f297, %f23, %f296, %f294;\n" " fma.rn.f32 %f298, %f32, %f296, %f295;\n" " ld.shared.f32 %f299, [%r18+2816];\n" " fma.rn.f32 %f300, %f24, %f299, %f297;\n" " fma.rn.f32 %f301, %f33, %f299, %f298;\n" " ld.shared.f32 %f302, [%r18+2944];\n" " fma.rn.f32 %f303, %f25, %f302, %f300;\n" " fma.rn.f32 %f304, %f34, %f302, %f301;\n" " fma.rn.f32 %f784, %f17, %f303, %f784;\n" " fma.rn.f32 %f776, %f26, %f304, %f776;\n" "\n" "$L__BB31_20:\n" " setp.lt.s32 %p13, %r7, 4;\n" " @%p13 bra $L__BB31_22;\n" "\n" " ld.shared.f32 %f305, [%r18+3072];\n" " mov.f32 %f306, 0f00000000;\n" " fma.rn.f32 %f307, %f18, %f305, %f306;\n" " fma.rn.f32 %f308, %f27, %f305, %f306;\n" " ld.shared.f32 %f309, [%r18+3200];\n" " fma.rn.f32 %f310, %f19, %f309, %f307;\n" " fma.rn.f32 %f311, %f28, %f309, %f308;\n" " ld.shared.f32 %f312, [%r18+3328];\n" " fma.rn.f32 %f313, %f20, %f312, %f310;\n" " fma.rn.f32 %f314, %f29, %f312, %f311;\n" " ld.shared.f32 %f315, [%r18+3456];\n" " fma.rn.f32 %f316, %f21, %f315, %f313;\n" " fma.rn.f32 %f317, %f30, %f315, %f314;\n" " ld.shared.f32 %f318, [%r18+3584];\n" " fma.rn.f32 %f319, %f22, %f318, %f316;\n" " fma.rn.f32 %f320, %f31, %f318, %f317;\n" " ld.shared.f32 %f321, [%r18+3712];\n" " fma.rn.f32 %f322, %f23, %f321, %f319;\n" " fma.rn.f32 %f323, %f32, %f321, %f320;\n" " ld.shared.f32 %f324, [%r18+3840];\n" " fma.rn.f32 %f325, %f24, %f324, %f322;\n" " fma.rn.f32 %f326, %f33, %f324, %f323;\n" " ld.shared.f32 %f327, [%r18+3968];\n" " fma.rn.f32 %f328, %f25, %f327, %f325;\n" " fma.rn.f32 %f329, %f34, %f327, %f326;\n" " fma.rn.f32 %f785, %f17, %f328, %f785;\n" " fma.rn.f32 %f777, %f26, %f329, %f777;\n" "\n" "$L__BB31_22:\n" " setp.lt.s32 %p14, %r7, 5;\n" " @%p14 bra $L__BB31_24;\n" "\n" " ld.shared.f32 %f330, [%r18+4096];\n" " mov.f32 %f331, 0f00000000;\n" " fma.rn.f32 %f332, %f18, %f330, %f331;\n" " fma.rn.f32 %f333, %f27, %f330, %f331;\n" " ld.shared.f32 %f334, [%r18+4224];\n" " fma.rn.f32 %f335, %f19, %f334, %f332;\n" " fma.rn.f32 %f336, %f28, %f334, %f333;\n" " ld.shared.f32 %f337, [%r18+4352];\n" " fma.rn.f32 %f338, %f20, %f337, %f335;\n" " fma.rn.f32 %f339, %f29, %f337, %f336;\n" " ld.shared.f32 %f340, [%r18+4480];\n" " fma.rn.f32 %f341, %f21, %f340, %f338;\n" " fma.rn.f32 %f342, %f30, %f340, %f339;\n" " ld.shared.f32 %f343, [%r18+4608];\n" " fma.rn.f32 %f344, %f22, %f343, %f341;\n" " fma.rn.f32 %f345, %f31, %f343, %f342;\n" " ld.shared.f32 %f346, [%r18+4736];\n" " fma.rn.f32 %f347, %f23, %f346, %f344;\n" " fma.rn.f32 %f348, %f32, %f346, %f345;\n" " ld.shared.f32 %f349, [%r18+4864];\n" " fma.rn.f32 %f350, %f24, %f349, %f347;\n" " fma.rn.f32 %f351, %f33, %f349, %f348;\n" " ld.shared.f32 %f352, [%r18+4992];\n" " fma.rn.f32 %f353, %f25, %f352, %f350;\n" " fma.rn.f32 %f354, %f34, %f352, %f351;\n" " fma.rn.f32 %f786, %f17, %f353, %f786;\n" " fma.rn.f32 %f778, %f26, %f354, %f778;\n" "\n" "$L__BB31_24:\n" " setp.lt.s32 %p15, %r7, 6;\n" " @%p15 bra $L__BB31_26;\n" "\n" " ld.shared.f32 %f355, [%r18+5120];\n" " mov.f32 %f356, 0f00000000;\n" " fma.rn.f32 %f357, %f18, %f355, %f356;\n" " fma.rn.f32 %f358, %f27, %f355, %f356;\n" " ld.shared.f32 %f359, [%r18+5248];\n" " fma.rn.f32 %f360, %f19, %f359, %f357;\n" " fma.rn.f32 %f361, %f28, %f359, %f358;\n" " ld.shared.f32 %f362, [%r18+5376];\n" " fma.rn.f32 %f363, %f20, %f362, %f360;\n" " fma.rn.f32 %f364, %f29, %f362, %f361;\n" " ld.shared.f32 %f365, [%r18+5504];\n" " fma.rn.f32 %f366, %f21, %f365, %f363;\n" " fma.rn.f32 %f367, %f30, %f365, %f364;\n" " ld.shared.f32 %f368, [%r18+5632];\n" " fma.rn.f32 %f369, %f22, %f368, %f366;\n" " fma.rn.f32 %f370, %f31, %f368, %f367;\n" " ld.shared.f32 %f371, [%r18+5760];\n" " fma.rn.f32 %f372, %f23, %f371, %f369;\n" " fma.rn.f32 %f373, %f32, %f371, %f370;\n" " ld.shared.f32 %f374, [%r18+5888];\n" " fma.rn.f32 %f375, %f24, %f374, %f372;\n" " fma.rn.f32 %f376, %f33, %f374, %f373;\n" " ld.shared.f32 %f377, [%r18+6016];\n" " fma.rn.f32 %f378, %f25, %f377, %f375;\n" " fma.rn.f32 %f379, %f34, %f377, %f376;\n" " fma.rn.f32 %f787, %f17, %f378, %f787;\n" " fma.rn.f32 %f779, %f26, %f379, %f779;\n" "\n" "$L__BB31_26:\n" " setp.lt.s32 %p16, %r7, 7;\n" " @%p16 bra $L__BB31_28;\n" "\n" " ld.shared.f32 %f380, [%r18+6144];\n" " mov.f32 %f381, 0f00000000;\n" " fma.rn.f32 %f382, %f18, %f380, %f381;\n" " fma.rn.f32 %f383, %f27, %f380, %f381;\n" " ld.shared.f32 %f384, [%r18+6272];\n" " fma.rn.f32 %f385, %f19, %f384, %f382;\n" " fma.rn.f32 %f386, %f28, %f384, %f383;\n" " ld.shared.f32 %f387, [%r18+6400];\n" " fma.rn.f32 %f388, %f20, %f387, %f385;\n" " fma.rn.f32 %f389, %f29, %f387, %f386;\n" " ld.shared.f32 %f390, [%r18+6528];\n" " fma.rn.f32 %f391, %f21, %f390, %f388;\n" " fma.rn.f32 %f392, %f30, %f390, %f389;\n" " ld.shared.f32 %f393, [%r18+6656];\n" " fma.rn.f32 %f394, %f22, %f393, %f391;\n" " fma.rn.f32 %f395, %f31, %f393, %f392;\n" " ld.shared.f32 %f396, [%r18+6784];\n" " fma.rn.f32 %f397, %f23, %f396, %f394;\n" " fma.rn.f32 %f398, %f32, %f396, %f395;\n" " ld.shared.f32 %f399, [%r18+6912];\n" " fma.rn.f32 %f400, %f24, %f399, %f397;\n" " fma.rn.f32 %f401, %f33, %f399, %f398;\n" " ld.shared.f32 %f402, [%r18+7040];\n" " fma.rn.f32 %f403, %f25, %f402, %f400;\n" " fma.rn.f32 %f404, %f34, %f402, %f401;\n" " fma.rn.f32 %f788, %f17, %f403, %f788;\n" " fma.rn.f32 %f780, %f26, %f404, %f780;\n" "\n" "$L__BB31_28:\n" " setp.lt.s32 %p17, %r7, 8;\n" " @%p17 bra $L__BB31_30;\n" "\n" " ld.shared.f32 %f405, [%r18+7168];\n" " mov.f32 %f406, 0f00000000;\n" " fma.rn.f32 %f407, %f18, %f405, %f406;\n" " fma.rn.f32 %f408, %f27, %f405, %f406;\n" " ld.shared.f32 %f409, [%r18+7296];\n" " fma.rn.f32 %f410, %f19, %f409, %f407;\n" " fma.rn.f32 %f411, %f28, %f409, %f408;\n" " ld.shared.f32 %f412, [%r18+7424];\n" " fma.rn.f32 %f413, %f20, %f412, %f410;\n" " fma.rn.f32 %f414, %f29, %f412, %f411;\n" " ld.shared.f32 %f415, [%r18+7552];\n" " fma.rn.f32 %f416, %f21, %f415, %f413;\n" " fma.rn.f32 %f417, %f30, %f415, %f414;\n" " ld.shared.f32 %f418, [%r18+7680];\n" " fma.rn.f32 %f419, %f22, %f418, %f416;\n" " fma.rn.f32 %f420, %f31, %f418, %f417;\n" " ld.shared.f32 %f421, [%r18+7808];\n" " fma.rn.f32 %f422, %f23, %f421, %f419;\n" " fma.rn.f32 %f423, %f32, %f421, %f420;\n" " ld.shared.f32 %f424, [%r18+7936];\n" " fma.rn.f32 %f425, %f24, %f424, %f422;\n" " fma.rn.f32 %f426, %f33, %f424, %f423;\n" " ld.shared.f32 %f427, [%r18+8064];\n" " fma.rn.f32 %f428, %f25, %f427, %f425;\n" " fma.rn.f32 %f429, %f34, %f427, %f426;\n" " fma.rn.f32 %f789, %f17, %f428, %f789;\n" " fma.rn.f32 %f781, %f26, %f429, %f781;\n" "\n" "$L__BB31_30:\n" " add.s64 %rd102, %rd102, 152;\n" " add.s64 %rd103, %rd103, 152;\n" "\n" "$L__BB31_31:\n" " add.s32 %r306, %r306, 1;\n" " setp.lt.s32 %p18, %r306, %r19;\n" " @%p18 bra $L__BB31_9;\n" "\n" "$L__BB31_32:\n" " mov.u32 %r303, %tid.x;\n" " mov.u32 %r302, %ctaid.z;\n" " shl.b32 %r301, %r302, 3;\n" " sub.s32 %r300, %r5, %r301;\n" " and.b32 %r299, %r303, 31;\n" " mov.u32 %r298, %ctaid.x;\n" " shr.u32 %r297, %r303, 5;\n" " shl.b32 %r296, %r298, 3;\n" " add.s32 %r295, %r296, %r297;\n" " ld.param.u32 %r294, [fc_moe_gateup_q3k_dedup_shx_param_7];\n" " setp.eq.s32 %p19, %r299, 0;\n" " setp.lt.s32 %p20, %r295, %r294;\n" " and.pred %p1, %p19, %p20;\n" " not.pred %p21, %p1;\n" " mov.u32 %r128, 16;\n" " mov.u32 %r137, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f430, %f782, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f433, %f782, %f430;\n" " mov.u32 %r130, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f432, %f433, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f435, %f433, %f432;\n" " mov.u32 %r132, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f434, %f435, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f437, %f435, %f434;\n" " mov.u32 %r134, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f436, %f437, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f439, %f437, %f436;\n" " mov.u32 %r136, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f438, %f439, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f99, %f439, %f438;\n" " // begin inline asm\n" " shfl.down.b32 %f440, %f774, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f443, %f774, %f440;\n" " // begin inline asm\n" " shfl.down.b32 %f442, %f443, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f445, %f443, %f442;\n" " // begin inline asm\n" " shfl.down.b32 %f444, %f445, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f447, %f445, %f444;\n" " // begin inline asm\n" " shfl.down.b32 %f446, %f447, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f449, %f447, %f446;\n" " // begin inline asm\n" " shfl.down.b32 %f448, %f449, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f100, %f449, %f448;\n" " setp.lt.s32 %p22, %r300, 1;\n" " or.pred %p23, %p21, %p22;\n" " @%p23 bra $L__BB31_34;\n" "\n" " neg.f32 %f450, %f99;\n" " mov.f32 %f451, 0f3F000000;\n" " mov.f32 %f452, 0f3BBB989D;\n" " fma.rn.f32 %f453, %f450, %f452, %f451;\n" " mov.f32 %f454, 0f3FB8AA3B;\n" " mov.f32 %f455, 0f437C0000;\n" " cvt.sat.f32.f32 %f456, %f453;\n" " mov.f32 %f457, 0f4B400001;\n" " fma.rm.f32 %f458, %f456, %f455, %f457;\n" " add.f32 %f459, %f458, 0fCB40007F;\n" " neg.f32 %f460, %f459;\n" " fma.rn.f32 %f461, %f450, %f454, %f460;\n" " mov.f32 %f462, 0f32A57060;\n" " fma.rn.f32 %f463, %f450, %f462, %f461;\n" " mov.b32 %r138, %f458;\n" " shl.b32 %r139, %r138, 23;\n" " mov.b32 %f464, %r139;\n" " ex2.approx.ftz.f32 %f465, %f463;\n" " fma.rn.f32 %f466, %f465, %f464, 0f3F800000;\n" " div.rn.f32 %f467, %f99, %f466;\n" " mul.f32 %f468, %f100, %f467;\n" " ld.shared.s32 %rd60, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj];\n" " mul.lo.s64 %rd61, %rd60, %rd3;\n" " add.s64 %rd62, %rd61, %rd4;\n" " shl.b64 %rd63, %rd62, 2;\n" " add.s64 %rd64, %rd1, %rd63;\n" " st.global.f32 [%rd64], %f468;\n" "\n" "$L__BB31_34:\n" " // begin inline asm\n" " shfl.down.b32 %f469, %f783, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f472, %f783, %f469;\n" " // begin inline asm\n" " shfl.down.b32 %f471, %f472, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f474, %f472, %f471;\n" " // begin inline asm\n" " shfl.down.b32 %f473, %f474, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f476, %f474, %f473;\n" " // begin inline asm\n" " shfl.down.b32 %f475, %f476, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f478, %f476, %f475;\n" " // begin inline asm\n" " shfl.down.b32 %f477, %f478, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f101, %f478, %f477;\n" " // begin inline asm\n" " shfl.down.b32 %f479, %f775, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f482, %f775, %f479;\n" " // begin inline asm\n" " shfl.down.b32 %f481, %f482, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f484, %f482, %f481;\n" " // begin inline asm\n" " shfl.down.b32 %f483, %f484, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f486, %f484, %f483;\n" " // begin inline asm\n" " shfl.down.b32 %f485, %f486, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f488, %f486, %f485;\n" " // begin inline asm\n" " shfl.down.b32 %f487, %f488, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f102, %f488, %f487;\n" " setp.lt.s32 %p24, %r7, 2;\n" " or.pred %p26, %p21, %p24;\n" " @%p26 bra $L__BB31_36;\n" "\n" " neg.f32 %f489, %f101;\n" " mov.f32 %f490, 0f3F000000;\n" " mov.f32 %f491, 0f3BBB989D;\n" " fma.rn.f32 %f492, %f489, %f491, %f490;\n" " mov.f32 %f493, 0f3FB8AA3B;\n" " mov.f32 %f494, 0f437C0000;\n" " cvt.sat.f32.f32 %f495, %f492;\n" " mov.f32 %f496, 0f4B400001;\n" " fma.rm.f32 %f497, %f495, %f494, %f496;\n" " add.f32 %f498, %f497, 0fCB40007F;\n" " neg.f32 %f499, %f498;\n" " fma.rn.f32 %f500, %f489, %f493, %f499;\n" " mov.f32 %f501, 0f32A57060;\n" " fma.rn.f32 %f502, %f489, %f501, %f500;\n" " mov.b32 %r160, %f497;\n" " shl.b32 %r161, %r160, 23;\n" " mov.b32 %f503, %r161;\n" " ex2.approx.ftz.f32 %f504, %f502;\n" " fma.rn.f32 %f505, %f504, %f503, 0f3F800000;\n" " div.rn.f32 %f506, %f101, %f505;\n" " mul.f32 %f507, %f102, %f506;\n" " ld.shared.s32 %rd65, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj+4];\n" " mul.lo.s64 %rd66, %rd65, %rd3;\n" " add.s64 %rd67, %rd66, %rd4;\n" " shl.b64 %rd68, %rd67, 2;\n" " add.s64 %rd69, %rd1, %rd68;\n" " st.global.f32 [%rd69], %f507;\n" "\n" "$L__BB31_36:\n" " // begin inline asm\n" " shfl.down.b32 %f508, %f784, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f511, %f784, %f508;\n" " // begin inline asm\n" " shfl.down.b32 %f510, %f511, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f513, %f511, %f510;\n" " // begin inline asm\n" " shfl.down.b32 %f512, %f513, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f515, %f513, %f512;\n" " // begin inline asm\n" " shfl.down.b32 %f514, %f515, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f517, %f515, %f514;\n" " // begin inline asm\n" " shfl.down.b32 %f516, %f517, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f103, %f517, %f516;\n" " // begin inline asm\n" " shfl.down.b32 %f518, %f776, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f521, %f776, %f518;\n" " // begin inline asm\n" " shfl.down.b32 %f520, %f521, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f523, %f521, %f520;\n" " // begin inline asm\n" " shfl.down.b32 %f522, %f523, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f525, %f523, %f522;\n" " // begin inline asm\n" " shfl.down.b32 %f524, %f525, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f527, %f525, %f524;\n" " // begin inline asm\n" " shfl.down.b32 %f526, %f527, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f104, %f527, %f526;\n" " setp.lt.s32 %p27, %r7, 3;\n" " or.pred %p29, %p21, %p27;\n" " @%p29 bra $L__BB31_38;\n" "\n" " neg.f32 %f528, %f103;\n" " mov.f32 %f529, 0f3F000000;\n" " mov.f32 %f530, 0f3BBB989D;\n" " fma.rn.f32 %f531, %f528, %f530, %f529;\n" " mov.f32 %f532, 0f3FB8AA3B;\n" " mov.f32 %f533, 0f437C0000;\n" " cvt.sat.f32.f32 %f534, %f531;\n" " mov.f32 %f535, 0f4B400001;\n" " fma.rm.f32 %f536, %f534, %f533, %f535;\n" " add.f32 %f537, %f536, 0fCB40007F;\n" " neg.f32 %f538, %f537;\n" " fma.rn.f32 %f539, %f528, %f532, %f538;\n" " mov.f32 %f540, 0f32A57060;\n" " fma.rn.f32 %f541, %f528, %f540, %f539;\n" " mov.b32 %r182, %f536;\n" " shl.b32 %r183, %r182, 23;\n" " mov.b32 %f542, %r183;\n" " ex2.approx.ftz.f32 %f543, %f541;\n" " fma.rn.f32 %f544, %f543, %f542, 0f3F800000;\n" " div.rn.f32 %f545, %f103, %f544;\n" " mul.f32 %f546, %f104, %f545;\n" " ld.shared.s32 %rd70, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj+8];\n" " mul.lo.s64 %rd71, %rd70, %rd3;\n" " add.s64 %rd72, %rd71, %rd4;\n" " shl.b64 %rd73, %rd72, 2;\n" " add.s64 %rd74, %rd1, %rd73;\n" " st.global.f32 [%rd74], %f546;\n" "\n" "$L__BB31_38:\n" " // begin inline asm\n" " shfl.down.b32 %f547, %f785, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f550, %f785, %f547;\n" " // begin inline asm\n" " shfl.down.b32 %f549, %f550, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f552, %f550, %f549;\n" " // begin inline asm\n" " shfl.down.b32 %f551, %f552, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f554, %f552, %f551;\n" " // begin inline asm\n" " shfl.down.b32 %f553, %f554, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f556, %f554, %f553;\n" " // begin inline asm\n" " shfl.down.b32 %f555, %f556, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f105, %f556, %f555;\n" " // begin inline asm\n" " shfl.down.b32 %f557, %f777, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f560, %f777, %f557;\n" " // begin inline asm\n" " shfl.down.b32 %f559, %f560, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f562, %f560, %f559;\n" " // begin inline asm\n" " shfl.down.b32 %f561, %f562, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f564, %f562, %f561;\n" " // begin inline asm\n" " shfl.down.b32 %f563, %f564, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f566, %f564, %f563;\n" " // begin inline asm\n" " shfl.down.b32 %f565, %f566, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f106, %f566, %f565;\n" " setp.lt.s32 %p30, %r7, 4;\n" " or.pred %p32, %p21, %p30;\n" " @%p32 bra $L__BB31_40;\n" "\n" " neg.f32 %f567, %f105;\n" " mov.f32 %f568, 0f3F000000;\n" " mov.f32 %f569, 0f3BBB989D;\n" " fma.rn.f32 %f570, %f567, %f569, %f568;\n" " mov.f32 %f571, 0f3FB8AA3B;\n" " mov.f32 %f572, 0f437C0000;\n" " cvt.sat.f32.f32 %f573, %f570;\n" " mov.f32 %f574, 0f4B400001;\n" " fma.rm.f32 %f575, %f573, %f572, %f574;\n" " add.f32 %f576, %f575, 0fCB40007F;\n" " neg.f32 %f577, %f576;\n" " fma.rn.f32 %f578, %f567, %f571, %f577;\n" " mov.f32 %f579, 0f32A57060;\n" " fma.rn.f32 %f580, %f567, %f579, %f578;\n" " mov.b32 %r204, %f575;\n" " shl.b32 %r205, %r204, 23;\n" " mov.b32 %f581, %r205;\n" " ex2.approx.ftz.f32 %f582, %f580;\n" " fma.rn.f32 %f583, %f582, %f581, 0f3F800000;\n" " div.rn.f32 %f584, %f105, %f583;\n" " mul.f32 %f585, %f106, %f584;\n" " ld.shared.s32 %rd75, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj+12];\n" " mul.lo.s64 %rd76, %rd75, %rd3;\n" " add.s64 %rd77, %rd76, %rd4;\n" " shl.b64 %rd78, %rd77, 2;\n" " add.s64 %rd79, %rd1, %rd78;\n" " st.global.f32 [%rd79], %f585;\n" "\n" "$L__BB31_40:\n" " // begin inline asm\n" " shfl.down.b32 %f586, %f786, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f589, %f786, %f586;\n" " // begin inline asm\n" " shfl.down.b32 %f588, %f589, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f591, %f589, %f588;\n" " // begin inline asm\n" " shfl.down.b32 %f590, %f591, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f593, %f591, %f590;\n" " // begin inline asm\n" " shfl.down.b32 %f592, %f593, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f595, %f593, %f592;\n" " // begin inline asm\n" " shfl.down.b32 %f594, %f595, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f107, %f595, %f594;\n" " // begin inline asm\n" " shfl.down.b32 %f596, %f778, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f599, %f778, %f596;\n" " // begin inline asm\n" " shfl.down.b32 %f598, %f599, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f601, %f599, %f598;\n" " // begin inline asm\n" " shfl.down.b32 %f600, %f601, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f603, %f601, %f600;\n" " // begin inline asm\n" " shfl.down.b32 %f602, %f603, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f605, %f603, %f602;\n" " // begin inline asm\n" " shfl.down.b32 %f604, %f605, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f108, %f605, %f604;\n" " setp.lt.s32 %p33, %r7, 5;\n" " or.pred %p35, %p21, %p33;\n" " @%p35 bra $L__BB31_42;\n" "\n" " neg.f32 %f606, %f107;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f608, 0f3BBB989D;\n" " fma.rn.f32 %f609, %f606, %f608, %f607;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f611, 0f437C0000;\n" " cvt.sat.f32.f32 %f612, %f609;\n" " mov.f32 %f613, 0f4B400001;\n" " fma.rm.f32 %f614, %f612, %f611, %f613;\n" " add.f32 %f615, %f614, 0fCB40007F;\n" " neg.f32 %f616, %f615;\n" " fma.rn.f32 %f617, %f606, %f610, %f616;\n" " mov.f32 %f618, 0f32A57060;\n" " fma.rn.f32 %f619, %f606, %f618, %f617;\n" " mov.b32 %r226, %f614;\n" " shl.b32 %r227, %r226, 23;\n" " mov.b32 %f620, %r227;\n" " ex2.approx.ftz.f32 %f621, %f619;\n" " fma.rn.f32 %f622, %f621, %f620, 0f3F800000;\n" " div.rn.f32 %f623, %f107, %f622;\n" " mul.f32 %f624, %f108, %f623;\n" " ld.shared.s32 %rd80, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj+16];\n" " mul.lo.s64 %rd81, %rd80, %rd3;\n" " add.s64 %rd82, %rd81, %rd4;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd84, %rd1, %rd83;\n" " st.global.f32 [%rd84], %f624;\n" "\n" "$L__BB31_42:\n" " // begin inline asm\n" " shfl.down.b32 %f625, %f787, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f628, %f787, %f625;\n" " // begin inline asm\n" " shfl.down.b32 %f627, %f628, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f630, %f628, %f627;\n" " // begin inline asm\n" " shfl.down.b32 %f629, %f630, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f632, %f630, %f629;\n" " // begin inline asm\n" " shfl.down.b32 %f631, %f632, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f634, %f632, %f631;\n" " // begin inline asm\n" " shfl.down.b32 %f633, %f634, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f109, %f634, %f633;\n" " // begin inline asm\n" " shfl.down.b32 %f635, %f779, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f638, %f779, %f635;\n" " // begin inline asm\n" " shfl.down.b32 %f637, %f638, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f640, %f638, %f637;\n" " // begin inline asm\n" " shfl.down.b32 %f639, %f640, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f642, %f640, %f639;\n" " // begin inline asm\n" " shfl.down.b32 %f641, %f642, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f644, %f642, %f641;\n" " // begin inline asm\n" " shfl.down.b32 %f643, %f644, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f110, %f644, %f643;\n" " setp.lt.s32 %p36, %r7, 6;\n" " or.pred %p38, %p21, %p36;\n" " @%p38 bra $L__BB31_44;\n" "\n" " neg.f32 %f645, %f109;\n" " mov.f32 %f646, 0f3F000000;\n" " mov.f32 %f647, 0f3BBB989D;\n" " fma.rn.f32 %f648, %f645, %f647, %f646;\n" " mov.f32 %f649, 0f3FB8AA3B;\n" " mov.f32 %f650, 0f437C0000;\n" " cvt.sat.f32.f32 %f651, %f648;\n" " mov.f32 %f652, 0f4B400001;\n" " fma.rm.f32 %f653, %f651, %f650, %f652;\n" " add.f32 %f654, %f653, 0fCB40007F;\n" " neg.f32 %f655, %f654;\n" " fma.rn.f32 %f656, %f645, %f649, %f655;\n" " mov.f32 %f657, 0f32A57060;\n" " fma.rn.f32 %f658, %f645, %f657, %f656;\n" " mov.b32 %r248, %f653;\n" " shl.b32 %r249, %r248, 23;\n" " mov.b32 %f659, %r249;\n" " ex2.approx.ftz.f32 %f660, %f658;\n" " fma.rn.f32 %f661, %f660, %f659, 0f3F800000;\n" " div.rn.f32 %f662, %f109, %f661;\n" " mul.f32 %f663, %f110, %f662;\n" " ld.shared.s32 %rd85, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj+20];\n" " mul.lo.s64 %rd86, %rd85, %rd3;\n" " add.s64 %rd87, %rd86, %rd4;\n" " shl.b64 %rd88, %rd87, 2;\n" " add.s64 %rd89, %rd1, %rd88;\n" " st.global.f32 [%rd89], %f663;\n" "\n" "$L__BB31_44:\n" " // begin inline asm\n" " shfl.down.b32 %f664, %f788, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f667, %f788, %f664;\n" " // begin inline asm\n" " shfl.down.b32 %f666, %f667, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f669, %f667, %f666;\n" " // begin inline asm\n" " shfl.down.b32 %f668, %f669, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f671, %f669, %f668;\n" " // begin inline asm\n" " shfl.down.b32 %f670, %f671, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f673, %f671, %f670;\n" " // begin inline asm\n" " shfl.down.b32 %f672, %f673, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f111, %f673, %f672;\n" " // begin inline asm\n" " shfl.down.b32 %f674, %f780, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f677, %f780, %f674;\n" " // begin inline asm\n" " shfl.down.b32 %f676, %f677, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f679, %f677, %f676;\n" " // begin inline asm\n" " shfl.down.b32 %f678, %f679, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f681, %f679, %f678;\n" " // begin inline asm\n" " shfl.down.b32 %f680, %f681, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f683, %f681, %f680;\n" " // begin inline asm\n" " shfl.down.b32 %f682, %f683, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f112, %f683, %f682;\n" " setp.lt.s32 %p39, %r7, 7;\n" " or.pred %p41, %p21, %p39;\n" " @%p41 bra $L__BB31_46;\n" "\n" " neg.f32 %f684, %f111;\n" " mov.f32 %f685, 0f3F000000;\n" " mov.f32 %f686, 0f3BBB989D;\n" " fma.rn.f32 %f687, %f684, %f686, %f685;\n" " mov.f32 %f688, 0f3FB8AA3B;\n" " mov.f32 %f689, 0f437C0000;\n" " cvt.sat.f32.f32 %f690, %f687;\n" " mov.f32 %f691, 0f4B400001;\n" " fma.rm.f32 %f692, %f690, %f689, %f691;\n" " add.f32 %f693, %f692, 0fCB40007F;\n" " neg.f32 %f694, %f693;\n" " fma.rn.f32 %f695, %f684, %f688, %f694;\n" " mov.f32 %f696, 0f32A57060;\n" " fma.rn.f32 %f697, %f684, %f696, %f695;\n" " mov.b32 %r270, %f692;\n" " shl.b32 %r271, %r270, 23;\n" " mov.b32 %f698, %r271;\n" " ex2.approx.ftz.f32 %f699, %f697;\n" " fma.rn.f32 %f700, %f699, %f698, 0f3F800000;\n" " div.rn.f32 %f701, %f111, %f700;\n" " mul.f32 %f702, %f112, %f701;\n" " ld.shared.s32 %rd90, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj+24];\n" " mul.lo.s64 %rd91, %rd90, %rd3;\n" " add.s64 %rd92, %rd91, %rd4;\n" " shl.b64 %rd93, %rd92, 2;\n" " add.s64 %rd94, %rd1, %rd93;\n" " st.global.f32 [%rd94], %f702;\n" "\n" "$L__BB31_46:\n" " // begin inline asm\n" " shfl.down.b32 %f703, %f789, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f706, %f789, %f703;\n" " // begin inline asm\n" " shfl.down.b32 %f705, %f706, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f708, %f706, %f705;\n" " // begin inline asm\n" " shfl.down.b32 %f707, %f708, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f710, %f708, %f707;\n" " // begin inline asm\n" " shfl.down.b32 %f709, %f710, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f712, %f710, %f709;\n" " // begin inline asm\n" " shfl.down.b32 %f711, %f712, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f113, %f712, %f711;\n" " // begin inline asm\n" " shfl.down.b32 %f713, %f781, %r128, %r137;\n" " // end inline asm\n" " add.f32 %f716, %f781, %f713;\n" " // begin inline asm\n" " shfl.down.b32 %f715, %f716, %r130, %r137;\n" " // end inline asm\n" " add.f32 %f718, %f716, %f715;\n" " // begin inline asm\n" " shfl.down.b32 %f717, %f718, %r132, %r137;\n" " // end inline asm\n" " add.f32 %f720, %f718, %f717;\n" " // begin inline asm\n" " shfl.down.b32 %f719, %f720, %r134, %r137;\n" " // end inline asm\n" " add.f32 %f722, %f720, %f719;\n" " // begin inline asm\n" " shfl.down.b32 %f721, %f722, %r136, %r137;\n" " // end inline asm\n" " add.f32 %f114, %f722, %f721;\n" " setp.lt.s32 %p42, %r7, 8;\n" " or.pred %p44, %p21, %p42;\n" " @%p44 bra $L__BB31_48;\n" "\n" " neg.f32 %f723, %f113;\n" " mov.f32 %f724, 0f3F000000;\n" " mov.f32 %f725, 0f3BBB989D;\n" " fma.rn.f32 %f726, %f723, %f725, %f724;\n" " mov.f32 %f727, 0f3FB8AA3B;\n" " mov.f32 %f728, 0f437C0000;\n" " cvt.sat.f32.f32 %f729, %f726;\n" " mov.f32 %f730, 0f4B400001;\n" " fma.rm.f32 %f731, %f729, %f728, %f730;\n" " add.f32 %f732, %f731, 0fCB40007F;\n" " neg.f32 %f733, %f732;\n" " fma.rn.f32 %f734, %f723, %f727, %f733;\n" " mov.f32 %f735, 0f32A57060;\n" " fma.rn.f32 %f736, %f723, %f735, %f734;\n" " mov.b32 %r292, %f731;\n" " shl.b32 %r293, %r292, 23;\n" " mov.b32 %f737, %r293;\n" " ex2.approx.ftz.f32 %f738, %f736;\n" " fma.rn.f32 %f739, %f738, %f737, 0f3F800000;\n" " div.rn.f32 %f740, %f113, %f739;\n" " mul.f32 %f741, %f114, %f740;\n" " ld.shared.s32 %rd95, [_ZZ27fc_moe_gateup_q3k_dedup_shxE5sh_fj+28];\n" " mul.lo.s64 %rd96, %rd95, %rd3;\n" " add.s64 %rd97, %rd96, %rd4;\n" " shl.b64 %rd98, %rd97, 2;\n" " add.s64 %rd99, %rd1, %rd98;\n" " st.global.f32 [%rd99], %f741;\n" "\n" "$L__BB31_48:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_dedup_build_wl\n" ".visible .entry fc_moe_dedup_build_wl(\n" " .param .u64 fc_moe_dedup_build_wl_param_0,\n" " .param .u64 fc_moe_dedup_build_wl_param_1,\n" " .param .u32 fc_moe_dedup_build_wl_param_2,\n" " .param .u32 fc_moe_dedup_build_wl_param_3\n" ")\n" "{\n" " .reg .pred %p<38>;\n" " .reg .b32 %r<236>;\n" " .reg .b64 %rd<35>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ21fc_moe_dedup_build_wlE8sh_wbase[4100];\n" "\n" " ld.param.u64 %rd13, [fc_moe_dedup_build_wl_param_0];\n" " ld.param.u64 %rd12, [fc_moe_dedup_build_wl_param_1];\n" " ld.param.u32 %r90, [fc_moe_dedup_build_wl_param_2];\n" " ld.param.u32 %r91, [fc_moe_dedup_build_wl_param_3];\n" " cvta.to.global.u64 %rd1, %rd12;\n" " cvta.to.global.u64 %rd2, %rd13;\n" " mov.u32 %r226, %tid.x;\n" " mov.u32 %r2, %ntid.x;\n" " setp.gt.s32 %p1, %r90, 1024;\n" " @%p1 bra $L__BB32_57;\n" "\n" " setp.ge.s32 %p2, %r226, %r91;\n" " @%p2 bra $L__BB32_8;\n" "\n" " not.b32 %r92, %r226;\n" " add.s32 %r93, %r92, %r91;\n" " div.u32 %r3, %r93, %r2;\n" " add.s32 %r94, %r3, 1;\n" " and.b32 %r203, %r94, 3;\n" " setp.eq.s32 %p3, %r203, 0;\n" " mov.u32 %r204, %r226;\n" " @%p3 bra $L__BB32_5;\n" "\n" " mov.u32 %r204, %r226;\n" "\n" "$L__BB32_4:\n" " .pragma \"nounroll\";\n" " mul.wide.s32 %rd14, %r204, 8;\n" " add.s64 %rd15, %rd1, %rd14;\n" " mov.u32 %r95, 0;\n" " mov.u32 %r96, -1;\n" " st.global.v2.u32 [%rd15], {%r96, %r95};\n" " add.s32 %r204, %r204, %r2;\n" " add.s32 %r203, %r203, -1;\n" " setp.ne.s32 %p4, %r203, 0;\n" " @%p4 bra $L__BB32_4;\n" "\n" "$L__BB32_5:\n" " setp.lt.u32 %p5, %r3, 3;\n" " @%p5 bra $L__BB32_8;\n" "\n" " shl.b32 %r10, %r2, 3;\n" " cvt.s64.s32 %rd18, %r10;\n" "\n" "$L__BB32_7:\n" " mul.wide.s32 %rd16, %r204, 8;\n" " add.s64 %rd17, %rd1, %rd16;\n" " mov.u32 %r97, 0;\n" " mov.u32 %r98, -1;\n" " st.global.v2.u32 [%rd17], {%r98, %r97};\n" " add.s64 %rd19, %rd17, %rd18;\n" " st.global.v2.u32 [%rd19], {%r98, %r97};\n" " add.s32 %r99, %r204, %r2;\n" " add.s32 %r100, %r99, %r2;\n" " add.s64 %rd20, %rd19, %rd18;\n" " st.global.v2.u32 [%rd20], {%r98, %r97};\n" " add.s32 %r101, %r100, %r2;\n" " add.s64 %rd21, %rd20, %rd18;\n" " st.global.v2.u32 [%rd21], {%r98, %r97};\n" " add.s32 %r204, %r101, %r2;\n" " setp.lt.s32 %p6, %r204, %r91;\n" " @%p6 bra $L__BB32_7;\n" "\n" "$L__BB32_8:\n" " setp.ge.s32 %p7, %r226, %r90;\n" " @%p7 bra $L__BB32_25;\n" "\n" " not.b32 %r102, %r226;\n" " add.s32 %r103, %r102, %r90;\n" " div.u32 %r13, %r103, %r2;\n" " add.s32 %r104, %r13, 1;\n" " and.b32 %r207, %r104, 3;\n" " setp.eq.s32 %p8, %r207, 0;\n" " mov.u32 %r209, %r226;\n" " @%p8 bra $L__BB32_14;\n" "\n" " mov.u32 %r209, %r226;\n" "\n" "$L__BB32_11:\n" " .pragma \"nounroll\";\n" " mul.wide.s32 %rd22, %r209, 4;\n" " add.s64 %rd23, %rd2, %rd22;\n" " ld.global.u32 %r17, [%rd23];\n" " setp.lt.s32 %p9, %r17, 1;\n" " mov.u32 %r208, 0;\n" " @%p9 bra $L__BB32_13;\n" "\n" " add.s32 %r106, %r17, 7;\n" " shr.s32 %r107, %r106, 31;\n" " shr.u32 %r108, %r107, 29;\n" " add.s32 %r109, %r106, %r108;\n" " shr.s32 %r208, %r109, 3;\n" "\n" "$L__BB32_13:\n" " shl.b32 %r110, %r209, 2;\n" " mov.u32 %r111, _ZZ21fc_moe_dedup_build_wlE8sh_wbase;\n" " add.s32 %r112, %r111, %r110;\n" " st.shared.u32 [%r112], %r208;\n" " add.s32 %r209, %r209, %r2;\n" " add.s32 %r207, %r207, -1;\n" " setp.ne.s32 %p10, %r207, 0;\n" " @%p10 bra $L__BB32_11;\n" "\n" "$L__BB32_14:\n" " setp.lt.u32 %p11, %r13, 3;\n" " @%p11 bra $L__BB32_25;\n" "\n" " shl.b32 %r25, %r2, 2;\n" " shl.b32 %r113, %r209, 2;\n" " mov.u32 %r114, _ZZ21fc_moe_dedup_build_wlE8sh_wbase;\n" " add.s32 %r211, %r114, %r113;\n" " cvt.s64.s32 %rd25, %r25;\n" "\n" "$L__BB32_16:\n" " mul.wide.s32 %rd24, %r209, 4;\n" " add.s64 %rd3, %rd2, %rd24;\n" " ld.global.u32 %r28, [%rd3];\n" " setp.lt.s32 %p12, %r28, 1;\n" " mov.u32 %r215, 0;\n" " mov.u32 %r212, %r215;\n" " @%p12 bra $L__BB32_18;\n" "\n" " add.s32 %r116, %r28, 7;\n" " shr.s32 %r117, %r116, 31;\n" " shr.u32 %r118, %r117, 29;\n" " add.s32 %r119, %r116, %r118;\n" " shr.s32 %r212, %r119, 3;\n" "\n" "$L__BB32_18:\n" " st.shared.u32 [%r211], %r212;\n" " add.s64 %rd4, %rd3, %rd25;\n" " ld.global.u32 %r31, [%rd4];\n" " setp.lt.s32 %p13, %r31, 1;\n" " mov.u32 %r213, %r215;\n" " @%p13 bra $L__BB32_20;\n" "\n" " add.s32 %r121, %r31, 7;\n" " shr.s32 %r122, %r121, 31;\n" " shr.u32 %r123, %r122, 29;\n" " add.s32 %r124, %r121, %r123;\n" " shr.s32 %r213, %r124, 3;\n" "\n" "$L__BB32_20:\n" " add.s32 %r34, %r211, %r25;\n" " st.shared.u32 [%r34], %r213;\n" " add.s64 %rd5, %rd4, %rd25;\n" " ld.global.u32 %r35, [%rd5];\n" " setp.lt.s32 %p14, %r35, 1;\n" " mov.u32 %r214, %r215;\n" " @%p14 bra $L__BB32_22;\n" "\n" " add.s32 %r126, %r35, 7;\n" " shr.s32 %r127, %r126, 31;\n" " shr.u32 %r128, %r127, 29;\n" " add.s32 %r129, %r126, %r128;\n" " shr.s32 %r214, %r129, 3;\n" "\n" "$L__BB32_22:\n" " add.s32 %r38, %r34, %r25;\n" " st.shared.u32 [%r38], %r214;\n" " add.s64 %rd28, %rd5, %rd25;\n" " ld.global.u32 %r39, [%rd28];\n" " setp.lt.s32 %p15, %r39, 1;\n" " @%p15 bra $L__BB32_24;\n" "\n" " add.s32 %r131, %r39, 7;\n" " shr.s32 %r132, %r131, 31;\n" " shr.u32 %r133, %r132, 29;\n" " add.s32 %r134, %r131, %r133;\n" " shr.s32 %r215, %r134, 3;\n" "\n" "$L__BB32_24:\n" " add.s32 %r135, %r38, %r25;\n" " add.s32 %r211, %r135, %r25;\n" " st.shared.u32 [%r135], %r215;\n" " add.s32 %r136, %r209, %r2;\n" " add.s32 %r137, %r136, %r2;\n" " add.s32 %r138, %r137, %r2;\n" " add.s32 %r209, %r138, %r2;\n" " setp.lt.s32 %p16, %r209, %r90;\n" " @%p16 bra $L__BB32_16;\n" "\n" "$L__BB32_25:\n" " bar.sync 0;\n" " setp.ne.s32 %p17, %r226, 0;\n" " @%p17 bra $L__BB32_33;\n" "\n" " setp.lt.s32 %p18, %r90, 1;\n" " mov.u32 %r225, 0;\n" " @%p18 bra $L__BB32_32;\n" "\n" " add.s32 %r143, %r90, -1;\n" " and.b32 %r224, %r90, 3;\n" " setp.lt.u32 %p19, %r143, 3;\n" " mov.u32 %r220, 0;\n" " mov.u32 %r225, %r220;\n" " @%p19 bra $L__BB32_30;\n" "\n" " sub.s32 %r218, %r90, %r224;\n" " mov.u32 %r220, 0;\n" " mov.u32 %r225, %r220;\n" "\n" "$L__BB32_29:\n" " shl.b32 %r146, %r220, 2;\n" " mov.u32 %r147, _ZZ21fc_moe_dedup_build_wlE8sh_wbase;\n" " add.s32 %r148, %r147, %r146;\n" " ld.shared.u32 %r149, [%r148];\n" " st.shared.u32 [%r148], %r225;\n" " add.s32 %r150, %r149, %r225;\n" " ld.shared.u32 %r151, [%r148+4];\n" " st.shared.u32 [%r148+4], %r150;\n" " add.s32 %r152, %r151, %r150;\n" " ld.shared.u32 %r153, [%r148+8];\n" " st.shared.u32 [%r148+8], %r152;\n" " add.s32 %r154, %r153, %r152;\n" " ld.shared.u32 %r155, [%r148+12];\n" " st.shared.u32 [%r148+12], %r154;\n" " add.s32 %r225, %r155, %r154;\n" " add.s32 %r220, %r220, 4;\n" " add.s32 %r218, %r218, -4;\n" " setp.ne.s32 %p20, %r218, 0;\n" " @%p20 bra $L__BB32_29;\n" "\n" "$L__BB32_30:\n" " setp.eq.s32 %p21, %r224, 0;\n" " @%p21 bra $L__BB32_32;\n" "\n" "$L__BB32_31:\n" " .pragma \"nounroll\";\n" " shl.b32 %r156, %r220, 2;\n" " mov.u32 %r157, _ZZ21fc_moe_dedup_build_wlE8sh_wbase;\n" " add.s32 %r158, %r157, %r156;\n" " ld.shared.u32 %r159, [%r158];\n" " st.shared.u32 [%r158], %r225;\n" " add.s32 %r225, %r159, %r225;\n" " add.s32 %r220, %r220, 1;\n" " add.s32 %r224, %r224, -1;\n" " setp.ne.s32 %p22, %r224, 0;\n" " @%p22 bra $L__BB32_31;\n" "\n" "$L__BB32_32:\n" " shl.b32 %r160, %r90, 2;\n" " mov.u32 %r161, _ZZ21fc_moe_dedup_build_wlE8sh_wbase;\n" " add.s32 %r162, %r161, %r160;\n" " st.shared.u32 [%r162], %r225;\n" "\n" "$L__BB32_33:\n" " bar.sync 0;\n" " @%p7 bra $L__BB32_57;\n" "\n" " add.s64 %rd6, %rd1, 16;\n" "\n" "$L__BB32_35:\n" " mul.wide.s32 %rd29, %r226, 4;\n" " add.s64 %rd30, %rd2, %rd29;\n" " ld.global.u32 %r63, [%rd30];\n" " setp.lt.s32 %p24, %r63, 1;\n" " @%p24 bra $L__BB32_56;\n" "\n" " shl.b32 %r164, %r226, 2;\n" " mov.u32 %r165, _ZZ21fc_moe_dedup_build_wlE8sh_wbase;\n" " add.s32 %r166, %r165, %r164;\n" " ld.shared.u32 %r235, [%r166];\n" " mov.u32 %r234, 0;\n" " add.s32 %r167, %r63, -1;\n" " shr.u32 %r65, %r167, 3;\n" " add.s32 %r168, %r65, 1;\n" " and.b32 %r66, %r168, 3;\n" " setp.lt.u32 %p25, %r167, 24;\n" " @%p25 bra $L__BB32_47;\n" "\n" " add.s32 %r230, %r63, -24;\n" " add.s32 %r172, %r66, -1;\n" " sub.s32 %r227, %r172, %r65;\n" " mul.wide.s32 %rd31, %r235, 8;\n" " add.s64 %rd34, %rd6, %rd31;\n" " mov.u32 %r228, 0;\n" " mov.u32 %r231, 1572864;\n" " mov.u32 %r229, %r63;\n" " mov.u32 %r234, %r228;\n" " mov.u32 %r233, %r235;\n" "\n" "$L__BB32_38:\n" " add.s64 %rd9, %rd34, -16;\n" " setp.ge.s32 %p26, %r233, %r91;\n" " @%p26 bra $L__BB32_40;\n" "\n" " min.s32 %r173, %r229, 8;\n" " or.b32 %r174, %r173, %r228;\n" " st.global.v2.u32 [%rd9], {%r226, %r174};\n" "\n" "$L__BB32_40:\n" " add.s32 %r76, %r233, 1;\n" " setp.ge.s32 %p27, %r76, %r91;\n" " @%p27 bra $L__BB32_42;\n" "\n" " add.s32 %r175, %r229, -8;\n" " add.s32 %r176, %r228, 524288;\n" " min.s32 %r177, %r175, 8;\n" " or.b32 %r178, %r177, %r176;\n" " st.global.v2.u32 [%rd9+8], {%r226, %r178};\n" "\n" "$L__BB32_42:\n" " add.s32 %r77, %r76, 1;\n" " setp.ge.s32 %p28, %r77, %r91;\n" " @%p28 bra $L__BB32_44;\n" "\n" " add.s32 %r179, %r229, -16;\n" " add.s32 %r180, %r231, -524288;\n" " min.s32 %r181, %r179, 8;\n" " or.b32 %r182, %r181, %r180;\n" " st.global.v2.u32 [%rd9+16], {%r226, %r182};\n" "\n" "$L__BB32_44:\n" " add.s32 %r78, %r77, 1;\n" " setp.ge.s32 %p29, %r78, %r91;\n" " @%p29 bra $L__BB32_46;\n" "\n" " min.s32 %r183, %r230, 8;\n" " or.b32 %r184, %r183, %r231;\n" " st.global.v2.u32 [%rd9+24], {%r226, %r184};\n" "\n" "$L__BB32_46:\n" " add.s32 %r234, %r234, 32;\n" " add.s32 %r235, %r233, 4;\n" " add.s32 %r231, %r231, 2097152;\n" " add.s32 %r230, %r230, -32;\n" " add.s32 %r229, %r229, -32;\n" " add.s32 %r228, %r228, 2097152;\n" " add.s64 %rd34, %rd34, 32;\n" " add.s32 %r227, %r227, 4;\n" " setp.ne.s32 %p30, %r227, 0;\n" " add.s32 %r233, %r78, 1;\n" " @%p30 bra $L__BB32_38;\n" "\n" "$L__BB32_47:\n" " setp.eq.s32 %p31, %r66, 0;\n" " @%p31 bra $L__BB32_56;\n" "\n" " mul.wide.s32 %rd33, %r235, 8;\n" " add.s64 %rd11, %rd1, %rd33;\n" " setp.ge.s32 %p32, %r235, %r91;\n" " @%p32 bra $L__BB32_50;\n" "\n" " sub.s32 %r185, %r63, %r234;\n" " shl.b32 %r186, %r234, 16;\n" " min.s32 %r187, %r185, 8;\n" " or.b32 %r188, %r187, %r186;\n" " st.global.v2.u32 [%rd11], {%r226, %r188};\n" "\n" "$L__BB32_50:\n" " setp.eq.s32 %p33, %r66, 1;\n" " @%p33 bra $L__BB32_56;\n" "\n" " add.s32 %r189, %r235, 1;\n" " setp.ge.s32 %p34, %r189, %r91;\n" " @%p34 bra $L__BB32_53;\n" "\n" " add.s32 %r190, %r234, 8;\n" " sub.s32 %r191, %r63, %r190;\n" " shl.b32 %r192, %r190, 16;\n" " min.s32 %r193, %r191, 8;\n" " or.b32 %r194, %r193, %r192;\n" " st.global.v2.u32 [%rd11+8], {%r226, %r194};\n" "\n" "$L__BB32_53:\n" " setp.eq.s32 %p35, %r66, 2;\n" " @%p35 bra $L__BB32_56;\n" "\n" " add.s32 %r195, %r235, 2;\n" " setp.ge.s32 %p36, %r195, %r91;\n" " @%p36 bra $L__BB32_56;\n" "\n" " add.s32 %r196, %r234, 16;\n" " sub.s32 %r197, %r63, %r196;\n" " shl.b32 %r198, %r196, 16;\n" " min.s32 %r199, %r197, 8;\n" " or.b32 %r200, %r199, %r198;\n" " st.global.v2.u32 [%rd11+16], {%r226, %r200};\n" "\n" "$L__BB32_56:\n" " add.s32 %r226, %r226, %r2;\n" " setp.lt.s32 %p37, %r226, %r90;\n" " @%p37 bra $L__BB32_35;\n" "\n" "$L__BB32_57:\n" " ret;\n" "\n" "}\n" " // .globl fc_moe_gateup_q3k_dedup_shxc\n" ".visible .entry fc_moe_gateup_q3k_dedup_shxc(\n" " .param .u64 fc_moe_gateup_q3k_dedup_shxc_param_0,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shxc_param_1,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shxc_param_2,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shxc_param_3,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shxc_param_4,\n" " .param .u64 fc_moe_gateup_q3k_dedup_shxc_param_5,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shxc_param_6,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shxc_param_7,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shxc_param_8,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shxc_param_9,\n" " .param .u32 fc_moe_gateup_q3k_dedup_shxc_param_10\n" ")\n" "{\n" " .reg .pred %p<45>;\n" " .reg .f32 %f<806>;\n" " .reg .b32 %r<307>;\n" " .reg .b64 %rd<103>;\n" " // demoted variable\n" " .shared .align 16 .b8 _ZZ28fc_moe_gateup_q3k_dedup_shxcE4sh_x[8192];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj[32];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ28fc_moe_gateup_q3k_dedup_shxcE6sh_tok[32];\n" "\n" " ld.param.u64 %rd17, [fc_moe_gateup_q3k_dedup_shxc_param_0];\n" " ld.param.u64 %rd18, [fc_moe_gateup_q3k_dedup_shxc_param_1];\n" " ld.param.u64 %rd19, [fc_moe_gateup_q3k_dedup_shxc_param_2];\n" " ld.param.u64 %rd20, [fc_moe_gateup_q3k_dedup_shxc_param_3];\n" " ld.param.u64 %rd21, [fc_moe_gateup_q3k_dedup_shxc_param_4];\n" " ld.param.u64 %rd22, [fc_moe_gateup_q3k_dedup_shxc_param_5];\n" " ld.param.u32 %r23, [fc_moe_gateup_q3k_dedup_shxc_param_6];\n" " ld.param.u32 %r24, [fc_moe_gateup_q3k_dedup_shxc_param_7];\n" " ld.param.u32 %r25, [fc_moe_gateup_q3k_dedup_shxc_param_8];\n" " ld.param.u32 %r26, [fc_moe_gateup_q3k_dedup_shxc_param_9];\n" " ld.param.u32 %r27, [fc_moe_gateup_q3k_dedup_shxc_param_10];\n" " cvta.to.global.u64 %rd1, %rd22;\n" " mov.u32 %r1, %tid.x;\n" " shr.u32 %r28, %r1, 5;\n" " and.b32 %r2, %r1, 31;\n" " mov.u32 %r29, %ctaid.x;\n" " shl.b32 %r30, %r29, 3;\n" " add.s32 %r3, %r30, %r28;\n" " mov.u32 %r31, %ctaid.y;\n" " cvta.to.global.u64 %rd23, %rd21;\n" " mul.wide.u32 %rd24, %r31, 8;\n" " add.s64 %rd25, %rd23, %rd24;\n" " ld.global.v2.u32 {%r32, %r33}, [%rd25];\n" " setp.lt.s32 %p2, %r32, 0;\n" " @%p2 bra $L__BB33_48;\n" "\n" " and.b32 %r8, %r33, 65535;\n" " setp.gt.s32 %p3, %r1, 7;\n" " @%p3 bra $L__BB33_7;\n" "\n" " setp.ge.s32 %p4, %r1, %r8;\n" " mov.u32 %r304, -1;\n" " @%p4 bra $L__BB33_4;\n" "\n" " mul.wide.s32 %rd26, %r32, %r27;\n" " cvt.s64.s32 %rd27, %r1;\n" " shr.s32 %r35, %r33, 16;\n" " cvt.s64.s32 %rd28, %r35;\n" " add.s64 %rd29, %rd28, %rd27;\n" " add.s64 %rd30, %rd29, %rd26;\n" " cvta.to.global.u64 %rd31, %rd20;\n" " shl.b64 %rd32, %rd30, 2;\n" " add.s64 %rd33, %rd31, %rd32;\n" " ld.global.u32 %r304, [%rd33];\n" "\n" "$L__BB33_4:\n" " shl.b32 %r37, %r1, 2;\n" " mov.u32 %r38, _ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj;\n" " add.s32 %r39, %r38, %r37;\n" " mov.u32 %r305, 0;\n" " st.shared.u32 [%r39], %r304;\n" " setp.lt.s32 %p5, %r304, 0;\n" " @%p5 bra $L__BB33_6;\n" "\n" " div.s32 %r305, %r304, %r23;\n" "\n" "$L__BB33_6:\n" " mov.u32 %r41, _ZZ28fc_moe_gateup_q3k_dedup_shxcE6sh_tok;\n" " add.s32 %r42, %r41, %r37;\n" " st.shared.u32 [%r42], %r305;\n" "\n" "$L__BB33_7:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd2, %r24;\n" " cvt.s64.s32 %rd3, %r3;\n" " setp.lt.s32 %p6, %r25, 256;\n" " mov.f32 %f774, 0f00000000;\n" " mov.f32 %f775, %f774;\n" " mov.f32 %f776, %f774;\n" " mov.f32 %f777, %f774;\n" " mov.f32 %f778, %f774;\n" " mov.f32 %f779, %f774;\n" " mov.f32 %f780, %f774;\n" " mov.f32 %f781, %f774;\n" " mov.f32 %f782, %f774;\n" " mov.f32 %f783, %f774;\n" " mov.f32 %f784, %f774;\n" " mov.f32 %f785, %f774;\n" " mov.f32 %f786, %f774;\n" " mov.f32 %f787, %f774;\n" " mov.f32 %f788, %f774;\n" " mov.f32 %f789, %f774;\n" " @%p6 bra $L__BB33_32;\n" "\n" " cvt.s64.s32 %rd34, %r26;\n" " cvt.s64.s32 %rd35, %r32;\n" " mul.lo.s64 %rd36, %rd35, %rd2;\n" " add.s64 %rd37, %rd36, %rd3;\n" " mul.lo.s64 %rd38, %rd37, %rd34;\n" " cvt.s64.s32 %rd4, %r25;\n" " cvt.u64.u32 %rd5, %r2;\n" " shr.u32 %r44, %r2, 4;\n" " cvt.u64.u32 %rd6, %r44;\n" " shl.b32 %r45, %r1, 2;\n" " and.b32 %r46, %r45, 252;\n" " shr.s32 %r13, %r1, 6;\n" " shl.b32 %r47, %r13, 2;\n" " mov.u32 %r48, _ZZ28fc_moe_gateup_q3k_dedup_shxcE6sh_tok;\n" " add.s32 %r14, %r48, %r47;\n" " cvt.u64.u32 %rd7, %r46;\n" " shl.b32 %r49, %r13, 8;\n" " or.b32 %r50, %r49, %r46;\n" " shl.b32 %r51, %r50, 2;\n" " mov.u32 %r52, _ZZ28fc_moe_gateup_q3k_dedup_shxcE4sh_x;\n" " add.s32 %r15, %r52, %r51;\n" " add.s32 %r53, %r1, 256;\n" " shr.s32 %r16, %r53, 6;\n" " shl.b32 %r54, %r53, 2;\n" " and.b32 %r55, %r54, 252;\n" " shl.b32 %r56, %r16, 2;\n" " add.s32 %r17, %r48, %r56;\n" " shl.b32 %r57, %r16, 8;\n" " or.b32 %r58, %r57, %r55;\n" " shl.b32 %r59, %r58, 2;\n" " add.s32 %r18, %r52, %r59;\n" " shl.b32 %r60, %r2, 2;\n" " add.s32 %r19, %r52, %r60;\n" " shr.s32 %r20, %r25, 8;\n" " cvta.to.global.u64 %rd39, %rd18;\n" " add.s64 %rd102, %rd39, %rd38;\n" " cvta.to.global.u64 %rd40, %rd17;\n" " add.s64 %rd101, %rd40, %rd38;\n" " mov.u32 %r306, 0;\n" " mov.f32 %f774, 0f00000000;\n" " setp.ge.s32 %p7, %r13, %r8;\n" " setp.ge.s32 %p8, %r16, %r8;\n" " setp.ge.s32 %p9, %r3, %r24;\n" " setp.eq.s32 %p10, %r8, 0;\n" " shl.b64 %rd53, %rd5, 2;\n" " shl.b64 %rd55, %rd6, 3;\n" " mov.f32 %f775, %f774;\n" " mov.f32 %f776, %f774;\n" " mov.f32 %f777, %f774;\n" " mov.f32 %f778, %f774;\n" " mov.f32 %f779, %f774;\n" " mov.f32 %f780, %f774;\n" " mov.f32 %f781, %f774;\n" " mov.f32 %f782, %f774;\n" " mov.f32 %f783, %f774;\n" " mov.f32 %f784, %f774;\n" " mov.f32 %f785, %f774;\n" " mov.f32 %f786, %f774;\n" " mov.f32 %f787, %f774;\n" " mov.f32 %f788, %f774;\n" " mov.f32 %f789, %f774;\n" "\n" "$L__BB33_9:\n" " bar.sync 0;\n" " mul.wide.u32 %rd12, %r306, 256;\n" " @%p7 bra $L__BB33_11;\n" "\n" " ld.shared.s32 %rd42, [%r14];\n" " mul.lo.s64 %rd43, %rd42, %rd4;\n" " add.s64 %rd44, %rd12, %rd7;\n" " add.s64 %rd45, %rd44, %rd43;\n" " shl.b64 %rd46, %rd45, 2;\n" " add.s64 %rd41, %rd19, %rd46;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f147,%f148,%f149,%f150}, [%rd41];\n" " // end inline asm\n" " st.shared.v4.f32 [%r15], {%f147, %f148, %f149, %f150};\n" "\n" "$L__BB33_11:\n" " @%p8 bra $L__BB33_13;\n" "\n" " ld.shared.s32 %rd48, [%r17];\n" " mul.lo.s64 %rd49, %rd48, %rd4;\n" " add.s64 %rd50, %rd12, %rd7;\n" " add.s64 %rd51, %rd50, %rd49;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd47, %rd19, %rd52;\n" " // begin inline asm\n" " ld.global.nc.v4.f32 {%f151,%f152,%f153,%f154}, [%rd47];\n" " // end inline asm\n" " st.shared.v4.f32 [%r18], {%f151, %f152, %f153, %f154};\n" "\n" "$L__BB33_13:\n" " bar.sync 0;\n" " @%p9 bra $L__BB33_31;\n" "\n" " add.s64 %rd54, %rd101, %rd53;\n" " ld.global.f32 %f17, [%rd101+144];\n" " add.s64 %rd56, %rd101, %rd55;\n" " ld.global.v2.u32 {%r61, %r62}, [%rd56+128];\n" " ld.global.u32 %r65, [%rd54];\n" " shr.u32 %r66, %r65, 16;\n" " and.b32 %r67, %r65, 15;\n" " or.b32 %r68, %r67, 1258291200;\n" " mov.b32 %f155, %r68;\n" " mov.f32 %f156, 0fCB000004;\n" " mov.f32 %f157, 0f3F800000;\n" " fma.rn.f32 %f158, %f155, %f157, %f156;\n" " and.b32 %r69, %r61, 255;\n" " cvt.rn.f32.u32 %f159, %r69;\n" " mov.f32 %f160, 0f42000000;\n" " sub.rn.f32 %f161, %f159, %f160;\n" " mul.rn.f32 %f18, %f161, %f158;\n" " and.b32 %r70, %r65, 240;\n" " or.b32 %r71, %r70, 1258291200;\n" " mov.b32 %f162, %r71;\n" " mov.f32 %f163, 0fC9000040;\n" " mov.f32 %f164, 0f3D800000;\n" " fma.rn.f32 %f165, %f162, %f164, %f163;\n" " bfe.u32 %r72, %r61, 8, 8;\n" " cvt.rn.f32.u32 %f166, %r72;\n" " sub.rn.f32 %f167, %f166, %f160;\n" " mul.rn.f32 %f19, %f167, %f165;\n" " and.b32 %r73, %r65, 3840;\n" " or.b32 %r74, %r73, 1258291200;\n" " mov.b32 %f168, %r74;\n" " mov.f32 %f169, 0fC7000400;\n" " mov.f32 %f170, 0f3B800000;\n" " fma.rn.f32 %f171, %f168, %f170, %f169;\n" " bfe.u32 %r75, %r61, 16, 8;\n" " cvt.rn.f32.u32 %f172, %r75;\n" " sub.rn.f32 %f173, %f172, %f160;\n" " mul.rn.f32 %f20, %f173, %f171;\n" " and.b32 %r76, %r65, 61440;\n" " or.b32 %r77, %r76, 1258291200;\n" " mov.b32 %f174, %r77;\n" " mov.f32 %f175, 0fC5004000;\n" " mov.f32 %f176, 0f39800000;\n" " fma.rn.f32 %f177, %f174, %f176, %f175;\n" " shr.u32 %r78, %r61, 24;\n" " cvt.rn.f32.u32 %f178, %r78;\n" " sub.rn.f32 %f179, %f178, %f160;\n" " mul.rn.f32 %f21, %f179, %f177;\n" " and.b32 %r79, %r65, 983040;\n" " or.b32 %r80, %r79, 1258291200;\n" " mov.b32 %f180, %r80;\n" " mov.f32 %f181, 0fC3040000;\n" " mov.f32 %f182, 0f37800000;\n" " fma.rn.f32 %f183, %f180, %f182, %f181;\n" " and.b32 %r81, %r62, 255;\n" " cvt.rn.f32.u32 %f184, %r81;\n" " sub.rn.f32 %f185, %f184, %f160;\n" " mul.rn.f32 %f22, %f185, %f183;\n" " and.b32 %r82, %r66, 240;\n" " or.b32 %r83, %r82, 1258291200;\n" " mov.b32 %f186, %r83;\n" " fma.rn.f32 %f187, %f186, %f164, %f163;\n" " bfe.u32 %r84, %r62, 8, 8;\n" " cvt.rn.f32.u32 %f188, %r84;\n" " sub.rn.f32 %f189, %f188, %f160;\n" " mul.rn.f32 %f23, %f189, %f187;\n" " and.b32 %r85, %r66, 3840;\n" " or.b32 %r86, %r85, 1258291200;\n" " mov.b32 %f190, %r86;\n" " fma.rn.f32 %f191, %f190, %f170, %f169;\n" " bfe.u32 %r87, %r62, 16, 8;\n" " cvt.rn.f32.u32 %f192, %r87;\n" " sub.rn.f32 %f193, %f192, %f160;\n" " mul.rn.f32 %f24, %f193, %f191;\n" " and.b32 %r88, %r66, 61440;\n" " or.b32 %r89, %r88, 1258291200;\n" " mov.b32 %f194, %r89;\n" " fma.rn.f32 %f195, %f194, %f176, %f175;\n" " shr.u32 %r90, %r62, 24;\n" " cvt.rn.f32.u32 %f196, %r90;\n" " sub.rn.f32 %f197, %f196, %f160;\n" " mul.rn.f32 %f25, %f197, %f195;\n" " add.s64 %rd57, %rd102, %rd53;\n" " ld.global.f32 %f26, [%rd102+144];\n" " add.s64 %rd58, %rd102, %rd55;\n" " ld.global.v2.u32 {%r91, %r92}, [%rd58+128];\n" " ld.global.u32 %r95, [%rd57];\n" " shr.u32 %r96, %r95, 16;\n" " and.b32 %r97, %r95, 15;\n" " or.b32 %r98, %r97, 1258291200;\n" " mov.b32 %f198, %r98;\n" " fma.rn.f32 %f199, %f198, %f157, %f156;\n" " and.b32 %r99, %r91, 255;\n" " cvt.rn.f32.u32 %f200, %r99;\n" " sub.rn.f32 %f201, %f200, %f160;\n" " mul.rn.f32 %f27, %f201, %f199;\n" " and.b32 %r100, %r95, 240;\n" " or.b32 %r101, %r100, 1258291200;\n" " mov.b32 %f202, %r101;\n" " fma.rn.f32 %f203, %f202, %f164, %f163;\n" " bfe.u32 %r102, %r91, 8, 8;\n" " cvt.rn.f32.u32 %f204, %r102;\n" " sub.rn.f32 %f205, %f204, %f160;\n" " mul.rn.f32 %f28, %f205, %f203;\n" " and.b32 %r103, %r95, 3840;\n" " or.b32 %r104, %r103, 1258291200;\n" " mov.b32 %f206, %r104;\n" " fma.rn.f32 %f207, %f206, %f170, %f169;\n" " bfe.u32 %r105, %r91, 16, 8;\n" " cvt.rn.f32.u32 %f208, %r105;\n" " sub.rn.f32 %f209, %f208, %f160;\n" " mul.rn.f32 %f29, %f209, %f207;\n" " and.b32 %r106, %r95, 61440;\n" " or.b32 %r107, %r106, 1258291200;\n" " mov.b32 %f210, %r107;\n" " fma.rn.f32 %f211, %f210, %f176, %f175;\n" " shr.u32 %r108, %r91, 24;\n" " cvt.rn.f32.u32 %f212, %r108;\n" " sub.rn.f32 %f213, %f212, %f160;\n" " mul.rn.f32 %f30, %f213, %f211;\n" " and.b32 %r109, %r95, 983040;\n" " or.b32 %r110, %r109, 1258291200;\n" " mov.b32 %f214, %r110;\n" " fma.rn.f32 %f215, %f214, %f182, %f181;\n" " and.b32 %r111, %r92, 255;\n" " cvt.rn.f32.u32 %f216, %r111;\n" " sub.rn.f32 %f217, %f216, %f160;\n" " mul.rn.f32 %f31, %f217, %f215;\n" " and.b32 %r112, %r96, 240;\n" " or.b32 %r113, %r112, 1258291200;\n" " mov.b32 %f218, %r113;\n" " fma.rn.f32 %f219, %f218, %f164, %f163;\n" " bfe.u32 %r114, %r92, 8, 8;\n" " cvt.rn.f32.u32 %f220, %r114;\n" " sub.rn.f32 %f221, %f220, %f160;\n" " mul.rn.f32 %f32, %f221, %f219;\n" " and.b32 %r115, %r96, 3840;\n" " or.b32 %r116, %r115, 1258291200;\n" " mov.b32 %f222, %r116;\n" " fma.rn.f32 %f223, %f222, %f170, %f169;\n" " bfe.u32 %r117, %r92, 16, 8;\n" " cvt.rn.f32.u32 %f224, %r117;\n" " sub.rn.f32 %f225, %f224, %f160;\n" " mul.rn.f32 %f33, %f225, %f223;\n" " and.b32 %r118, %r96, 61440;\n" " or.b32 %r119, %r118, 1258291200;\n" " mov.b32 %f226, %r119;\n" " fma.rn.f32 %f227, %f226, %f176, %f175;\n" " shr.u32 %r120, %r92, 24;\n" " cvt.rn.f32.u32 %f228, %r120;\n" " sub.rn.f32 %f229, %f228, %f160;\n" " mul.rn.f32 %f34, %f229, %f227;\n" " @%p10 bra $L__BB33_16;\n" "\n" " ld.shared.f32 %f230, [%r19];\n" " mov.f32 %f231, 0f00000000;\n" " fma.rn.f32 %f232, %f18, %f230, %f231;\n" " fma.rn.f32 %f233, %f27, %f230, %f231;\n" " ld.shared.f32 %f234, [%r19+128];\n" " fma.rn.f32 %f235, %f19, %f234, %f232;\n" " fma.rn.f32 %f236, %f28, %f234, %f233;\n" " ld.shared.f32 %f237, [%r19+256];\n" " fma.rn.f32 %f238, %f20, %f237, %f235;\n" " fma.rn.f32 %f239, %f29, %f237, %f236;\n" " ld.shared.f32 %f240, [%r19+384];\n" " fma.rn.f32 %f241, %f21, %f240, %f238;\n" " fma.rn.f32 %f242, %f30, %f240, %f239;\n" " ld.shared.f32 %f243, [%r19+512];\n" " fma.rn.f32 %f244, %f22, %f243, %f241;\n" " fma.rn.f32 %f245, %f31, %f243, %f242;\n" " ld.shared.f32 %f246, [%r19+640];\n" " fma.rn.f32 %f247, %f23, %f246, %f244;\n" " fma.rn.f32 %f248, %f32, %f246, %f245;\n" " ld.shared.f32 %f249, [%r19+768];\n" " fma.rn.f32 %f250, %f24, %f249, %f247;\n" " fma.rn.f32 %f251, %f33, %f249, %f248;\n" " ld.shared.f32 %f252, [%r19+896];\n" " fma.rn.f32 %f253, %f25, %f252, %f250;\n" " fma.rn.f32 %f254, %f34, %f252, %f251;\n" " fma.rn.f32 %f782, %f17, %f253, %f782;\n" " fma.rn.f32 %f774, %f26, %f254, %f774;\n" "\n" "$L__BB33_16:\n" " setp.lt.u32 %p11, %r8, 2;\n" " @%p11 bra $L__BB33_18;\n" "\n" " ld.shared.f32 %f255, [%r19+1024];\n" " mov.f32 %f256, 0f00000000;\n" " fma.rn.f32 %f257, %f18, %f255, %f256;\n" " fma.rn.f32 %f258, %f27, %f255, %f256;\n" " ld.shared.f32 %f259, [%r19+1152];\n" " fma.rn.f32 %f260, %f19, %f259, %f257;\n" " fma.rn.f32 %f261, %f28, %f259, %f258;\n" " ld.shared.f32 %f262, [%r19+1280];\n" " fma.rn.f32 %f263, %f20, %f262, %f260;\n" " fma.rn.f32 %f264, %f29, %f262, %f261;\n" " ld.shared.f32 %f265, [%r19+1408];\n" " fma.rn.f32 %f266, %f21, %f265, %f263;\n" " fma.rn.f32 %f267, %f30, %f265, %f264;\n" " ld.shared.f32 %f268, [%r19+1536];\n" " fma.rn.f32 %f269, %f22, %f268, %f266;\n" " fma.rn.f32 %f270, %f31, %f268, %f267;\n" " ld.shared.f32 %f271, [%r19+1664];\n" " fma.rn.f32 %f272, %f23, %f271, %f269;\n" " fma.rn.f32 %f273, %f32, %f271, %f270;\n" " ld.shared.f32 %f274, [%r19+1792];\n" " fma.rn.f32 %f275, %f24, %f274, %f272;\n" " fma.rn.f32 %f276, %f33, %f274, %f273;\n" " ld.shared.f32 %f277, [%r19+1920];\n" " fma.rn.f32 %f278, %f25, %f277, %f275;\n" " fma.rn.f32 %f279, %f34, %f277, %f276;\n" " fma.rn.f32 %f783, %f17, %f278, %f783;\n" " fma.rn.f32 %f775, %f26, %f279, %f775;\n" "\n" "$L__BB33_18:\n" " setp.lt.u32 %p12, %r8, 3;\n" " @%p12 bra $L__BB33_20;\n" "\n" " ld.shared.f32 %f280, [%r19+2048];\n" " mov.f32 %f281, 0f00000000;\n" " fma.rn.f32 %f282, %f18, %f280, %f281;\n" " fma.rn.f32 %f283, %f27, %f280, %f281;\n" " ld.shared.f32 %f284, [%r19+2176];\n" " fma.rn.f32 %f285, %f19, %f284, %f282;\n" " fma.rn.f32 %f286, %f28, %f284, %f283;\n" " ld.shared.f32 %f287, [%r19+2304];\n" " fma.rn.f32 %f288, %f20, %f287, %f285;\n" " fma.rn.f32 %f289, %f29, %f287, %f286;\n" " ld.shared.f32 %f290, [%r19+2432];\n" " fma.rn.f32 %f291, %f21, %f290, %f288;\n" " fma.rn.f32 %f292, %f30, %f290, %f289;\n" " ld.shared.f32 %f293, [%r19+2560];\n" " fma.rn.f32 %f294, %f22, %f293, %f291;\n" " fma.rn.f32 %f295, %f31, %f293, %f292;\n" " ld.shared.f32 %f296, [%r19+2688];\n" " fma.rn.f32 %f297, %f23, %f296, %f294;\n" " fma.rn.f32 %f298, %f32, %f296, %f295;\n" " ld.shared.f32 %f299, [%r19+2816];\n" " fma.rn.f32 %f300, %f24, %f299, %f297;\n" " fma.rn.f32 %f301, %f33, %f299, %f298;\n" " ld.shared.f32 %f302, [%r19+2944];\n" " fma.rn.f32 %f303, %f25, %f302, %f300;\n" " fma.rn.f32 %f304, %f34, %f302, %f301;\n" " fma.rn.f32 %f784, %f17, %f303, %f784;\n" " fma.rn.f32 %f776, %f26, %f304, %f776;\n" "\n" "$L__BB33_20:\n" " setp.lt.u32 %p13, %r8, 4;\n" " @%p13 bra $L__BB33_22;\n" "\n" " ld.shared.f32 %f305, [%r19+3072];\n" " mov.f32 %f306, 0f00000000;\n" " fma.rn.f32 %f307, %f18, %f305, %f306;\n" " fma.rn.f32 %f308, %f27, %f305, %f306;\n" " ld.shared.f32 %f309, [%r19+3200];\n" " fma.rn.f32 %f310, %f19, %f309, %f307;\n" " fma.rn.f32 %f311, %f28, %f309, %f308;\n" " ld.shared.f32 %f312, [%r19+3328];\n" " fma.rn.f32 %f313, %f20, %f312, %f310;\n" " fma.rn.f32 %f314, %f29, %f312, %f311;\n" " ld.shared.f32 %f315, [%r19+3456];\n" " fma.rn.f32 %f316, %f21, %f315, %f313;\n" " fma.rn.f32 %f317, %f30, %f315, %f314;\n" " ld.shared.f32 %f318, [%r19+3584];\n" " fma.rn.f32 %f319, %f22, %f318, %f316;\n" " fma.rn.f32 %f320, %f31, %f318, %f317;\n" " ld.shared.f32 %f321, [%r19+3712];\n" " fma.rn.f32 %f322, %f23, %f321, %f319;\n" " fma.rn.f32 %f323, %f32, %f321, %f320;\n" " ld.shared.f32 %f324, [%r19+3840];\n" " fma.rn.f32 %f325, %f24, %f324, %f322;\n" " fma.rn.f32 %f326, %f33, %f324, %f323;\n" " ld.shared.f32 %f327, [%r19+3968];\n" " fma.rn.f32 %f328, %f25, %f327, %f325;\n" " fma.rn.f32 %f329, %f34, %f327, %f326;\n" " fma.rn.f32 %f785, %f17, %f328, %f785;\n" " fma.rn.f32 %f777, %f26, %f329, %f777;\n" "\n" "$L__BB33_22:\n" " setp.lt.u32 %p14, %r8, 5;\n" " @%p14 bra $L__BB33_24;\n" "\n" " ld.shared.f32 %f330, [%r19+4096];\n" " mov.f32 %f331, 0f00000000;\n" " fma.rn.f32 %f332, %f18, %f330, %f331;\n" " fma.rn.f32 %f333, %f27, %f330, %f331;\n" " ld.shared.f32 %f334, [%r19+4224];\n" " fma.rn.f32 %f335, %f19, %f334, %f332;\n" " fma.rn.f32 %f336, %f28, %f334, %f333;\n" " ld.shared.f32 %f337, [%r19+4352];\n" " fma.rn.f32 %f338, %f20, %f337, %f335;\n" " fma.rn.f32 %f339, %f29, %f337, %f336;\n" " ld.shared.f32 %f340, [%r19+4480];\n" " fma.rn.f32 %f341, %f21, %f340, %f338;\n" " fma.rn.f32 %f342, %f30, %f340, %f339;\n" " ld.shared.f32 %f343, [%r19+4608];\n" " fma.rn.f32 %f344, %f22, %f343, %f341;\n" " fma.rn.f32 %f345, %f31, %f343, %f342;\n" " ld.shared.f32 %f346, [%r19+4736];\n" " fma.rn.f32 %f347, %f23, %f346, %f344;\n" " fma.rn.f32 %f348, %f32, %f346, %f345;\n" " ld.shared.f32 %f349, [%r19+4864];\n" " fma.rn.f32 %f350, %f24, %f349, %f347;\n" " fma.rn.f32 %f351, %f33, %f349, %f348;\n" " ld.shared.f32 %f352, [%r19+4992];\n" " fma.rn.f32 %f353, %f25, %f352, %f350;\n" " fma.rn.f32 %f354, %f34, %f352, %f351;\n" " fma.rn.f32 %f786, %f17, %f353, %f786;\n" " fma.rn.f32 %f778, %f26, %f354, %f778;\n" "\n" "$L__BB33_24:\n" " setp.lt.u32 %p15, %r8, 6;\n" " @%p15 bra $L__BB33_26;\n" "\n" " ld.shared.f32 %f355, [%r19+5120];\n" " mov.f32 %f356, 0f00000000;\n" " fma.rn.f32 %f357, %f18, %f355, %f356;\n" " fma.rn.f32 %f358, %f27, %f355, %f356;\n" " ld.shared.f32 %f359, [%r19+5248];\n" " fma.rn.f32 %f360, %f19, %f359, %f357;\n" " fma.rn.f32 %f361, %f28, %f359, %f358;\n" " ld.shared.f32 %f362, [%r19+5376];\n" " fma.rn.f32 %f363, %f20, %f362, %f360;\n" " fma.rn.f32 %f364, %f29, %f362, %f361;\n" " ld.shared.f32 %f365, [%r19+5504];\n" " fma.rn.f32 %f366, %f21, %f365, %f363;\n" " fma.rn.f32 %f367, %f30, %f365, %f364;\n" " ld.shared.f32 %f368, [%r19+5632];\n" " fma.rn.f32 %f369, %f22, %f368, %f366;\n" " fma.rn.f32 %f370, %f31, %f368, %f367;\n" " ld.shared.f32 %f371, [%r19+5760];\n" " fma.rn.f32 %f372, %f23, %f371, %f369;\n" " fma.rn.f32 %f373, %f32, %f371, %f370;\n" " ld.shared.f32 %f374, [%r19+5888];\n" " fma.rn.f32 %f375, %f24, %f374, %f372;\n" " fma.rn.f32 %f376, %f33, %f374, %f373;\n" " ld.shared.f32 %f377, [%r19+6016];\n" " fma.rn.f32 %f378, %f25, %f377, %f375;\n" " fma.rn.f32 %f379, %f34, %f377, %f376;\n" " fma.rn.f32 %f787, %f17, %f378, %f787;\n" " fma.rn.f32 %f779, %f26, %f379, %f779;\n" "\n" "$L__BB33_26:\n" " setp.lt.u32 %p16, %r8, 7;\n" " @%p16 bra $L__BB33_28;\n" "\n" " ld.shared.f32 %f380, [%r19+6144];\n" " mov.f32 %f381, 0f00000000;\n" " fma.rn.f32 %f382, %f18, %f380, %f381;\n" " fma.rn.f32 %f383, %f27, %f380, %f381;\n" " ld.shared.f32 %f384, [%r19+6272];\n" " fma.rn.f32 %f385, %f19, %f384, %f382;\n" " fma.rn.f32 %f386, %f28, %f384, %f383;\n" " ld.shared.f32 %f387, [%r19+6400];\n" " fma.rn.f32 %f388, %f20, %f387, %f385;\n" " fma.rn.f32 %f389, %f29, %f387, %f386;\n" " ld.shared.f32 %f390, [%r19+6528];\n" " fma.rn.f32 %f391, %f21, %f390, %f388;\n" " fma.rn.f32 %f392, %f30, %f390, %f389;\n" " ld.shared.f32 %f393, [%r19+6656];\n" " fma.rn.f32 %f394, %f22, %f393, %f391;\n" " fma.rn.f32 %f395, %f31, %f393, %f392;\n" " ld.shared.f32 %f396, [%r19+6784];\n" " fma.rn.f32 %f397, %f23, %f396, %f394;\n" " fma.rn.f32 %f398, %f32, %f396, %f395;\n" " ld.shared.f32 %f399, [%r19+6912];\n" " fma.rn.f32 %f400, %f24, %f399, %f397;\n" " fma.rn.f32 %f401, %f33, %f399, %f398;\n" " ld.shared.f32 %f402, [%r19+7040];\n" " fma.rn.f32 %f403, %f25, %f402, %f400;\n" " fma.rn.f32 %f404, %f34, %f402, %f401;\n" " fma.rn.f32 %f788, %f17, %f403, %f788;\n" " fma.rn.f32 %f780, %f26, %f404, %f780;\n" "\n" "$L__BB33_28:\n" " setp.lt.u32 %p17, %r8, 8;\n" " @%p17 bra $L__BB33_30;\n" "\n" " ld.shared.f32 %f405, [%r19+7168];\n" " mov.f32 %f406, 0f00000000;\n" " fma.rn.f32 %f407, %f18, %f405, %f406;\n" " fma.rn.f32 %f408, %f27, %f405, %f406;\n" " ld.shared.f32 %f409, [%r19+7296];\n" " fma.rn.f32 %f410, %f19, %f409, %f407;\n" " fma.rn.f32 %f411, %f28, %f409, %f408;\n" " ld.shared.f32 %f412, [%r19+7424];\n" " fma.rn.f32 %f413, %f20, %f412, %f410;\n" " fma.rn.f32 %f414, %f29, %f412, %f411;\n" " ld.shared.f32 %f415, [%r19+7552];\n" " fma.rn.f32 %f416, %f21, %f415, %f413;\n" " fma.rn.f32 %f417, %f30, %f415, %f414;\n" " ld.shared.f32 %f418, [%r19+7680];\n" " fma.rn.f32 %f419, %f22, %f418, %f416;\n" " fma.rn.f32 %f420, %f31, %f418, %f417;\n" " ld.shared.f32 %f421, [%r19+7808];\n" " fma.rn.f32 %f422, %f23, %f421, %f419;\n" " fma.rn.f32 %f423, %f32, %f421, %f420;\n" " ld.shared.f32 %f424, [%r19+7936];\n" " fma.rn.f32 %f425, %f24, %f424, %f422;\n" " fma.rn.f32 %f426, %f33, %f424, %f423;\n" " ld.shared.f32 %f427, [%r19+8064];\n" " fma.rn.f32 %f428, %f25, %f427, %f425;\n" " fma.rn.f32 %f429, %f34, %f427, %f426;\n" " fma.rn.f32 %f789, %f17, %f428, %f789;\n" " fma.rn.f32 %f781, %f26, %f429, %f781;\n" "\n" "$L__BB33_30:\n" " add.s64 %rd101, %rd101, 152;\n" " add.s64 %rd102, %rd102, 152;\n" "\n" "$L__BB33_31:\n" " add.s32 %r306, %r306, 1;\n" " setp.lt.s32 %p18, %r306, %r20;\n" " @%p18 bra $L__BB33_9;\n" "\n" "$L__BB33_32:\n" " mov.u32 %r303, %tid.x;\n" " and.b32 %r302, %r303, 31;\n" " mov.u32 %r301, %ctaid.x;\n" " shr.u32 %r300, %r303, 5;\n" " shl.b32 %r299, %r301, 3;\n" " add.s32 %r298, %r299, %r300;\n" " ld.param.u32 %r297, [fc_moe_gateup_q3k_dedup_shxc_param_7];\n" " setp.eq.s32 %p19, %r302, 0;\n" " setp.lt.s32 %p20, %r298, %r297;\n" " and.pred %p1, %p19, %p20;\n" " not.pred %p21, %p1;\n" " mov.u32 %r131, 16;\n" " mov.u32 %r140, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f430, %f782, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f433, %f782, %f430;\n" " mov.u32 %r133, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f432, %f433, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f435, %f433, %f432;\n" " mov.u32 %r135, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f434, %f435, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f437, %f435, %f434;\n" " mov.u32 %r137, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f436, %f437, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f439, %f437, %f436;\n" " mov.u32 %r139, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f438, %f439, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f99, %f439, %f438;\n" " // begin inline asm\n" " shfl.down.b32 %f440, %f774, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f443, %f774, %f440;\n" " // begin inline asm\n" " shfl.down.b32 %f442, %f443, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f445, %f443, %f442;\n" " // begin inline asm\n" " shfl.down.b32 %f444, %f445, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f447, %f445, %f444;\n" " // begin inline asm\n" " shfl.down.b32 %f446, %f447, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f449, %f447, %f446;\n" " // begin inline asm\n" " shfl.down.b32 %f448, %f449, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f100, %f449, %f448;\n" " setp.eq.s32 %p22, %r8, 0;\n" " or.pred %p23, %p21, %p22;\n" " @%p23 bra $L__BB33_34;\n" "\n" " neg.f32 %f450, %f99;\n" " mov.f32 %f451, 0f3F000000;\n" " mov.f32 %f452, 0f3BBB989D;\n" " fma.rn.f32 %f453, %f450, %f452, %f451;\n" " mov.f32 %f454, 0f3FB8AA3B;\n" " mov.f32 %f455, 0f437C0000;\n" " cvt.sat.f32.f32 %f456, %f453;\n" " mov.f32 %f457, 0f4B400001;\n" " fma.rm.f32 %f458, %f456, %f455, %f457;\n" " add.f32 %f459, %f458, 0fCB40007F;\n" " neg.f32 %f460, %f459;\n" " fma.rn.f32 %f461, %f450, %f454, %f460;\n" " mov.f32 %f462, 0f32A57060;\n" " fma.rn.f32 %f463, %f450, %f462, %f461;\n" " mov.b32 %r141, %f458;\n" " shl.b32 %r142, %r141, 23;\n" " mov.b32 %f464, %r142;\n" " ex2.approx.ftz.f32 %f465, %f463;\n" " fma.rn.f32 %f466, %f465, %f464, 0f3F800000;\n" " div.rn.f32 %f467, %f99, %f466;\n" " mul.f32 %f468, %f100, %f467;\n" " ld.shared.s32 %rd59, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj];\n" " mul.lo.s64 %rd60, %rd59, %rd2;\n" " add.s64 %rd61, %rd60, %rd3;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd63, %rd1, %rd62;\n" " st.global.f32 [%rd63], %f468;\n" "\n" "$L__BB33_34:\n" " // begin inline asm\n" " shfl.down.b32 %f469, %f783, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f472, %f783, %f469;\n" " // begin inline asm\n" " shfl.down.b32 %f471, %f472, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f474, %f472, %f471;\n" " // begin inline asm\n" " shfl.down.b32 %f473, %f474, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f476, %f474, %f473;\n" " // begin inline asm\n" " shfl.down.b32 %f475, %f476, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f478, %f476, %f475;\n" " // begin inline asm\n" " shfl.down.b32 %f477, %f478, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f101, %f478, %f477;\n" " // begin inline asm\n" " shfl.down.b32 %f479, %f775, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f482, %f775, %f479;\n" " // begin inline asm\n" " shfl.down.b32 %f481, %f482, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f484, %f482, %f481;\n" " // begin inline asm\n" " shfl.down.b32 %f483, %f484, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f486, %f484, %f483;\n" " // begin inline asm\n" " shfl.down.b32 %f485, %f486, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f488, %f486, %f485;\n" " // begin inline asm\n" " shfl.down.b32 %f487, %f488, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f102, %f488, %f487;\n" " setp.lt.u32 %p24, %r8, 2;\n" " or.pred %p26, %p21, %p24;\n" " @%p26 bra $L__BB33_36;\n" "\n" " neg.f32 %f489, %f101;\n" " mov.f32 %f490, 0f3F000000;\n" " mov.f32 %f491, 0f3BBB989D;\n" " fma.rn.f32 %f492, %f489, %f491, %f490;\n" " mov.f32 %f493, 0f3FB8AA3B;\n" " mov.f32 %f494, 0f437C0000;\n" " cvt.sat.f32.f32 %f495, %f492;\n" " mov.f32 %f496, 0f4B400001;\n" " fma.rm.f32 %f497, %f495, %f494, %f496;\n" " add.f32 %f498, %f497, 0fCB40007F;\n" " neg.f32 %f499, %f498;\n" " fma.rn.f32 %f500, %f489, %f493, %f499;\n" " mov.f32 %f501, 0f32A57060;\n" " fma.rn.f32 %f502, %f489, %f501, %f500;\n" " mov.b32 %r163, %f497;\n" " shl.b32 %r164, %r163, 23;\n" " mov.b32 %f503, %r164;\n" " ex2.approx.ftz.f32 %f504, %f502;\n" " fma.rn.f32 %f505, %f504, %f503, 0f3F800000;\n" " div.rn.f32 %f506, %f101, %f505;\n" " mul.f32 %f507, %f102, %f506;\n" " ld.shared.s32 %rd64, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj+4];\n" " mul.lo.s64 %rd65, %rd64, %rd2;\n" " add.s64 %rd66, %rd65, %rd3;\n" " shl.b64 %rd67, %rd66, 2;\n" " add.s64 %rd68, %rd1, %rd67;\n" " st.global.f32 [%rd68], %f507;\n" "\n" "$L__BB33_36:\n" " // begin inline asm\n" " shfl.down.b32 %f508, %f784, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f511, %f784, %f508;\n" " // begin inline asm\n" " shfl.down.b32 %f510, %f511, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f513, %f511, %f510;\n" " // begin inline asm\n" " shfl.down.b32 %f512, %f513, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f515, %f513, %f512;\n" " // begin inline asm\n" " shfl.down.b32 %f514, %f515, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f517, %f515, %f514;\n" " // begin inline asm\n" " shfl.down.b32 %f516, %f517, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f103, %f517, %f516;\n" " // begin inline asm\n" " shfl.down.b32 %f518, %f776, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f521, %f776, %f518;\n" " // begin inline asm\n" " shfl.down.b32 %f520, %f521, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f523, %f521, %f520;\n" " // begin inline asm\n" " shfl.down.b32 %f522, %f523, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f525, %f523, %f522;\n" " // begin inline asm\n" " shfl.down.b32 %f524, %f525, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f527, %f525, %f524;\n" " // begin inline asm\n" " shfl.down.b32 %f526, %f527, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f104, %f527, %f526;\n" " setp.lt.u32 %p27, %r8, 3;\n" " or.pred %p29, %p21, %p27;\n" " @%p29 bra $L__BB33_38;\n" "\n" " neg.f32 %f528, %f103;\n" " mov.f32 %f529, 0f3F000000;\n" " mov.f32 %f530, 0f3BBB989D;\n" " fma.rn.f32 %f531, %f528, %f530, %f529;\n" " mov.f32 %f532, 0f3FB8AA3B;\n" " mov.f32 %f533, 0f437C0000;\n" " cvt.sat.f32.f32 %f534, %f531;\n" " mov.f32 %f535, 0f4B400001;\n" " fma.rm.f32 %f536, %f534, %f533, %f535;\n" " add.f32 %f537, %f536, 0fCB40007F;\n" " neg.f32 %f538, %f537;\n" " fma.rn.f32 %f539, %f528, %f532, %f538;\n" " mov.f32 %f540, 0f32A57060;\n" " fma.rn.f32 %f541, %f528, %f540, %f539;\n" " mov.b32 %r185, %f536;\n" " shl.b32 %r186, %r185, 23;\n" " mov.b32 %f542, %r186;\n" " ex2.approx.ftz.f32 %f543, %f541;\n" " fma.rn.f32 %f544, %f543, %f542, 0f3F800000;\n" " div.rn.f32 %f545, %f103, %f544;\n" " mul.f32 %f546, %f104, %f545;\n" " ld.shared.s32 %rd69, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj+8];\n" " mul.lo.s64 %rd70, %rd69, %rd2;\n" " add.s64 %rd71, %rd70, %rd3;\n" " shl.b64 %rd72, %rd71, 2;\n" " add.s64 %rd73, %rd1, %rd72;\n" " st.global.f32 [%rd73], %f546;\n" "\n" "$L__BB33_38:\n" " // begin inline asm\n" " shfl.down.b32 %f547, %f785, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f550, %f785, %f547;\n" " // begin inline asm\n" " shfl.down.b32 %f549, %f550, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f552, %f550, %f549;\n" " // begin inline asm\n" " shfl.down.b32 %f551, %f552, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f554, %f552, %f551;\n" " // begin inline asm\n" " shfl.down.b32 %f553, %f554, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f556, %f554, %f553;\n" " // begin inline asm\n" " shfl.down.b32 %f555, %f556, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f105, %f556, %f555;\n" " // begin inline asm\n" " shfl.down.b32 %f557, %f777, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f560, %f777, %f557;\n" " // begin inline asm\n" " shfl.down.b32 %f559, %f560, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f562, %f560, %f559;\n" " // begin inline asm\n" " shfl.down.b32 %f561, %f562, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f564, %f562, %f561;\n" " // begin inline asm\n" " shfl.down.b32 %f563, %f564, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f566, %f564, %f563;\n" " // begin inline asm\n" " shfl.down.b32 %f565, %f566, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f106, %f566, %f565;\n" " setp.lt.u32 %p30, %r8, 4;\n" " or.pred %p32, %p21, %p30;\n" " @%p32 bra $L__BB33_40;\n" "\n" " neg.f32 %f567, %f105;\n" " mov.f32 %f568, 0f3F000000;\n" " mov.f32 %f569, 0f3BBB989D;\n" " fma.rn.f32 %f570, %f567, %f569, %f568;\n" " mov.f32 %f571, 0f3FB8AA3B;\n" " mov.f32 %f572, 0f437C0000;\n" " cvt.sat.f32.f32 %f573, %f570;\n" " mov.f32 %f574, 0f4B400001;\n" " fma.rm.f32 %f575, %f573, %f572, %f574;\n" " add.f32 %f576, %f575, 0fCB40007F;\n" " neg.f32 %f577, %f576;\n" " fma.rn.f32 %f578, %f567, %f571, %f577;\n" " mov.f32 %f579, 0f32A57060;\n" " fma.rn.f32 %f580, %f567, %f579, %f578;\n" " mov.b32 %r207, %f575;\n" " shl.b32 %r208, %r207, 23;\n" " mov.b32 %f581, %r208;\n" " ex2.approx.ftz.f32 %f582, %f580;\n" " fma.rn.f32 %f583, %f582, %f581, 0f3F800000;\n" " div.rn.f32 %f584, %f105, %f583;\n" " mul.f32 %f585, %f106, %f584;\n" " ld.shared.s32 %rd74, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj+12];\n" " mul.lo.s64 %rd75, %rd74, %rd2;\n" " add.s64 %rd76, %rd75, %rd3;\n" " shl.b64 %rd77, %rd76, 2;\n" " add.s64 %rd78, %rd1, %rd77;\n" " st.global.f32 [%rd78], %f585;\n" "\n" "$L__BB33_40:\n" " // begin inline asm\n" " shfl.down.b32 %f586, %f786, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f589, %f786, %f586;\n" " // begin inline asm\n" " shfl.down.b32 %f588, %f589, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f591, %f589, %f588;\n" " // begin inline asm\n" " shfl.down.b32 %f590, %f591, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f593, %f591, %f590;\n" " // begin inline asm\n" " shfl.down.b32 %f592, %f593, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f595, %f593, %f592;\n" " // begin inline asm\n" " shfl.down.b32 %f594, %f595, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f107, %f595, %f594;\n" " // begin inline asm\n" " shfl.down.b32 %f596, %f778, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f599, %f778, %f596;\n" " // begin inline asm\n" " shfl.down.b32 %f598, %f599, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f601, %f599, %f598;\n" " // begin inline asm\n" " shfl.down.b32 %f600, %f601, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f603, %f601, %f600;\n" " // begin inline asm\n" " shfl.down.b32 %f602, %f603, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f605, %f603, %f602;\n" " // begin inline asm\n" " shfl.down.b32 %f604, %f605, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f108, %f605, %f604;\n" " setp.lt.u32 %p33, %r8, 5;\n" " or.pred %p35, %p21, %p33;\n" " @%p35 bra $L__BB33_42;\n" "\n" " neg.f32 %f606, %f107;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f608, 0f3BBB989D;\n" " fma.rn.f32 %f609, %f606, %f608, %f607;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f611, 0f437C0000;\n" " cvt.sat.f32.f32 %f612, %f609;\n" " mov.f32 %f613, 0f4B400001;\n" " fma.rm.f32 %f614, %f612, %f611, %f613;\n" " add.f32 %f615, %f614, 0fCB40007F;\n" " neg.f32 %f616, %f615;\n" " fma.rn.f32 %f617, %f606, %f610, %f616;\n" " mov.f32 %f618, 0f32A57060;\n" " fma.rn.f32 %f619, %f606, %f618, %f617;\n" " mov.b32 %r229, %f614;\n" " shl.b32 %r230, %r229, 23;\n" " mov.b32 %f620, %r230;\n" " ex2.approx.ftz.f32 %f621, %f619;\n" " fma.rn.f32 %f622, %f621, %f620, 0f3F800000;\n" " div.rn.f32 %f623, %f107, %f622;\n" " mul.f32 %f624, %f108, %f623;\n" " ld.shared.s32 %rd79, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj+16];\n" " mul.lo.s64 %rd80, %rd79, %rd2;\n" " add.s64 %rd81, %rd80, %rd3;\n" " shl.b64 %rd82, %rd81, 2;\n" " add.s64 %rd83, %rd1, %rd82;\n" " st.global.f32 [%rd83], %f624;\n" "\n" "$L__BB33_42:\n" " // begin inline asm\n" " shfl.down.b32 %f625, %f787, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f628, %f787, %f625;\n" " // begin inline asm\n" " shfl.down.b32 %f627, %f628, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f630, %f628, %f627;\n" " // begin inline asm\n" " shfl.down.b32 %f629, %f630, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f632, %f630, %f629;\n" " // begin inline asm\n" " shfl.down.b32 %f631, %f632, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f634, %f632, %f631;\n" " // begin inline asm\n" " shfl.down.b32 %f633, %f634, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f109, %f634, %f633;\n" " // begin inline asm\n" " shfl.down.b32 %f635, %f779, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f638, %f779, %f635;\n" " // begin inline asm\n" " shfl.down.b32 %f637, %f638, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f640, %f638, %f637;\n" " // begin inline asm\n" " shfl.down.b32 %f639, %f640, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f642, %f640, %f639;\n" " // begin inline asm\n" " shfl.down.b32 %f641, %f642, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f644, %f642, %f641;\n" " // begin inline asm\n" " shfl.down.b32 %f643, %f644, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f110, %f644, %f643;\n" " setp.lt.u32 %p36, %r8, 6;\n" " or.pred %p38, %p21, %p36;\n" " @%p38 bra $L__BB33_44;\n" "\n" " neg.f32 %f645, %f109;\n" " mov.f32 %f646, 0f3F000000;\n" " mov.f32 %f647, 0f3BBB989D;\n" " fma.rn.f32 %f648, %f645, %f647, %f646;\n" " mov.f32 %f649, 0f3FB8AA3B;\n" " mov.f32 %f650, 0f437C0000;\n" " cvt.sat.f32.f32 %f651, %f648;\n" " mov.f32 %f652, 0f4B400001;\n" " fma.rm.f32 %f653, %f651, %f650, %f652;\n" " add.f32 %f654, %f653, 0fCB40007F;\n" " neg.f32 %f655, %f654;\n" " fma.rn.f32 %f656, %f645, %f649, %f655;\n" " mov.f32 %f657, 0f32A57060;\n" " fma.rn.f32 %f658, %f645, %f657, %f656;\n" " mov.b32 %r251, %f653;\n" " shl.b32 %r252, %r251, 23;\n" " mov.b32 %f659, %r252;\n" " ex2.approx.ftz.f32 %f660, %f658;\n" " fma.rn.f32 %f661, %f660, %f659, 0f3F800000;\n" " div.rn.f32 %f662, %f109, %f661;\n" " mul.f32 %f663, %f110, %f662;\n" " ld.shared.s32 %rd84, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj+20];\n" " mul.lo.s64 %rd85, %rd84, %rd2;\n" " add.s64 %rd86, %rd85, %rd3;\n" " shl.b64 %rd87, %rd86, 2;\n" " add.s64 %rd88, %rd1, %rd87;\n" " st.global.f32 [%rd88], %f663;\n" "\n" "$L__BB33_44:\n" " // begin inline asm\n" " shfl.down.b32 %f664, %f788, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f667, %f788, %f664;\n" " // begin inline asm\n" " shfl.down.b32 %f666, %f667, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f669, %f667, %f666;\n" " // begin inline asm\n" " shfl.down.b32 %f668, %f669, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f671, %f669, %f668;\n" " // begin inline asm\n" " shfl.down.b32 %f670, %f671, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f673, %f671, %f670;\n" " // begin inline asm\n" " shfl.down.b32 %f672, %f673, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f111, %f673, %f672;\n" " // begin inline asm\n" " shfl.down.b32 %f674, %f780, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f677, %f780, %f674;\n" " // begin inline asm\n" " shfl.down.b32 %f676, %f677, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f679, %f677, %f676;\n" " // begin inline asm\n" " shfl.down.b32 %f678, %f679, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f681, %f679, %f678;\n" " // begin inline asm\n" " shfl.down.b32 %f680, %f681, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f683, %f681, %f680;\n" " // begin inline asm\n" " shfl.down.b32 %f682, %f683, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f112, %f683, %f682;\n" " setp.lt.u32 %p39, %r8, 7;\n" " or.pred %p41, %p21, %p39;\n" " @%p41 bra $L__BB33_46;\n" "\n" " neg.f32 %f684, %f111;\n" " mov.f32 %f685, 0f3F000000;\n" " mov.f32 %f686, 0f3BBB989D;\n" " fma.rn.f32 %f687, %f684, %f686, %f685;\n" " mov.f32 %f688, 0f3FB8AA3B;\n" " mov.f32 %f689, 0f437C0000;\n" " cvt.sat.f32.f32 %f690, %f687;\n" " mov.f32 %f691, 0f4B400001;\n" " fma.rm.f32 %f692, %f690, %f689, %f691;\n" " add.f32 %f693, %f692, 0fCB40007F;\n" " neg.f32 %f694, %f693;\n" " fma.rn.f32 %f695, %f684, %f688, %f694;\n" " mov.f32 %f696, 0f32A57060;\n" " fma.rn.f32 %f697, %f684, %f696, %f695;\n" " mov.b32 %r273, %f692;\n" " shl.b32 %r274, %r273, 23;\n" " mov.b32 %f698, %r274;\n" " ex2.approx.ftz.f32 %f699, %f697;\n" " fma.rn.f32 %f700, %f699, %f698, 0f3F800000;\n" " div.rn.f32 %f701, %f111, %f700;\n" " mul.f32 %f702, %f112, %f701;\n" " ld.shared.s32 %rd89, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj+24];\n" " mul.lo.s64 %rd90, %rd89, %rd2;\n" " add.s64 %rd91, %rd90, %rd3;\n" " shl.b64 %rd92, %rd91, 2;\n" " add.s64 %rd93, %rd1, %rd92;\n" " st.global.f32 [%rd93], %f702;\n" "\n" "$L__BB33_46:\n" " // begin inline asm\n" " shfl.down.b32 %f703, %f789, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f706, %f789, %f703;\n" " // begin inline asm\n" " shfl.down.b32 %f705, %f706, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f708, %f706, %f705;\n" " // begin inline asm\n" " shfl.down.b32 %f707, %f708, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f710, %f708, %f707;\n" " // begin inline asm\n" " shfl.down.b32 %f709, %f710, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f712, %f710, %f709;\n" " // begin inline asm\n" " shfl.down.b32 %f711, %f712, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f113, %f712, %f711;\n" " // begin inline asm\n" " shfl.down.b32 %f713, %f781, %r131, %r140;\n" " // end inline asm\n" " add.f32 %f716, %f781, %f713;\n" " // begin inline asm\n" " shfl.down.b32 %f715, %f716, %r133, %r140;\n" " // end inline asm\n" " add.f32 %f718, %f716, %f715;\n" " // begin inline asm\n" " shfl.down.b32 %f717, %f718, %r135, %r140;\n" " // end inline asm\n" " add.f32 %f720, %f718, %f717;\n" " // begin inline asm\n" " shfl.down.b32 %f719, %f720, %r137, %r140;\n" " // end inline asm\n" " add.f32 %f722, %f720, %f719;\n" " // begin inline asm\n" " shfl.down.b32 %f721, %f722, %r139, %r140;\n" " // end inline asm\n" " add.f32 %f114, %f722, %f721;\n" " setp.lt.u32 %p42, %r8, 8;\n" " or.pred %p44, %p21, %p42;\n" " @%p44 bra $L__BB33_48;\n" "\n" " neg.f32 %f723, %f113;\n" " mov.f32 %f724, 0f3F000000;\n" " mov.f32 %f725, 0f3BBB989D;\n" " fma.rn.f32 %f726, %f723, %f725, %f724;\n" " mov.f32 %f727, 0f3FB8AA3B;\n" " mov.f32 %f728, 0f437C0000;\n" " cvt.sat.f32.f32 %f729, %f726;\n" " mov.f32 %f730, 0f4B400001;\n" " fma.rm.f32 %f731, %f729, %f728, %f730;\n" " add.f32 %f732, %f731, 0fCB40007F;\n" " neg.f32 %f733, %f732;\n" " fma.rn.f32 %f734, %f723, %f727, %f733;\n" " mov.f32 %f735, 0f32A57060;\n" " fma.rn.f32 %f736, %f723, %f735, %f734;\n" " mov.b32 %r295, %f731;\n" " shl.b32 %r296, %r295, 23;\n" " mov.b32 %f737, %r296;\n" " ex2.approx.ftz.f32 %f738, %f736;\n" " fma.rn.f32 %f739, %f738, %f737, 0f3F800000;\n" " div.rn.f32 %f740, %f113, %f739;\n" " mul.f32 %f741, %f114, %f740;\n" " ld.shared.s32 %rd94, [_ZZ28fc_moe_gateup_q3k_dedup_shxcE5sh_fj+28];\n" " mul.lo.s64 %rd95, %rd94, %rd2;\n" " add.s64 %rd96, %rd95, %rd3;\n" " shl.b64 %rd97, %rd96, 2;\n" " add.s64 %rd98, %rd1, %rd97;\n" " st.global.f32 [%rd98], %f741;\n" "\n" "$L__BB33_48:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_SSM_GATED_DELTA[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_ssm_recurrence\n" ".extern .shared .align 16 .b8 sdata[];\n" ".extern .shared .align 16 .b8 sh[];\n" "\n" ".visible .entry fc_ssm_recurrence(\n" " .param .u64 fc_ssm_recurrence_param_0,\n" " .param .u64 fc_ssm_recurrence_param_1,\n" " .param .u64 fc_ssm_recurrence_param_2,\n" " .param .u64 fc_ssm_recurrence_param_3,\n" " .param .u64 fc_ssm_recurrence_param_4,\n" " .param .u64 fc_ssm_recurrence_param_5,\n" " .param .u64 fc_ssm_recurrence_param_6,\n" " .param .u32 fc_ssm_recurrence_param_7,\n" " .param .f32 fc_ssm_recurrence_param_8\n" ")\n" "{\n" " .reg .pred %p<9>;\n" " .reg .f32 %f<40>;\n" " .reg .b32 %r<41>;\n" " .reg .b64 %rd<52>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_ssm_recurrence_param_0];\n" " ld.param.u64 %rd2, [fc_ssm_recurrence_param_1];\n" " ld.param.u64 %rd3, [fc_ssm_recurrence_param_2];\n" " ld.param.u64 %rd4, [fc_ssm_recurrence_param_3];\n" " ld.param.u64 %rd5, [fc_ssm_recurrence_param_4];\n" " ld.param.u64 %rd6, [fc_ssm_recurrence_param_5];\n" " ld.param.u64 %rd7, [fc_ssm_recurrence_param_6];\n" " ld.param.u32 %r9, [fc_ssm_recurrence_param_7];\n" " ld.param.f32 %f3, [fc_ssm_recurrence_param_8];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r9;\n" " @%p1 bra $L__BB0_12;\n" "\n" " mov.u32 %r10, %ctaid.y;\n" " cvt.s64.s32 %rd8, %r9;\n" " mul.wide.s32 %rd9, %r9, %r10;\n" " mov.u32 %r11, %ctaid.x;\n" " cvt.s64.s32 %rd10, %r11;\n" " add.s64 %rd11, %rd9, %rd10;\n" " mul.lo.s64 %rd12, %rd11, %rd8;\n" " cvta.to.global.u64 %rd13, %rd5;\n" " mul.wide.s32 %rd14, %r10, 4;\n" " add.s64 %rd15, %rd13, %rd14;\n" " ld.global.f32 %f4, [%rd15];\n" " mov.f32 %f5, 0f3F000000;\n" " mov.f32 %f6, 0f3BBB989D;\n" " fma.rn.f32 %f7, %f4, %f6, %f5;\n" " mov.f32 %f8, 0f3FB8AA3B;\n" " mov.f32 %f9, 0f437C0000;\n" " cvt.sat.f32.f32 %f10, %f7;\n" " mov.f32 %f11, 0f4B400001;\n" " fma.rm.f32 %f12, %f10, %f9, %f11;\n" " add.f32 %f13, %f12, 0fCB40007F;\n" " neg.f32 %f14, %f13;\n" " fma.rn.f32 %f15, %f4, %f8, %f14;\n" " mov.f32 %f16, 0f32A57060;\n" " fma.rn.f32 %f17, %f4, %f16, %f15;\n" " mov.b32 %r12, %f12;\n" " shl.b32 %r13, %r12, 23;\n" " mov.b32 %f18, %r13;\n" " ex2.approx.ftz.f32 %f19, %f17;\n" " mul.f32 %f20, %f19, %f18;\n" " cvta.to.global.u64 %rd16, %rd6;\n" " add.s64 %rd17, %rd16, %rd14;\n" " ld.global.f32 %f1, [%rd17];\n" " cvt.s64.s32 %rd18, %r1;\n" " add.s64 %rd19, %rd12, %rd18;\n" " cvta.to.global.u64 %rd20, %rd1;\n" " shl.b64 %rd21, %rd19, 2;\n" " add.s64 %rd22, %rd20, %rd21;\n" " ld.global.f32 %f21, [%rd22];\n" " mul.f32 %f2, %f20, %f21;\n" " add.s64 %rd23, %rd9, %rd18;\n" " cvta.to.global.u64 %rd24, %rd3;\n" " shl.b64 %rd25, %rd23, 2;\n" " add.s64 %rd26, %rd24, %rd25;\n" " ld.global.f32 %f22, [%rd26];\n" " mul.f32 %f23, %f2, %f22;\n" " shl.b32 %r14, %r1, 2;\n" " mov.u32 %r15, sdata;\n" " add.s32 %r16, %r15, %r14;\n" " st.shared.f32 [%r16], %f23;\n" " bar.sync 0;\n" " shr.u32 %r17, %r9, 31;\n" " add.s32 %r18, %r9, %r17;\n" " shr.s32 %r40, %r18, 1;\n" " setp.lt.s32 %p2, %r9, 2;\n" " @%p2 bra $L__BB0_6;\n" "\n" " mov.u32 %r39, %r40;\n" "\n" "$L__BB0_3:\n" " setp.ge.s32 %p3, %r1, %r39;\n" " @%p3 bra $L__BB0_5;\n" "\n" " shl.b32 %r23, %r39, 2;\n" " add.s32 %r24, %r16, %r23;\n" " ld.shared.f32 %f24, [%r16];\n" " ld.shared.f32 %f25, [%r24];\n" " add.f32 %f26, %f25, %f24;\n" " st.shared.f32 [%r16], %f26;\n" "\n" "$L__BB0_5:\n" " bar.sync 0;\n" " shr.s32 %r5, %r39, 1;\n" " setp.gt.s32 %p4, %r39, 1;\n" " mov.u32 %r39, %r5;\n" " @%p4 bra $L__BB0_3;\n" "\n" "$L__BB0_6:\n" " cvta.to.global.u64 %rd31, %rd4;\n" " shl.b64 %rd32, %rd11, 2;\n" " add.s64 %rd33, %rd31, %rd32;\n" " ld.global.f32 %f27, [%rd33];\n" " ld.shared.f32 %f28, [sdata];\n" " sub.f32 %f29, %f27, %f28;\n" " mul.f32 %f30, %f1, %f29;\n" " bar.sync 0;\n" " ld.global.f32 %f31, [%rd26];\n" " fma.rn.f32 %f32, %f31, %f30, %f2;\n" " st.global.f32 [%rd22], %f32;\n" " cvta.to.global.u64 %rd44, %rd2;\n" " add.s64 %rd45, %rd44, %rd25;\n" " ld.global.f32 %f33, [%rd45];\n" " mul.f32 %f34, %f32, %f33;\n" " st.shared.f32 [%r16], %f34;\n" " bar.sync 0;\n" " @%p2 bra $L__BB0_10;\n" "\n" "$L__BB0_7:\n" " setp.ge.s32 %p6, %r1, %r40;\n" " @%p6 bra $L__BB0_9;\n" "\n" " shl.b32 %r34, %r40, 2;\n" " add.s32 %r35, %r16, %r34;\n" " ld.shared.f32 %f35, [%r16];\n" " ld.shared.f32 %f36, [%r35];\n" " add.f32 %f37, %f36, %f35;\n" " st.shared.f32 [%r16], %f37;\n" "\n" "$L__BB0_9:\n" " bar.sync 0;\n" " shr.s32 %r8, %r40, 1;\n" " setp.gt.s32 %p7, %r40, 1;\n" " mov.u32 %r40, %r8;\n" " @%p7 bra $L__BB0_7;\n" "\n" "$L__BB0_10:\n" " setp.ne.s32 %p8, %r1, 0;\n" " @%p8 bra $L__BB0_12;\n" "\n" " ld.shared.f32 %f38, [sdata];\n" " mul.f32 %f39, %f38, %f3;\n" " cvta.to.global.u64 %rd49, %rd7;\n" " add.s64 %rd51, %rd49, %rd32;\n" " st.global.f32 [%rd51], %f39;\n" "\n" "$L__BB0_12:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_conv1d_silu\n" ".visible .entry fc_ssm_conv1d_silu(\n" " .param .u64 fc_ssm_conv1d_silu_param_0,\n" " .param .u64 fc_ssm_conv1d_silu_param_1,\n" " .param .u64 fc_ssm_conv1d_silu_param_2,\n" " .param .u64 fc_ssm_conv1d_silu_param_3,\n" " .param .u32 fc_ssm_conv1d_silu_param_4,\n" " .param .u32 fc_ssm_conv1d_silu_param_5\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<54>;\n" " .reg .b32 %r<42>;\n" " .reg .b64 %rd<63>;\n" "\n" "\n" " ld.param.u64 %rd25, [fc_ssm_conv1d_silu_param_0];\n" " ld.param.u64 %rd27, [fc_ssm_conv1d_silu_param_1];\n" " ld.param.u64 %rd28, [fc_ssm_conv1d_silu_param_2];\n" " ld.param.u64 %rd26, [fc_ssm_conv1d_silu_param_3];\n" " ld.param.u32 %r23, [fc_ssm_conv1d_silu_param_4];\n" " ld.param.u32 %r22, [fc_ssm_conv1d_silu_param_5];\n" " cvta.to.global.u64 %rd1, %rd28;\n" " cvta.to.global.u64 %rd2, %rd27;\n" " mov.u32 %r24, %ntid.x;\n" " mov.u32 %r25, %ctaid.x;\n" " mov.u32 %r26, %tid.x;\n" " mad.lo.s32 %r1, %r25, %r24, %r26;\n" " setp.ge.s32 %p1, %r1, %r23;\n" " @%p1 bra $L__BB1_16;\n" "\n" " cvta.to.global.u64 %rd29, %rd25;\n" " cvt.s64.s32 %rd3, %r1;\n" " mul.wide.s32 %rd4, %r22, %r1;\n" " add.s32 %r2, %r22, -1;\n" " cvt.s64.s32 %rd30, %r2;\n" " mul.wide.s32 %rd5, %r2, %r1;\n" " mul.wide.s32 %rd31, %r1, 4;\n" " add.s64 %rd6, %rd29, %rd31;\n" " add.s64 %rd32, %rd4, %rd30;\n" " shl.b64 %rd33, %rd32, 2;\n" " add.s64 %rd34, %rd2, %rd33;\n" " ld.global.f32 %f9, [%rd34];\n" " ld.global.f32 %f10, [%rd6];\n" " mul.f32 %f53, %f10, %f9;\n" " add.s32 %r3, %r22, -2;\n" " setp.lt.s32 %p2, %r22, 2;\n" " @%p2 bra $L__BB1_8;\n" "\n" " and.b32 %r37, %r2, 3;\n" " setp.lt.u32 %p3, %r3, 3;\n" " mov.u32 %r36, 0;\n" " @%p3 bra $L__BB1_5;\n" "\n" " sub.s32 %r34, %r22, %r37;\n" " shl.b64 %rd35, %rd5, 2;\n" " add.s64 %rd36, %rd1, %rd35;\n" " add.s64 %rd58, %rd36, 8;\n" " shl.b64 %rd37, %rd4, 2;\n" " add.s64 %rd38, %rd2, %rd37;\n" " add.s64 %rd57, %rd38, 8;\n" " mov.u32 %r36, 0;\n" "\n" "$L__BB1_4:\n" " ld.global.f32 %f12, [%rd57+-8];\n" " ld.global.f32 %f13, [%rd58+-8];\n" " fma.rn.f32 %f14, %f13, %f12, %f53;\n" " ld.global.f32 %f15, [%rd57+-4];\n" " ld.global.f32 %f16, [%rd58+-4];\n" " fma.rn.f32 %f17, %f16, %f15, %f14;\n" " ld.global.f32 %f18, [%rd57];\n" " ld.global.f32 %f19, [%rd58];\n" " fma.rn.f32 %f20, %f19, %f18, %f17;\n" " ld.global.f32 %f21, [%rd57+4];\n" " ld.global.f32 %f22, [%rd58+4];\n" " fma.rn.f32 %f53, %f22, %f21, %f20;\n" " add.s32 %r36, %r36, 4;\n" " add.s64 %rd58, %rd58, 16;\n" " add.s64 %rd57, %rd57, 16;\n" " add.s32 %r34, %r34, -4;\n" " setp.ne.s32 %p4, %r34, 1;\n" " @%p4 bra $L__BB1_4;\n" "\n" "$L__BB1_5:\n" " setp.eq.s32 %p5, %r37, 0;\n" " @%p5 bra $L__BB1_8;\n" "\n" " cvt.s64.s32 %rd39, %r36;\n" " add.s64 %rd40, %rd4, %rd39;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd60, %rd2, %rd41;\n" " add.s64 %rd42, %rd5, %rd39;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd59, %rd1, %rd43;\n" "\n" "$L__BB1_7:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f23, [%rd60];\n" " ld.global.f32 %f24, [%rd59];\n" " fma.rn.f32 %f53, %f24, %f23, %f53;\n" " add.s64 %rd60, %rd60, 4;\n" " add.s64 %rd59, %rd59, 4;\n" " add.s32 %r37, %r37, -1;\n" " setp.ne.s32 %p6, %r37, 0;\n" " @%p6 bra $L__BB1_7;\n" "\n" "$L__BB1_8:\n" " neg.f32 %f25, %f53;\n" " mov.f32 %f26, 0f3F000000;\n" " mov.f32 %f27, 0f3BBB989D;\n" " fma.rn.f32 %f28, %f25, %f27, %f26;\n" " mov.f32 %f29, 0f3FB8AA3B;\n" " mov.f32 %f30, 0f437C0000;\n" " cvt.sat.f32.f32 %f31, %f28;\n" " mov.f32 %f32, 0f4B400001;\n" " fma.rm.f32 %f33, %f31, %f30, %f32;\n" " add.f32 %f34, %f33, 0fCB40007F;\n" " neg.f32 %f35, %f34;\n" " fma.rn.f32 %f36, %f25, %f29, %f35;\n" " mov.f32 %f37, 0f32A57060;\n" " fma.rn.f32 %f38, %f25, %f37, %f36;\n" " mov.b32 %r29, %f33;\n" " shl.b32 %r30, %r29, 23;\n" " mov.b32 %f39, %r30;\n" " ex2.approx.ftz.f32 %f40, %f38;\n" " fma.rn.f32 %f41, %f40, %f39, 0f3F800000;\n" " div.rn.f32 %f42, %f53, %f41;\n" " cvta.to.global.u64 %rd44, %rd26;\n" " shl.b64 %rd45, %rd3, 2;\n" " add.s64 %rd46, %rd44, %rd45;\n" " st.global.f32 [%rd46], %f42;\n" " setp.lt.s32 %p7, %r22, 3;\n" " @%p7 bra $L__BB1_15;\n" "\n" " add.s32 %r32, %r22, -3;\n" " and.b32 %r41, %r3, 3;\n" " setp.lt.u32 %p8, %r32, 3;\n" " mov.u32 %r40, 0;\n" " @%p8 bra $L__BB1_12;\n" "\n" " sub.s32 %r38, %r22, %r41;\n" " shl.b64 %rd47, %rd5, 2;\n" " add.s64 %rd48, %rd1, %rd47;\n" " add.s64 %rd61, %rd48, 8;\n" " mov.u32 %r40, 0;\n" "\n" "$L__BB1_11:\n" " ld.global.f32 %f43, [%rd61+-4];\n" " st.global.f32 [%rd61+-8], %f43;\n" " ld.global.f32 %f44, [%rd61];\n" " st.global.f32 [%rd61+-4], %f44;\n" " ld.global.f32 %f45, [%rd61+4];\n" " st.global.f32 [%rd61], %f45;\n" " add.s32 %r40, %r40, 4;\n" " ld.global.f32 %f46, [%rd61+8];\n" " st.global.f32 [%rd61+4], %f46;\n" " add.s64 %rd61, %rd61, 16;\n" " add.s32 %r38, %r38, -4;\n" " setp.ne.s32 %p9, %r38, 2;\n" " @%p9 bra $L__BB1_11;\n" "\n" "$L__BB1_12:\n" " setp.eq.s32 %p10, %r41, 0;\n" " @%p10 bra $L__BB1_15;\n" "\n" " cvt.u64.u32 %rd49, %r40;\n" " add.s64 %rd50, %rd5, %rd49;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd52, %rd1, %rd51;\n" " add.s64 %rd62, %rd52, 4;\n" "\n" "$L__BB1_14:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f47, [%rd62];\n" " st.global.f32 [%rd62+-4], %f47;\n" " add.s64 %rd62, %rd62, 4;\n" " add.s32 %r41, %r41, -1;\n" " setp.ne.s32 %p11, %r41, 0;\n" " @%p11 bra $L__BB1_14;\n" "\n" "$L__BB1_15:\n" " ld.global.f32 %f48, [%rd6];\n" " cvt.s64.s32 %rd53, %r3;\n" " add.s64 %rd54, %rd5, %rd53;\n" " shl.b64 %rd55, %rd54, 2;\n" " add.s64 %rd56, %rd1, %rd55;\n" " st.global.f32 [%rd56], %f48;\n" "\n" "$L__BB1_16:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_gated_norm\n" ".visible .entry fc_ssm_gated_norm(\n" " .param .u64 fc_ssm_gated_norm_param_0,\n" " .param .u64 fc_ssm_gated_norm_param_1,\n" " .param .u64 fc_ssm_gated_norm_param_2,\n" " .param .u64 fc_ssm_gated_norm_param_3,\n" " .param .u32 fc_ssm_gated_norm_param_4,\n" " .param .f32 fc_ssm_gated_norm_param_5\n" ")\n" "{\n" " .reg .pred %p<14>;\n" " .reg .f32 %f<137>;\n" " .reg .b32 %r<64>;\n" " .reg .b64 %rd<70>;\n" "\n" "\n" " ld.param.u64 %rd27, [fc_ssm_gated_norm_param_0];\n" " ld.param.u64 %rd25, [fc_ssm_gated_norm_param_1];\n" " ld.param.u64 %rd26, [fc_ssm_gated_norm_param_2];\n" " ld.param.u64 %rd28, [fc_ssm_gated_norm_param_3];\n" " ld.param.u32 %r25, [fc_ssm_gated_norm_param_4];\n" " ld.param.f32 %f9, [fc_ssm_gated_norm_param_5];\n" " cvta.to.global.u64 %rd1, %rd28;\n" " cvta.to.global.u64 %rd2, %rd27;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r26, %ctaid.x;\n" " mul.wide.s32 %rd3, %r25, %r26;\n" " mov.u32 %r62, %tid.x;\n" " setp.ge.s32 %p1, %r62, %r25;\n" " mov.f32 %f136, 0f00000000;\n" " @%p1 bra $L__BB2_7;\n" "\n" " not.b32 %r27, %r62;\n" " add.s32 %r28, %r27, %r25;\n" " div.u32 %r3, %r28, %r1;\n" " add.s32 %r29, %r3, 1;\n" " and.b32 %r56, %r29, 3;\n" " setp.eq.s32 %p2, %r56, 0;\n" " mov.f32 %f136, 0f00000000;\n" " mov.u32 %r57, %r62;\n" " @%p2 bra $L__BB2_4;\n" "\n" " cvt.s64.s32 %rd29, %r62;\n" " add.s64 %rd30, %rd3, %rd29;\n" " shl.b64 %rd31, %rd30, 2;\n" " add.s64 %rd65, %rd2, %rd31;\n" " mul.wide.s32 %rd5, %r1, 4;\n" " mov.f32 %f136, 0f00000000;\n" " mov.u32 %r57, %r62;\n" "\n" "$L__BB2_3:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f14, [%rd65];\n" " fma.rn.f32 %f136, %f14, %f14, %f136;\n" " add.s32 %r57, %r57, %r1;\n" " add.s64 %rd65, %rd65, %rd5;\n" " add.s32 %r56, %r56, -1;\n" " setp.ne.s32 %p3, %r56, 0;\n" " @%p3 bra $L__BB2_3;\n" "\n" "$L__BB2_4:\n" " setp.lt.u32 %p4, %r3, 3;\n" " @%p4 bra $L__BB2_7;\n" "\n" " mul.wide.s32 %rd8, %r1, 4;\n" "\n" "$L__BB2_6:\n" " cvt.s64.s32 %rd32, %r57;\n" " add.s64 %rd33, %rd3, %rd32;\n" " shl.b64 %rd34, %rd33, 2;\n" " add.s64 %rd35, %rd2, %rd34;\n" " ld.global.f32 %f15, [%rd35];\n" " fma.rn.f32 %f16, %f15, %f15, %f136;\n" " add.s64 %rd36, %rd35, %rd8;\n" " ld.global.f32 %f17, [%rd36];\n" " fma.rn.f32 %f18, %f17, %f17, %f16;\n" " add.s32 %r30, %r57, %r1;\n" " add.s32 %r31, %r30, %r1;\n" " add.s64 %rd37, %rd36, %rd8;\n" " ld.global.f32 %f19, [%rd37];\n" " fma.rn.f32 %f20, %f19, %f19, %f18;\n" " add.s32 %r32, %r31, %r1;\n" " add.s64 %rd38, %rd37, %rd8;\n" " ld.global.f32 %f21, [%rd38];\n" " fma.rn.f32 %f136, %f21, %f21, %f20;\n" " add.s32 %r57, %r32, %r1;\n" " setp.lt.s32 %p5, %r57, %r25;\n" " @%p5 bra $L__BB2_6;\n" "\n" "$L__BB2_7:\n" " shl.b32 %r33, %r62, 2;\n" " mov.u32 %r34, sdata;\n" " add.s32 %r12, %r34, %r33;\n" " st.shared.f32 [%r12], %f136;\n" " bar.sync 0;\n" " setp.lt.s32 %p6, %r1, 2;\n" " @%p6 bra $L__BB2_12;\n" "\n" " shr.u32 %r35, %r1, 31;\n" " add.s32 %r36, %r1, %r35;\n" " shr.s32 %r59, %r36, 1;\n" "\n" "$L__BB2_9:\n" " setp.ge.s32 %p7, %r62, %r59;\n" " @%p7 bra $L__BB2_11;\n" "\n" " shl.b32 %r37, %r59, 2;\n" " add.s32 %r38, %r12, %r37;\n" " ld.shared.f32 %f22, [%r12];\n" " ld.shared.f32 %f23, [%r38];\n" " add.f32 %f24, %f23, %f22;\n" " st.shared.f32 [%r12], %f24;\n" "\n" "$L__BB2_11:\n" " bar.sync 0;\n" " shr.s32 %r15, %r59, 1;\n" " setp.gt.s32 %p8, %r59, 1;\n" " mov.u32 %r59, %r15;\n" " @%p8 bra $L__BB2_9;\n" "\n" "$L__BB2_12:\n" " cvt.rn.f32.s32 %f25, %r25;\n" " ld.shared.f32 %f26, [sdata];\n" " div.rn.f32 %f27, %f26, %f25;\n" " add.f32 %f28, %f27, %f9;\n" " sqrt.rn.f32 %f29, %f28;\n" " rcp.rn.f32 %f8, %f29;\n" " @%p1 bra $L__BB2_19;\n" "\n" " not.b32 %r39, %r62;\n" " add.s32 %r40, %r39, %r25;\n" " div.u32 %r16, %r40, %r1;\n" " add.s32 %r41, %r16, 1;\n" " and.b32 %r61, %r41, 3;\n" " setp.eq.s32 %p10, %r61, 0;\n" " @%p10 bra $L__BB2_16;\n" "\n" " cvt.s64.s32 %rd39, %r62;\n" " cvta.to.global.u64 %rd40, %rd25;\n" " mul.wide.s32 %rd41, %r62, 4;\n" " add.s64 %rd69, %rd40, %rd41;\n" " mul.wide.s32 %rd10, %r1, 4;\n" " add.s64 %rd42, %rd3, %rd39;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd68, %rd1, %rd43;\n" " cvta.to.global.u64 %rd44, %rd26;\n" " add.s64 %rd67, %rd44, %rd43;\n" " add.s64 %rd66, %rd2, %rd43;\n" " mov.f32 %f36, 0f3F000000;\n" " mov.f32 %f37, 0f3BBB989D;\n" " mov.f32 %f39, 0f3FB8AA3B;\n" " mov.f32 %f40, 0f437C0000;\n" "\n" "$L__BB2_15:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f30, [%rd66];\n" " mul.f32 %f31, %f8, %f30;\n" " ld.global.f32 %f32, [%rd69];\n" " mul.f32 %f33, %f31, %f32;\n" " ld.global.f32 %f34, [%rd67];\n" " neg.f32 %f35, %f34;\n" " fma.rn.f32 %f38, %f35, %f37, %f36;\n" " cvt.sat.f32.f32 %f41, %f38;\n" " mov.f32 %f42, 0f4B400001;\n" " fma.rm.f32 %f43, %f41, %f40, %f42;\n" " add.f32 %f44, %f43, 0fCB40007F;\n" " neg.f32 %f45, %f44;\n" " fma.rn.f32 %f46, %f35, %f39, %f45;\n" " mov.f32 %f47, 0f32A57060;\n" " fma.rn.f32 %f48, %f35, %f47, %f46;\n" " mov.b32 %r42, %f43;\n" " shl.b32 %r43, %r42, 23;\n" " mov.b32 %f49, %r43;\n" " ex2.approx.ftz.f32 %f50, %f48;\n" " fma.rn.f32 %f51, %f50, %f49, 0f3F800000;\n" " div.rn.f32 %f52, %f34, %f51;\n" " mul.f32 %f53, %f33, %f52;\n" " st.global.f32 [%rd68], %f53;\n" " add.s32 %r62, %r62, %r1;\n" " add.s64 %rd69, %rd69, %rd10;\n" " add.s64 %rd68, %rd68, %rd10;\n" " add.s64 %rd67, %rd67, %rd10;\n" " add.s64 %rd66, %rd66, %rd10;\n" " add.s32 %r61, %r61, -1;\n" " setp.ne.s32 %p11, %r61, 0;\n" " @%p11 bra $L__BB2_15;\n" "\n" "$L__BB2_16:\n" " setp.lt.u32 %p12, %r16, 3;\n" " @%p12 bra $L__BB2_19;\n" "\n" " mul.wide.s32 %rd22, %r1, 4;\n" " cvta.to.global.u64 %rd23, %rd25;\n" " cvta.to.global.u64 %rd24, %rd26;\n" " mov.f32 %f60, 0f3F000000;\n" " mov.f32 %f61, 0f3BBB989D;\n" " mov.f32 %f63, 0f3FB8AA3B;\n" " mov.f32 %f64, 0f437C0000;\n" "\n" "$L__BB2_18:\n" " cvt.s64.s32 %rd45, %r62;\n" " add.s64 %rd46, %rd3, %rd45;\n" " shl.b64 %rd47, %rd46, 2;\n" " add.s64 %rd48, %rd2, %rd47;\n" " ld.global.f32 %f54, [%rd48];\n" " mul.f32 %f55, %f8, %f54;\n" " mul.wide.s32 %rd49, %r62, 4;\n" " add.s64 %rd50, %rd23, %rd49;\n" " ld.global.f32 %f56, [%rd50];\n" " mul.f32 %f57, %f55, %f56;\n" " add.s64 %rd51, %rd24, %rd47;\n" " ld.global.f32 %f58, [%rd51];\n" " neg.f32 %f59, %f58;\n" " fma.rn.f32 %f62, %f59, %f61, %f60;\n" " cvt.sat.f32.f32 %f65, %f62;\n" " mov.f32 %f66, 0f4B400001;\n" " fma.rm.f32 %f67, %f65, %f64, %f66;\n" " add.f32 %f68, %f67, 0fCB40007F;\n" " neg.f32 %f69, %f68;\n" " fma.rn.f32 %f70, %f59, %f63, %f69;\n" " mov.f32 %f71, 0f32A57060;\n" " fma.rn.f32 %f72, %f59, %f71, %f70;\n" " mov.b32 %r44, %f67;\n" " shl.b32 %r45, %r44, 23;\n" " mov.b32 %f73, %r45;\n" " ex2.approx.ftz.f32 %f74, %f72;\n" " fma.rn.f32 %f75, %f74, %f73, 0f3F800000;\n" " div.rn.f32 %f76, %f58, %f75;\n" " mul.f32 %f77, %f57, %f76;\n" " add.s64 %rd52, %rd1, %rd47;\n" " st.global.f32 [%rd52], %f77;\n" " add.s64 %rd53, %rd48, %rd22;\n" " ld.global.f32 %f78, [%rd53];\n" " mul.f32 %f79, %f8, %f78;\n" " add.s64 %rd54, %rd50, %rd22;\n" " ld.global.f32 %f80, [%rd54];\n" " mul.f32 %f81, %f79, %f80;\n" " add.s64 %rd55, %rd51, %rd22;\n" " ld.global.f32 %f82, [%rd55];\n" " neg.f32 %f83, %f82;\n" " fma.rn.f32 %f84, %f83, %f61, %f60;\n" " cvt.sat.f32.f32 %f85, %f84;\n" " fma.rm.f32 %f86, %f85, %f64, %f66;\n" " add.f32 %f87, %f86, 0fCB40007F;\n" " neg.f32 %f88, %f87;\n" " fma.rn.f32 %f89, %f83, %f63, %f88;\n" " fma.rn.f32 %f90, %f83, %f71, %f89;\n" " mov.b32 %r46, %f86;\n" " shl.b32 %r47, %r46, 23;\n" " mov.b32 %f91, %r47;\n" " ex2.approx.ftz.f32 %f92, %f90;\n" " fma.rn.f32 %f93, %f92, %f91, 0f3F800000;\n" " div.rn.f32 %f94, %f82, %f93;\n" " mul.f32 %f95, %f81, %f94;\n" " add.s64 %rd56, %rd52, %rd22;\n" " st.global.f32 [%rd56], %f95;\n" " add.s32 %r48, %r62, %r1;\n" " add.s32 %r49, %r48, %r1;\n" " add.s64 %rd57, %rd53, %rd22;\n" " ld.global.f32 %f96, [%rd57];\n" " mul.f32 %f97, %f8, %f96;\n" " add.s64 %rd58, %rd54, %rd22;\n" " ld.global.f32 %f98, [%rd58];\n" " mul.f32 %f99, %f97, %f98;\n" " add.s64 %rd59, %rd55, %rd22;\n" " ld.global.f32 %f100, [%rd59];\n" " neg.f32 %f101, %f100;\n" " fma.rn.f32 %f102, %f101, %f61, %f60;\n" " cvt.sat.f32.f32 %f103, %f102;\n" " fma.rm.f32 %f104, %f103, %f64, %f66;\n" " add.f32 %f105, %f104, 0fCB40007F;\n" " neg.f32 %f106, %f105;\n" " fma.rn.f32 %f107, %f101, %f63, %f106;\n" " fma.rn.f32 %f108, %f101, %f71, %f107;\n" " mov.b32 %r50, %f104;\n" " shl.b32 %r51, %r50, 23;\n" " mov.b32 %f109, %r51;\n" " ex2.approx.ftz.f32 %f110, %f108;\n" " fma.rn.f32 %f111, %f110, %f109, 0f3F800000;\n" " div.rn.f32 %f112, %f100, %f111;\n" " mul.f32 %f113, %f99, %f112;\n" " add.s64 %rd60, %rd56, %rd22;\n" " st.global.f32 [%rd60], %f113;\n" " add.s32 %r52, %r49, %r1;\n" " add.s64 %rd61, %rd57, %rd22;\n" " ld.global.f32 %f114, [%rd61];\n" " mul.f32 %f115, %f8, %f114;\n" " add.s64 %rd62, %rd58, %rd22;\n" " ld.global.f32 %f116, [%rd62];\n" " mul.f32 %f117, %f115, %f116;\n" " add.s64 %rd63, %rd59, %rd22;\n" " ld.global.f32 %f118, [%rd63];\n" " neg.f32 %f119, %f118;\n" " fma.rn.f32 %f120, %f119, %f61, %f60;\n" " cvt.sat.f32.f32 %f121, %f120;\n" " fma.rm.f32 %f122, %f121, %f64, %f66;\n" " add.f32 %f123, %f122, 0fCB40007F;\n" " neg.f32 %f124, %f123;\n" " fma.rn.f32 %f125, %f119, %f63, %f124;\n" " fma.rn.f32 %f126, %f119, %f71, %f125;\n" " mov.b32 %r53, %f122;\n" " shl.b32 %r54, %r53, 23;\n" " mov.b32 %f127, %r54;\n" " ex2.approx.ftz.f32 %f128, %f126;\n" " fma.rn.f32 %f129, %f128, %f127, 0f3F800000;\n" " div.rn.f32 %f130, %f118, %f129;\n" " mul.f32 %f131, %f117, %f130;\n" " add.s64 %rd64, %rd60, %rd22;\n" " st.global.f32 [%rd64], %f131;\n" " add.s32 %r62, %r52, %r1;\n" " setp.lt.s32 %p13, %r62, %r25;\n" " @%p13 bra $L__BB2_18;\n" "\n" "$L__BB2_19:\n" " ret;\n" "\n" "}\n" " // .globl fc_sigmoid\n" ".visible .entry fc_sigmoid(\n" " .param .u64 fc_sigmoid_param_0,\n" " .param .u32 fc_sigmoid_param_1\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<20>;\n" " .reg .b32 %r<8>;\n" " .reg .b64 %rd<5>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_sigmoid_param_0];\n" " ld.param.u32 %r2, [fc_sigmoid_param_1];\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r1, %r3, %r4, %r5;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB3_2;\n" "\n" " cvta.to.global.u64 %rd2, %rd1;\n" " mul.wide.s32 %rd3, %r1, 4;\n" " add.s64 %rd4, %rd2, %rd3;\n" " ld.global.f32 %f1, [%rd4];\n" " neg.f32 %f2, %f1;\n" " mov.f32 %f3, 0f3F000000;\n" " mov.f32 %f4, 0f3BBB989D;\n" " fma.rn.f32 %f5, %f2, %f4, %f3;\n" " mov.f32 %f6, 0f3FB8AA3B;\n" " mov.f32 %f7, 0f437C0000;\n" " cvt.sat.f32.f32 %f8, %f5;\n" " mov.f32 %f9, 0f4B400001;\n" " fma.rm.f32 %f10, %f8, %f7, %f9;\n" " add.f32 %f11, %f10, 0fCB40007F;\n" " neg.f32 %f12, %f11;\n" " fma.rn.f32 %f13, %f2, %f6, %f12;\n" " mov.f32 %f14, 0f32A57060;\n" " fma.rn.f32 %f15, %f2, %f14, %f13;\n" " mov.b32 %r6, %f10;\n" " shl.b32 %r7, %r6, 23;\n" " mov.b32 %f16, %r7;\n" " ex2.approx.ftz.f32 %f17, %f15;\n" " fma.rn.f32 %f18, %f17, %f16, 0f3F800000;\n" " rcp.rn.f32 %f19, %f18;\n" " st.global.f32 [%rd4], %f19;\n" "\n" "$L__BB3_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_decay_gate\n" ".visible .entry fc_ssm_decay_gate(\n" " .param .u64 fc_ssm_decay_gate_param_0,\n" " .param .u64 fc_ssm_decay_gate_param_1,\n" " .param .u64 fc_ssm_decay_gate_param_2,\n" " .param .u64 fc_ssm_decay_gate_param_3,\n" " .param .u32 fc_ssm_decay_gate_param_4\n" ")\n" "{\n" " .reg .pred %p<6>;\n" " .reg .f32 %f<60>;\n" " .reg .b32 %r<19>;\n" " .reg .b64 %rd<15>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_ssm_decay_gate_param_0];\n" " ld.param.u64 %rd2, [fc_ssm_decay_gate_param_1];\n" " ld.param.u64 %rd3, [fc_ssm_decay_gate_param_2];\n" " ld.param.u64 %rd4, [fc_ssm_decay_gate_param_3];\n" " ld.param.u32 %r3, [fc_ssm_decay_gate_param_4];\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %ctaid.x;\n" " mov.u32 %r6, %tid.x;\n" " mad.lo.s32 %r1, %r5, %r4, %r6;\n" " setp.ge.s32 %p1, %r1, %r3;\n" " @%p1 bra $L__BB4_7;\n" "\n" " cvta.to.global.u64 %rd5, %rd1;\n" " mul.wide.s32 %rd6, %r1, 4;\n" " add.s64 %rd7, %rd5, %rd6;\n" " cvta.to.global.u64 %rd8, %rd2;\n" " add.s64 %rd9, %rd8, %rd6;\n" " ld.global.f32 %f8, [%rd9];\n" " ld.global.f32 %f9, [%rd7];\n" " add.f32 %f59, %f9, %f8;\n" " setp.gt.f32 %p2, %f59, 0f41A00000;\n" " @%p2 bra $L__BB4_6;\n" "\n" " mov.f32 %f10, 0f3F000000;\n" " mov.f32 %f11, 0f3BBB989D;\n" " fma.rn.f32 %f12, %f59, %f11, %f10;\n" " mov.f32 %f13, 0f3FB8AA3B;\n" " mov.f32 %f14, 0f437C0000;\n" " cvt.sat.f32.f32 %f15, %f12;\n" " mov.f32 %f16, 0f4B400001;\n" " fma.rm.f32 %f17, %f15, %f14, %f16;\n" " add.f32 %f18, %f17, 0fCB40007F;\n" " neg.f32 %f19, %f18;\n" " fma.rn.f32 %f20, %f59, %f13, %f19;\n" " mov.f32 %f21, 0f32A57060;\n" " fma.rn.f32 %f22, %f59, %f21, %f20;\n" " mov.b32 %r7, %f17;\n" " shl.b32 %r8, %r7, 23;\n" " mov.b32 %f23, %r8;\n" " ex2.approx.ftz.f32 %f24, %f22;\n" " mul.f32 %f2, %f24, %f23;\n" " mov.f32 %f25, 0f3F800000;\n" " add.rz.f32 %f26, %f2, %f25;\n" " mov.b32 %r9, %f26;\n" " add.s32 %r10, %r9, -1061158912;\n" " and.b32 %r11, %r10, -8388608;\n" " mov.b32 %r2, %f2;\n" " sub.s32 %r12, %r2, %r11;\n" " mov.b32 %f27, %r12;\n" " mov.u32 %r13, 1082130432;\n" " sub.s32 %r14, %r13, %r11;\n" " mov.b32 %f28, %r14;\n" " mov.f32 %f29, 0fBF800000;\n" " mov.f32 %f30, 0f3E800000;\n" " fma.rn.f32 %f31, %f30, %f28, %f29;\n" " add.f32 %f32, %f31, %f27;\n" " cvt.rn.f32.s32 %f33, %r11;\n" " mul.f32 %f34, %f33, 0f34000000;\n" " mov.f32 %f35, 0f3DD80012;\n" " mov.f32 %f36, 0fBD39BF78;\n" " fma.rn.f32 %f37, %f36, %f32, %f35;\n" " mov.f32 %f38, 0fBE0778E0;\n" " fma.rn.f32 %f39, %f37, %f32, %f38;\n" " mov.f32 %f40, 0f3E146475;\n" " fma.rn.f32 %f41, %f39, %f32, %f40;\n" " mov.f32 %f42, 0fBE2A68DD;\n" " fma.rn.f32 %f43, %f41, %f32, %f42;\n" " mov.f32 %f44, 0f3E4CAF9E;\n" " fma.rn.f32 %f45, %f43, %f32, %f44;\n" " mov.f32 %f46, 0fBE800042;\n" " fma.rn.f32 %f47, %f45, %f32, %f46;\n" " mov.f32 %f48, 0f3EAAAAE6;\n" " fma.rn.f32 %f49, %f47, %f32, %f48;\n" " mov.f32 %f50, 0fBF000000;\n" " fma.rn.f32 %f51, %f49, %f32, %f50;\n" " mul.f32 %f52, %f32, %f51;\n" " fma.rn.f32 %f53, %f52, %f32, %f32;\n" " mov.f32 %f54, 0f3F317218;\n" " fma.rn.f32 %f59, %f34, %f54, %f53;\n" " setp.lt.u32 %p3, %r2, 2139095040;\n" " @%p3 bra $L__BB4_6;\n" "\n" " setp.lt.s32 %p4, %r2, -1082130431;\n" " @%p4 bra $L__BB4_5;\n" "\n" " mov.f32 %f55, 0f7F800000;\n" " fma.rn.f32 %f59, %f2, %f55, %f55;\n" "\n" "$L__BB4_5:\n" " setp.eq.f32 %p5, %f2, 0f00000000;\n" " selp.f32 %f59, 0f80000000, %f59, %p5;\n" "\n" "$L__BB4_6:\n" " cvta.to.global.u64 %rd10, %rd3;\n" " add.s64 %rd12, %rd10, %rd6;\n" " ld.global.f32 %f56, [%rd12];\n" " mul.f32 %f57, %f59, %f56;\n" " cvta.to.global.u64 %rd13, %rd4;\n" " add.s64 %rd14, %rd13, %rd6;\n" " st.global.f32 [%rd14], %f57;\n" "\n" "$L__BB4_7:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_qknorm_gqa\n" ".visible .entry fc_ssm_qknorm_gqa(\n" " .param .u64 fc_ssm_qknorm_gqa_param_0,\n" " .param .u64 fc_ssm_qknorm_gqa_param_1,\n" " .param .u64 fc_ssm_qknorm_gqa_param_2,\n" " .param .u32 fc_ssm_qknorm_gqa_param_3,\n" " .param .u32 fc_ssm_qknorm_gqa_param_4,\n" " .param .u32 fc_ssm_qknorm_gqa_param_5,\n" " .param .u32 fc_ssm_qknorm_gqa_param_6,\n" " .param .f32 fc_ssm_qknorm_gqa_param_7\n" ")\n" "{\n" " .reg .pred %p<10>;\n" " .reg .f32 %f<22>;\n" " .reg .b32 %r<42>;\n" " .reg .b64 %rd<44>;\n" "\n" "\n" " ld.param.u64 %rd14, [fc_ssm_qknorm_gqa_param_0];\n" " ld.param.u64 %rd15, [fc_ssm_qknorm_gqa_param_1];\n" " ld.param.u64 %rd16, [fc_ssm_qknorm_gqa_param_2];\n" " ld.param.u32 %r17, [fc_ssm_qknorm_gqa_param_3];\n" " ld.param.u32 %r18, [fc_ssm_qknorm_gqa_param_4];\n" " ld.param.u32 %r19, [fc_ssm_qknorm_gqa_param_5];\n" " ld.param.u32 %r20, [fc_ssm_qknorm_gqa_param_6];\n" " ld.param.f32 %f5, [fc_ssm_qknorm_gqa_param_7];\n" " cvta.to.global.u64 %rd1, %rd16;\n" " cvta.to.global.u64 %rd2, %rd15;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r17;\n" " @%p1 bra $L__BB5_12;\n" "\n" " mov.u32 %r21, %tid.x;\n" " cvt.s64.s32 %rd3, %r19;\n" " cvt.s64.s32 %rd42, %r1;\n" " mul.wide.s32 %rd17, %r19, %r1;\n" " cvt.s64.s32 %rd5, %r21;\n" " add.s64 %rd18, %rd17, %rd5;\n" " cvta.to.global.u64 %rd19, %rd14;\n" " shl.b64 %rd20, %rd18, 2;\n" " add.s64 %rd21, %rd19, %rd20;\n" " mul.wide.s32 %rd22, %r20, 4;\n" " add.s64 %rd23, %rd21, %rd22;\n" " ld.global.f32 %f1, [%rd23];\n" " ld.global.f32 %f2, [%rd21];\n" " mul.f32 %f6, %f2, %f2;\n" " shl.b32 %r22, %r21, 2;\n" " mov.u32 %r23, sh;\n" " add.s32 %r2, %r23, %r22;\n" " st.shared.f32 [%r2], %f6;\n" " mul.f32 %f7, %f1, %f1;\n" " mov.u32 %r3, %ntid.x;\n" " shl.b32 %r24, %r3, 2;\n" " add.s32 %r4, %r2, %r24;\n" " st.shared.f32 [%r4], %f7;\n" " bar.sync 0;\n" " shr.u32 %r37, %r3, 1;\n" " setp.eq.s32 %p2, %r37, 0;\n" " @%p2 bra $L__BB5_5;\n" "\n" "$L__BB5_2:\n" " cvt.u32.u64 %r25, %rd5;\n" " setp.ge.s32 %p3, %r25, %r37;\n" " @%p3 bra $L__BB5_4;\n" "\n" " shl.b32 %r26, %r37, 2;\n" " add.s32 %r27, %r2, %r26;\n" " ld.shared.f32 %f8, [%r2];\n" " ld.shared.f32 %f9, [%r27];\n" " add.f32 %f10, %f9, %f8;\n" " st.shared.f32 [%r2], %f10;\n" " ld.shared.f32 %f11, [%r4];\n" " add.s32 %r28, %r4, %r26;\n" " ld.shared.f32 %f12, [%r28];\n" " add.f32 %f13, %f12, %f11;\n" " st.shared.f32 [%r4], %f13;\n" "\n" "$L__BB5_4:\n" " bar.sync 0;\n" " shr.s32 %r7, %r37, 1;\n" " setp.gt.s32 %p4, %r37, 1;\n" " mov.u32 %r37, %r7;\n" " @%p4 bra $L__BB5_2;\n" "\n" "$L__BB5_5:\n" " ld.shared.f32 %f14, [sh];\n" " sqrt.rn.f32 %f15, %f14;\n" " max.f32 %f16, %f15, %f5;\n" " rcp.rn.f32 %f17, %f16;\n" " add.s32 %r31, %r23, %r24;\n" " ld.shared.f32 %f18, [%r31];\n" " sqrt.rn.f32 %f19, %f18;\n" " max.f32 %f20, %f19, %f5;\n" " rcp.rn.f32 %f21, %f20;\n" " mul.f32 %f3, %f2, %f17;\n" " mul.f32 %f4, %f1, %f21;\n" " setp.lt.s32 %p5, %r18, 1;\n" " @%p5 bra $L__BB5_12;\n" "\n" " add.s32 %r33, %r18, -1;\n" " and.b32 %r41, %r18, 3;\n" " setp.lt.u32 %p6, %r33, 3;\n" " mov.u32 %r40, 0;\n" " @%p6 bra $L__BB5_9;\n" "\n" " sub.s32 %r39, %r18, %r41;\n" " shl.b32 %r35, %r17, 2;\n" " cvt.s64.s32 %rd6, %r35;\n" " cvt.s64.s32 %rd24, %r17;\n" " mul.lo.s64 %rd25, %rd3, %rd24;\n" " shl.b64 %rd7, %rd25, 2;\n" " mov.u32 %r40, 0;\n" "\n" "$L__BB5_8:\n" " mul.lo.s64 %rd26, %rd42, %rd3;\n" " add.s64 %rd27, %rd26, %rd5;\n" " shl.b64 %rd28, %rd27, 2;\n" " add.s64 %rd29, %rd2, %rd28;\n" " st.global.f32 [%rd29], %f3;\n" " add.s64 %rd30, %rd1, %rd28;\n" " st.global.f32 [%rd30], %f4;\n" " add.s64 %rd31, %rd29, %rd7;\n" " st.global.f32 [%rd31], %f3;\n" " add.s64 %rd32, %rd30, %rd7;\n" " st.global.f32 [%rd32], %f4;\n" " add.s64 %rd33, %rd31, %rd7;\n" " st.global.f32 [%rd33], %f3;\n" " add.s64 %rd34, %rd32, %rd7;\n" " st.global.f32 [%rd34], %f4;\n" " add.s64 %rd35, %rd33, %rd7;\n" " st.global.f32 [%rd35], %f3;\n" " add.s64 %rd36, %rd34, %rd7;\n" " st.global.f32 [%rd36], %f4;\n" " add.s32 %r40, %r40, 4;\n" " add.s64 %rd42, %rd42, %rd6;\n" " add.s32 %r39, %r39, -4;\n" " setp.ne.s32 %p7, %r39, 0;\n" " @%p7 bra $L__BB5_8;\n" "\n" "$L__BB5_9:\n" " setp.eq.s32 %p8, %r41, 0;\n" " @%p8 bra $L__BB5_12;\n" "\n" " mad.lo.s32 %r36, %r40, %r17, %r1;\n" " cvt.s64.s32 %rd43, %r36;\n" " cvt.s64.s32 %rd11, %r17;\n" "\n" "$L__BB5_11:\n" " .pragma \"nounroll\";\n" " mul.lo.s64 %rd37, %rd43, %rd3;\n" " add.s64 %rd38, %rd37, %rd5;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd40, %rd2, %rd39;\n" " st.global.f32 [%rd40], %f3;\n" " add.s64 %rd41, %rd1, %rd39;\n" " st.global.f32 [%rd41], %f4;\n" " add.s64 %rd43, %rd43, %rd11;\n" " add.s32 %r41, %r41, -1;\n" " setp.ne.s32 %p9, %r41, 0;\n" " @%p9 bra $L__BB5_11;\n" "\n" "$L__BB5_12:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_conv1d_silu_seq\n" ".visible .entry fc_ssm_conv1d_silu_seq(\n" " .param .u64 fc_ssm_conv1d_silu_seq_param_0,\n" " .param .u64 fc_ssm_conv1d_silu_seq_param_1,\n" " .param .u64 fc_ssm_conv1d_silu_seq_param_2,\n" " .param .u64 fc_ssm_conv1d_silu_seq_param_3,\n" " .param .u32 fc_ssm_conv1d_silu_seq_param_4,\n" " .param .u32 fc_ssm_conv1d_silu_seq_param_5,\n" " .param .u32 fc_ssm_conv1d_silu_seq_param_6,\n" " .param .u64 fc_ssm_conv1d_silu_seq_param_7\n" ")\n" "{\n" " .local .align 4 .b8 __local_depot6[12];\n" " .reg .b64 %SP;\n" " .reg .b64 %SPL;\n" " .reg .pred %p<50>;\n" " .reg .f32 %f<361>;\n" " .reg .b32 %r<154>;\n" " .reg .b64 %rd<158>;\n" "\n" "\n" " mov.u64 %SPL, __local_depot6;\n" " ld.param.u64 %rd61, [fc_ssm_conv1d_silu_seq_param_0];\n" " ld.param.u64 %rd59, [fc_ssm_conv1d_silu_seq_param_1];\n" " ld.param.u64 %rd62, [fc_ssm_conv1d_silu_seq_param_2];\n" " ld.param.u64 %rd63, [fc_ssm_conv1d_silu_seq_param_3];\n" " ld.param.u32 %r68, [fc_ssm_conv1d_silu_seq_param_4];\n" " ld.param.u32 %r69, [fc_ssm_conv1d_silu_seq_param_5];\n" " ld.param.u32 %r70, [fc_ssm_conv1d_silu_seq_param_6];\n" " ld.param.u64 %rd60, [fc_ssm_conv1d_silu_seq_param_7];\n" " cvta.to.global.u64 %rd1, %rd60;\n" " cvta.to.global.u64 %rd2, %rd63;\n" " cvta.to.global.u64 %rd3, %rd61;\n" " cvta.to.global.u64 %rd4, %rd62;\n" " add.u64 %rd5, %SPL, 0;\n" " mov.u32 %r71, %ntid.x;\n" " mov.u32 %r72, %ctaid.x;\n" " mov.u32 %r73, %tid.x;\n" " mad.lo.s32 %r1, %r72, %r71, %r73;\n" " setp.ge.s32 %p1, %r1, %r68;\n" " @%p1 bra $L__BB6_68;\n" "\n" " add.s32 %r74, %r69, -1;\n" " cvt.s64.s32 %rd6, %r1;\n" " mul.wide.s32 %rd65, %r69, %r1;\n" " cvta.to.global.u64 %rd66, %rd59;\n" " shl.b64 %rd67, %rd65, 2;\n" " add.s64 %rd7, %rd66, %rd67;\n" " cvt.s64.s32 %rd8, %r74;\n" " mul.wide.s32 %rd9, %r74, %r1;\n" " shl.b64 %rd68, %rd9, 2;\n" " add.s64 %rd10, %rd4, %rd68;\n" " setp.eq.s32 %p2, %r69, 4;\n" " @%p2 bra $L__BB6_53;\n" " bra.uni $L__BB6_2;\n" "\n" "$L__BB6_53:\n" " ld.global.f32 %f12, [%rd7];\n" " ld.global.f32 %f13, [%rd7+4];\n" " ld.global.f32 %f14, [%rd7+8];\n" " ld.global.f32 %f15, [%rd7+12];\n" " ld.global.f32 %f358, [%rd10];\n" " ld.global.f32 %f359, [%rd10+4];\n" " ld.global.f32 %f360, [%rd10+8];\n" " setp.lt.s32 %p40, %r70, 1;\n" " @%p40 bra $L__BB6_67;\n" "\n" " cvt.s64.s32 %rd34, %r68;\n" " add.s32 %r50, %r70, -1;\n" " and.b32 %r149, %r70, 3;\n" " setp.eq.s64 %p41, %rd60, 0;\n" " @%p41 bra $L__BB6_61;\n" "\n" " setp.lt.u32 %p42, %r50, 3;\n" " mov.u32 %r148, 0;\n" " @%p42 bra $L__BB6_58;\n" "\n" " sub.s32 %r147, %r70, %r149;\n" " mul.lo.s64 %rd117, %rd6, 12;\n" " add.s64 %rd151, %rd1, %rd117;\n" " mul.lo.s64 %rd36, %rd34, 12;\n" " shl.b64 %rd118, %rd6, 2;\n" " add.s64 %rd152, %rd3, %rd118;\n" " shl.b64 %rd38, %rd34, 2;\n" " add.s64 %rd153, %rd2, %rd118;\n" " mov.u32 %r148, 0;\n" " mov.f32 %f149, 0f3F000000;\n" " mov.f32 %f150, 0f3BBB989D;\n" " mov.f32 %f152, 0f3FB8AA3B;\n" " mov.f32 %f153, 0f437C0000;\n" "\n" "$L__BB6_57:\n" " ld.global.f32 %f143, [%rd152];\n" " mul.rn.f32 %f144, %f143, %f15;\n" " fma.rn.f32 %f145, %f358, %f12, %f144;\n" " fma.rn.f32 %f146, %f359, %f13, %f145;\n" " fma.rn.f32 %f147, %f360, %f14, %f146;\n" " neg.f32 %f148, %f147;\n" " fma.rn.f32 %f151, %f148, %f150, %f149;\n" " cvt.sat.f32.f32 %f154, %f151;\n" " mov.f32 %f155, 0f4B400001;\n" " fma.rm.f32 %f156, %f154, %f153, %f155;\n" " add.f32 %f157, %f156, 0fCB40007F;\n" " neg.f32 %f158, %f157;\n" " fma.rn.f32 %f159, %f148, %f152, %f158;\n" " mov.f32 %f160, 0f32A57060;\n" " fma.rn.f32 %f161, %f148, %f160, %f159;\n" " mov.b32 %r102, %f156;\n" " shl.b32 %r103, %r102, 23;\n" " mov.b32 %f162, %r103;\n" " ex2.approx.ftz.f32 %f163, %f161;\n" " fma.rn.f32 %f164, %f163, %f162, 0f3F800000;\n" " div.rn.f32 %f165, %f147, %f164;\n" " st.global.f32 [%rd153], %f165;\n" " st.global.f32 [%rd151], %f359;\n" " st.global.f32 [%rd151+4], %f360;\n" " st.global.f32 [%rd151+8], %f143;\n" " add.s64 %rd119, %rd152, %rd38;\n" " ld.global.f32 %f358, [%rd119];\n" " mul.rn.f32 %f166, %f358, %f15;\n" " fma.rn.f32 %f167, %f359, %f12, %f166;\n" " fma.rn.f32 %f168, %f360, %f13, %f167;\n" " fma.rn.f32 %f169, %f143, %f14, %f168;\n" " neg.f32 %f170, %f169;\n" " fma.rn.f32 %f171, %f170, %f150, %f149;\n" " cvt.sat.f32.f32 %f172, %f171;\n" " fma.rm.f32 %f173, %f172, %f153, %f155;\n" " add.f32 %f174, %f173, 0fCB40007F;\n" " neg.f32 %f175, %f174;\n" " fma.rn.f32 %f176, %f170, %f152, %f175;\n" " fma.rn.f32 %f177, %f170, %f160, %f176;\n" " mov.b32 %r104, %f173;\n" " shl.b32 %r105, %r104, 23;\n" " mov.b32 %f178, %r105;\n" " ex2.approx.ftz.f32 %f179, %f177;\n" " fma.rn.f32 %f180, %f179, %f178, 0f3F800000;\n" " div.rn.f32 %f181, %f169, %f180;\n" " add.s64 %rd120, %rd153, %rd38;\n" " st.global.f32 [%rd120], %f181;\n" " add.s64 %rd121, %rd151, %rd36;\n" " st.global.f32 [%rd121], %f360;\n" " st.global.f32 [%rd121+4], %f143;\n" " st.global.f32 [%rd121+8], %f358;\n" " add.s64 %rd122, %rd119, %rd38;\n" " ld.global.f32 %f359, [%rd122];\n" " mul.rn.f32 %f182, %f359, %f15;\n" " fma.rn.f32 %f183, %f360, %f12, %f182;\n" " fma.rn.f32 %f184, %f143, %f13, %f183;\n" " fma.rn.f32 %f185, %f358, %f14, %f184;\n" " neg.f32 %f186, %f185;\n" " fma.rn.f32 %f187, %f186, %f150, %f149;\n" " cvt.sat.f32.f32 %f188, %f187;\n" " fma.rm.f32 %f189, %f188, %f153, %f155;\n" " add.f32 %f190, %f189, 0fCB40007F;\n" " neg.f32 %f191, %f190;\n" " fma.rn.f32 %f192, %f186, %f152, %f191;\n" " fma.rn.f32 %f193, %f186, %f160, %f192;\n" " mov.b32 %r106, %f189;\n" " shl.b32 %r107, %r106, 23;\n" " mov.b32 %f194, %r107;\n" " ex2.approx.ftz.f32 %f195, %f193;\n" " fma.rn.f32 %f196, %f195, %f194, 0f3F800000;\n" " div.rn.f32 %f197, %f185, %f196;\n" " add.s64 %rd123, %rd120, %rd38;\n" " st.global.f32 [%rd123], %f197;\n" " add.s64 %rd124, %rd121, %rd36;\n" " st.global.f32 [%rd124], %f143;\n" " st.global.f32 [%rd124+4], %f358;\n" " st.global.f32 [%rd124+8], %f359;\n" " add.s64 %rd125, %rd122, %rd38;\n" " add.s64 %rd152, %rd125, %rd38;\n" " ld.global.f32 %f360, [%rd125];\n" " mul.rn.f32 %f198, %f360, %f15;\n" " fma.rn.f32 %f199, %f143, %f12, %f198;\n" " fma.rn.f32 %f200, %f358, %f13, %f199;\n" " fma.rn.f32 %f201, %f359, %f14, %f200;\n" " neg.f32 %f202, %f201;\n" " fma.rn.f32 %f203, %f202, %f150, %f149;\n" " cvt.sat.f32.f32 %f204, %f203;\n" " fma.rm.f32 %f205, %f204, %f153, %f155;\n" " add.f32 %f206, %f205, 0fCB40007F;\n" " neg.f32 %f207, %f206;\n" " fma.rn.f32 %f208, %f202, %f152, %f207;\n" " fma.rn.f32 %f209, %f202, %f160, %f208;\n" " mov.b32 %r108, %f205;\n" " shl.b32 %r109, %r108, 23;\n" " mov.b32 %f210, %r109;\n" " ex2.approx.ftz.f32 %f211, %f209;\n" " fma.rn.f32 %f212, %f211, %f210, 0f3F800000;\n" " div.rn.f32 %f213, %f201, %f212;\n" " add.s64 %rd126, %rd123, %rd38;\n" " add.s64 %rd153, %rd126, %rd38;\n" " st.global.f32 [%rd126], %f213;\n" " add.s64 %rd127, %rd124, %rd36;\n" " add.s64 %rd151, %rd127, %rd36;\n" " st.global.f32 [%rd127], %f358;\n" " st.global.f32 [%rd127+4], %f359;\n" " st.global.f32 [%rd127+8], %f360;\n" " add.s32 %r148, %r148, 4;\n" " add.s32 %r147, %r147, -4;\n" " setp.ne.s32 %p43, %r147, 0;\n" " @%p43 bra $L__BB6_57;\n" "\n" "$L__BB6_58:\n" " setp.eq.s32 %p44, %r149, 0;\n" " @%p44 bra $L__BB6_67;\n" "\n" " cvt.s64.s32 %rd154, %r148;\n" " mov.f32 %f345, %f358;\n" "\n" "$L__BB6_60:\n" " .pragma \"nounroll\";\n" " mov.f32 %f358, %f359;\n" " mov.f32 %f359, %f360;\n" " mul.lo.s64 %rd128, %rd154, %rd34;\n" " add.s64 %rd129, %rd128, %rd6;\n" " shl.b64 %rd130, %rd129, 2;\n" " add.s64 %rd131, %rd3, %rd130;\n" " ld.global.f32 %f360, [%rd131];\n" " mul.rn.f32 %f214, %f360, %f15;\n" " fma.rn.f32 %f215, %f345, %f12, %f214;\n" " fma.rn.f32 %f216, %f358, %f13, %f215;\n" " fma.rn.f32 %f217, %f359, %f14, %f216;\n" " neg.f32 %f218, %f217;\n" " mov.f32 %f219, 0f3F000000;\n" " mov.f32 %f220, 0f3BBB989D;\n" " fma.rn.f32 %f221, %f218, %f220, %f219;\n" " mov.f32 %f222, 0f3FB8AA3B;\n" " mov.f32 %f223, 0f437C0000;\n" " cvt.sat.f32.f32 %f224, %f221;\n" " mov.f32 %f225, 0f4B400001;\n" " fma.rm.f32 %f226, %f224, %f223, %f225;\n" " add.f32 %f227, %f226, 0fCB40007F;\n" " neg.f32 %f228, %f227;\n" " fma.rn.f32 %f229, %f218, %f222, %f228;\n" " mov.f32 %f230, 0f32A57060;\n" " fma.rn.f32 %f231, %f218, %f230, %f229;\n" " mov.b32 %r110, %f226;\n" " shl.b32 %r111, %r110, 23;\n" " mov.b32 %f232, %r111;\n" " ex2.approx.ftz.f32 %f233, %f231;\n" " fma.rn.f32 %f234, %f233, %f232, 0f3F800000;\n" " div.rn.f32 %f235, %f217, %f234;\n" " add.s64 %rd132, %rd2, %rd130;\n" " st.global.f32 [%rd132], %f235;\n" " mul.lo.s64 %rd133, %rd129, 12;\n" " add.s64 %rd134, %rd1, %rd133;\n" " st.global.f32 [%rd134], %f358;\n" " st.global.f32 [%rd134+4], %f359;\n" " st.global.f32 [%rd134+8], %f360;\n" " add.s64 %rd154, %rd154, 1;\n" " add.s32 %r149, %r149, -1;\n" " setp.eq.s32 %p45, %r149, 0;\n" " mov.f32 %f345, %f358;\n" " @%p45 bra $L__BB6_67;\n" " bra.uni $L__BB6_60;\n" "\n" "$L__BB6_2:\n" " setp.lt.s32 %p3, %r69, 2;\n" " @%p3 bra $L__BB6_9;\n" "\n" " cvt.u32.u64 %r76, %rd8;\n" " and.b32 %r127, %r76, 3;\n" " add.s32 %r77, %r69, -2;\n" " setp.lt.u32 %p4, %r77, 3;\n" " mov.u32 %r126, 0;\n" " @%p4 bra $L__BB6_6;\n" "\n" " sub.s32 %r125, %r76, %r127;\n" " mov.u32 %r126, 0;\n" "\n" "$L__BB6_5:\n" " mul.wide.s32 %rd69, %r126, 4;\n" " add.s64 %rd70, %rd10, %rd69;\n" " ld.global.f32 %f54, [%rd70];\n" " add.s64 %rd71, %rd5, %rd69;\n" " st.local.f32 [%rd71], %f54;\n" " ld.global.f32 %f55, [%rd70+4];\n" " st.local.f32 [%rd71+4], %f55;\n" " ld.global.f32 %f56, [%rd70+8];\n" " st.local.f32 [%rd71+8], %f56;\n" " ld.global.f32 %f57, [%rd70+12];\n" " st.local.f32 [%rd71+12], %f57;\n" " add.s32 %r126, %r126, 4;\n" " add.s32 %r125, %r125, -4;\n" " setp.ne.s32 %p5, %r125, 0;\n" " @%p5 bra $L__BB6_5;\n" "\n" "$L__BB6_6:\n" " setp.eq.s32 %p6, %r127, 0;\n" " @%p6 bra $L__BB6_9;\n" "\n" " cvt.s64.s32 %rd72, %r126;\n" " add.s64 %rd73, %rd9, %rd72;\n" " shl.b64 %rd74, %rd73, 2;\n" " add.s64 %rd148, %rd4, %rd74;\n" " mul.wide.s32 %rd75, %r126, 4;\n" " add.s64 %rd147, %rd5, %rd75;\n" "\n" "$L__BB6_8:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f58, [%rd148];\n" " st.local.f32 [%rd147], %f58;\n" " add.s64 %rd148, %rd148, 4;\n" " add.s64 %rd147, %rd147, 4;\n" " add.s32 %r127, %r127, -1;\n" " setp.ne.s32 %p7, %r127, 0;\n" " @%p7 bra $L__BB6_8;\n" "\n" "$L__BB6_9:\n" " setp.lt.s32 %p8, %r70, 1;\n" " @%p8 bra $L__BB6_46;\n" "\n" " setp.gt.s32 %p9, %r69, 1;\n" " cvt.s64.s32 %rd17, %r68;\n" " shl.b64 %rd76, %rd8, 2;\n" " add.s64 %rd18, %rd7, %rd76;\n" " add.s32 %r11, %r69, -2;\n" " mul.wide.s32 %rd77, %r11, 4;\n" " add.s64 %rd19, %rd5, %rd77;\n" " add.s32 %r12, %r69, -3;\n" " and.b32 %r13, %r11, 3;\n" " sub.s32 %r14, %r11, %r13;\n" " @%p9 bra $L__BB6_21;\n" " bra.uni $L__BB6_11;\n" "\n" "$L__BB6_21:\n" " cvt.u32.u64 %r86, %rd8;\n" " and.b32 %r22, %r86, 3;\n" " sub.s32 %r23, %r86, %r22;\n" " mov.u32 %r85, 0;\n" " setp.lt.u32 %p17, %r11, 3;\n" " setp.eq.s32 %p19, %r22, 0;\n" " setp.lt.s32 %p22, %r69, 3;\n" " setp.lt.u32 %p23, %r12, 3;\n" " mov.u32 %r132, %r85;\n" "\n" "$L__BB6_22:\n" " cvt.s64.s32 %rd87, %r132;\n" " mul.lo.s64 %rd88, %rd87, %rd17;\n" " add.s64 %rd21, %rd88, %rd6;\n" " shl.b64 %rd89, %rd21, 2;\n" " add.s64 %rd90, %rd3, %rd89;\n" " ld.global.f32 %f87, [%rd18];\n" " ld.global.f32 %f2, [%rd90];\n" " mul.f32 %f333, %f2, %f87;\n" " mov.u32 %r135, %r85;\n" " @%p17 bra $L__BB6_25;\n" "\n" " mov.u32 %r135, 0;\n" " mov.u32 %r134, %r23;\n" "\n" "$L__BB6_24:\n" " mul.wide.s32 %rd91, %r135, 4;\n" " add.s64 %rd92, %rd5, %rd91;\n" " add.s64 %rd93, %rd7, %rd91;\n" " ld.global.f32 %f88, [%rd93];\n" " ld.local.f32 %f89, [%rd92];\n" " fma.rn.f32 %f90, %f89, %f88, %f333;\n" " ld.global.f32 %f91, [%rd93+4];\n" " ld.local.f32 %f92, [%rd92+4];\n" " fma.rn.f32 %f93, %f92, %f91, %f90;\n" " ld.global.f32 %f94, [%rd93+8];\n" " ld.local.f32 %f95, [%rd92+8];\n" " fma.rn.f32 %f96, %f95, %f94, %f93;\n" " ld.global.f32 %f97, [%rd93+12];\n" " ld.local.f32 %f98, [%rd92+12];\n" " fma.rn.f32 %f333, %f98, %f97, %f96;\n" " add.s32 %r135, %r135, 4;\n" " add.s32 %r134, %r134, -4;\n" " setp.ne.s32 %p18, %r134, 0;\n" " @%p18 bra $L__BB6_24;\n" "\n" "$L__BB6_25:\n" " @%p19 bra $L__BB6_29;\n" "\n" " setp.eq.s32 %p20, %r22, 1;\n" " mul.wide.s32 %rd94, %r135, 4;\n" " add.s64 %rd22, %rd5, %rd94;\n" " add.s64 %rd23, %rd7, %rd94;\n" " ld.global.f32 %f99, [%rd23];\n" " ld.local.f32 %f100, [%rd22];\n" " fma.rn.f32 %f333, %f100, %f99, %f333;\n" " @%p20 bra $L__BB6_29;\n" "\n" " setp.eq.s32 %p21, %r22, 2;\n" " ld.global.f32 %f101, [%rd23+4];\n" " ld.local.f32 %f102, [%rd22+4];\n" " fma.rn.f32 %f333, %f102, %f101, %f333;\n" " @%p21 bra $L__BB6_29;\n" "\n" " ld.global.f32 %f103, [%rd23+8];\n" " ld.local.f32 %f104, [%rd22+8];\n" " fma.rn.f32 %f333, %f104, %f103, %f333;\n" "\n" "$L__BB6_29:\n" " neg.f32 %f105, %f333;\n" " mov.f32 %f106, 0f3F000000;\n" " mov.f32 %f107, 0f3BBB989D;\n" " fma.rn.f32 %f108, %f105, %f107, %f106;\n" " mov.f32 %f109, 0f3FB8AA3B;\n" " mov.f32 %f110, 0f437C0000;\n" " cvt.sat.f32.f32 %f111, %f108;\n" " mov.f32 %f112, 0f4B400001;\n" " fma.rm.f32 %f113, %f111, %f110, %f112;\n" " add.f32 %f114, %f113, 0fCB40007F;\n" " neg.f32 %f115, %f114;\n" " fma.rn.f32 %f116, %f105, %f109, %f115;\n" " mov.f32 %f117, 0f32A57060;\n" " fma.rn.f32 %f118, %f105, %f117, %f116;\n" " mov.b32 %r89, %f113;\n" " shl.b32 %r90, %r89, 23;\n" " mov.b32 %f119, %r90;\n" " ex2.approx.ftz.f32 %f120, %f118;\n" " fma.rn.f32 %f121, %f120, %f119, 0f3F800000;\n" " div.rn.f32 %f122, %f333, %f121;\n" " add.s64 %rd96, %rd2, %rd89;\n" " st.global.f32 [%rd96], %f122;\n" " @%p22 bra $L__BB6_37;\n" "\n" " mov.u32 %r138, 0;\n" " @%p23 bra $L__BB6_33;\n" "\n" " mov.u32 %r138, 0;\n" " mov.u32 %r137, %r14;\n" "\n" "$L__BB6_32:\n" " mul.wide.s32 %rd97, %r138, 4;\n" " add.s64 %rd98, %rd5, %rd97;\n" " ld.local.f32 %f123, [%rd98+4];\n" " st.local.f32 [%rd98], %f123;\n" " ld.local.f32 %f124, [%rd98+8];\n" " st.local.f32 [%rd98+4], %f124;\n" " ld.local.f32 %f125, [%rd98+12];\n" " st.local.f32 [%rd98+8], %f125;\n" " add.s32 %r138, %r138, 4;\n" " ld.local.f32 %f126, [%rd98+16];\n" " st.local.f32 [%rd98+12], %f126;\n" " add.s32 %r137, %r137, -4;\n" " setp.ne.s32 %p24, %r137, 0;\n" " @%p24 bra $L__BB6_32;\n" "\n" "$L__BB6_33:\n" " setp.eq.s32 %p25, %r13, 0;\n" " @%p25 bra $L__BB6_37;\n" "\n" " setp.eq.s32 %p26, %r13, 1;\n" " mul.wide.s32 %rd99, %r138, 4;\n" " add.s64 %rd24, %rd5, %rd99;\n" " ld.local.f32 %f127, [%rd24+4];\n" " st.local.f32 [%rd24], %f127;\n" " @%p26 bra $L__BB6_37;\n" "\n" " setp.eq.s32 %p27, %r13, 2;\n" " ld.local.f32 %f128, [%rd24+8];\n" " st.local.f32 [%rd24+4], %f128;\n" " @%p27 bra $L__BB6_37;\n" "\n" " ld.local.f32 %f129, [%rd24+12];\n" " st.local.f32 [%rd24+8], %f129;\n" "\n" "$L__BB6_37:\n" " setp.eq.s64 %p28, %rd60, 0;\n" " st.local.f32 [%rd19], %f2;\n" " @%p28 bra $L__BB6_45;\n" "\n" " mul.lo.s64 %rd25, %rd21, %rd8;\n" " mov.u32 %r141, 0;\n" " @%p17 bra $L__BB6_41;\n" "\n" " mov.u32 %r141, 0;\n" " mov.u32 %r140, %r23;\n" "\n" "$L__BB6_40:\n" " cvt.s64.s32 %rd100, %r141;\n" " mul.wide.s32 %rd101, %r141, 4;\n" " add.s64 %rd102, %rd5, %rd101;\n" " ld.local.f32 %f130, [%rd102];\n" " add.s64 %rd103, %rd25, %rd100;\n" " shl.b64 %rd104, %rd103, 2;\n" " add.s64 %rd105, %rd1, %rd104;\n" " st.global.f32 [%rd105], %f130;\n" " ld.local.f32 %f131, [%rd102+4];\n" " st.global.f32 [%rd105+4], %f131;\n" " ld.local.f32 %f132, [%rd102+8];\n" " st.global.f32 [%rd105+8], %f132;\n" " ld.local.f32 %f133, [%rd102+12];\n" " st.global.f32 [%rd105+12], %f133;\n" " add.s32 %r141, %r141, 4;\n" " add.s32 %r140, %r140, -4;\n" " setp.ne.s32 %p30, %r140, 0;\n" " @%p30 bra $L__BB6_40;\n" "\n" "$L__BB6_41:\n" " @%p19 bra $L__BB6_45;\n" "\n" " setp.eq.s32 %p32, %r22, 1;\n" " cvt.s64.s32 %rd106, %r141;\n" " mul.wide.s32 %rd107, %r141, 4;\n" " add.s64 %rd26, %rd5, %rd107;\n" " ld.local.f32 %f134, [%rd26];\n" " add.s64 %rd108, %rd25, %rd106;\n" " shl.b64 %rd109, %rd108, 2;\n" " add.s64 %rd27, %rd1, %rd109;\n" " st.global.f32 [%rd27], %f134;\n" " @%p32 bra $L__BB6_45;\n" "\n" " setp.eq.s32 %p33, %r22, 2;\n" " ld.local.f32 %f135, [%rd26+4];\n" " st.global.f32 [%rd27+4], %f135;\n" " @%p33 bra $L__BB6_45;\n" "\n" " ld.local.f32 %f136, [%rd26+8];\n" " st.global.f32 [%rd27+8], %f136;\n" "\n" "$L__BB6_45:\n" " add.s32 %r132, %r132, 1;\n" " setp.lt.s32 %p34, %r132, %r70;\n" " @%p34 bra $L__BB6_22;\n" " bra.uni $L__BB6_46;\n" "\n" "$L__BB6_11:\n" " mov.u32 %r128, 0;\n" " mov.f32 %f62, 0f3F000000;\n" " mov.f32 %f63, 0f3BBB989D;\n" " mov.f32 %f65, 0f3FB8AA3B;\n" " mov.f32 %f66, 0f437C0000;\n" " setp.lt.s32 %p10, %r69, 3;\n" " setp.lt.u32 %p11, %r12, 3;\n" " setp.eq.s32 %p13, %r13, 0;\n" " setp.eq.s32 %p14, %r13, 1;\n" "\n" "$L__BB6_12:\n" " cvt.s64.s32 %rd78, %r128;\n" " mul.lo.s64 %rd79, %rd78, %rd17;\n" " add.s64 %rd80, %rd79, %rd6;\n" " shl.b64 %rd81, %rd80, 2;\n" " add.s64 %rd82, %rd3, %rd81;\n" " ld.global.f32 %f59, [%rd18];\n" " ld.global.f32 %f1, [%rd82];\n" " mul.f32 %f60, %f1, %f59;\n" " neg.f32 %f61, %f60;\n" " fma.rn.f32 %f64, %f61, %f63, %f62;\n" " cvt.sat.f32.f32 %f67, %f64;\n" " mov.f32 %f68, 0f4B400001;\n" " fma.rm.f32 %f69, %f67, %f66, %f68;\n" " add.f32 %f70, %f69, 0fCB40007F;\n" " neg.f32 %f71, %f70;\n" " fma.rn.f32 %f72, %f61, %f65, %f71;\n" " mov.f32 %f73, 0f32A57060;\n" " fma.rn.f32 %f74, %f61, %f73, %f72;\n" " mov.b32 %r81, %f69;\n" " shl.b32 %r82, %r81, 23;\n" " mov.b32 %f75, %r82;\n" " ex2.approx.ftz.f32 %f76, %f74;\n" " fma.rn.f32 %f77, %f76, %f75, 0f3F800000;\n" " div.rn.f32 %f78, %f60, %f77;\n" " add.s64 %rd83, %rd2, %rd81;\n" " st.global.f32 [%rd83], %f78;\n" " @%p10 bra $L__BB6_20;\n" "\n" " mov.u32 %r131, 0;\n" " @%p11 bra $L__BB6_16;\n" "\n" " mov.u32 %r131, 0;\n" " mov.u32 %r130, %r14;\n" "\n" "$L__BB6_15:\n" " mul.wide.s32 %rd84, %r131, 4;\n" " add.s64 %rd85, %rd5, %rd84;\n" " ld.local.f32 %f79, [%rd85+4];\n" " st.local.f32 [%rd85], %f79;\n" " ld.local.f32 %f80, [%rd85+8];\n" " st.local.f32 [%rd85+4], %f80;\n" " ld.local.f32 %f81, [%rd85+12];\n" " st.local.f32 [%rd85+8], %f81;\n" " add.s32 %r131, %r131, 4;\n" " ld.local.f32 %f82, [%rd85+16];\n" " st.local.f32 [%rd85+12], %f82;\n" " add.s32 %r130, %r130, -4;\n" " setp.ne.s32 %p12, %r130, 0;\n" " @%p12 bra $L__BB6_15;\n" "\n" "$L__BB6_16:\n" " @%p13 bra $L__BB6_20;\n" "\n" " mul.wide.s32 %rd86, %r131, 4;\n" " add.s64 %rd20, %rd5, %rd86;\n" " ld.local.f32 %f83, [%rd20+4];\n" " st.local.f32 [%rd20], %f83;\n" " @%p14 bra $L__BB6_20;\n" "\n" " setp.eq.s32 %p15, %r13, 2;\n" " ld.local.f32 %f84, [%rd20+8];\n" " st.local.f32 [%rd20+4], %f84;\n" " @%p15 bra $L__BB6_20;\n" "\n" " ld.local.f32 %f85, [%rd20+12];\n" " st.local.f32 [%rd20+8], %f85;\n" "\n" "$L__BB6_20:\n" " st.local.f32 [%rd19], %f1;\n" " add.s32 %r128, %r128, 1;\n" " setp.lt.s32 %p16, %r128, %r70;\n" " @%p16 bra $L__BB6_12;\n" "\n" "$L__BB6_46:\n" " @%p3 bra $L__BB6_68;\n" "\n" " cvt.u32.u64 %r96, %rd8;\n" " and.b32 %r145, %r96, 3;\n" " add.s32 %r97, %r69, -2;\n" " setp.lt.u32 %p36, %r97, 3;\n" " mov.u32 %r144, 0;\n" " @%p36 bra $L__BB6_50;\n" "\n" " sub.s32 %r143, %r96, %r145;\n" " mov.u32 %r144, 0;\n" "\n" "$L__BB6_49:\n" " mul.wide.s32 %rd110, %r144, 4;\n" " add.s64 %rd111, %rd5, %rd110;\n" " ld.local.f32 %f137, [%rd111];\n" " add.s64 %rd112, %rd10, %rd110;\n" " st.global.f32 [%rd112], %f137;\n" " ld.local.f32 %f138, [%rd111+4];\n" " st.global.f32 [%rd112+4], %f138;\n" " ld.local.f32 %f139, [%rd111+8];\n" " st.global.f32 [%rd112+8], %f139;\n" " ld.local.f32 %f140, [%rd111+12];\n" " st.global.f32 [%rd112+12], %f140;\n" " add.s32 %r144, %r144, 4;\n" " add.s32 %r143, %r143, -4;\n" " setp.ne.s32 %p37, %r143, 0;\n" " @%p37 bra $L__BB6_49;\n" "\n" "$L__BB6_50:\n" " setp.eq.s32 %p38, %r145, 0;\n" " @%p38 bra $L__BB6_68;\n" "\n" " cvt.s64.s32 %rd113, %r144;\n" " add.s64 %rd114, %rd9, %rd113;\n" " shl.b64 %rd115, %rd114, 2;\n" " add.s64 %rd150, %rd4, %rd115;\n" " mul.wide.s32 %rd116, %r144, 4;\n" " add.s64 %rd149, %rd5, %rd116;\n" "\n" "$L__BB6_52:\n" " .pragma \"nounroll\";\n" " ld.local.f32 %f141, [%rd149];\n" " st.global.f32 [%rd150], %f141;\n" " add.s64 %rd150, %rd150, 4;\n" " add.s64 %rd149, %rd149, 4;\n" " add.s32 %r145, %r145, -1;\n" " setp.eq.s32 %p39, %r145, 0;\n" " @%p39 bra $L__BB6_68;\n" " bra.uni $L__BB6_52;\n" "\n" "$L__BB6_61:\n" " setp.lt.u32 %p46, %r50, 3;\n" " mov.u32 %r152, 0;\n" " @%p46 bra $L__BB6_64;\n" "\n" " sub.s32 %r151, %r70, %r149;\n" " shl.b64 %rd135, %rd6, 2;\n" " add.s64 %rd155, %rd3, %rd135;\n" " shl.b64 %rd50, %rd34, 2;\n" " add.s64 %rd156, %rd2, %rd135;\n" " mov.u32 %r152, 0;\n" " mov.f32 %f243, 0f3F000000;\n" " mov.f32 %f244, 0f3BBB989D;\n" " mov.f32 %f246, 0f3FB8AA3B;\n" " mov.f32 %f247, 0f437C0000;\n" "\n" "$L__BB6_63:\n" " ld.global.f32 %f237, [%rd155];\n" " mul.rn.f32 %f238, %f237, %f15;\n" " fma.rn.f32 %f239, %f358, %f12, %f238;\n" " fma.rn.f32 %f240, %f359, %f13, %f239;\n" " fma.rn.f32 %f241, %f360, %f14, %f240;\n" " neg.f32 %f242, %f241;\n" " fma.rn.f32 %f245, %f242, %f244, %f243;\n" " cvt.sat.f32.f32 %f248, %f245;\n" " mov.f32 %f249, 0f4B400001;\n" " fma.rm.f32 %f250, %f248, %f247, %f249;\n" " add.f32 %f251, %f250, 0fCB40007F;\n" " neg.f32 %f252, %f251;\n" " fma.rn.f32 %f253, %f242, %f246, %f252;\n" " mov.f32 %f254, 0f32A57060;\n" " fma.rn.f32 %f255, %f242, %f254, %f253;\n" " mov.b32 %r114, %f250;\n" " shl.b32 %r115, %r114, 23;\n" " mov.b32 %f256, %r115;\n" " ex2.approx.ftz.f32 %f257, %f255;\n" " fma.rn.f32 %f258, %f257, %f256, 0f3F800000;\n" " div.rn.f32 %f259, %f241, %f258;\n" " st.global.f32 [%rd156], %f259;\n" " add.s64 %rd136, %rd155, %rd50;\n" " ld.global.f32 %f358, [%rd136];\n" " mul.rn.f32 %f260, %f358, %f15;\n" " fma.rn.f32 %f261, %f359, %f12, %f260;\n" " fma.rn.f32 %f262, %f360, %f13, %f261;\n" " fma.rn.f32 %f263, %f237, %f14, %f262;\n" " neg.f32 %f264, %f263;\n" " fma.rn.f32 %f265, %f264, %f244, %f243;\n" " cvt.sat.f32.f32 %f266, %f265;\n" " fma.rm.f32 %f267, %f266, %f247, %f249;\n" " add.f32 %f268, %f267, 0fCB40007F;\n" " neg.f32 %f269, %f268;\n" " fma.rn.f32 %f270, %f264, %f246, %f269;\n" " fma.rn.f32 %f271, %f264, %f254, %f270;\n" " mov.b32 %r116, %f267;\n" " shl.b32 %r117, %r116, 23;\n" " mov.b32 %f272, %r117;\n" " ex2.approx.ftz.f32 %f273, %f271;\n" " fma.rn.f32 %f274, %f273, %f272, 0f3F800000;\n" " div.rn.f32 %f275, %f263, %f274;\n" " add.s64 %rd137, %rd156, %rd50;\n" " st.global.f32 [%rd137], %f275;\n" " add.s64 %rd138, %rd136, %rd50;\n" " ld.global.f32 %f359, [%rd138];\n" " mul.rn.f32 %f276, %f359, %f15;\n" " fma.rn.f32 %f277, %f360, %f12, %f276;\n" " fma.rn.f32 %f278, %f237, %f13, %f277;\n" " fma.rn.f32 %f279, %f358, %f14, %f278;\n" " neg.f32 %f280, %f279;\n" " fma.rn.f32 %f281, %f280, %f244, %f243;\n" " cvt.sat.f32.f32 %f282, %f281;\n" " fma.rm.f32 %f283, %f282, %f247, %f249;\n" " add.f32 %f284, %f283, 0fCB40007F;\n" " neg.f32 %f285, %f284;\n" " fma.rn.f32 %f286, %f280, %f246, %f285;\n" " fma.rn.f32 %f287, %f280, %f254, %f286;\n" " mov.b32 %r118, %f283;\n" " shl.b32 %r119, %r118, 23;\n" " mov.b32 %f288, %r119;\n" " ex2.approx.ftz.f32 %f289, %f287;\n" " fma.rn.f32 %f290, %f289, %f288, 0f3F800000;\n" " div.rn.f32 %f291, %f279, %f290;\n" " add.s64 %rd139, %rd137, %rd50;\n" " st.global.f32 [%rd139], %f291;\n" " add.s64 %rd140, %rd138, %rd50;\n" " add.s64 %rd155, %rd140, %rd50;\n" " ld.global.f32 %f360, [%rd140];\n" " mul.rn.f32 %f292, %f360, %f15;\n" " fma.rn.f32 %f293, %f237, %f12, %f292;\n" " fma.rn.f32 %f294, %f358, %f13, %f293;\n" " fma.rn.f32 %f295, %f359, %f14, %f294;\n" " neg.f32 %f296, %f295;\n" " fma.rn.f32 %f297, %f296, %f244, %f243;\n" " cvt.sat.f32.f32 %f298, %f297;\n" " fma.rm.f32 %f299, %f298, %f247, %f249;\n" " add.f32 %f300, %f299, 0fCB40007F;\n" " neg.f32 %f301, %f300;\n" " fma.rn.f32 %f302, %f296, %f246, %f301;\n" " fma.rn.f32 %f303, %f296, %f254, %f302;\n" " mov.b32 %r120, %f299;\n" " shl.b32 %r121, %r120, 23;\n" " mov.b32 %f304, %r121;\n" " ex2.approx.ftz.f32 %f305, %f303;\n" " fma.rn.f32 %f306, %f305, %f304, 0f3F800000;\n" " div.rn.f32 %f307, %f295, %f306;\n" " add.s64 %rd141, %rd139, %rd50;\n" " add.s64 %rd156, %rd141, %rd50;\n" " st.global.f32 [%rd141], %f307;\n" " add.s32 %r152, %r152, 4;\n" " add.s32 %r151, %r151, -4;\n" " setp.ne.s32 %p47, %r151, 0;\n" " @%p47 bra $L__BB6_63;\n" "\n" "$L__BB6_64:\n" " setp.eq.s32 %p48, %r149, 0;\n" " @%p48 bra $L__BB6_67;\n" "\n" " cvt.s64.s32 %rd157, %r152;\n" " mov.f32 %f357, %f358;\n" "\n" "$L__BB6_66:\n" " .pragma \"nounroll\";\n" " mov.f32 %f358, %f359;\n" " mov.f32 %f359, %f360;\n" " mul.lo.s64 %rd142, %rd157, %rd34;\n" " add.s64 %rd143, %rd142, %rd6;\n" " shl.b64 %rd144, %rd143, 2;\n" " add.s64 %rd145, %rd3, %rd144;\n" " ld.global.f32 %f360, [%rd145];\n" " mul.rn.f32 %f308, %f360, %f15;\n" " fma.rn.f32 %f309, %f357, %f12, %f308;\n" " fma.rn.f32 %f310, %f358, %f13, %f309;\n" " fma.rn.f32 %f311, %f359, %f14, %f310;\n" " neg.f32 %f312, %f311;\n" " mov.f32 %f313, 0f3F000000;\n" " mov.f32 %f314, 0f3BBB989D;\n" " fma.rn.f32 %f315, %f312, %f314, %f313;\n" " mov.f32 %f316, 0f3FB8AA3B;\n" " mov.f32 %f317, 0f437C0000;\n" " cvt.sat.f32.f32 %f318, %f315;\n" " mov.f32 %f319, 0f4B400001;\n" " fma.rm.f32 %f320, %f318, %f317, %f319;\n" " add.f32 %f321, %f320, 0fCB40007F;\n" " neg.f32 %f322, %f321;\n" " fma.rn.f32 %f323, %f312, %f316, %f322;\n" " mov.f32 %f324, 0f32A57060;\n" " fma.rn.f32 %f325, %f312, %f324, %f323;\n" " mov.b32 %r122, %f320;\n" " shl.b32 %r123, %r122, 23;\n" " mov.b32 %f326, %r123;\n" " ex2.approx.ftz.f32 %f327, %f325;\n" " fma.rn.f32 %f328, %f327, %f326, 0f3F800000;\n" " div.rn.f32 %f329, %f311, %f328;\n" " add.s64 %rd146, %rd2, %rd144;\n" " st.global.f32 [%rd146], %f329;\n" " add.s64 %rd157, %rd157, 1;\n" " add.s32 %r149, %r149, -1;\n" " setp.ne.s32 %p49, %r149, 0;\n" " mov.f32 %f357, %f358;\n" " @%p49 bra $L__BB6_66;\n" "\n" "$L__BB6_67:\n" " st.global.f32 [%rd10], %f358;\n" " st.global.f32 [%rd10+4], %f359;\n" " st.global.f32 [%rd10+8], %f360;\n" "\n" "$L__BB6_68:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_recurrence_seq\n" ".visible .entry fc_ssm_recurrence_seq(\n" " .param .u64 fc_ssm_recurrence_seq_param_0,\n" " .param .u64 fc_ssm_recurrence_seq_param_1,\n" " .param .u64 fc_ssm_recurrence_seq_param_2,\n" " .param .u64 fc_ssm_recurrence_seq_param_3,\n" " .param .u64 fc_ssm_recurrence_seq_param_4,\n" " .param .u64 fc_ssm_recurrence_seq_param_5,\n" " .param .u64 fc_ssm_recurrence_seq_param_6,\n" " .param .u32 fc_ssm_recurrence_seq_param_7,\n" " .param .u32 fc_ssm_recurrence_seq_param_8,\n" " .param .u32 fc_ssm_recurrence_seq_param_9,\n" " .param .f32 fc_ssm_recurrence_seq_param_10,\n" " .param .u64 fc_ssm_recurrence_seq_param_11\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<47>;\n" " .reg .b32 %r<30>;\n" " .reg .b64 %rd<50>;\n" "\n" "\n" " ld.param.u64 %rd21, [fc_ssm_recurrence_seq_param_0];\n" " ld.param.u64 %rd22, [fc_ssm_recurrence_seq_param_1];\n" " ld.param.u64 %rd23, [fc_ssm_recurrence_seq_param_2];\n" " ld.param.u64 %rd24, [fc_ssm_recurrence_seq_param_3];\n" " ld.param.u64 %rd25, [fc_ssm_recurrence_seq_param_4];\n" " ld.param.u64 %rd26, [fc_ssm_recurrence_seq_param_5];\n" " ld.param.u64 %rd27, [fc_ssm_recurrence_seq_param_6];\n" " ld.param.u32 %r10, [fc_ssm_recurrence_seq_param_7];\n" " ld.param.u32 %r11, [fc_ssm_recurrence_seq_param_8];\n" " ld.param.u32 %r12, [fc_ssm_recurrence_seq_param_9];\n" " ld.param.f32 %f7, [fc_ssm_recurrence_seq_param_10];\n" " ld.param.u64 %rd28, [fc_ssm_recurrence_seq_param_11];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r10;\n" " @%p1 bra $L__BB7_19;\n" "\n" " mov.u32 %r13, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r13;\n" " cvt.s64.s32 %rd2, %r10;\n" " mul.wide.s32 %rd3, %r10, %r13;\n" " mov.u32 %r14, %ctaid.x;\n" " cvt.s64.s32 %rd4, %r14;\n" " add.s64 %rd29, %rd3, %rd4;\n" " mul.lo.s64 %rd30, %rd29, %rd2;\n" " cvt.s64.s32 %rd5, %r1;\n" " add.s64 %rd6, %rd30, %rd5;\n" " cvta.to.global.u64 %rd31, %rd21;\n" " shl.b64 %rd32, %rd6, 2;\n" " add.s64 %rd7, %rd31, %rd32;\n" " ld.global.f32 %f46, [%rd7];\n" " setp.lt.s32 %p2, %r12, 1;\n" " @%p2 bra $L__BB7_18;\n" "\n" " shl.b32 %r16, %r1, 2;\n" " mov.u32 %r17, sdata;\n" " add.s32 %r2, %r17, %r16;\n" " shr.u32 %r18, %r10, 31;\n" " add.s32 %r19, %r10, %r18;\n" " shr.s32 %r3, %r19, 1;\n" " cvta.to.global.u64 %rd8, %rd27;\n" " cvta.to.global.u64 %rd9, %rd22;\n" " cvta.to.global.u64 %rd10, %rd28;\n" " cvta.to.global.u64 %rd11, %rd24;\n" " cvta.to.global.u64 %rd12, %rd23;\n" " cvta.to.global.u64 %rd13, %rd26;\n" " cvta.to.global.u64 %rd14, %rd25;\n" " cvt.s64.s32 %rd15, %r11;\n" " mov.u32 %r27, 0;\n" " setp.lt.s32 %p3, %r10, 2;\n" " setp.eq.s64 %p6, %rd28, 0;\n" " setp.ne.s32 %p10, %r1, 0;\n" "\n" "$L__BB7_3:\n" " mov.f32 %f44, 0f3FB8AA3B;\n" " mov.f32 %f43, 0f3F000000;\n" " mov.f32 %f42, 0f3BBB989D;\n" " cvt.s64.s32 %rd33, %r27;\n" " mul.lo.s64 %rd34, %rd33, %rd15;\n" " mul.lo.s64 %rd16, %rd34, %rd2;\n" " add.s64 %rd17, %rd16, %rd3;\n" " add.s64 %rd35, %rd34, %rd1;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd37, %rd14, %rd36;\n" " ld.global.f32 %f8, [%rd37];\n" " fma.rn.f32 %f11, %f8, %f42, %f43;\n" " mov.f32 %f13, 0f437C0000;\n" " cvt.sat.f32.f32 %f14, %f11;\n" " mov.f32 %f15, 0f4B400001;\n" " fma.rm.f32 %f16, %f14, %f13, %f15;\n" " add.f32 %f17, %f16, 0fCB40007F;\n" " neg.f32 %f18, %f17;\n" " fma.rn.f32 %f19, %f8, %f44, %f18;\n" " mov.f32 %f20, 0f32A57060;\n" " fma.rn.f32 %f21, %f8, %f20, %f19;\n" " mov.b32 %r20, %f16;\n" " shl.b32 %r21, %r20, 23;\n" " mov.b32 %f22, %r21;\n" " ex2.approx.ftz.f32 %f23, %f21;\n" " mul.f32 %f24, %f23, %f22;\n" " add.s64 %rd38, %rd13, %rd36;\n" " ld.global.f32 %f3, [%rd38];\n" " mul.f32 %f4, %f46, %f24;\n" " add.s64 %rd18, %rd17, %rd5;\n" " shl.b64 %rd39, %rd18, 2;\n" " add.s64 %rd19, %rd12, %rd39;\n" " ld.global.f32 %f25, [%rd19];\n" " mul.f32 %f26, %f4, %f25;\n" " st.shared.f32 [%r2], %f26;\n" " bar.sync 0;\n" " @%p3 bra $L__BB7_8;\n" "\n" " mov.u32 %r28, %r3;\n" "\n" "$L__BB7_5:\n" " setp.ge.s32 %p4, %r1, %r28;\n" " @%p4 bra $L__BB7_7;\n" "\n" " shl.b32 %r22, %r28, 2;\n" " add.s32 %r23, %r2, %r22;\n" " ld.shared.f32 %f27, [%r2];\n" " ld.shared.f32 %f28, [%r23];\n" " add.f32 %f29, %f28, %f27;\n" " st.shared.f32 [%r2], %f29;\n" "\n" "$L__BB7_7:\n" " bar.sync 0;\n" " shr.s32 %r6, %r28, 1;\n" " setp.gt.s32 %p5, %r28, 1;\n" " mov.u32 %r28, %r6;\n" " @%p5 bra $L__BB7_5;\n" "\n" "$L__BB7_8:\n" " add.s64 %rd20, %rd17, %rd4;\n" " shl.b64 %rd40, %rd20, 2;\n" " add.s64 %rd41, %rd11, %rd40;\n" " ld.global.f32 %f30, [%rd41];\n" " ld.shared.f32 %f31, [sdata];\n" " sub.f32 %f32, %f30, %f31;\n" " mul.f32 %f33, %f3, %f32;\n" " bar.sync 0;\n" " ld.global.f32 %f34, [%rd19];\n" " fma.rn.f32 %f46, %f34, %f33, %f4;\n" " @%p6 bra $L__BB7_10;\n" "\n" " mul.lo.s64 %rd42, %rd16, %rd2;\n" " add.s64 %rd43, %rd6, %rd42;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd45, %rd10, %rd44;\n" " st.global.f32 [%rd45], %f46;\n" "\n" "$L__BB7_10:\n" " add.s64 %rd47, %rd9, %rd39;\n" " ld.global.f32 %f35, [%rd47];\n" " mul.f32 %f36, %f46, %f35;\n" " st.shared.f32 [%r2], %f36;\n" " bar.sync 0;\n" " @%p3 bra $L__BB7_15;\n" "\n" " mov.u32 %r29, %r3;\n" "\n" "$L__BB7_12:\n" " setp.ge.s32 %p8, %r1, %r29;\n" " @%p8 bra $L__BB7_14;\n" "\n" " shl.b32 %r24, %r29, 2;\n" " add.s32 %r25, %r2, %r24;\n" " ld.shared.f32 %f37, [%r2];\n" " ld.shared.f32 %f38, [%r25];\n" " add.f32 %f39, %f38, %f37;\n" " st.shared.f32 [%r2], %f39;\n" "\n" "$L__BB7_14:\n" " bar.sync 0;\n" " shr.s32 %r8, %r29, 1;\n" " setp.gt.s32 %p9, %r29, 1;\n" " mov.u32 %r29, %r8;\n" " @%p9 bra $L__BB7_12;\n" "\n" "$L__BB7_15:\n" " @%p10 bra $L__BB7_17;\n" "\n" " ld.shared.f32 %f40, [sdata];\n" " mul.f32 %f41, %f40, %f7;\n" " add.s64 %rd49, %rd8, %rd40;\n" " st.global.f32 [%rd49], %f41;\n" "\n" "$L__BB7_17:\n" " ld.param.u32 %r26, [fc_ssm_recurrence_seq_param_9];\n" " bar.sync 0;\n" " add.s32 %r27, %r27, 1;\n" " setp.lt.s32 %p11, %r27, %r26;\n" " @%p11 bra $L__BB7_3;\n" "\n" "$L__BB7_18:\n" " st.global.f32 [%rd7], %f46;\n" "\n" "$L__BB7_19:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_recurrence_seq_unroll128\n" ".visible .entry fc_ssm_recurrence_seq_unroll128(\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_0,\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_1,\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_2,\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_3,\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_4,\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_5,\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_6,\n" " .param .u32 fc_ssm_recurrence_seq_unroll128_param_7,\n" " .param .u32 fc_ssm_recurrence_seq_unroll128_param_8,\n" " .param .u32 fc_ssm_recurrence_seq_unroll128_param_9,\n" " .param .f32 fc_ssm_recurrence_seq_unroll128_param_10,\n" " .param .u64 fc_ssm_recurrence_seq_unroll128_param_11\n" ")\n" "{\n" " .reg .pred %p<21>;\n" " .reg .f32 %f<80>;\n" " .reg .b32 %r<16>;\n" " .reg .b64 %rd<61>;\n" "\n" "\n" " ld.param.u64 %rd25, [fc_ssm_recurrence_seq_unroll128_param_0];\n" " ld.param.u64 %rd26, [fc_ssm_recurrence_seq_unroll128_param_1];\n" " ld.param.u64 %rd27, [fc_ssm_recurrence_seq_unroll128_param_2];\n" " ld.param.u64 %rd28, [fc_ssm_recurrence_seq_unroll128_param_3];\n" " ld.param.u64 %rd29, [fc_ssm_recurrence_seq_unroll128_param_4];\n" " ld.param.u64 %rd30, [fc_ssm_recurrence_seq_unroll128_param_5];\n" " ld.param.u64 %rd31, [fc_ssm_recurrence_seq_unroll128_param_6];\n" " ld.param.u32 %r7, [fc_ssm_recurrence_seq_unroll128_param_7];\n" " ld.param.u32 %r5, [fc_ssm_recurrence_seq_unroll128_param_8];\n" " ld.param.u32 %r6, [fc_ssm_recurrence_seq_unroll128_param_9];\n" " ld.param.f32 %f7, [fc_ssm_recurrence_seq_unroll128_param_10];\n" " ld.param.u64 %rd32, [fc_ssm_recurrence_seq_unroll128_param_11];\n" " setp.ne.s32 %p1, %r7, 128;\n" " @%p1 bra $L__BB8_38;\n" "\n" " mov.u32 %r1, %tid.x;\n" " setp.gt.s32 %p2, %r1, 127;\n" " @%p2 bra $L__BB8_38;\n" "\n" " mov.u32 %r8, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r8;\n" " mul.wide.s32 %rd33, %r8, 128;\n" " mov.u32 %r9, %ctaid.x;\n" " cvt.s64.s32 %rd2, %r9;\n" " add.s64 %rd34, %rd33, %rd2;\n" " shl.b64 %rd35, %rd34, 7;\n" " cvt.s64.s32 %rd3, %r1;\n" " add.s64 %rd36, %rd35, %rd3;\n" " cvta.to.global.u64 %rd37, %rd25;\n" " shl.b64 %rd38, %rd36, 2;\n" " add.s64 %rd4, %rd37, %rd38;\n" " ld.global.f32 %f79, [%rd4];\n" " setp.lt.s32 %p3, %r6, 1;\n" " @%p3 bra $L__BB8_37;\n" "\n" " shl.b32 %r11, %r1, 2;\n" " mov.u32 %r12, sdata;\n" " add.s32 %r2, %r12, %r11;\n" " cvta.to.global.u64 %rd39, %rd30;\n" " shl.b64 %rd40, %rd1, 2;\n" " add.s64 %rd60, %rd39, %rd40;\n" " mul.wide.s32 %rd6, %r5, 4;\n" " cvta.to.global.u64 %rd41, %rd29;\n" " add.s64 %rd59, %rd41, %rd40;\n" " cvta.to.global.u64 %rd42, %rd31;\n" " shl.b64 %rd43, %rd2, 2;\n" " add.s64 %rd8, %rd42, %rd43;\n" " shl.b64 %rd58, %rd1, 9;\n" " mul.wide.s32 %rd10, %r5, 512;\n" " cvta.to.global.u64 %rd44, %rd28;\n" " add.s64 %rd11, %rd44, %rd43;\n" " cvta.to.global.u64 %rd45, %rd26;\n" " shl.b64 %rd46, %rd3, 2;\n" " add.s64 %rd12, %rd45, %rd46;\n" " cvta.to.global.u64 %rd47, %rd27;\n" " add.s64 %rd13, %rd47, %rd46;\n" " shl.b64 %rd48, %rd2, 7;\n" " shl.b64 %rd49, %rd1, 14;\n" " add.s64 %rd50, %rd49, %rd48;\n" " add.s64 %rd51, %rd50, %rd3;\n" " cvta.to.global.u64 %rd52, %rd32;\n" " shl.b64 %rd53, %rd51, 2;\n" " add.s64 %rd57, %rd52, %rd53;\n" " mul.wide.s32 %rd15, %r5, 65536;\n" " mov.u32 %r15, 0;\n" " mov.f32 %f9, 0f3F000000;\n" " mov.f32 %f10, 0f3BBB989D;\n" " mov.f32 %f12, 0f3FB8AA3B;\n" " setp.gt.s32 %p4, %r1, 63;\n" " setp.gt.s32 %p5, %r1, 31;\n" " setp.gt.s32 %p6, %r1, 15;\n" " setp.gt.s32 %p7, %r1, 7;\n" "\n" "$L__BB8_4:\n" " ld.global.f32 %f8, [%rd59];\n" " fma.rn.f32 %f11, %f8, %f10, %f9;\n" " mov.f32 %f13, 0f437C0000;\n" " cvt.sat.f32.f32 %f14, %f11;\n" " mov.f32 %f15, 0f4B400001;\n" " fma.rm.f32 %f16, %f14, %f13, %f15;\n" " add.f32 %f17, %f16, 0fCB40007F;\n" " neg.f32 %f18, %f17;\n" " fma.rn.f32 %f19, %f8, %f12, %f18;\n" " mov.f32 %f20, 0f32A57060;\n" " fma.rn.f32 %f21, %f8, %f20, %f19;\n" " mov.b32 %r13, %f16;\n" " shl.b32 %r14, %r13, 23;\n" " mov.b32 %f22, %r14;\n" " ex2.approx.ftz.f32 %f23, %f21;\n" " mul.f32 %f24, %f23, %f22;\n" " ld.global.f32 %f3, [%rd60];\n" " mul.f32 %f4, %f79, %f24;\n" " add.s64 %rd20, %rd13, %rd58;\n" " ld.global.f32 %f25, [%rd20];\n" " mul.f32 %f26, %f4, %f25;\n" " st.shared.f32 [%r2], %f26;\n" " bar.sync 0;\n" " @%p4 bra $L__BB8_6;\n" "\n" " ld.shared.f32 %f27, [%r2];\n" " ld.shared.f32 %f28, [%r2+256];\n" " add.f32 %f29, %f28, %f27;\n" " st.shared.f32 [%r2], %f29;\n" "\n" "$L__BB8_6:\n" " bar.sync 0;\n" " @%p5 bra $L__BB8_8;\n" "\n" " ld.shared.f32 %f30, [%r2];\n" " ld.shared.f32 %f31, [%r2+128];\n" " add.f32 %f32, %f31, %f30;\n" " st.shared.f32 [%r2], %f32;\n" "\n" "$L__BB8_8:\n" " bar.sync 0;\n" " @%p6 bra $L__BB8_10;\n" "\n" " ld.shared.f32 %f33, [%r2];\n" " ld.shared.f32 %f34, [%r2+64];\n" " add.f32 %f35, %f34, %f33;\n" " st.shared.f32 [%r2], %f35;\n" "\n" "$L__BB8_10:\n" " bar.sync 0;\n" " @%p7 bra $L__BB8_12;\n" "\n" " ld.shared.f32 %f36, [%r2];\n" " ld.shared.f32 %f37, [%r2+32];\n" " add.f32 %f38, %f37, %f36;\n" " st.shared.f32 [%r2], %f38;\n" "\n" "$L__BB8_12:\n" " setp.gt.s32 %p8, %r1, 3;\n" " bar.sync 0;\n" " @%p8 bra $L__BB8_14;\n" "\n" " ld.shared.f32 %f39, [%r2];\n" " ld.shared.f32 %f40, [%r2+16];\n" " add.f32 %f41, %f40, %f39;\n" " st.shared.f32 [%r2], %f41;\n" "\n" "$L__BB8_14:\n" " setp.gt.s32 %p9, %r1, 1;\n" " bar.sync 0;\n" " @%p9 bra $L__BB8_16;\n" "\n" " ld.shared.f32 %f42, [%r2];\n" " ld.shared.f32 %f43, [%r2+8];\n" " add.f32 %f44, %f43, %f42;\n" " st.shared.f32 [%r2], %f44;\n" "\n" "$L__BB8_16:\n" " setp.gt.s32 %p10, %r1, 0;\n" " bar.sync 0;\n" " @%p10 bra $L__BB8_18;\n" "\n" " ld.shared.f32 %f45, [%r2];\n" " ld.shared.f32 %f46, [%r2+4];\n" " add.f32 %f47, %f46, %f45;\n" " st.shared.f32 [%r2], %f47;\n" "\n" "$L__BB8_18:\n" " bar.sync 0;\n" " add.s64 %rd54, %rd11, %rd58;\n" " ld.global.f32 %f48, [%rd54];\n" " ld.shared.f32 %f49, [sdata];\n" " sub.f32 %f50, %f48, %f49;\n" " mul.f32 %f51, %f3, %f50;\n" " bar.sync 0;\n" " ld.global.f32 %f52, [%rd20];\n" " fma.rn.f32 %f79, %f52, %f51, %f4;\n" " setp.eq.s64 %p11, %rd32, 0;\n" " @%p11 bra $L__BB8_20;\n" "\n" " st.global.f32 [%rd57], %f79;\n" "\n" "$L__BB8_20:\n" " add.s64 %rd55, %rd12, %rd58;\n" " ld.global.f32 %f53, [%rd55];\n" " mul.f32 %f54, %f79, %f53;\n" " st.shared.f32 [%r2], %f54;\n" " bar.sync 0;\n" " @%p4 bra $L__BB8_22;\n" "\n" " ld.shared.f32 %f55, [%r2];\n" " ld.shared.f32 %f56, [%r2+256];\n" " add.f32 %f57, %f56, %f55;\n" " st.shared.f32 [%r2], %f57;\n" "\n" "$L__BB8_22:\n" " bar.sync 0;\n" " @%p5 bra $L__BB8_24;\n" "\n" " ld.shared.f32 %f58, [%r2];\n" " ld.shared.f32 %f59, [%r2+128];\n" " add.f32 %f60, %f59, %f58;\n" " st.shared.f32 [%r2], %f60;\n" "\n" "$L__BB8_24:\n" " bar.sync 0;\n" " @%p6 bra $L__BB8_26;\n" "\n" " ld.shared.f32 %f61, [%r2];\n" " ld.shared.f32 %f62, [%r2+64];\n" " add.f32 %f63, %f62, %f61;\n" " st.shared.f32 [%r2], %f63;\n" "\n" "$L__BB8_26:\n" " bar.sync 0;\n" " @%p7 bra $L__BB8_28;\n" "\n" " ld.shared.f32 %f64, [%r2];\n" " ld.shared.f32 %f65, [%r2+32];\n" " add.f32 %f66, %f65, %f64;\n" " st.shared.f32 [%r2], %f66;\n" "\n" "$L__BB8_28:\n" " bar.sync 0;\n" " @%p8 bra $L__BB8_30;\n" "\n" " ld.shared.f32 %f67, [%r2];\n" " ld.shared.f32 %f68, [%r2+16];\n" " add.f32 %f69, %f68, %f67;\n" " st.shared.f32 [%r2], %f69;\n" "\n" "$L__BB8_30:\n" " bar.sync 0;\n" " @%p9 bra $L__BB8_32;\n" "\n" " ld.shared.f32 %f70, [%r2];\n" " ld.shared.f32 %f71, [%r2+8];\n" " add.f32 %f72, %f71, %f70;\n" " st.shared.f32 [%r2], %f72;\n" "\n" "$L__BB8_32:\n" " bar.sync 0;\n" " @%p10 bra $L__BB8_34;\n" "\n" " ld.shared.f32 %f73, [%r2];\n" " ld.shared.f32 %f74, [%r2+4];\n" " add.f32 %f75, %f74, %f73;\n" " st.shared.f32 [%r2], %f75;\n" "\n" "$L__BB8_34:\n" " setp.ne.s32 %p19, %r1, 0;\n" " bar.sync 0;\n" " @%p19 bra $L__BB8_36;\n" "\n" " ld.shared.f32 %f76, [sdata];\n" " mul.f32 %f77, %f76, %f7;\n" " add.s64 %rd56, %rd8, %rd58;\n" " st.global.f32 [%rd56], %f77;\n" "\n" "$L__BB8_36:\n" " bar.sync 0;\n" " add.s64 %rd60, %rd60, %rd6;\n" " add.s64 %rd59, %rd59, %rd6;\n" " add.s64 %rd58, %rd58, %rd10;\n" " add.s64 %rd57, %rd57, %rd15;\n" " add.s32 %r15, %r15, 1;\n" " setp.lt.s32 %p20, %r15, %r6;\n" " @%p20 bra $L__BB8_4;\n" "\n" "$L__BB8_37:\n" " st.global.f32 [%rd4], %f79;\n" "\n" "$L__BB8_38:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_recurrence_seq_warp\n" ".visible .entry fc_ssm_recurrence_seq_warp(\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_0,\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_1,\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_2,\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_3,\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_4,\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_5,\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_6,\n" " .param .u32 fc_ssm_recurrence_seq_warp_param_7,\n" " .param .u32 fc_ssm_recurrence_seq_warp_param_8,\n" " .param .u32 fc_ssm_recurrence_seq_warp_param_9,\n" " .param .f32 fc_ssm_recurrence_seq_warp_param_10,\n" " .param .u64 fc_ssm_recurrence_seq_warp_param_11\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<151>;\n" " .reg .b32 %r<90>;\n" " .reg .b64 %rd<85>;\n" "\n" "\n" " ld.param.u64 %rd34, [fc_ssm_recurrence_seq_warp_param_0];\n" " ld.param.u64 %rd35, [fc_ssm_recurrence_seq_warp_param_1];\n" " ld.param.u64 %rd36, [fc_ssm_recurrence_seq_warp_param_2];\n" " ld.param.u64 %rd37, [fc_ssm_recurrence_seq_warp_param_3];\n" " ld.param.u64 %rd40, [fc_ssm_recurrence_seq_warp_param_4];\n" " ld.param.u64 %rd41, [fc_ssm_recurrence_seq_warp_param_5];\n" " ld.param.u64 %rd38, [fc_ssm_recurrence_seq_warp_param_6];\n" " ld.param.u32 %r30, [fc_ssm_recurrence_seq_warp_param_7];\n" " ld.param.u32 %r28, [fc_ssm_recurrence_seq_warp_param_8];\n" " ld.param.u32 %r29, [fc_ssm_recurrence_seq_warp_param_9];\n" " ld.param.u64 %rd39, [fc_ssm_recurrence_seq_warp_param_11];\n" " cvta.to.global.u64 %rd1, %rd41;\n" " cvta.to.global.u64 %rd2, %rd40;\n" " setp.ne.s32 %p1, %r30, 128;\n" " @%p1 bra $L__BB9_17;\n" "\n" " mov.u32 %r85, %tid.x;\n" " and.b32 %r2, %r85, 31;\n" " shr.u32 %r31, %r85, 5;\n" " mov.u32 %r32, %ctaid.x;\n" " shl.b32 %r33, %r32, 2;\n" " add.s32 %r3, %r33, %r31;\n" " mov.u32 %r4, %ctaid.y;\n" " setp.ge.s32 %p2, %r85, %r29;\n" " @%p2 bra $L__BB9_8;\n" "\n" " cvt.s64.s32 %rd3, %r28;\n" " cvt.s64.s32 %rd4, %r4;\n" " not.b32 %r34, %r85;\n" " add.s32 %r5, %r34, %r29;\n" " shr.u32 %r35, %r5, 7;\n" " add.s32 %r36, %r35, 1;\n" " and.b32 %r84, %r36, 3;\n" " setp.eq.s32 %p3, %r84, 0;\n" " @%p3 bra $L__BB9_5;\n" "\n" " add.s32 %r37, %r85, %r29;\n" " shl.b32 %r38, %r37, 2;\n" " mov.u32 %r39, sh;\n" " add.s32 %r83, %r39, %r38;\n" " shl.b32 %r40, %r85, 2;\n" " add.s32 %r82, %r39, %r40;\n" " cvt.s64.s32 %rd79, %r85;\n" " mov.f32 %f24, 0f3F000000;\n" " mov.f32 %f25, 0f3BBB989D;\n" " mov.f32 %f27, 0f3FB8AA3B;\n" " mov.f32 %f28, 0f437C0000;\n" "\n" "$L__BB9_4:\n" " .pragma \"nounroll\";\n" " mul.lo.s64 %rd42, %rd79, %rd3;\n" " add.s64 %rd43, %rd42, %rd4;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd45, %rd2, %rd44;\n" " ld.global.f32 %f23, [%rd45];\n" " fma.rn.f32 %f26, %f23, %f25, %f24;\n" " cvt.sat.f32.f32 %f29, %f26;\n" " mov.f32 %f30, 0f4B400001;\n" " fma.rm.f32 %f31, %f29, %f28, %f30;\n" " add.f32 %f32, %f31, 0fCB40007F;\n" " neg.f32 %f33, %f32;\n" " fma.rn.f32 %f34, %f23, %f27, %f33;\n" " mov.f32 %f35, 0f32A57060;\n" " fma.rn.f32 %f36, %f23, %f35, %f34;\n" " mov.b32 %r41, %f31;\n" " shl.b32 %r42, %r41, 23;\n" " mov.b32 %f37, %r42;\n" " ex2.approx.ftz.f32 %f38, %f36;\n" " mul.f32 %f39, %f38, %f37;\n" " st.shared.f32 [%r82], %f39;\n" " add.s64 %rd46, %rd1, %rd44;\n" " ld.global.f32 %f40, [%rd46];\n" " st.shared.f32 [%r83], %f40;\n" " add.s64 %rd79, %rd79, 128;\n" " cvt.u32.u64 %r85, %rd79;\n" " add.s32 %r83, %r83, 512;\n" " add.s32 %r82, %r82, 512;\n" " add.s32 %r84, %r84, -1;\n" " setp.ne.s32 %p4, %r84, 0;\n" " @%p4 bra $L__BB9_4;\n" "\n" "$L__BB9_5:\n" " setp.lt.u32 %p5, %r5, 384;\n" " @%p5 bra $L__BB9_8;\n" "\n" " cvt.s64.s32 %rd80, %r85;\n" " add.s32 %r43, %r85, %r29;\n" " shl.b32 %r44, %r43, 2;\n" " mov.u32 %r45, sh;\n" " add.s32 %r46, %r45, %r44;\n" " add.s32 %r87, %r46, 1024;\n" " shl.b32 %r47, %r85, 2;\n" " add.s32 %r48, %r45, %r47;\n" " add.s32 %r86, %r48, 1024;\n" " mul.lo.s64 %rd47, %rd80, %rd3;\n" " add.s64 %rd48, %rd47, %rd4;\n" " shl.b64 %rd49, %rd48, 2;\n" " add.s64 %rd81, %rd2, %rd49;\n" " shl.b64 %rd10, %rd3, 9;\n" " add.s64 %rd82, %rd1, %rd49;\n" " mov.f32 %f42, 0f3F000000;\n" " mov.f32 %f43, 0f3BBB989D;\n" " mov.f32 %f45, 0f3FB8AA3B;\n" " mov.f32 %f46, 0f437C0000;\n" "\n" "$L__BB9_7:\n" " ld.global.f32 %f41, [%rd81];\n" " fma.rn.f32 %f44, %f41, %f43, %f42;\n" " cvt.sat.f32.f32 %f47, %f44;\n" " mov.f32 %f48, 0f4B400001;\n" " fma.rm.f32 %f49, %f47, %f46, %f48;\n" " add.f32 %f50, %f49, 0fCB40007F;\n" " neg.f32 %f51, %f50;\n" " fma.rn.f32 %f52, %f41, %f45, %f51;\n" " mov.f32 %f53, 0f32A57060;\n" " fma.rn.f32 %f54, %f41, %f53, %f52;\n" " mov.b32 %r49, %f49;\n" " shl.b32 %r50, %r49, 23;\n" " mov.b32 %f55, %r50;\n" " ex2.approx.ftz.f32 %f56, %f54;\n" " mul.f32 %f57, %f56, %f55;\n" " st.shared.f32 [%r86+-1024], %f57;\n" " ld.global.f32 %f58, [%rd82];\n" " st.shared.f32 [%r87+-1024], %f58;\n" " add.s64 %rd50, %rd81, %rd10;\n" " ld.global.f32 %f59, [%rd50];\n" " fma.rn.f32 %f60, %f59, %f43, %f42;\n" " cvt.sat.f32.f32 %f61, %f60;\n" " fma.rm.f32 %f62, %f61, %f46, %f48;\n" " add.f32 %f63, %f62, 0fCB40007F;\n" " neg.f32 %f64, %f63;\n" " fma.rn.f32 %f65, %f59, %f45, %f64;\n" " fma.rn.f32 %f66, %f59, %f53, %f65;\n" " mov.b32 %r51, %f62;\n" " shl.b32 %r52, %r51, 23;\n" " mov.b32 %f67, %r52;\n" " ex2.approx.ftz.f32 %f68, %f66;\n" " mul.f32 %f69, %f68, %f67;\n" " st.shared.f32 [%r86+-512], %f69;\n" " add.s64 %rd51, %rd82, %rd10;\n" " ld.global.f32 %f70, [%rd51];\n" " st.shared.f32 [%r87+-512], %f70;\n" " add.s64 %rd52, %rd50, %rd10;\n" " ld.global.f32 %f71, [%rd52];\n" " fma.rn.f32 %f72, %f71, %f43, %f42;\n" " cvt.sat.f32.f32 %f73, %f72;\n" " fma.rm.f32 %f74, %f73, %f46, %f48;\n" " add.f32 %f75, %f74, 0fCB40007F;\n" " neg.f32 %f76, %f75;\n" " fma.rn.f32 %f77, %f71, %f45, %f76;\n" " fma.rn.f32 %f78, %f71, %f53, %f77;\n" " mov.b32 %r53, %f74;\n" " shl.b32 %r54, %r53, 23;\n" " mov.b32 %f79, %r54;\n" " ex2.approx.ftz.f32 %f80, %f78;\n" " mul.f32 %f81, %f80, %f79;\n" " st.shared.f32 [%r86], %f81;\n" " add.s64 %rd53, %rd51, %rd10;\n" " ld.global.f32 %f82, [%rd53];\n" " st.shared.f32 [%r87], %f82;\n" " add.s64 %rd54, %rd52, %rd10;\n" " add.s64 %rd81, %rd54, %rd10;\n" " ld.global.f32 %f83, [%rd54];\n" " fma.rn.f32 %f84, %f83, %f43, %f42;\n" " cvt.sat.f32.f32 %f85, %f84;\n" " fma.rm.f32 %f86, %f85, %f46, %f48;\n" " add.f32 %f87, %f86, 0fCB40007F;\n" " neg.f32 %f88, %f87;\n" " fma.rn.f32 %f89, %f83, %f45, %f88;\n" " fma.rn.f32 %f90, %f83, %f53, %f89;\n" " mov.b32 %r55, %f86;\n" " shl.b32 %r56, %r55, 23;\n" " mov.b32 %f91, %r56;\n" " ex2.approx.ftz.f32 %f92, %f90;\n" " mul.f32 %f93, %f92, %f91;\n" " st.shared.f32 [%r86+512], %f93;\n" " add.s64 %rd55, %rd53, %rd10;\n" " add.s64 %rd82, %rd55, %rd10;\n" " ld.global.f32 %f94, [%rd55];\n" " st.shared.f32 [%r87+512], %f94;\n" " add.s64 %rd80, %rd80, 512;\n" " cvt.u32.u64 %r57, %rd80;\n" " add.s32 %r87, %r87, 2048;\n" " add.s32 %r86, %r86, 2048;\n" " setp.lt.s32 %p6, %r57, %r29;\n" " @%p6 bra $L__BB9_7;\n" "\n" "$L__BB9_8:\n" " bar.sync 0;\n" " setp.gt.s32 %p7, %r3, 127;\n" " @%p7 bra $L__BB9_17;\n" "\n" " cvt.s64.s32 %rd18, %r4;\n" " mul.wide.s32 %rd56, %r4, 128;\n" " cvt.s64.s32 %rd19, %r3;\n" " add.s64 %rd57, %rd56, %rd19;\n" " shl.b64 %rd58, %rd57, 7;\n" " cvt.u64.u32 %rd20, %r2;\n" " or.b64 %rd59, %rd58, %rd20;\n" " cvta.to.global.u64 %rd60, %rd34;\n" " shl.b64 %rd61, %rd59, 2;\n" " add.s64 %rd21, %rd60, %rd61;\n" " ld.global.f32 %f147, [%rd21];\n" " ld.global.f32 %f148, [%rd21+128];\n" " ld.global.f32 %f149, [%rd21+256];\n" " ld.global.f32 %f150, [%rd21+384];\n" " setp.lt.s32 %p8, %r29, 1;\n" " @%p8 bra $L__BB9_16;\n" "\n" " cvta.to.global.u64 %rd62, %rd38;\n" " shl.b64 %rd63, %rd19, 2;\n" " add.s64 %rd22, %rd62, %rd63;\n" " shl.b64 %rd84, %rd18, 9;\n" " mul.wide.s32 %rd24, %r28, 512;\n" " cvta.to.global.u64 %rd64, %rd37;\n" " add.s64 %rd25, %rd64, %rd63;\n" " shl.b32 %r23, %r29, 2;\n" " cvta.to.global.u64 %rd65, %rd35;\n" " shl.b64 %rd66, %rd20, 2;\n" " add.s64 %rd26, %rd65, %rd66;\n" " shl.b64 %rd67, %rd19, 7;\n" " shl.b64 %rd68, %rd18, 14;\n" " cvta.to.global.u64 %rd69, %rd39;\n" " or.b64 %rd70, %rd20, %rd67;\n" " add.s64 %rd71, %rd70, %rd68;\n" " shl.b64 %rd72, %rd71, 2;\n" " add.s64 %rd73, %rd69, %rd72;\n" " add.s64 %rd83, %rd73, 256;\n" " mul.wide.s32 %rd28, %r28, 65536;\n" " cvta.to.global.u64 %rd74, %rd36;\n" " add.s64 %rd29, %rd74, %rd66;\n" " mov.u32 %r89, 0;\n" " mov.u32 %r88, sh;\n" " setp.eq.s64 %p9, %rd39, 0;\n" " setp.ne.s32 %p10, %r2, 0;\n" "\n" "$L__BB9_11:\n" " add.s32 %r70, %r88, %r23;\n" " ld.shared.f32 %f105, [%r70];\n" " add.s64 %rd75, %rd29, %rd84;\n" " ld.shared.f32 %f106, [%r88];\n" " mul.rn.f32 %f107, %f147, %f106;\n" " mul.rn.f32 %f108, %f148, %f106;\n" " mul.rn.f32 %f109, %f149, %f106;\n" " mul.rn.f32 %f110, %f150, %f106;\n" " ld.global.f32 %f111, [%rd75];\n" " mul.rn.f32 %f112, %f107, %f111;\n" " ld.global.f32 %f113, [%rd75+128];\n" " mul.rn.f32 %f114, %f108, %f113;\n" " ld.global.f32 %f115, [%rd75+256];\n" " mul.rn.f32 %f116, %f109, %f115;\n" " ld.global.f32 %f117, [%rd75+384];\n" " mul.rn.f32 %f118, %f110, %f117;\n" " add.rn.f32 %f119, %f112, %f116;\n" " add.rn.f32 %f120, %f114, %f118;\n" " add.rn.f32 %f96, %f119, %f120;\n" " mov.u32 %r60, 16;\n" " mov.u32 %r69, 31;\n" " // begin inline asm\n" " shfl.bfly.b32 %f95, %f96, %r60, %r69;\n" " // end inline asm\n" " add.rn.f32 %f98, %f96, %f95;\n" " mov.u32 %r62, 8;\n" " // begin inline asm\n" " shfl.bfly.b32 %f97, %f98, %r62, %r69;\n" " // end inline asm\n" " add.rn.f32 %f100, %f98, %f97;\n" " mov.u32 %r64, 4;\n" " // begin inline asm\n" " shfl.bfly.b32 %f99, %f100, %r64, %r69;\n" " // end inline asm\n" " add.rn.f32 %f102, %f100, %f99;\n" " mov.u32 %r66, 2;\n" " // begin inline asm\n" " shfl.bfly.b32 %f101, %f102, %r66, %r69;\n" " // end inline asm\n" " add.rn.f32 %f104, %f102, %f101;\n" " mov.u32 %r68, 1;\n" " // begin inline asm\n" " shfl.bfly.b32 %f103, %f104, %r68, %r69;\n" " // end inline asm\n" " add.rn.f32 %f121, %f104, %f103;\n" " add.s64 %rd76, %rd25, %rd84;\n" " ld.global.f32 %f122, [%rd76];\n" " sub.rn.f32 %f123, %f122, %f121;\n" " mul.rn.f32 %f124, %f123, %f105;\n" " fma.rn.f32 %f147, %f111, %f124, %f107;\n" " add.s64 %rd77, %rd26, %rd84;\n" " ld.global.f32 %f10, [%rd77];\n" " fma.rn.f32 %f148, %f113, %f124, %f108;\n" " ld.global.f32 %f12, [%rd77+128];\n" " fma.rn.f32 %f149, %f115, %f124, %f109;\n" " ld.global.f32 %f14, [%rd77+256];\n" " fma.rn.f32 %f150, %f117, %f124, %f110;\n" " ld.global.f32 %f16, [%rd77+384];\n" " @%p9 bra $L__BB9_13;\n" "\n" " st.global.f32 [%rd83+-256], %f147;\n" " st.global.f32 [%rd83+-128], %f148;\n" " st.global.f32 [%rd83], %f149;\n" " st.global.f32 [%rd83+128], %f150;\n" "\n" "$L__BB9_13:\n" " mul.rn.f32 %f135, %f149, %f14;\n" " mul.rn.f32 %f136, %f147, %f10;\n" " add.rn.f32 %f137, %f136, %f135;\n" " mul.rn.f32 %f138, %f150, %f16;\n" " mul.rn.f32 %f139, %f148, %f12;\n" " add.rn.f32 %f140, %f139, %f138;\n" " add.rn.f32 %f126, %f137, %f140;\n" " // begin inline asm\n" " shfl.bfly.b32 %f125, %f126, %r60, %r69;\n" " // end inline asm\n" " add.rn.f32 %f128, %f126, %f125;\n" " // begin inline asm\n" " shfl.bfly.b32 %f127, %f128, %r62, %r69;\n" " // end inline asm\n" " add.rn.f32 %f130, %f128, %f127;\n" " // begin inline asm\n" " shfl.bfly.b32 %f129, %f130, %r64, %r69;\n" " // end inline asm\n" " add.rn.f32 %f132, %f130, %f129;\n" " // begin inline asm\n" " shfl.bfly.b32 %f131, %f132, %r66, %r69;\n" " // end inline asm\n" " add.rn.f32 %f134, %f132, %f131;\n" " // begin inline asm\n" " shfl.bfly.b32 %f133, %f134, %r68, %r69;\n" " // end inline asm\n" " add.rn.f32 %f17, %f134, %f133;\n" " @%p10 bra $L__BB9_15;\n" "\n" " ld.param.f32 %f142, [fc_ssm_recurrence_seq_warp_param_10];\n" " add.s64 %rd78, %rd22, %rd84;\n" " mul.rn.f32 %f141, %f17, %f142;\n" " st.global.f32 [%rd78], %f141;\n" "\n" "$L__BB9_15:\n" " ld.param.u32 %r81, [fc_ssm_recurrence_seq_warp_param_9];\n" " add.s64 %rd84, %rd84, %rd24;\n" " add.s32 %r88, %r88, 4;\n" " add.s64 %rd83, %rd83, %rd28;\n" " add.s32 %r89, %r89, 1;\n" " setp.lt.s32 %p11, %r89, %r81;\n" " @%p11 bra $L__BB9_11;\n" "\n" "$L__BB9_16:\n" " st.global.f32 [%rd21], %f147;\n" " st.global.f32 [%rd21+128], %f148;\n" " st.global.f32 [%rd21+256], %f149;\n" " st.global.f32 [%rd21+384], %f150;\n" "\n" "$L__BB9_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_conv1d_silu_slots\n" ".visible .entry fc_ssm_conv1d_silu_slots(\n" " .param .u64 fc_ssm_conv1d_silu_slots_param_0,\n" " .param .u64 fc_ssm_conv1d_silu_slots_param_1,\n" " .param .u64 fc_ssm_conv1d_silu_slots_param_2,\n" " .param .u64 fc_ssm_conv1d_silu_slots_param_3,\n" " .param .u32 fc_ssm_conv1d_silu_slots_param_4,\n" " .param .u32 fc_ssm_conv1d_silu_slots_param_5\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<54>;\n" " .reg .b32 %r<43>;\n" " .reg .b64 %rd<65>;\n" "\n" "\n" " ld.param.u64 %rd25, [fc_ssm_conv1d_silu_slots_param_0];\n" " ld.param.u64 %rd27, [fc_ssm_conv1d_silu_slots_param_1];\n" " ld.param.u64 %rd28, [fc_ssm_conv1d_silu_slots_param_2];\n" " ld.param.u64 %rd26, [fc_ssm_conv1d_silu_slots_param_3];\n" " ld.param.u32 %r22, [fc_ssm_conv1d_silu_slots_param_4];\n" " ld.param.u32 %r23, [fc_ssm_conv1d_silu_slots_param_5];\n" " cvta.to.global.u64 %rd1, %rd28;\n" " cvta.to.global.u64 %rd2, %rd27;\n" " mov.u32 %r24, %ntid.x;\n" " mov.u32 %r25, %ctaid.x;\n" " mov.u32 %r26, %tid.x;\n" " mad.lo.s32 %r1, %r25, %r24, %r26;\n" " setp.ge.s32 %p1, %r1, %r22;\n" " @%p1 bra $L__BB10_16;\n" "\n" " mov.u32 %r27, %ctaid.y;\n" " cvt.s64.s32 %rd29, %r1;\n" " mul.wide.s32 %rd3, %r23, %r1;\n" " mul.wide.s32 %rd30, %r22, %r27;\n" " add.s64 %rd4, %rd30, %rd29;\n" " add.s32 %r2, %r23, -1;\n" " cvt.s64.s32 %rd31, %r2;\n" " mul.lo.s64 %rd5, %rd4, %rd31;\n" " cvta.to.global.u64 %rd32, %rd25;\n" " shl.b64 %rd33, %rd4, 2;\n" " add.s64 %rd6, %rd32, %rd33;\n" " add.s64 %rd34, %rd3, %rd31;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd36, %rd2, %rd35;\n" " ld.global.f32 %f9, [%rd36];\n" " ld.global.f32 %f10, [%rd6];\n" " mul.f32 %f53, %f10, %f9;\n" " add.s32 %r3, %r23, -2;\n" " setp.lt.s32 %p2, %r23, 2;\n" " @%p2 bra $L__BB10_8;\n" "\n" " and.b32 %r38, %r2, 3;\n" " setp.lt.u32 %p3, %r3, 3;\n" " mov.u32 %r37, 0;\n" " @%p3 bra $L__BB10_5;\n" "\n" " sub.s32 %r35, %r23, %r38;\n" " shl.b64 %rd37, %rd5, 2;\n" " add.s64 %rd38, %rd1, %rd37;\n" " add.s64 %rd60, %rd38, 8;\n" " shl.b64 %rd39, %rd3, 2;\n" " add.s64 %rd40, %rd2, %rd39;\n" " add.s64 %rd59, %rd40, 8;\n" " mov.u32 %r37, 0;\n" "\n" "$L__BB10_4:\n" " ld.global.f32 %f12, [%rd59+-8];\n" " ld.global.f32 %f13, [%rd60+-8];\n" " fma.rn.f32 %f14, %f13, %f12, %f53;\n" " ld.global.f32 %f15, [%rd59+-4];\n" " ld.global.f32 %f16, [%rd60+-4];\n" " fma.rn.f32 %f17, %f16, %f15, %f14;\n" " ld.global.f32 %f18, [%rd59];\n" " ld.global.f32 %f19, [%rd60];\n" " fma.rn.f32 %f20, %f19, %f18, %f17;\n" " ld.global.f32 %f21, [%rd59+4];\n" " ld.global.f32 %f22, [%rd60+4];\n" " fma.rn.f32 %f53, %f22, %f21, %f20;\n" " add.s32 %r37, %r37, 4;\n" " add.s64 %rd60, %rd60, 16;\n" " add.s64 %rd59, %rd59, 16;\n" " add.s32 %r35, %r35, -4;\n" " setp.ne.s32 %p4, %r35, 1;\n" " @%p4 bra $L__BB10_4;\n" "\n" "$L__BB10_5:\n" " setp.eq.s32 %p5, %r38, 0;\n" " @%p5 bra $L__BB10_8;\n" "\n" " cvt.s64.s32 %rd41, %r37;\n" " add.s64 %rd42, %rd3, %rd41;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd62, %rd2, %rd43;\n" " add.s64 %rd44, %rd5, %rd41;\n" " shl.b64 %rd45, %rd44, 2;\n" " add.s64 %rd61, %rd1, %rd45;\n" "\n" "$L__BB10_7:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f23, [%rd62];\n" " ld.global.f32 %f24, [%rd61];\n" " fma.rn.f32 %f53, %f24, %f23, %f53;\n" " add.s64 %rd62, %rd62, 4;\n" " add.s64 %rd61, %rd61, 4;\n" " add.s32 %r38, %r38, -1;\n" " setp.ne.s32 %p6, %r38, 0;\n" " @%p6 bra $L__BB10_7;\n" "\n" "$L__BB10_8:\n" " neg.f32 %f25, %f53;\n" " mov.f32 %f26, 0f3F000000;\n" " mov.f32 %f27, 0f3BBB989D;\n" " fma.rn.f32 %f28, %f25, %f27, %f26;\n" " mov.f32 %f29, 0f3FB8AA3B;\n" " mov.f32 %f30, 0f437C0000;\n" " cvt.sat.f32.f32 %f31, %f28;\n" " mov.f32 %f32, 0f4B400001;\n" " fma.rm.f32 %f33, %f31, %f30, %f32;\n" " add.f32 %f34, %f33, 0fCB40007F;\n" " neg.f32 %f35, %f34;\n" " fma.rn.f32 %f36, %f25, %f29, %f35;\n" " mov.f32 %f37, 0f32A57060;\n" " fma.rn.f32 %f38, %f25, %f37, %f36;\n" " mov.b32 %r30, %f33;\n" " shl.b32 %r31, %r30, 23;\n" " mov.b32 %f39, %r31;\n" " ex2.approx.ftz.f32 %f40, %f38;\n" " fma.rn.f32 %f41, %f40, %f39, 0f3F800000;\n" " div.rn.f32 %f42, %f53, %f41;\n" " cvta.to.global.u64 %rd46, %rd26;\n" " add.s64 %rd48, %rd46, %rd33;\n" " st.global.f32 [%rd48], %f42;\n" " setp.lt.s32 %p7, %r23, 3;\n" " @%p7 bra $L__BB10_15;\n" "\n" " add.s32 %r33, %r23, -3;\n" " and.b32 %r42, %r3, 3;\n" " setp.lt.u32 %p8, %r33, 3;\n" " mov.u32 %r41, 0;\n" " @%p8 bra $L__BB10_12;\n" "\n" " sub.s32 %r39, %r23, %r42;\n" " shl.b64 %rd49, %rd5, 2;\n" " add.s64 %rd50, %rd1, %rd49;\n" " add.s64 %rd63, %rd50, 8;\n" " mov.u32 %r41, 0;\n" "\n" "$L__BB10_11:\n" " ld.global.f32 %f43, [%rd63+-4];\n" " st.global.f32 [%rd63+-8], %f43;\n" " ld.global.f32 %f44, [%rd63];\n" " st.global.f32 [%rd63+-4], %f44;\n" " ld.global.f32 %f45, [%rd63+4];\n" " st.global.f32 [%rd63], %f45;\n" " add.s32 %r41, %r41, 4;\n" " ld.global.f32 %f46, [%rd63+8];\n" " st.global.f32 [%rd63+4], %f46;\n" " add.s64 %rd63, %rd63, 16;\n" " add.s32 %r39, %r39, -4;\n" " setp.ne.s32 %p9, %r39, 2;\n" " @%p9 bra $L__BB10_11;\n" "\n" "$L__BB10_12:\n" " setp.eq.s32 %p10, %r42, 0;\n" " @%p10 bra $L__BB10_15;\n" "\n" " cvt.u64.u32 %rd51, %r41;\n" " add.s64 %rd52, %rd5, %rd51;\n" " shl.b64 %rd53, %rd52, 2;\n" " add.s64 %rd54, %rd1, %rd53;\n" " add.s64 %rd64, %rd54, 4;\n" "\n" "$L__BB10_14:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f47, [%rd64];\n" " st.global.f32 [%rd64+-4], %f47;\n" " add.s64 %rd64, %rd64, 4;\n" " add.s32 %r42, %r42, -1;\n" " setp.ne.s32 %p11, %r42, 0;\n" " @%p11 bra $L__BB10_14;\n" "\n" "$L__BB10_15:\n" " ld.global.f32 %f48, [%rd6];\n" " cvt.s64.s32 %rd55, %r3;\n" " add.s64 %rd56, %rd5, %rd55;\n" " shl.b64 %rd57, %rd56, 2;\n" " add.s64 %rd58, %rd1, %rd57;\n" " st.global.f32 [%rd58], %f48;\n" "\n" "$L__BB10_16:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_recurrence_slots\n" ".visible .entry fc_ssm_recurrence_slots(\n" " .param .u64 fc_ssm_recurrence_slots_param_0,\n" " .param .u64 fc_ssm_recurrence_slots_param_1,\n" " .param .u64 fc_ssm_recurrence_slots_param_2,\n" " .param .u64 fc_ssm_recurrence_slots_param_3,\n" " .param .u64 fc_ssm_recurrence_slots_param_4,\n" " .param .u64 fc_ssm_recurrence_slots_param_5,\n" " .param .u64 fc_ssm_recurrence_slots_param_6,\n" " .param .u32 fc_ssm_recurrence_slots_param_7,\n" " .param .u32 fc_ssm_recurrence_slots_param_8,\n" " .param .f32 fc_ssm_recurrence_slots_param_9\n" ")\n" "{\n" " .reg .pred %p<9>;\n" " .reg .f32 %f<40>;\n" " .reg .b32 %r<42>;\n" " .reg .b64 %rd<60>;\n" "\n" "\n" " ld.param.u64 %rd2, [fc_ssm_recurrence_slots_param_0];\n" " ld.param.u64 %rd3, [fc_ssm_recurrence_slots_param_1];\n" " ld.param.u64 %rd4, [fc_ssm_recurrence_slots_param_2];\n" " ld.param.u64 %rd5, [fc_ssm_recurrence_slots_param_3];\n" " ld.param.u64 %rd6, [fc_ssm_recurrence_slots_param_4];\n" " ld.param.u64 %rd7, [fc_ssm_recurrence_slots_param_5];\n" " ld.param.u64 %rd8, [fc_ssm_recurrence_slots_param_6];\n" " ld.param.u32 %r8, [fc_ssm_recurrence_slots_param_7];\n" " ld.param.u32 %r9, [fc_ssm_recurrence_slots_param_8];\n" " ld.param.f32 %f3, [fc_ssm_recurrence_slots_param_9];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r8;\n" " @%p1 bra $L__BB11_12;\n" "\n" " mov.u32 %r10, %ctaid.z;\n" " mul.wide.s32 %rd9, %r9, %r10;\n" " cvt.s64.s32 %rd10, %r8;\n" " mul.lo.s64 %rd11, %rd9, %rd10;\n" " mov.u32 %r11, %ctaid.y;\n" " cvt.s64.s32 %rd12, %r11;\n" " mul.wide.s32 %rd13, %r8, %r11;\n" " mov.u32 %r12, %ctaid.x;\n" " cvt.s64.s32 %rd14, %r12;\n" " add.s64 %rd15, %rd13, %rd14;\n" " add.s64 %rd16, %rd15, %rd11;\n" " mul.lo.s64 %rd17, %rd16, %rd10;\n" " add.s64 %rd18, %rd11, %rd13;\n" " add.s64 %rd19, %rd9, %rd12;\n" " cvta.to.global.u64 %rd20, %rd6;\n" " shl.b64 %rd21, %rd19, 2;\n" " add.s64 %rd22, %rd20, %rd21;\n" " ld.global.f32 %f4, [%rd22];\n" " mov.f32 %f5, 0f3F000000;\n" " mov.f32 %f6, 0f3BBB989D;\n" " fma.rn.f32 %f7, %f4, %f6, %f5;\n" " mov.f32 %f8, 0f3FB8AA3B;\n" " mov.f32 %f9, 0f437C0000;\n" " cvt.sat.f32.f32 %f10, %f7;\n" " mov.f32 %f11, 0f4B400001;\n" " fma.rm.f32 %f12, %f10, %f9, %f11;\n" " add.f32 %f13, %f12, 0fCB40007F;\n" " neg.f32 %f14, %f13;\n" " fma.rn.f32 %f15, %f4, %f8, %f14;\n" " mov.f32 %f16, 0f32A57060;\n" " fma.rn.f32 %f17, %f4, %f16, %f15;\n" " mov.b32 %r13, %f12;\n" " shl.b32 %r14, %r13, 23;\n" " mov.b32 %f18, %r14;\n" " ex2.approx.ftz.f32 %f19, %f17;\n" " mul.f32 %f20, %f19, %f18;\n" " cvta.to.global.u64 %rd23, %rd7;\n" " add.s64 %rd24, %rd23, %rd21;\n" " ld.global.f32 %f1, [%rd24];\n" " cvt.s64.s32 %rd25, %r1;\n" " add.s64 %rd26, %rd17, %rd25;\n" " cvta.to.global.u64 %rd27, %rd2;\n" " shl.b64 %rd28, %rd26, 2;\n" " add.s64 %rd29, %rd27, %rd28;\n" " ld.global.f32 %f21, [%rd29];\n" " mul.f32 %f2, %f20, %f21;\n" " add.s64 %rd30, %rd18, %rd25;\n" " cvta.to.global.u64 %rd31, %rd4;\n" " shl.b64 %rd32, %rd30, 2;\n" " add.s64 %rd33, %rd31, %rd32;\n" " ld.global.f32 %f22, [%rd33];\n" " mul.f32 %f23, %f2, %f22;\n" " shl.b32 %r15, %r1, 2;\n" " mov.u32 %r16, sdata;\n" " add.s32 %r17, %r16, %r15;\n" " st.shared.f32 [%r17], %f23;\n" " bar.sync 0;\n" " shr.u32 %r18, %r8, 31;\n" " add.s32 %r19, %r8, %r18;\n" " shr.s32 %r41, %r19, 1;\n" " setp.lt.s32 %p2, %r8, 2;\n" " @%p2 bra $L__BB11_6;\n" "\n" " mov.u32 %r40, %r41;\n" "\n" "$L__BB11_3:\n" " setp.ge.s32 %p3, %r1, %r40;\n" " @%p3 bra $L__BB11_5;\n" "\n" " shl.b32 %r25, %r40, 2;\n" " add.s32 %r26, %r17, %r25;\n" " ld.shared.f32 %f24, [%r17];\n" " ld.shared.f32 %f25, [%r26];\n" " add.f32 %f26, %f25, %f24;\n" " st.shared.f32 [%r17], %f26;\n" "\n" "$L__BB11_5:\n" " bar.sync 0;\n" " shr.s32 %r4, %r40, 1;\n" " setp.gt.s32 %p4, %r40, 1;\n" " mov.u32 %r40, %r4;\n" " @%p4 bra $L__BB11_3;\n" "\n" "$L__BB11_6:\n" " add.s64 %rd1, %rd18, %rd14;\n" " cvta.to.global.u64 %rd40, %rd5;\n" " shl.b64 %rd41, %rd1, 2;\n" " add.s64 %rd42, %rd40, %rd41;\n" " ld.global.f32 %f27, [%rd42];\n" " ld.shared.f32 %f28, [sdata];\n" " sub.f32 %f29, %f27, %f28;\n" " mul.f32 %f30, %f1, %f29;\n" " bar.sync 0;\n" " ld.global.f32 %f31, [%rd33];\n" " fma.rn.f32 %f32, %f31, %f30, %f2;\n" " st.global.f32 [%rd29], %f32;\n" " cvta.to.global.u64 %rd55, %rd3;\n" " add.s64 %rd56, %rd55, %rd32;\n" " ld.global.f32 %f33, [%rd56];\n" " mul.f32 %f34, %f32, %f33;\n" " st.shared.f32 [%r17], %f34;\n" " bar.sync 0;\n" " @%p2 bra $L__BB11_10;\n" "\n" "$L__BB11_7:\n" " setp.ge.s32 %p6, %r1, %r41;\n" " @%p6 bra $L__BB11_9;\n" "\n" " shl.b32 %r37, %r41, 2;\n" " add.s32 %r38, %r17, %r37;\n" " ld.shared.f32 %f35, [%r17];\n" " ld.shared.f32 %f36, [%r38];\n" " add.f32 %f37, %f36, %f35;\n" " st.shared.f32 [%r17], %f37;\n" "\n" "$L__BB11_9:\n" " bar.sync 0;\n" " shr.s32 %r7, %r41, 1;\n" " setp.gt.s32 %p7, %r41, 1;\n" " mov.u32 %r41, %r7;\n" " @%p7 bra $L__BB11_7;\n" "\n" "$L__BB11_10:\n" " setp.ne.s32 %p8, %r1, 0;\n" " @%p8 bra $L__BB11_12;\n" "\n" " ld.shared.f32 %f38, [sdata];\n" " mul.f32 %f39, %f38, %f3;\n" " cvta.to.global.u64 %rd57, %rd8;\n" " add.s64 %rd59, %rd57, %rd41;\n" " st.global.f32 [%rd59], %f39;\n" "\n" "$L__BB11_12:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_recurrence_slots_2pass\n" ".visible .entry fc_ssm_recurrence_slots_2pass(\n" " .param .u64 fc_ssm_recurrence_slots_2pass_param_0,\n" " .param .u64 fc_ssm_recurrence_slots_2pass_param_1,\n" " .param .u64 fc_ssm_recurrence_slots_2pass_param_2,\n" " .param .u64 fc_ssm_recurrence_slots_2pass_param_3,\n" " .param .u64 fc_ssm_recurrence_slots_2pass_param_4,\n" " .param .u64 fc_ssm_recurrence_slots_2pass_param_5,\n" " .param .u64 fc_ssm_recurrence_slots_2pass_param_6,\n" " .param .u32 fc_ssm_recurrence_slots_2pass_param_7,\n" " .param .u32 fc_ssm_recurrence_slots_2pass_param_8,\n" " .param .f32 fc_ssm_recurrence_slots_2pass_param_9\n" ")\n" "{\n" " .reg .pred %p<6>;\n" " .reg .f32 %f<45>;\n" " .reg .b32 %r<31>;\n" " .reg .b64 %rd<42>;\n" "\n" "\n" " ld.param.u64 %rd5, [fc_ssm_recurrence_slots_2pass_param_0];\n" " ld.param.u64 %rd6, [fc_ssm_recurrence_slots_2pass_param_1];\n" " ld.param.u64 %rd7, [fc_ssm_recurrence_slots_2pass_param_2];\n" " ld.param.u64 %rd8, [fc_ssm_recurrence_slots_2pass_param_3];\n" " ld.param.u64 %rd9, [fc_ssm_recurrence_slots_2pass_param_4];\n" " ld.param.u64 %rd10, [fc_ssm_recurrence_slots_2pass_param_5];\n" " ld.param.u64 %rd11, [fc_ssm_recurrence_slots_2pass_param_6];\n" " ld.param.u32 %r9, [fc_ssm_recurrence_slots_2pass_param_7];\n" " ld.param.u32 %r10, [fc_ssm_recurrence_slots_2pass_param_8];\n" " ld.param.f32 %f5, [fc_ssm_recurrence_slots_2pass_param_9];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r9;\n" " @%p1 bra $L__BB12_8;\n" "\n" " mov.u32 %r11, %ctaid.z;\n" " mul.wide.s32 %rd12, %r10, %r11;\n" " cvt.s64.s32 %rd13, %r9;\n" " mul.lo.s64 %rd14, %rd12, %rd13;\n" " mov.u32 %r12, %ctaid.y;\n" " cvt.s64.s32 %rd15, %r12;\n" " mul.wide.s32 %rd16, %r9, %r12;\n" " mov.u32 %r13, %ctaid.x;\n" " cvt.s64.s32 %rd1, %r13;\n" " add.s64 %rd17, %rd16, %rd1;\n" " add.s64 %rd18, %rd17, %rd14;\n" " mul.lo.s64 %rd19, %rd18, %rd13;\n" " add.s64 %rd2, %rd14, %rd16;\n" " add.s64 %rd20, %rd12, %rd15;\n" " cvta.to.global.u64 %rd21, %rd9;\n" " shl.b64 %rd22, %rd20, 2;\n" " add.s64 %rd23, %rd21, %rd22;\n" " ld.global.f32 %f6, [%rd23];\n" " mov.f32 %f7, 0f3F000000;\n" " mov.f32 %f8, 0f3BBB989D;\n" " fma.rn.f32 %f9, %f6, %f8, %f7;\n" " mov.f32 %f10, 0f3FB8AA3B;\n" " mov.f32 %f11, 0f437C0000;\n" " cvt.sat.f32.f32 %f12, %f9;\n" " mov.f32 %f13, 0f4B400001;\n" " fma.rm.f32 %f14, %f12, %f11, %f13;\n" " add.f32 %f15, %f14, 0fCB40007F;\n" " neg.f32 %f16, %f15;\n" " fma.rn.f32 %f17, %f6, %f10, %f16;\n" " mov.f32 %f18, 0f32A57060;\n" " fma.rn.f32 %f19, %f6, %f18, %f17;\n" " mov.b32 %r14, %f14;\n" " shl.b32 %r15, %r14, 23;\n" " mov.b32 %f20, %r15;\n" " ex2.approx.ftz.f32 %f21, %f19;\n" " mul.f32 %f22, %f21, %f20;\n" " cvta.to.global.u64 %rd24, %rd10;\n" " add.s64 %rd25, %rd24, %rd22;\n" " ld.global.f32 %f1, [%rd25];\n" " cvt.s64.s32 %rd26, %r1;\n" " add.s64 %rd27, %rd19, %rd26;\n" " cvta.to.global.u64 %rd28, %rd5;\n" " shl.b64 %rd29, %rd27, 2;\n" " add.s64 %rd3, %rd28, %rd29;\n" " ld.global.f32 %f23, [%rd3];\n" " mul.f32 %f2, %f22, %f23;\n" " add.s64 %rd30, %rd2, %rd26;\n" " cvta.to.global.u64 %rd31, %rd7;\n" " shl.b64 %rd32, %rd30, 2;\n" " add.s64 %rd33, %rd31, %rd32;\n" " cvta.to.global.u64 %rd34, %rd6;\n" " add.s64 %rd35, %rd34, %rd32;\n" " ld.global.f32 %f24, [%rd35];\n" " ld.global.f32 %f3, [%rd33];\n" " mul.f32 %f25, %f2, %f3;\n" " shl.b32 %r16, %r1, 2;\n" " mov.u32 %r17, sdata;\n" " add.s32 %r2, %r17, %r16;\n" " st.shared.f32 [%r2], %f25;\n" " mul.f32 %f26, %f2, %f24;\n" " shl.b32 %r18, %r9, 2;\n" " add.s32 %r3, %r2, %r18;\n" " st.shared.f32 [%r3], %f26;\n" " mul.f32 %f27, %f3, %f24;\n" " add.s32 %r4, %r3, %r18;\n" " st.shared.f32 [%r4], %f27;\n" " bar.sync 0;\n" " setp.lt.s32 %p2, %r9, 2;\n" " @%p2 bra $L__BB12_6;\n" "\n" " shr.u32 %r19, %r9, 31;\n" " add.s32 %r20, %r9, %r19;\n" " shr.s32 %r30, %r20, 1;\n" "\n" "$L__BB12_3:\n" " setp.ge.s32 %p3, %r1, %r30;\n" " @%p3 bra $L__BB12_5;\n" "\n" " shl.b32 %r21, %r30, 2;\n" " add.s32 %r22, %r2, %r21;\n" " ld.shared.f32 %f28, [%r2];\n" " ld.shared.f32 %f29, [%r22];\n" " add.f32 %f30, %f29, %f28;\n" " st.shared.f32 [%r2], %f30;\n" " add.s32 %r23, %r22, %r18;\n" " ld.shared.f32 %f31, [%r3];\n" " ld.shared.f32 %f32, [%r23];\n" " add.f32 %f33, %f32, %f31;\n" " st.shared.f32 [%r3], %f33;\n" " add.s32 %r24, %r23, %r18;\n" " ld.shared.f32 %f34, [%r4];\n" " ld.shared.f32 %f35, [%r24];\n" " add.f32 %f36, %f35, %f34;\n" " st.shared.f32 [%r4], %f36;\n" "\n" "$L__BB12_5:\n" " bar.sync 0;\n" " shr.s32 %r8, %r30, 1;\n" " setp.gt.s32 %p4, %r30, 1;\n" " mov.u32 %r30, %r8;\n" " @%p4 bra $L__BB12_3;\n" "\n" "$L__BB12_6:\n" " ld.shared.f32 %f37, [sdata];\n" " add.s64 %rd4, %rd2, %rd1;\n" " cvta.to.global.u64 %rd36, %rd8;\n" " shl.b64 %rd37, %rd4, 2;\n" " add.s64 %rd38, %rd36, %rd37;\n" " ld.global.f32 %f38, [%rd38];\n" " sub.f32 %f39, %f38, %f37;\n" " mul.f32 %f4, %f1, %f39;\n" " fma.rn.f32 %f40, %f3, %f4, %f2;\n" " st.global.f32 [%rd3], %f40;\n" " setp.ne.s32 %p5, %r1, 0;\n" " @%p5 bra $L__BB12_8;\n" "\n" " add.s32 %r27, %r17, %r18;\n" " ld.shared.f32 %f41, [%r27];\n" " shl.b32 %r28, %r9, 3;\n" " add.s32 %r29, %r17, %r28;\n" " ld.shared.f32 %f42, [%r29];\n" " fma.rn.f32 %f43, %f42, %f4, %f41;\n" " mul.f32 %f44, %f43, %f5;\n" " cvta.to.global.u64 %rd39, %rd11;\n" " add.s64 %rd41, %rd39, %rd37;\n" " st.global.f32 [%rd41], %f44;\n" "\n" "$L__BB12_8:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_decay_gate_nx\n" ".visible .entry fc_ssm_decay_gate_nx(\n" " .param .u64 fc_ssm_decay_gate_nx_param_0,\n" " .param .u64 fc_ssm_decay_gate_nx_param_1,\n" " .param .u64 fc_ssm_decay_gate_nx_param_2,\n" " .param .u64 fc_ssm_decay_gate_nx_param_3,\n" " .param .u32 fc_ssm_decay_gate_nx_param_4,\n" " .param .u32 fc_ssm_decay_gate_nx_param_5\n" ")\n" "{\n" " .reg .pred %p<6>;\n" " .reg .f32 %f<60>;\n" " .reg .b32 %r<17>;\n" " .reg .b64 %rd<17>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_ssm_decay_gate_nx_param_0];\n" " ld.param.u64 %rd2, [fc_ssm_decay_gate_nx_param_1];\n" " ld.param.u64 %rd3, [fc_ssm_decay_gate_nx_param_2];\n" " ld.param.u64 %rd4, [fc_ssm_decay_gate_nx_param_3];\n" " ld.param.u32 %r4, [fc_ssm_decay_gate_nx_param_4];\n" " ld.param.u32 %r5, [fc_ssm_decay_gate_nx_param_5];\n" " mov.u32 %r6, %ntid.x;\n" " mov.u32 %r7, %ctaid.x;\n" " mov.u32 %r8, %tid.x;\n" " mad.lo.s32 %r1, %r7, %r6, %r8;\n" " setp.ge.s32 %p1, %r1, %r5;\n" " @%p1 bra $L__BB13_7;\n" "\n" " cvta.to.global.u64 %rd5, %rd1;\n" " mul.wide.s32 %rd6, %r1, 4;\n" " add.s64 %rd7, %rd5, %rd6;\n" " rem.s32 %r2, %r1, %r4;\n" " cvta.to.global.u64 %rd8, %rd2;\n" " mul.wide.s32 %rd9, %r2, 4;\n" " add.s64 %rd10, %rd8, %rd9;\n" " ld.global.f32 %f8, [%rd10];\n" " ld.global.f32 %f9, [%rd7];\n" " add.f32 %f59, %f9, %f8;\n" " setp.gt.f32 %p2, %f59, 0f41A00000;\n" " @%p2 bra $L__BB13_6;\n" "\n" " mov.f32 %f10, 0f3F000000;\n" " mov.f32 %f11, 0f3BBB989D;\n" " fma.rn.f32 %f12, %f59, %f11, %f10;\n" " mov.f32 %f13, 0f3FB8AA3B;\n" " mov.f32 %f14, 0f437C0000;\n" " cvt.sat.f32.f32 %f15, %f12;\n" " mov.f32 %f16, 0f4B400001;\n" " fma.rm.f32 %f17, %f15, %f14, %f16;\n" " add.f32 %f18, %f17, 0fCB40007F;\n" " neg.f32 %f19, %f18;\n" " fma.rn.f32 %f20, %f59, %f13, %f19;\n" " mov.f32 %f21, 0f32A57060;\n" " fma.rn.f32 %f22, %f59, %f21, %f20;\n" " mov.b32 %r9, %f17;\n" " shl.b32 %r10, %r9, 23;\n" " mov.b32 %f23, %r10;\n" " ex2.approx.ftz.f32 %f24, %f22;\n" " mul.f32 %f2, %f24, %f23;\n" " mov.f32 %f25, 0f3F800000;\n" " add.rz.f32 %f26, %f2, %f25;\n" " mov.b32 %r11, %f26;\n" " add.s32 %r12, %r11, -1061158912;\n" " and.b32 %r13, %r12, -8388608;\n" " mov.b32 %r3, %f2;\n" " sub.s32 %r14, %r3, %r13;\n" " mov.b32 %f27, %r14;\n" " mov.u32 %r15, 1082130432;\n" " sub.s32 %r16, %r15, %r13;\n" " mov.b32 %f28, %r16;\n" " mov.f32 %f29, 0fBF800000;\n" " mov.f32 %f30, 0f3E800000;\n" " fma.rn.f32 %f31, %f30, %f28, %f29;\n" " add.f32 %f32, %f31, %f27;\n" " cvt.rn.f32.s32 %f33, %r13;\n" " mul.f32 %f34, %f33, 0f34000000;\n" " mov.f32 %f35, 0f3DD80012;\n" " mov.f32 %f36, 0fBD39BF78;\n" " fma.rn.f32 %f37, %f36, %f32, %f35;\n" " mov.f32 %f38, 0fBE0778E0;\n" " fma.rn.f32 %f39, %f37, %f32, %f38;\n" " mov.f32 %f40, 0f3E146475;\n" " fma.rn.f32 %f41, %f39, %f32, %f40;\n" " mov.f32 %f42, 0fBE2A68DD;\n" " fma.rn.f32 %f43, %f41, %f32, %f42;\n" " mov.f32 %f44, 0f3E4CAF9E;\n" " fma.rn.f32 %f45, %f43, %f32, %f44;\n" " mov.f32 %f46, 0fBE800042;\n" " fma.rn.f32 %f47, %f45, %f32, %f46;\n" " mov.f32 %f48, 0f3EAAAAE6;\n" " fma.rn.f32 %f49, %f47, %f32, %f48;\n" " mov.f32 %f50, 0fBF000000;\n" " fma.rn.f32 %f51, %f49, %f32, %f50;\n" " mul.f32 %f52, %f32, %f51;\n" " fma.rn.f32 %f53, %f52, %f32, %f32;\n" " mov.f32 %f54, 0f3F317218;\n" " fma.rn.f32 %f59, %f34, %f54, %f53;\n" " setp.lt.u32 %p3, %r3, 2139095040;\n" " @%p3 bra $L__BB13_6;\n" "\n" " setp.lt.s32 %p4, %r3, -1082130431;\n" " @%p4 bra $L__BB13_5;\n" "\n" " mov.f32 %f55, 0f7F800000;\n" " fma.rn.f32 %f59, %f2, %f55, %f55;\n" "\n" "$L__BB13_5:\n" " setp.eq.f32 %p5, %f2, 0f00000000;\n" " selp.f32 %f59, 0f80000000, %f59, %p5;\n" "\n" "$L__BB13_6:\n" " cvta.to.global.u64 %rd11, %rd4;\n" " cvta.to.global.u64 %rd12, %rd3;\n" " add.s64 %rd14, %rd12, %rd9;\n" " ld.global.f32 %f56, [%rd14];\n" " mul.f32 %f57, %f59, %f56;\n" " add.s64 %rd16, %rd11, %rd6;\n" " st.global.f32 [%rd16], %f57;\n" "\n" "$L__BB13_7:\n" " ret;\n" "\n" "}\n" " // .globl fc_ssm_qknorm_gqa_nx\n" ".visible .entry fc_ssm_qknorm_gqa_nx(\n" " .param .u64 fc_ssm_qknorm_gqa_nx_param_0,\n" " .param .u64 fc_ssm_qknorm_gqa_nx_param_1,\n" " .param .u64 fc_ssm_qknorm_gqa_nx_param_2,\n" " .param .u64 fc_ssm_qknorm_gqa_nx_param_3,\n" " .param .u32 fc_ssm_qknorm_gqa_nx_param_4,\n" " .param .u32 fc_ssm_qknorm_gqa_nx_param_5,\n" " .param .u32 fc_ssm_qknorm_gqa_nx_param_6,\n" " .param .u32 fc_ssm_qknorm_gqa_nx_param_7,\n" " .param .u32 fc_ssm_qknorm_gqa_nx_param_8,\n" " .param .f32 fc_ssm_qknorm_gqa_nx_param_9\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<23>;\n" " .reg .b32 %r<49>;\n" " .reg .b64 %rd<62>;\n" "\n" "\n" " ld.param.u64 %rd21, [fc_ssm_qknorm_gqa_nx_param_0];\n" " ld.param.u64 %rd22, [fc_ssm_qknorm_gqa_nx_param_1];\n" " ld.param.u64 %rd23, [fc_ssm_qknorm_gqa_nx_param_2];\n" " ld.param.u64 %rd20, [fc_ssm_qknorm_gqa_nx_param_3];\n" " ld.param.u32 %r21, [fc_ssm_qknorm_gqa_nx_param_4];\n" " ld.param.u32 %r22, [fc_ssm_qknorm_gqa_nx_param_5];\n" " ld.param.u32 %r23, [fc_ssm_qknorm_gqa_nx_param_6];\n" " ld.param.u32 %r24, [fc_ssm_qknorm_gqa_nx_param_7];\n" " ld.param.u32 %r25, [fc_ssm_qknorm_gqa_nx_param_8];\n" " ld.param.f32 %f5, [fc_ssm_qknorm_gqa_nx_param_9];\n" " cvta.to.global.u64 %rd1, %rd23;\n" " cvta.to.global.u64 %rd2, %rd22;\n" " cvta.to.global.u64 %rd3, %rd21;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p1, %r1, %r21;\n" " @%p1 bra $L__BB14_15;\n" "\n" " mov.u32 %r26, %ctaid.y;\n" " shl.b32 %r27, %r24, 1;\n" " add.s32 %r28, %r27, %r25;\n" " mul.wide.s32 %rd24, %r28, %r26;\n" " cvt.s64.s32 %rd25, %r27;\n" " add.s64 %rd4, %rd24, %rd25;\n" " mul.wide.s32 %rd5, %r25, %r26;\n" " cvt.s64.s32 %rd6, %r23;\n" " cvt.s64.s32 %rd7, %r1;\n" " mul.wide.s32 %rd26, %r23, %r1;\n" " mov.u32 %r48, %tid.x;\n" " cvt.s64.s32 %rd8, %r48;\n" " add.s64 %rd27, %rd26, %rd8;\n" " add.s64 %rd28, %rd24, %rd27;\n" " shl.b64 %rd29, %rd28, 2;\n" " add.s64 %rd30, %rd3, %rd29;\n" " mul.wide.s32 %rd31, %r24, 4;\n" " add.s64 %rd32, %rd30, %rd31;\n" " ld.global.f32 %f1, [%rd32];\n" " ld.global.f32 %f2, [%rd30];\n" " mul.f32 %f6, %f2, %f2;\n" " shl.b32 %r29, %r48, 2;\n" " mov.u32 %r30, sh;\n" " add.s32 %r3, %r30, %r29;\n" " st.shared.f32 [%r3], %f6;\n" " mul.f32 %f7, %f1, %f1;\n" " mov.u32 %r4, %ntid.x;\n" " shl.b32 %r31, %r4, 2;\n" " add.s32 %r5, %r3, %r31;\n" " st.shared.f32 [%r5], %f7;\n" " bar.sync 0;\n" " shr.u32 %r43, %r4, 1;\n" " setp.eq.s32 %p2, %r43, 0;\n" " @%p2 bra $L__BB14_5;\n" "\n" "$L__BB14_2:\n" " setp.ge.s32 %p3, %r48, %r43;\n" " @%p3 bra $L__BB14_4;\n" "\n" " shl.b32 %r32, %r43, 2;\n" " add.s32 %r33, %r3, %r32;\n" " ld.shared.f32 %f8, [%r3];\n" " ld.shared.f32 %f9, [%r33];\n" " add.f32 %f10, %f9, %f8;\n" " st.shared.f32 [%r3], %f10;\n" " ld.shared.f32 %f11, [%r5];\n" " add.s32 %r34, %r5, %r32;\n" " ld.shared.f32 %f12, [%r34];\n" " add.f32 %f13, %f12, %f11;\n" " st.shared.f32 [%r5], %f13;\n" "\n" "$L__BB14_4:\n" " bar.sync 0;\n" " shr.s32 %r8, %r43, 1;\n" " setp.gt.s32 %p4, %r43, 1;\n" " mov.u32 %r43, %r8;\n" " @%p4 bra $L__BB14_2;\n" "\n" "$L__BB14_5:\n" " ld.shared.f32 %f14, [sh];\n" " sqrt.rn.f32 %f15, %f14;\n" " max.f32 %f16, %f15, %f5;\n" " rcp.rn.f32 %f17, %f16;\n" " add.s32 %r37, %r30, %r31;\n" " ld.shared.f32 %f18, [%r37];\n" " sqrt.rn.f32 %f19, %f18;\n" " max.f32 %f20, %f19, %f5;\n" " rcp.rn.f32 %f21, %f20;\n" " mul.f32 %f3, %f2, %f17;\n" " mul.f32 %f4, %f1, %f21;\n" " setp.lt.s32 %p5, %r22, 1;\n" " @%p5 bra $L__BB14_12;\n" "\n" " add.s64 %rd9, %rd5, %rd8;\n" " and.b32 %r47, %r22, 3;\n" " add.s32 %r39, %r22, -1;\n" " setp.lt.u32 %p6, %r39, 3;\n" " mov.u32 %r46, 0;\n" " @%p6 bra $L__BB14_9;\n" "\n" " sub.s32 %r45, %r22, %r47;\n" " shl.b32 %r41, %r21, 2;\n" " cvt.s64.s32 %rd10, %r41;\n" " cvt.s64.s32 %rd33, %r21;\n" " mul.lo.s64 %rd34, %rd6, %rd33;\n" " shl.b64 %rd11, %rd34, 2;\n" " mov.u32 %r46, 0;\n" " mov.u64 %rd60, %rd7;\n" "\n" "$L__BB14_8:\n" " mul.lo.s64 %rd35, %rd60, %rd6;\n" " add.s64 %rd36, %rd9, %rd35;\n" " shl.b64 %rd37, %rd36, 2;\n" " add.s64 %rd38, %rd2, %rd37;\n" " st.global.f32 [%rd38], %f3;\n" " add.s64 %rd39, %rd1, %rd37;\n" " st.global.f32 [%rd39], %f4;\n" " add.s64 %rd40, %rd38, %rd11;\n" " st.global.f32 [%rd40], %f3;\n" " add.s64 %rd41, %rd39, %rd11;\n" " st.global.f32 [%rd41], %f4;\n" " add.s64 %rd42, %rd40, %rd11;\n" " st.global.f32 [%rd42], %f3;\n" " add.s64 %rd43, %rd41, %rd11;\n" " st.global.f32 [%rd43], %f4;\n" " add.s64 %rd44, %rd42, %rd11;\n" " st.global.f32 [%rd44], %f3;\n" " add.s64 %rd45, %rd43, %rd11;\n" " st.global.f32 [%rd45], %f4;\n" " add.s32 %r46, %r46, 4;\n" " add.s64 %rd60, %rd60, %rd10;\n" " add.s32 %r45, %r45, -4;\n" " setp.ne.s32 %p7, %r45, 0;\n" " @%p7 bra $L__BB14_8;\n" "\n" "$L__BB14_9:\n" " setp.eq.s32 %p8, %r47, 0;\n" " @%p8 bra $L__BB14_12;\n" "\n" " mad.lo.s32 %r42, %r46, %r21, %r1;\n" " cvt.s64.s32 %rd61, %r42;\n" " cvt.s64.s32 %rd15, %r21;\n" "\n" "$L__BB14_11:\n" " .pragma \"nounroll\";\n" " mul.lo.s64 %rd46, %rd61, %rd6;\n" " add.s64 %rd47, %rd9, %rd46;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd49, %rd2, %rd48;\n" " st.global.f32 [%rd49], %f3;\n" " add.s64 %rd50, %rd1, %rd48;\n" " st.global.f32 [%rd50], %f4;\n" " add.s64 %rd61, %rd61, %rd15;\n" " add.s32 %r47, %r47, -1;\n" " setp.ne.s32 %p9, %r47, 0;\n" " @%p9 bra $L__BB14_11;\n" "\n" "$L__BB14_12:\n" " div.s32 %r18, %r25, %r21;\n" " setp.ge.s32 %p10, %r48, %r18;\n" " @%p10 bra $L__BB14_15;\n" "\n" " cvt.s64.s32 %rd51, %r18;\n" " mul.lo.s64 %rd18, %rd51, %rd7;\n" " cvta.to.global.u64 %rd19, %rd20;\n" "\n" "$L__BB14_14:\n" " cvt.s64.s32 %rd52, %r48;\n" " add.s64 %rd53, %rd18, %rd52;\n" " add.s64 %rd54, %rd4, %rd53;\n" " shl.b64 %rd55, %rd54, 2;\n" " add.s64 %rd56, %rd3, %rd55;\n" " ld.global.f32 %f22, [%rd56];\n" " add.s64 %rd57, %rd53, %rd5;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd59, %rd19, %rd58;\n" " st.global.f32 [%rd59], %f22;\n" " add.s32 %r48, %r48, %r4;\n" " setp.lt.s32 %p11, %r48, %r18;\n" " @%p11 bra $L__BB14_14;\n" "\n" "$L__BB14_15:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_FUSED_LAYER[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_fused_multi\n" ".extern .shared .align 16 .b8 sh[];\n" ".extern .shared .align 16 .b8 sdata[];\n" ".global .align 4 .b8 __cudart_i2opi_f[24] = {65, 144, 67, 60, 153, 149, 98, 219, 192, 221, 52, 245, 209, 87, 39, 252, 41, 21, 68, 78, 110, 131, 249, 162};\n" "\n" ".visible .entry fc_fused_multi(\n" " .param .u64 fc_fused_multi_param_0,\n" " .param .u32 fc_fused_multi_param_1,\n" " .param .u64 fc_fused_multi_param_2\n" ")\n" "{\n" " .reg .pred %p<116>;\n" " .reg .b16 %rs<61>;\n" " .reg .f32 %f<2040>;\n" " .reg .b32 %r<969>;\n" " .reg .b64 %rd<419>;\n" "\n" "\n" " ld.param.u64 %rd119, [fc_fused_multi_param_0];\n" " ld.param.u32 %r83, [fc_fused_multi_param_1];\n" " ld.param.u64 %rd118, [fc_fused_multi_param_2];\n" " cvta.to.global.u64 %rd1, %rd118;\n" " cvta.to.global.u64 %rd2, %rd119;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.lt.s32 %p1, %r83, 2;\n" " mov.u32 %r953, 0;\n" " @%p1 bra $L__BB0_6;\n" "\n" " add.s32 %r2, %r83, -1;\n" " and.b32 %r952, %r2, 3;\n" " add.s32 %r88, %r83, -2;\n" " setp.lt.u32 %p2, %r88, 3;\n" " mov.u32 %r949, 1;\n" " mov.u32 %r953, 0;\n" " @%p2 bra $L__BB0_4;\n" "\n" " sub.s32 %r946, %r2, %r952;\n" " mov.u32 %r949, 1;\n" " mov.u32 %r953, 0;\n" "\n" "$L__BB0_3:\n" " mul.wide.s32 %rd120, %r949, 80;\n" " add.s64 %rd121, %rd2, %rd120;\n" " ld.global.u32 %r91, [%rd121+72];\n" " setp.lt.u32 %p3, %r1, %r91;\n" " selp.b32 %r92, %r953, %r949, %p3;\n" " ld.global.u32 %r93, [%rd121+152];\n" " setp.lt.u32 %p4, %r1, %r93;\n" " add.s32 %r94, %r949, 1;\n" " selp.b32 %r95, %r92, %r94, %p4;\n" " ld.global.u32 %r96, [%rd121+232];\n" " setp.lt.u32 %p5, %r1, %r96;\n" " add.s32 %r97, %r949, 2;\n" " selp.b32 %r98, %r95, %r97, %p5;\n" " ld.global.u32 %r99, [%rd121+312];\n" " setp.lt.u32 %p6, %r1, %r99;\n" " add.s32 %r100, %r949, 3;\n" " selp.b32 %r953, %r98, %r100, %p6;\n" " add.s32 %r949, %r949, 4;\n" " add.s32 %r946, %r946, -4;\n" " setp.ne.s32 %p7, %r946, 0;\n" " @%p7 bra $L__BB0_3;\n" "\n" "$L__BB0_4:\n" " setp.eq.s32 %p8, %r952, 0;\n" " @%p8 bra $L__BB0_6;\n" "\n" "$L__BB0_5:\n" " .pragma \"nounroll\";\n" " mul.wide.s32 %rd122, %r949, 80;\n" " add.s64 %rd123, %rd2, %rd122;\n" " ld.global.u32 %r101, [%rd123+72];\n" " setp.lt.u32 %p9, %r1, %r101;\n" " selp.b32 %r953, %r953, %r949, %p9;\n" " add.s32 %r949, %r949, 1;\n" " add.s32 %r952, %r952, -1;\n" " setp.ne.s32 %p10, %r952, 0;\n" " @%p10 bra $L__BB0_5;\n" "\n" "$L__BB0_6:\n" " mul.wide.s32 %rd124, %r953, 80;\n" " add.s64 %rd125, %rd2, %rd124;\n" " add.s64 %rd3, %rd125, 48;\n" " ld.global.v2.u32 {%r102, %r103}, [%rd125+48];\n" " ld.global.v2.u32 {%r104, %r105}, [%rd125+56];\n" " ld.global.u32 %r106, [%rd125+72];\n" " sub.s32 %r25, %r1, %r106;\n" " setp.eq.s32 %p11, %r105, 1;\n" " @%p11 bra $L__BB0_138;\n" " bra.uni $L__BB0_7;\n" "\n" "$L__BB0_138:\n" " setp.ge.s32 %p97, %r25, %r102;\n" " @%p97 bra $L__BB0_167;\n" "\n" " mov.u32 %r67, %tid.x;\n" " mov.u32 %r68, %ntid.x;\n" " cvt.s64.s32 %rd104, %r25;\n" " mul.wide.s32 %rd105, %r25, %r104;\n" " ld.global.u64 %rd106, [%rd3+-48];\n" " setp.ge.s32 %p98, %r67, %r103;\n" " mov.f32 %f2037, 0f00000000;\n" " @%p98 bra $L__BB0_146;\n" "\n" " not.b32 %r897, %r67;\n" " add.s32 %r898, %r103, %r897;\n" " div.u32 %r69, %r898, %r68;\n" " add.s32 %r899, %r69, 1;\n" " and.b32 %r965, %r899, 3;\n" " setp.eq.s32 %p99, %r965, 0;\n" " mov.f32 %f2037, 0f00000000;\n" " mov.u32 %r966, %r67;\n" " @%p99 bra $L__BB0_143;\n" "\n" " mul.wide.s32 %rd318, %r67, 4;\n" " add.s64 %rd418, %rd1, %rd318;\n" " mul.wide.s32 %rd108, %r68, 4;\n" " add.s64 %rd319, %rd318, %rd105;\n" " add.s64 %rd417, %rd106, %rd319;\n" " mov.f32 %f2037, 0f00000000;\n" " mov.u32 %r966, %r67;\n" "\n" "$L__BB0_142:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f1881, [%rd418];\n" " ld.f32 %f1882, [%rd417];\n" " fma.rn.f32 %f2037, %f1882, %f1881, %f2037;\n" " add.s32 %r966, %r966, %r68;\n" " add.s64 %rd418, %rd418, %rd108;\n" " add.s64 %rd417, %rd417, %rd108;\n" " add.s32 %r965, %r965, -1;\n" " setp.ne.s32 %p100, %r965, 0;\n" " @%p100 bra $L__BB0_142;\n" "\n" "$L__BB0_143:\n" " setp.lt.u32 %p101, %r69, 3;\n" " add.s64 %rd114, %rd106, %rd105;\n" " @%p101 bra $L__BB0_146;\n" "\n" " mul.wide.s32 %rd115, %r68, 4;\n" "\n" "$L__BB0_145:\n" " mul.wide.s32 %rd320, %r966, 4;\n" " add.s64 %rd321, %rd114, %rd320;\n" " add.s64 %rd322, %rd1, %rd320;\n" " ld.global.f32 %f1883, [%rd322];\n" " ld.f32 %f1884, [%rd321];\n" " fma.rn.f32 %f1885, %f1884, %f1883, %f2037;\n" " add.s64 %rd323, %rd321, %rd115;\n" " add.s64 %rd324, %rd322, %rd115;\n" " ld.global.f32 %f1886, [%rd324];\n" " ld.f32 %f1887, [%rd323];\n" " fma.rn.f32 %f1888, %f1887, %f1886, %f1885;\n" " add.s32 %r900, %r966, %r68;\n" " add.s32 %r901, %r900, %r68;\n" " add.s64 %rd325, %rd323, %rd115;\n" " add.s64 %rd326, %rd324, %rd115;\n" " ld.global.f32 %f1889, [%rd326];\n" " ld.f32 %f1890, [%rd325];\n" " fma.rn.f32 %f1891, %f1890, %f1889, %f1888;\n" " add.s32 %r902, %r901, %r68;\n" " add.s64 %rd327, %rd325, %rd115;\n" " add.s64 %rd328, %rd326, %rd115;\n" " ld.global.f32 %f1892, [%rd328];\n" " ld.f32 %f1893, [%rd327];\n" " fma.rn.f32 %f2037, %f1893, %f1892, %f1891;\n" " add.s32 %r966, %r902, %r68;\n" " setp.lt.s32 %p102, %r966, %r103;\n" " @%p102 bra $L__BB0_145;\n" "\n" "$L__BB0_146:\n" " shl.b32 %r903, %r67, 2;\n" " mov.u32 %r904, sh;\n" " add.s32 %r78, %r904, %r903;\n" " st.shared.f32 [%r78], %f2037;\n" " bar.sync 0;\n" " setp.lt.s32 %p103, %r68, 2;\n" " @%p103 bra $L__BB0_151;\n" "\n" " shr.u32 %r905, %r68, 31;\n" " add.s32 %r906, %r68, %r905;\n" " shr.s32 %r968, %r906, 1;\n" "\n" "$L__BB0_148:\n" " setp.ge.s32 %p104, %r67, %r968;\n" " @%p104 bra $L__BB0_150;\n" "\n" " shl.b32 %r907, %r968, 2;\n" " add.s32 %r908, %r78, %r907;\n" " ld.shared.f32 %f1894, [%r78];\n" " ld.shared.f32 %f1895, [%r908];\n" " add.f32 %f1896, %f1895, %f1894;\n" " st.shared.f32 [%r78], %f1896;\n" "\n" "$L__BB0_150:\n" " bar.sync 0;\n" " shr.s32 %r81, %r968, 1;\n" " setp.gt.s32 %p105, %r968, 1;\n" " mov.u32 %r968, %r81;\n" " @%p105 bra $L__BB0_148;\n" "\n" "$L__BB0_151:\n" " setp.ne.s32 %p106, %r67, 0;\n" " @%p106 bra $L__BB0_167;\n" "\n" " ld.shared.f32 %f129, [sh];\n" " ld.global.u32 %r909, [%rd3+20];\n" " ld.global.u64 %rd329, [%rd3+-32];\n" " shl.b64 %rd330, %rd104, 2;\n" " add.s64 %rd116, %rd329, %rd330;\n" " setp.gt.s32 %p107, %r909, 2;\n" " @%p107 bra $L__BB0_156;\n" "\n" " setp.eq.s32 %p110, %r909, 1;\n" " @%p110 bra $L__BB0_166;\n" "\n" " setp.eq.s32 %p111, %r909, 2;\n" " @%p111 bra $L__BB0_160;\n" " bra.uni $L__BB0_155;\n" "\n" "$L__BB0_160:\n" " ld.global.u64 %rd117, [%rd3+-16];\n" " ld.global.u64 %rd340, [%rd3+-24];\n" " add.s64 %rd342, %rd340, %rd330;\n" " ld.f32 %f1905, [%rd342];\n" " add.f32 %f2039, %f129, %f1905;\n" " setp.gt.f32 %p112, %f2039, 0f41A00000;\n" " @%p112 bra $L__BB0_165;\n" "\n" " mov.f32 %f1906, 0f3F000000;\n" " mov.f32 %f1907, 0f3BBB989D;\n" " fma.rn.f32 %f1908, %f2039, %f1907, %f1906;\n" " mov.f32 %f1909, 0f3FB8AA3B;\n" " mov.f32 %f1910, 0f437C0000;\n" " cvt.sat.f32.f32 %f1911, %f1908;\n" " mov.f32 %f1912, 0f4B400001;\n" " fma.rm.f32 %f1913, %f1911, %f1910, %f1912;\n" " add.f32 %f1914, %f1913, 0fCB40007F;\n" " neg.f32 %f1915, %f1914;\n" " fma.rn.f32 %f1916, %f2039, %f1909, %f1915;\n" " mov.f32 %f1917, 0f32A57060;\n" " fma.rn.f32 %f1918, %f2039, %f1917, %f1916;\n" " mov.b32 %r910, %f1913;\n" " shl.b32 %r911, %r910, 23;\n" " mov.b32 %f1919, %r911;\n" " ex2.approx.ftz.f32 %f1920, %f1918;\n" " mul.f32 %f131, %f1920, %f1919;\n" " mov.f32 %f1921, 0f3F800000;\n" " add.rz.f32 %f1922, %f131, %f1921;\n" " mov.b32 %r912, %f1922;\n" " add.s32 %r913, %r912, -1061158912;\n" " and.b32 %r914, %r913, -8388608;\n" " mov.b32 %r82, %f131;\n" " sub.s32 %r915, %r82, %r914;\n" " mov.b32 %f1923, %r915;\n" " mov.u32 %r916, 1082130432;\n" " sub.s32 %r917, %r916, %r914;\n" " mov.b32 %f1924, %r917;\n" " mov.f32 %f1925, 0fBF800000;\n" " mov.f32 %f1926, 0f3E800000;\n" " fma.rn.f32 %f1927, %f1926, %f1924, %f1925;\n" " add.f32 %f1928, %f1927, %f1923;\n" " cvt.rn.f32.s32 %f1929, %r914;\n" " mul.f32 %f1930, %f1929, 0f34000000;\n" " mov.f32 %f1931, 0f3DD80012;\n" " mov.f32 %f1932, 0fBD39BF78;\n" " fma.rn.f32 %f1933, %f1932, %f1928, %f1931;\n" " mov.f32 %f1934, 0fBE0778E0;\n" " fma.rn.f32 %f1935, %f1933, %f1928, %f1934;\n" " mov.f32 %f1936, 0f3E146475;\n" " fma.rn.f32 %f1937, %f1935, %f1928, %f1936;\n" " mov.f32 %f1938, 0fBE2A68DD;\n" " fma.rn.f32 %f1939, %f1937, %f1928, %f1938;\n" " mov.f32 %f1940, 0f3E4CAF9E;\n" " fma.rn.f32 %f1941, %f1939, %f1928, %f1940;\n" " mov.f32 %f1942, 0fBE800042;\n" " fma.rn.f32 %f1943, %f1941, %f1928, %f1942;\n" " mov.f32 %f1944, 0f3EAAAAE6;\n" " fma.rn.f32 %f1945, %f1943, %f1928, %f1944;\n" " mov.f32 %f1946, 0fBF000000;\n" " fma.rn.f32 %f1947, %f1945, %f1928, %f1946;\n" " mul.f32 %f1948, %f1928, %f1947;\n" " fma.rn.f32 %f1949, %f1948, %f1928, %f1928;\n" " mov.f32 %f1950, 0f3F317218;\n" " fma.rn.f32 %f2039, %f1930, %f1950, %f1949;\n" " setp.lt.u32 %p113, %r82, 2139095040;\n" " @%p113 bra $L__BB0_165;\n" "\n" " setp.lt.s32 %p114, %r82, -1082130431;\n" " @%p114 bra $L__BB0_164;\n" "\n" " mov.f32 %f1951, 0f7F800000;\n" " fma.rn.f32 %f2039, %f131, %f1951, %f1951;\n" "\n" "$L__BB0_164:\n" " setp.eq.f32 %p115, %f131, 0f00000000;\n" " selp.f32 %f2039, 0f80000000, %f2039, %p115;\n" "\n" "$L__BB0_165:\n" " add.s64 %rd344, %rd117, %rd330;\n" " ld.f32 %f1952, [%rd344];\n" " mul.f32 %f1953, %f2039, %f1952;\n" " st.f32 [%rd116], %f1953;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_7:\n" " mov.u32 %r26, %tid.x;\n" " and.b32 %r27, %r26, 31;\n" " shr.u32 %r107, %r26, 4;\n" " and.b32 %r108, %r107, 268435454;\n" " shl.b32 %r109, %r25, 4;\n" " add.s32 %r110, %r109, %r108;\n" " or.b32 %r28, %r110, 1;\n" " setp.ge.s32 %p12, %r28, %r102;\n" " ld.global.u32 %r29, [%rd3+16];\n" " cvt.s64.s32 %rd4, %r110;\n" " cvt.s64.s32 %rd5, %r104;\n" " mul.wide.s32 %rd6, %r110, %r104;\n" " ld.global.u64 %rd126, [%rd3+-48];\n" " add.s64 %rd413, %rd126, %rd6;\n" " mov.u64 %rd414, %rd413;\n" " @%p12 bra $L__BB0_9;\n" "\n" " add.s64 %rd414, %rd413, %rd5;\n" "\n" "$L__BB0_9:\n" " ld.global.u64 %rd127, [%rd3+-40];\n" " add.s64 %rd393, %rd127, %rd6;\n" " mov.u64 %rd394, %rd393;\n" " @%p12 bra $L__BB0_11;\n" "\n" " add.s64 %rd394, %rd393, %rd5;\n" "\n" "$L__BB0_11:\n" " setp.lt.s32 %p14, %r103, 1;\n" " mov.f32 %f2017, 0f00000000;\n" " mov.f32 %f2018, %f2017;\n" " mov.f32 %f2019, %f2017;\n" " mov.f32 %f2020, %f2017;\n" " @%p14 bra $L__BB0_100;\n" "\n" " setp.eq.s32 %p15, %r29, 14;\n" " selp.b64 %rd128, 216, 152, %p15;\n" " setp.eq.s32 %p16, %r29, 13;\n" " selp.b64 %rd13, 184, %rd128, %p16;\n" " add.s32 %r113, %r27, 4;\n" " cvt.u64.u32 %rd14, %r113;\n" " cvt.u64.u32 %rd15, %r27;\n" " shr.u32 %r114, %r27, 4;\n" " cvt.u64.u32 %rd16, %r114;\n" " or.b32 %r115, %r27, 128;\n" " cvt.u64.u32 %rd17, %r115;\n" " mov.u32 %r116, -2;\n" " sub.s32 %r30, %r116, %r26;\n" " not.b32 %r31, %r103;\n" " cvt.s64.s32 %rd18, %r26;\n" " shl.b32 %r117, %r26, 2;\n" " mov.u32 %r118, sh;\n" " add.s32 %r32, %r118, %r117;\n" " add.s32 %r33, %r26, 256;\n" " cvt.s64.s32 %rd19, %r33;\n" " add.s32 %r34, %r26, 512;\n" " cvt.s64.s32 %rd20, %r34;\n" " add.s32 %r35, %r26, 768;\n" " mov.f32 %f2020, 0f00000000;\n" " mov.u32 %r954, 0;\n" " setp.eq.s32 %p24, %r29, 8;\n" " setp.ne.s32 %p56, %r105, 2;\n" " setp.eq.s32 %p26, %r29, 11;\n" " mov.u32 %r955, %r954;\n" " mov.f32 %f2019, %f2020;\n" " mov.f32 %f2018, %f2020;\n" " mov.f32 %f2017, %f2020;\n" "\n" "$L__BB0_13:\n" " shl.b32 %r119, %r954, 11;\n" " add.s32 %r120, %r119, %r31;\n" " max.s32 %r121, %r120, -2049;\n" " sub.s32 %r38, %r30, %r121;\n" " sub.s32 %r122, %r103, %r955;\n" " min.s32 %r39, %r122, 2048;\n" " bar.sync 0;\n" " setp.ge.s32 %p17, %r26, %r39;\n" " @%p17 bra $L__BB0_21;\n" "\n" " shr.u32 %r123, %r38, 8;\n" " add.s32 %r124, %r123, 1;\n" " cvt.s64.s32 %rd25, %r955;\n" " and.b32 %r40, %r124, 3;\n" " setp.eq.s32 %p18, %r40, 0;\n" " mov.u32 %r956, %r26;\n" " @%p18 bra $L__BB0_18;\n" "\n" " add.s64 %rd130, %rd18, %rd25;\n" " shl.b64 %rd131, %rd130, 2;\n" " add.s64 %rd129, %rd118, %rd131;\n" " // begin inline asm\n" " ld.global.nc.f32 %f145, [%rd129];\n" " // end inline asm\n" " st.shared.f32 [%r32], %f145;\n" " setp.eq.s32 %p19, %r40, 1;\n" " mov.u32 %r956, %r33;\n" " @%p19 bra $L__BB0_18;\n" "\n" " add.s64 %rd133, %rd19, %rd25;\n" " shl.b64 %rd134, %rd133, 2;\n" " add.s64 %rd132, %rd118, %rd134;\n" " // begin inline asm\n" " ld.global.nc.f32 %f146, [%rd132];\n" " // end inline asm\n" " st.shared.f32 [%r32+1024], %f146;\n" " setp.eq.s32 %p20, %r40, 2;\n" " mov.u32 %r956, %r34;\n" " @%p20 bra $L__BB0_18;\n" "\n" " add.s64 %rd136, %rd20, %rd25;\n" " shl.b64 %rd137, %rd136, 2;\n" " add.s64 %rd135, %rd118, %rd137;\n" " // begin inline asm\n" " ld.global.nc.f32 %f147, [%rd135];\n" " // end inline asm\n" " st.shared.f32 [%r32+2048], %f147;\n" " mov.u32 %r956, %r35;\n" "\n" "$L__BB0_18:\n" " shl.b32 %r933, %r954, 11;\n" " add.s32 %r932, %r933, %r31;\n" " max.s32 %r931, %r932, -2049;\n" " sub.s32 %r930, %r30, %r931;\n" " setp.lt.u32 %p21, %r930, 768;\n" " @%p21 bra $L__BB0_21;\n" "\n" " cvt.s64.s32 %rd138, %r956;\n" " add.s64 %rd139, %rd138, %rd25;\n" " shl.b64 %rd140, %rd139, 2;\n" " add.s64 %rd381, %rd118, %rd140;\n" " shl.b32 %r125, %r956, 2;\n" " mov.u32 %r126, sh;\n" " add.s32 %r127, %r126, %r125;\n" " add.s32 %r957, %r127, 2048;\n" "\n" "$L__BB0_20:\n" " sub.s32 %r935, %r103, %r955;\n" " min.s32 %r934, %r935, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f148, [%rd381];\n" " // end inline asm\n" " st.shared.f32 [%r957+-2048], %f148;\n" " add.s64 %rd142, %rd381, 1024;\n" " // begin inline asm\n" " ld.global.nc.f32 %f149, [%rd142];\n" " // end inline asm\n" " st.shared.f32 [%r957+-1024], %f149;\n" " add.s64 %rd143, %rd381, 2048;\n" " // begin inline asm\n" " ld.global.nc.f32 %f150, [%rd143];\n" " // end inline asm\n" " st.shared.f32 [%r957], %f150;\n" " add.s64 %rd144, %rd381, 3072;\n" " // begin inline asm\n" " ld.global.nc.f32 %f151, [%rd144];\n" " // end inline asm\n" " st.shared.f32 [%r957+1024], %f151;\n" " add.s64 %rd381, %rd381, 4096;\n" " add.s32 %r957, %r957, 4096;\n" " add.s32 %r956, %r956, 1024;\n" " setp.lt.s32 %p22, %r956, %r934;\n" " @%p22 bra $L__BB0_20;\n" "\n" "$L__BB0_21:\n" " cvt.u32.u64 %r128, %rd4;\n" " setp.ge.s32 %p23, %r128, %r102;\n" " bar.sync 0;\n" " @%p23 bra $L__BB0_99;\n" "\n" " @%p24 bra $L__BB0_43;\n" " bra.uni $L__BB0_23;\n" "\n" "$L__BB0_43:\n" " sub.s32 %r941, %r103, %r955;\n" " min.s32 %r940, %r941, 2048;\n" " setp.lt.s32 %p35, %r940, 32;\n" " @%p35 bra $L__BB0_99;\n" "\n" " sub.s32 %r943, %r103, %r955;\n" " min.s32 %r942, %r943, 2048;\n" " shr.s32 %r797, %r942, 5;\n" " max.s32 %r53, %r797, 1;\n" " add.s32 %r798, %r53, -1;\n" " and.b32 %r54, %r53, 3;\n" " setp.lt.u32 %p36, %r798, 3;\n" " mov.u32 %r963, sh;\n" " mov.u64 %rd404, %rd414;\n" " mov.u64 %rd403, %rd413;\n" " @%p36 bra $L__BB0_75;\n" "\n" " sub.s32 %r962, %r53, %r54;\n" " add.s64 %rd392, %rd413, %rd14;\n" " mov.u32 %r963, sh;\n" " mov.u64 %rd404, %rd414;\n" " mov.u64 %rd403, %rd413;\n" "\n" "$L__BB0_46:\n" " // begin inline asm\n" " ld.global.nc.f32 %f1539, [%rd403];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.s8 %r800, [%rd392];\n" " // end inline asm\n" " cvt.u16.u32 %rs5, %r800;\n" " cvt.s16.s8 %rs6, %rs5;\n" " cvt.rn.f32.s16 %f1540, %rs6;\n" " mul.rn.f32 %f1541, %f1539, %f1540;\n" " shl.b32 %r801, %r27, 2;\n" " add.s32 %r58, %r963, %r801;\n" " ld.shared.f32 %f1542, [%r58];\n" " fma.rn.f32 %f33, %f1541, %f1542, %f2017;\n" " @%p12 bra $L__BB0_48;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1543, [%rd404];\n" " // end inline asm\n" " add.s64 %rd233, %rd404, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r802, [%rd233];\n" " // end inline asm\n" " cvt.u16.u32 %rs7, %r802;\n" " cvt.s16.s8 %rs8, %rs7;\n" " cvt.rn.f32.s16 %f1544, %rs8;\n" " mul.rn.f32 %f1545, %f1543, %f1544;\n" " ld.shared.f32 %f1546, [%r58];\n" " fma.rn.f32 %f2019, %f1545, %f1546, %f2019;\n" "\n" "$L__BB0_48:\n" " @%p56 bra $L__BB0_51;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1547, [%rd393];\n" " // end inline asm\n" " add.s64 %rd235, %rd393, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r803, [%rd235];\n" " // end inline asm\n" " cvt.u16.u32 %rs9, %r803;\n" " cvt.s16.s8 %rs10, %rs9;\n" " cvt.rn.f32.s16 %f1548, %rs10;\n" " mul.rn.f32 %f1549, %f1547, %f1548;\n" " ld.shared.f32 %f1550, [%r58];\n" " fma.rn.f32 %f2018, %f1549, %f1550, %f2018;\n" " @%p12 bra $L__BB0_51;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1551, [%rd394];\n" " // end inline asm\n" " add.s64 %rd237, %rd394, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r804, [%rd237];\n" " // end inline asm\n" " cvt.u16.u32 %rs11, %r804;\n" " cvt.s16.s8 %rs12, %rs11;\n" " cvt.rn.f32.s16 %f1552, %rs12;\n" " mul.rn.f32 %f1553, %f1551, %f1552;\n" " ld.shared.f32 %f1554, [%r58];\n" " fma.rn.f32 %f2020, %f1553, %f1554, %f2020;\n" "\n" "$L__BB0_51:\n" " @%p56 bra $L__BB0_53;\n" "\n" " add.s64 %rd393, %rd393, 36;\n" " add.s64 %rd394, %rd394, 36;\n" "\n" "$L__BB0_53:\n" " add.s64 %rd238, %rd403, 36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1555, [%rd238];\n" " // end inline asm\n" " add.s64 %rd239, %rd392, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r805, [%rd239];\n" " // end inline asm\n" " cvt.u16.u32 %rs13, %r805;\n" " cvt.s16.s8 %rs14, %rs13;\n" " cvt.rn.f32.s16 %f1556, %rs14;\n" " mul.rn.f32 %f1557, %f1555, %f1556;\n" " ld.shared.f32 %f1558, [%r58+128];\n" " fma.rn.f32 %f40, %f1557, %f1558, %f33;\n" " @%p12 bra $L__BB0_55;\n" "\n" " add.s64 %rd370, %rd404, 36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1559, [%rd370];\n" " // end inline asm\n" " add.s64 %rd241, %rd370, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r806, [%rd241];\n" " // end inline asm\n" " cvt.u16.u32 %rs15, %r806;\n" " cvt.s16.s8 %rs16, %rs15;\n" " cvt.rn.f32.s16 %f1560, %rs16;\n" " mul.rn.f32 %f1561, %f1559, %f1560;\n" " ld.shared.f32 %f1562, [%r58+128];\n" " fma.rn.f32 %f2019, %f1561, %f1562, %f2019;\n" "\n" "$L__BB0_55:\n" " @%p56 bra $L__BB0_58;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1563, [%rd393];\n" " // end inline asm\n" " add.s64 %rd243, %rd393, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r807, [%rd243];\n" " // end inline asm\n" " cvt.u16.u32 %rs17, %r807;\n" " cvt.s16.s8 %rs18, %rs17;\n" " cvt.rn.f32.s16 %f1564, %rs18;\n" " mul.rn.f32 %f1565, %f1563, %f1564;\n" " ld.shared.f32 %f1566, [%r58+128];\n" " fma.rn.f32 %f2018, %f1565, %f1566, %f2018;\n" " @%p12 bra $L__BB0_58;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1567, [%rd394];\n" " // end inline asm\n" " add.s64 %rd245, %rd394, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r808, [%rd245];\n" " // end inline asm\n" " cvt.u16.u32 %rs19, %r808;\n" " cvt.s16.s8 %rs20, %rs19;\n" " cvt.rn.f32.s16 %f1568, %rs20;\n" " mul.rn.f32 %f1569, %f1567, %f1568;\n" " ld.shared.f32 %f1570, [%r58+128];\n" " fma.rn.f32 %f2020, %f1569, %f1570, %f2020;\n" "\n" "$L__BB0_58:\n" " @%p56 bra $L__BB0_60;\n" "\n" " add.s64 %rd393, %rd393, 36;\n" " add.s64 %rd394, %rd394, 36;\n" "\n" "$L__BB0_60:\n" " add.s64 %rd246, %rd403, 72;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1571, [%rd246];\n" " // end inline asm\n" " add.s64 %rd247, %rd392, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r809, [%rd247];\n" " // end inline asm\n" " cvt.u16.u32 %rs21, %r809;\n" " cvt.s16.s8 %rs22, %rs21;\n" " cvt.rn.f32.s16 %f1572, %rs22;\n" " mul.rn.f32 %f1573, %f1571, %f1572;\n" " ld.shared.f32 %f1574, [%r58+256];\n" " fma.rn.f32 %f47, %f1573, %f1574, %f40;\n" " @%p12 bra $L__BB0_62;\n" "\n" " add.s64 %rd371, %rd404, 72;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1575, [%rd371];\n" " // end inline asm\n" " add.s64 %rd249, %rd371, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r810, [%rd249];\n" " // end inline asm\n" " cvt.u16.u32 %rs23, %r810;\n" " cvt.s16.s8 %rs24, %rs23;\n" " cvt.rn.f32.s16 %f1576, %rs24;\n" " mul.rn.f32 %f1577, %f1575, %f1576;\n" " ld.shared.f32 %f1578, [%r58+256];\n" " fma.rn.f32 %f2019, %f1577, %f1578, %f2019;\n" "\n" "$L__BB0_62:\n" " @%p56 bra $L__BB0_65;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1579, [%rd393];\n" " // end inline asm\n" " add.s64 %rd251, %rd393, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r811, [%rd251];\n" " // end inline asm\n" " cvt.u16.u32 %rs25, %r811;\n" " cvt.s16.s8 %rs26, %rs25;\n" " cvt.rn.f32.s16 %f1580, %rs26;\n" " mul.rn.f32 %f1581, %f1579, %f1580;\n" " ld.shared.f32 %f1582, [%r58+256];\n" " fma.rn.f32 %f2018, %f1581, %f1582, %f2018;\n" " @%p12 bra $L__BB0_65;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1583, [%rd394];\n" " // end inline asm\n" " add.s64 %rd253, %rd394, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r812, [%rd253];\n" " // end inline asm\n" " cvt.u16.u32 %rs27, %r812;\n" " cvt.s16.s8 %rs28, %rs27;\n" " cvt.rn.f32.s16 %f1584, %rs28;\n" " mul.rn.f32 %f1585, %f1583, %f1584;\n" " ld.shared.f32 %f1586, [%r58+256];\n" " fma.rn.f32 %f2020, %f1585, %f1586, %f2020;\n" "\n" "$L__BB0_65:\n" " @%p56 bra $L__BB0_67;\n" "\n" " add.s64 %rd393, %rd393, 36;\n" " add.s64 %rd394, %rd394, 36;\n" "\n" "$L__BB0_67:\n" " add.s64 %rd254, %rd403, 108;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1587, [%rd254];\n" " // end inline asm\n" " add.s64 %rd255, %rd392, 108;\n" " // begin inline asm\n" " ld.global.nc.s8 %r813, [%rd255];\n" " // end inline asm\n" " cvt.u16.u32 %rs29, %r813;\n" " cvt.s16.s8 %rs30, %rs29;\n" " cvt.rn.f32.s16 %f1588, %rs30;\n" " mul.rn.f32 %f1589, %f1587, %f1588;\n" " ld.shared.f32 %f1590, [%r58+384];\n" " fma.rn.f32 %f2017, %f1589, %f1590, %f47;\n" " @%p12 bra $L__BB0_69;\n" "\n" " add.s64 %rd256, %rd404, 108;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1591, [%rd256];\n" " // end inline asm\n" " add.s64 %rd257, %rd256, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r814, [%rd257];\n" " // end inline asm\n" " cvt.u16.u32 %rs31, %r814;\n" " cvt.s16.s8 %rs32, %rs31;\n" " cvt.rn.f32.s16 %f1592, %rs32;\n" " mul.rn.f32 %f1593, %f1591, %f1592;\n" " ld.shared.f32 %f1594, [%r58+384];\n" " fma.rn.f32 %f2019, %f1593, %f1594, %f2019;\n" "\n" "$L__BB0_69:\n" " @%p56 bra $L__BB0_72;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1595, [%rd393];\n" " // end inline asm\n" " add.s64 %rd259, %rd393, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r815, [%rd259];\n" " // end inline asm\n" " cvt.u16.u32 %rs33, %r815;\n" " cvt.s16.s8 %rs34, %rs33;\n" " cvt.rn.f32.s16 %f1596, %rs34;\n" " mul.rn.f32 %f1597, %f1595, %f1596;\n" " ld.shared.f32 %f1598, [%r58+384];\n" " fma.rn.f32 %f2018, %f1597, %f1598, %f2018;\n" " @%p12 bra $L__BB0_72;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1599, [%rd394];\n" " // end inline asm\n" " add.s64 %rd261, %rd394, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r816, [%rd261];\n" " // end inline asm\n" " cvt.u16.u32 %rs35, %r816;\n" " cvt.s16.s8 %rs36, %rs35;\n" " cvt.rn.f32.s16 %f1600, %rs36;\n" " mul.rn.f32 %f1601, %f1599, %f1600;\n" " ld.shared.f32 %f1602, [%r58+384];\n" " fma.rn.f32 %f2020, %f1601, %f1602, %f2020;\n" "\n" "$L__BB0_72:\n" " @%p56 bra $L__BB0_74;\n" "\n" " add.s64 %rd393, %rd393, 36;\n" " add.s64 %rd394, %rd394, 36;\n" "\n" "$L__BB0_74:\n" " add.s64 %rd404, %rd404, 144;\n" " add.s32 %r963, %r963, 512;\n" " add.s32 %r962, %r962, -4;\n" " setp.ne.s32 %p53, %r962, 0;\n" " add.s64 %rd403, %rd403, 144;\n" " add.s64 %rd392, %rd392, 144;\n" " @%p53 bra $L__BB0_46;\n" "\n" "$L__BB0_75:\n" " setp.eq.s32 %p54, %r54, 0;\n" " mov.u64 %rd413, %rd403;\n" " mov.u64 %rd414, %rd404;\n" " @%p54 bra $L__BB0_99;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1603, [%rd403];\n" " // end inline asm\n" " add.s64 %rd263, %rd403, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r817, [%rd263];\n" " // end inline asm\n" " cvt.u16.u32 %rs37, %r817;\n" " cvt.s16.s8 %rs38, %rs37;\n" " cvt.rn.f32.s16 %f1604, %rs38;\n" " mul.rn.f32 %f1605, %f1603, %f1604;\n" " shl.b32 %r818, %r27, 2;\n" " add.s32 %r62, %r963, %r818;\n" " ld.shared.f32 %f1606, [%r62];\n" " fma.rn.f32 %f2017, %f1605, %f1606, %f2017;\n" " @%p12 bra $L__BB0_78;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1607, [%rd404];\n" " // end inline asm\n" " add.s64 %rd265, %rd404, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r819, [%rd265];\n" " // end inline asm\n" " cvt.u16.u32 %rs39, %r819;\n" " cvt.s16.s8 %rs40, %rs39;\n" " cvt.rn.f32.s16 %f1608, %rs40;\n" " mul.rn.f32 %f1609, %f1607, %f1608;\n" " ld.shared.f32 %f1610, [%r62];\n" " fma.rn.f32 %f2019, %f1609, %f1610, %f2019;\n" "\n" "$L__BB0_78:\n" " @%p56 bra $L__BB0_81;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1611, [%rd393];\n" " // end inline asm\n" " add.s64 %rd267, %rd393, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r820, [%rd267];\n" " // end inline asm\n" " cvt.u16.u32 %rs41, %r820;\n" " cvt.s16.s8 %rs42, %rs41;\n" " cvt.rn.f32.s16 %f1612, %rs42;\n" " mul.rn.f32 %f1613, %f1611, %f1612;\n" " ld.shared.f32 %f1614, [%r62];\n" " fma.rn.f32 %f2018, %f1613, %f1614, %f2018;\n" " @%p12 bra $L__BB0_81;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1615, [%rd394];\n" " // end inline asm\n" " add.s64 %rd269, %rd394, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r821, [%rd269];\n" " // end inline asm\n" " cvt.u16.u32 %rs43, %r821;\n" " cvt.s16.s8 %rs44, %rs43;\n" " cvt.rn.f32.s16 %f1616, %rs44;\n" " mul.rn.f32 %f1617, %f1615, %f1616;\n" " ld.shared.f32 %f1618, [%r62];\n" " fma.rn.f32 %f2020, %f1617, %f1618, %f2020;\n" "\n" "$L__BB0_81:\n" " @%p56 bra $L__BB0_83;\n" "\n" " add.s64 %rd393, %rd393, 36;\n" " add.s64 %rd394, %rd394, 36;\n" "\n" "$L__BB0_83:\n" " add.s64 %rd413, %rd403, 36;\n" " add.s64 %rd414, %rd404, 36;\n" " setp.eq.s32 %p59, %r54, 1;\n" " @%p59 bra $L__BB0_99;\n" "\n" " add.s64 %rd372, %rd403, %rd14;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1619, [%rd413];\n" " // end inline asm\n" " add.s64 %rd271, %rd372, 36;\n" " // begin inline asm\n" " ld.global.nc.s8 %r822, [%rd271];\n" " // end inline asm\n" " cvt.u16.u32 %rs45, %r822;\n" " cvt.s16.s8 %rs46, %rs45;\n" " cvt.rn.f32.s16 %f1620, %rs46;\n" " mul.rn.f32 %f1621, %f1619, %f1620;\n" " ld.shared.f32 %f1622, [%r62+128];\n" " fma.rn.f32 %f2017, %f1621, %f1622, %f2017;\n" " @%p12 bra $L__BB0_86;\n" "\n" " add.s64 %rd374, %rd404, 36;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1623, [%rd374];\n" " // end inline asm\n" " add.s64 %rd273, %rd374, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r823, [%rd273];\n" " // end inline asm\n" " cvt.u16.u32 %rs47, %r823;\n" " cvt.s16.s8 %rs48, %rs47;\n" " cvt.rn.f32.s16 %f1624, %rs48;\n" " mul.rn.f32 %f1625, %f1623, %f1624;\n" " ld.shared.f32 %f1626, [%r62+128];\n" " fma.rn.f32 %f2019, %f1625, %f1626, %f2019;\n" "\n" "$L__BB0_86:\n" " @%p56 bra $L__BB0_89;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1627, [%rd393];\n" " // end inline asm\n" " add.s64 %rd275, %rd393, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r824, [%rd275];\n" " // end inline asm\n" " cvt.u16.u32 %rs49, %r824;\n" " cvt.s16.s8 %rs50, %rs49;\n" " cvt.rn.f32.s16 %f1628, %rs50;\n" " mul.rn.f32 %f1629, %f1627, %f1628;\n" " ld.shared.f32 %f1630, [%r62+128];\n" " fma.rn.f32 %f2018, %f1629, %f1630, %f2018;\n" " @%p12 bra $L__BB0_89;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1631, [%rd394];\n" " // end inline asm\n" " add.s64 %rd277, %rd394, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r825, [%rd277];\n" " // end inline asm\n" " cvt.u16.u32 %rs51, %r825;\n" " cvt.s16.s8 %rs52, %rs51;\n" " cvt.rn.f32.s16 %f1632, %rs52;\n" " mul.rn.f32 %f1633, %f1631, %f1632;\n" " ld.shared.f32 %f1634, [%r62+128];\n" " fma.rn.f32 %f2020, %f1633, %f1634, %f2020;\n" "\n" "$L__BB0_89:\n" " @%p56 bra $L__BB0_91;\n" "\n" " add.s64 %rd393, %rd393, 36;\n" " add.s64 %rd394, %rd394, 36;\n" "\n" "$L__BB0_91:\n" " add.s64 %rd413, %rd403, 72;\n" " add.s64 %rd414, %rd404, 72;\n" " setp.eq.s32 %p64, %r54, 2;\n" " @%p64 bra $L__BB0_99;\n" "\n" " add.s64 %rd373, %rd403, %rd14;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1635, [%rd413];\n" " // end inline asm\n" " add.s64 %rd279, %rd373, 72;\n" " // begin inline asm\n" " ld.global.nc.s8 %r826, [%rd279];\n" " // end inline asm\n" " cvt.u16.u32 %rs53, %r826;\n" " cvt.s16.s8 %rs54, %rs53;\n" " cvt.rn.f32.s16 %f1636, %rs54;\n" " mul.rn.f32 %f1637, %f1635, %f1636;\n" " ld.shared.f32 %f1638, [%r62+256];\n" " fma.rn.f32 %f2017, %f1637, %f1638, %f2017;\n" " @%p12 bra $L__BB0_94;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1639, [%rd414];\n" " // end inline asm\n" " add.s64 %rd281, %rd414, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r827, [%rd281];\n" " // end inline asm\n" " cvt.u16.u32 %rs55, %r827;\n" " cvt.s16.s8 %rs56, %rs55;\n" " cvt.rn.f32.s16 %f1640, %rs56;\n" " mul.rn.f32 %f1641, %f1639, %f1640;\n" " ld.shared.f32 %f1642, [%r62+256];\n" " fma.rn.f32 %f2019, %f1641, %f1642, %f2019;\n" "\n" "$L__BB0_94:\n" " @%p56 bra $L__BB0_97;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1643, [%rd393];\n" " // end inline asm\n" " add.s64 %rd283, %rd393, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r828, [%rd283];\n" " // end inline asm\n" " cvt.u16.u32 %rs57, %r828;\n" " cvt.s16.s8 %rs58, %rs57;\n" " cvt.rn.f32.s16 %f1644, %rs58;\n" " mul.rn.f32 %f1645, %f1643, %f1644;\n" " ld.shared.f32 %f1646, [%r62+256];\n" " fma.rn.f32 %f2018, %f1645, %f1646, %f2018;\n" " @%p12 bra $L__BB0_97;\n" "\n" " // begin inline asm\n" " ld.global.nc.f32 %f1647, [%rd394];\n" " // end inline asm\n" " add.s64 %rd285, %rd394, %rd14;\n" " // begin inline asm\n" " ld.global.nc.s8 %r829, [%rd285];\n" " // end inline asm\n" " cvt.u16.u32 %rs59, %r829;\n" " cvt.s16.s8 %rs60, %rs59;\n" " cvt.rn.f32.s16 %f1648, %rs60;\n" " mul.rn.f32 %f1649, %f1647, %f1648;\n" " ld.shared.f32 %f1650, [%r62+256];\n" " fma.rn.f32 %f2020, %f1649, %f1650, %f2020;\n" "\n" "$L__BB0_97:\n" " add.s64 %rd413, %rd403, 108;\n" " add.s64 %rd414, %rd404, 108;\n" " @%p56 bra $L__BB0_99;\n" "\n" " add.s64 %rd393, %rd393, 36;\n" " add.s64 %rd394, %rd394, 36;\n" " bra.uni $L__BB0_99;\n" "\n" "$L__BB0_23:\n" " sub.s32 %r937, %r103, %r955;\n" " min.s32 %r936, %r937, 2048;\n" " setp.lt.s32 %p25, %r936, 256;\n" " @%p25 bra $L__BB0_99;\n" "\n" " mov.u32 %r959, 0;\n" " mov.u32 %r960, %r118;\n" "\n" "$L__BB0_25:\n" " shl.b32 %r131, %r27, 2;\n" " add.s32 %r50, %r960, %r131;\n" " @%p26 bra $L__BB0_30;\n" "\n" " setp.eq.s32 %p27, %r29, 12;\n" " @%p27 bra $L__BB0_29;\n" "\n" " setp.ne.s32 %p28, %r29, 13;\n" " @%p28 bra $L__BB0_31;\n" "\n" " shl.b64 %rd351, %rd15, 2;\n" " add.s64 %rd350, %rd414, %rd351;\n" " add.s64 %rd349, %rd413, %rd351;\n" " // begin inline asm\n" " ld.global.nc.u32 %r132, [%rd349];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r133, [%rd350];\n" " // end inline asm\n" " add.s64 %rd148, %rd413, %rd17;\n" " // begin inline asm\n" " ld.global.nc.u8 %r134, [%rd148];\n" " // end inline asm\n" " add.s64 %rd149, %rd414, %rd17;\n" " // begin inline asm\n" " ld.global.nc.u8 %r135, [%rd149];\n" " // end inline asm\n" " add.s64 %rd150, %rd413, 160;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r136,%r137}, [%rd150];\n" " // end inline asm\n" " add.s64 %rd151, %rd414, 160;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r138,%r139}, [%rd151];\n" " // end inline asm\n" " add.s64 %rd152, %rd413, 168;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r140,%r141}, [%rd152];\n" " // end inline asm\n" " add.s64 %rd153, %rd414, 168;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r142,%r143}, [%rd153];\n" " // end inline asm\n" " add.s64 %rd154, %rd413, 176;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f152,%f153}, [%rd154];\n" " // end inline asm\n" " add.s64 %rd155, %rd414, 176;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f154,%f155}, [%rd155];\n" " // end inline asm\n" " shr.u32 %r144, %r132, 16;\n" " shr.u32 %r145, %r133, 16;\n" " and.b32 %r146, %r132, 15;\n" " or.b32 %r147, %r146, 1258291200;\n" " mov.b32 %f156, %r147;\n" " mov.f32 %f157, 0fCB000000;\n" " mov.f32 %f158, 0f3F800000;\n" " fma.rn.f32 %f159, %f156, %f158, %f157;\n" " and.b32 %r148, %r134, 1;\n" " or.b32 %r149, %r148, 1258291200;\n" " mov.b32 %f160, %r149;\n" " mov.f32 %f161, 0fCD000000;\n" " mov.f32 %f162, 0f41800000;\n" " fma.rn.f32 %f163, %f160, %f162, %f161;\n" " add.rn.f32 %f164, %f159, %f163;\n" " and.b32 %r150, %r133, 15;\n" " or.b32 %r151, %r150, 1258291200;\n" " mov.b32 %f165, %r151;\n" " fma.rn.f32 %f166, %f165, %f158, %f157;\n" " and.b32 %r152, %r135, 1;\n" " or.b32 %r153, %r152, 1258291200;\n" " mov.b32 %f167, %r153;\n" " fma.rn.f32 %f168, %f167, %f162, %f161;\n" " add.rn.f32 %f169, %f166, %f168;\n" " and.b32 %r154, %r136, 255;\n" " cvt.rn.f32.u32 %f170, %r154;\n" " mul.rn.f32 %f171, %f152, %f170;\n" " mul.rn.f32 %f172, %f171, %f164;\n" " and.b32 %r155, %r140, 255;\n" " cvt.rn.f32.u32 %f173, %r155;\n" " mul.rn.f32 %f174, %f153, %f173;\n" " sub.rn.f32 %f175, %f172, %f174;\n" " and.b32 %r156, %r138, 255;\n" " cvt.rn.f32.u32 %f176, %r156;\n" " mul.rn.f32 %f177, %f154, %f176;\n" " mul.rn.f32 %f178, %f177, %f169;\n" " and.b32 %r157, %r142, 255;\n" " cvt.rn.f32.u32 %f179, %r157;\n" " mul.rn.f32 %f180, %f155, %f179;\n" " sub.rn.f32 %f181, %f178, %f180;\n" " ld.shared.f32 %f182, [%r50];\n" " mov.f32 %f183, 0f00000000;\n" " fma.rn.f32 %f184, %f175, %f182, %f183;\n" " fma.rn.f32 %f185, %f181, %f182, %f183;\n" " and.b32 %r158, %r132, 240;\n" " or.b32 %r159, %r158, 1258291200;\n" " mov.b32 %f186, %r159;\n" " mov.f32 %f187, 0fC9000000;\n" " mov.f32 %f188, 0f3D800000;\n" " fma.rn.f32 %f189, %f186, %f188, %f187;\n" " and.b32 %r160, %r134, 2;\n" " or.b32 %r161, %r160, 1258291200;\n" " mov.b32 %f190, %r161;\n" " mov.f32 %f191, 0fCC800000;\n" " mov.f32 %f192, 0f41000000;\n" " fma.rn.f32 %f193, %f190, %f192, %f191;\n" " add.rn.f32 %f194, %f189, %f193;\n" " and.b32 %r162, %r133, 240;\n" " or.b32 %r163, %r162, 1258291200;\n" " mov.b32 %f195, %r163;\n" " fma.rn.f32 %f196, %f195, %f188, %f187;\n" " and.b32 %r164, %r135, 2;\n" " or.b32 %r165, %r164, 1258291200;\n" " mov.b32 %f197, %r165;\n" " fma.rn.f32 %f198, %f197, %f192, %f191;\n" " add.rn.f32 %f199, %f196, %f198;\n" " bfe.u32 %r166, %r136, 8, 8;\n" " cvt.rn.f32.u32 %f200, %r166;\n" " mul.rn.f32 %f201, %f152, %f200;\n" " mul.rn.f32 %f202, %f201, %f194;\n" " bfe.u32 %r167, %r140, 8, 8;\n" " cvt.rn.f32.u32 %f203, %r167;\n" " mul.rn.f32 %f204, %f153, %f203;\n" " sub.rn.f32 %f205, %f202, %f204;\n" " bfe.u32 %r168, %r138, 8, 8;\n" " cvt.rn.f32.u32 %f206, %r168;\n" " mul.rn.f32 %f207, %f154, %f206;\n" " mul.rn.f32 %f208, %f207, %f199;\n" " bfe.u32 %r169, %r142, 8, 8;\n" " cvt.rn.f32.u32 %f209, %r169;\n" " mul.rn.f32 %f210, %f155, %f209;\n" " sub.rn.f32 %f211, %f208, %f210;\n" " ld.shared.f32 %f212, [%r50+128];\n" " fma.rn.f32 %f213, %f205, %f212, %f184;\n" " fma.rn.f32 %f214, %f211, %f212, %f185;\n" " and.b32 %r170, %r132, 3840;\n" " or.b32 %r171, %r170, 1258291200;\n" " mov.b32 %f215, %r171;\n" " mov.f32 %f216, 0fC7000000;\n" " mov.f32 %f217, 0f3B800000;\n" " fma.rn.f32 %f218, %f215, %f217, %f216;\n" " and.b32 %r172, %r134, 4;\n" " or.b32 %r173, %r172, 1258291200;\n" " mov.b32 %f219, %r173;\n" " mov.f32 %f220, 0fCC000000;\n" " mov.f32 %f221, 0f40800000;\n" " fma.rn.f32 %f222, %f219, %f221, %f220;\n" " add.rn.f32 %f223, %f218, %f222;\n" " and.b32 %r174, %r133, 3840;\n" " or.b32 %r175, %r174, 1258291200;\n" " mov.b32 %f224, %r175;\n" " fma.rn.f32 %f225, %f224, %f217, %f216;\n" " and.b32 %r176, %r135, 4;\n" " or.b32 %r177, %r176, 1258291200;\n" " mov.b32 %f226, %r177;\n" " fma.rn.f32 %f227, %f226, %f221, %f220;\n" " add.rn.f32 %f228, %f225, %f227;\n" " bfe.u32 %r178, %r136, 16, 8;\n" " cvt.rn.f32.u32 %f229, %r178;\n" " mul.rn.f32 %f230, %f152, %f229;\n" " mul.rn.f32 %f231, %f230, %f223;\n" " bfe.u32 %r179, %r140, 16, 8;\n" " cvt.rn.f32.u32 %f232, %r179;\n" " mul.rn.f32 %f233, %f153, %f232;\n" " sub.rn.f32 %f234, %f231, %f233;\n" " bfe.u32 %r180, %r138, 16, 8;\n" " cvt.rn.f32.u32 %f235, %r180;\n" " mul.rn.f32 %f236, %f154, %f235;\n" " mul.rn.f32 %f237, %f236, %f228;\n" " bfe.u32 %r181, %r142, 16, 8;\n" " cvt.rn.f32.u32 %f238, %r181;\n" " mul.rn.f32 %f239, %f155, %f238;\n" " sub.rn.f32 %f240, %f237, %f239;\n" " ld.shared.f32 %f241, [%r50+256];\n" " fma.rn.f32 %f242, %f234, %f241, %f213;\n" " fma.rn.f32 %f243, %f240, %f241, %f214;\n" " and.b32 %r182, %r132, 61440;\n" " or.b32 %r183, %r182, 1258291200;\n" " mov.b32 %f244, %r183;\n" " mov.f32 %f245, 0fC5000000;\n" " mov.f32 %f246, 0f39800000;\n" " fma.rn.f32 %f247, %f244, %f246, %f245;\n" " and.b32 %r184, %r134, 8;\n" " or.b32 %r185, %r184, 1258291200;\n" " mov.b32 %f248, %r185;\n" " mov.f32 %f249, 0fCB800000;\n" " mov.f32 %f250, 0f40000000;\n" " fma.rn.f32 %f251, %f248, %f250, %f249;\n" " add.rn.f32 %f252, %f247, %f251;\n" " and.b32 %r186, %r133, 61440;\n" " or.b32 %r187, %r186, 1258291200;\n" " mov.b32 %f253, %r187;\n" " fma.rn.f32 %f254, %f253, %f246, %f245;\n" " and.b32 %r188, %r135, 8;\n" " or.b32 %r189, %r188, 1258291200;\n" " mov.b32 %f255, %r189;\n" " fma.rn.f32 %f256, %f255, %f250, %f249;\n" " add.rn.f32 %f257, %f254, %f256;\n" " shr.u32 %r190, %r136, 24;\n" " cvt.rn.f32.u32 %f258, %r190;\n" " mul.rn.f32 %f259, %f152, %f258;\n" " mul.rn.f32 %f260, %f259, %f252;\n" " shr.u32 %r191, %r140, 24;\n" " cvt.rn.f32.u32 %f261, %r191;\n" " mul.rn.f32 %f262, %f153, %f261;\n" " sub.rn.f32 %f263, %f260, %f262;\n" " shr.u32 %r192, %r138, 24;\n" " cvt.rn.f32.u32 %f264, %r192;\n" " mul.rn.f32 %f265, %f154, %f264;\n" " mul.rn.f32 %f266, %f265, %f257;\n" " shr.u32 %r193, %r142, 24;\n" " cvt.rn.f32.u32 %f267, %r193;\n" " mul.rn.f32 %f268, %f155, %f267;\n" " sub.rn.f32 %f269, %f266, %f268;\n" " ld.shared.f32 %f270, [%r50+384];\n" " fma.rn.f32 %f271, %f263, %f270, %f242;\n" " fma.rn.f32 %f272, %f269, %f270, %f243;\n" " and.b32 %r194, %r132, 983040;\n" " or.b32 %r195, %r194, 1258291200;\n" " mov.b32 %f273, %r195;\n" " mov.f32 %f274, 0fC3000000;\n" " mov.f32 %f275, 0f37800000;\n" " fma.rn.f32 %f276, %f273, %f275, %f274;\n" " and.b32 %r196, %r134, 16;\n" " or.b32 %r197, %r196, 1258291200;\n" " mov.b32 %f277, %r197;\n" " fma.rn.f32 %f278, %f277, %f158, %f157;\n" " add.rn.f32 %f279, %f276, %f278;\n" " and.b32 %r198, %r133, 983040;\n" " or.b32 %r199, %r198, 1258291200;\n" " mov.b32 %f280, %r199;\n" " fma.rn.f32 %f281, %f280, %f275, %f274;\n" " and.b32 %r200, %r135, 16;\n" " or.b32 %r201, %r200, 1258291200;\n" " mov.b32 %f282, %r201;\n" " fma.rn.f32 %f283, %f282, %f158, %f157;\n" " add.rn.f32 %f284, %f281, %f283;\n" " and.b32 %r202, %r137, 255;\n" " cvt.rn.f32.u32 %f285, %r202;\n" " mul.rn.f32 %f286, %f152, %f285;\n" " mul.rn.f32 %f287, %f286, %f279;\n" " and.b32 %r203, %r141, 255;\n" " cvt.rn.f32.u32 %f288, %r203;\n" " mul.rn.f32 %f289, %f153, %f288;\n" " sub.rn.f32 %f290, %f287, %f289;\n" " and.b32 %r204, %r139, 255;\n" " cvt.rn.f32.u32 %f291, %r204;\n" " mul.rn.f32 %f292, %f154, %f291;\n" " mul.rn.f32 %f293, %f292, %f284;\n" " and.b32 %r205, %r143, 255;\n" " cvt.rn.f32.u32 %f294, %r205;\n" " mul.rn.f32 %f295, %f155, %f294;\n" " sub.rn.f32 %f296, %f293, %f295;\n" " ld.shared.f32 %f297, [%r50+512];\n" " fma.rn.f32 %f298, %f290, %f297, %f271;\n" " fma.rn.f32 %f299, %f296, %f297, %f272;\n" " and.b32 %r206, %r144, 240;\n" " or.b32 %r207, %r206, 1258291200;\n" " mov.b32 %f300, %r207;\n" " fma.rn.f32 %f301, %f300, %f188, %f187;\n" " and.b32 %r208, %r134, 32;\n" " or.b32 %r209, %r208, 1258291200;\n" " mov.b32 %f302, %r209;\n" " mov.f32 %f303, 0fCA800000;\n" " mov.f32 %f304, 0f3F000000;\n" " fma.rn.f32 %f305, %f302, %f304, %f303;\n" " add.rn.f32 %f306, %f301, %f305;\n" " and.b32 %r210, %r145, 240;\n" " or.b32 %r211, %r210, 1258291200;\n" " mov.b32 %f307, %r211;\n" " fma.rn.f32 %f308, %f307, %f188, %f187;\n" " and.b32 %r212, %r135, 32;\n" " or.b32 %r213, %r212, 1258291200;\n" " mov.b32 %f309, %r213;\n" " fma.rn.f32 %f310, %f309, %f304, %f303;\n" " add.rn.f32 %f311, %f308, %f310;\n" " bfe.u32 %r214, %r137, 8, 8;\n" " cvt.rn.f32.u32 %f312, %r214;\n" " mul.rn.f32 %f313, %f152, %f312;\n" " mul.rn.f32 %f314, %f313, %f306;\n" " bfe.u32 %r215, %r141, 8, 8;\n" " cvt.rn.f32.u32 %f315, %r215;\n" " mul.rn.f32 %f316, %f153, %f315;\n" " sub.rn.f32 %f317, %f314, %f316;\n" " bfe.u32 %r216, %r139, 8, 8;\n" " cvt.rn.f32.u32 %f318, %r216;\n" " mul.rn.f32 %f319, %f154, %f318;\n" " mul.rn.f32 %f320, %f319, %f311;\n" " bfe.u32 %r217, %r143, 8, 8;\n" " cvt.rn.f32.u32 %f321, %r217;\n" " mul.rn.f32 %f322, %f155, %f321;\n" " sub.rn.f32 %f323, %f320, %f322;\n" " ld.shared.f32 %f324, [%r50+640];\n" " fma.rn.f32 %f325, %f317, %f324, %f298;\n" " fma.rn.f32 %f326, %f323, %f324, %f299;\n" " and.b32 %r218, %r144, 3840;\n" " or.b32 %r219, %r218, 1258291200;\n" " mov.b32 %f327, %r219;\n" " fma.rn.f32 %f328, %f327, %f217, %f216;\n" " and.b32 %r220, %r134, 64;\n" " or.b32 %r221, %r220, 1258291200;\n" " mov.b32 %f329, %r221;\n" " mov.f32 %f330, 0fCA000000;\n" " mov.f32 %f331, 0f3E800000;\n" " fma.rn.f32 %f332, %f329, %f331, %f330;\n" " add.rn.f32 %f333, %f328, %f332;\n" " and.b32 %r222, %r145, 3840;\n" " or.b32 %r223, %r222, 1258291200;\n" " mov.b32 %f334, %r223;\n" " fma.rn.f32 %f335, %f334, %f217, %f216;\n" " and.b32 %r224, %r135, 64;\n" " or.b32 %r225, %r224, 1258291200;\n" " mov.b32 %f336, %r225;\n" " fma.rn.f32 %f337, %f336, %f331, %f330;\n" " add.rn.f32 %f338, %f335, %f337;\n" " bfe.u32 %r226, %r137, 16, 8;\n" " cvt.rn.f32.u32 %f339, %r226;\n" " mul.rn.f32 %f340, %f152, %f339;\n" " mul.rn.f32 %f341, %f340, %f333;\n" " bfe.u32 %r227, %r141, 16, 8;\n" " cvt.rn.f32.u32 %f342, %r227;\n" " mul.rn.f32 %f343, %f153, %f342;\n" " sub.rn.f32 %f344, %f341, %f343;\n" " bfe.u32 %r228, %r139, 16, 8;\n" " cvt.rn.f32.u32 %f345, %r228;\n" " mul.rn.f32 %f346, %f154, %f345;\n" " mul.rn.f32 %f347, %f346, %f338;\n" " bfe.u32 %r229, %r143, 16, 8;\n" " cvt.rn.f32.u32 %f348, %r229;\n" " mul.rn.f32 %f349, %f155, %f348;\n" " sub.rn.f32 %f350, %f347, %f349;\n" " ld.shared.f32 %f351, [%r50+768];\n" " fma.rn.f32 %f352, %f344, %f351, %f325;\n" " fma.rn.f32 %f353, %f350, %f351, %f326;\n" " and.b32 %r230, %r144, 61440;\n" " or.b32 %r231, %r230, 1258291200;\n" " mov.b32 %f354, %r231;\n" " fma.rn.f32 %f355, %f354, %f246, %f245;\n" " and.b32 %r232, %r134, 128;\n" " or.b32 %r233, %r232, 1258291200;\n" " mov.b32 %f356, %r233;\n" " mov.f32 %f357, 0fC9800000;\n" " mov.f32 %f358, 0f3E000000;\n" " fma.rn.f32 %f359, %f356, %f358, %f357;\n" " add.rn.f32 %f360, %f355, %f359;\n" " and.b32 %r234, %r145, 61440;\n" " or.b32 %r235, %r234, 1258291200;\n" " mov.b32 %f361, %r235;\n" " fma.rn.f32 %f362, %f361, %f246, %f245;\n" " and.b32 %r236, %r135, 128;\n" " or.b32 %r237, %r236, 1258291200;\n" " mov.b32 %f363, %r237;\n" " fma.rn.f32 %f364, %f363, %f358, %f357;\n" " add.rn.f32 %f365, %f362, %f364;\n" " shr.u32 %r238, %r137, 24;\n" " cvt.rn.f32.u32 %f366, %r238;\n" " mul.rn.f32 %f367, %f152, %f366;\n" " mul.rn.f32 %f368, %f367, %f360;\n" " shr.u32 %r239, %r141, 24;\n" " cvt.rn.f32.u32 %f369, %r239;\n" " mul.rn.f32 %f370, %f153, %f369;\n" " sub.rn.f32 %f371, %f368, %f370;\n" " shr.u32 %r240, %r139, 24;\n" " cvt.rn.f32.u32 %f372, %r240;\n" " mul.rn.f32 %f373, %f154, %f372;\n" " mul.rn.f32 %f374, %f373, %f365;\n" " shr.u32 %r241, %r143, 24;\n" " cvt.rn.f32.u32 %f375, %r241;\n" " mul.rn.f32 %f376, %f155, %f375;\n" " sub.rn.f32 %f377, %f374, %f376;\n" " ld.shared.f32 %f378, [%r50+896];\n" " fma.rn.f32 %f379, %f371, %f378, %f352;\n" " fma.rn.f32 %f380, %f377, %f378, %f353;\n" " add.rn.f32 %f2017, %f2017, %f379;\n" " add.rn.f32 %f2019, %f2019, %f380;\n" " bra.uni $L__BB0_32;\n" "\n" "$L__BB0_30:\n" " shl.b64 %rd357, %rd15, 2;\n" " add.s64 %rd356, %rd414, %rd357;\n" " add.s64 %rd355, %rd413, %rd357;\n" " // begin inline asm\n" " ld.global.nc.u32 %r318, [%rd355];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r319, [%rd356];\n" " // end inline asm\n" " add.s64 %rd166, %rd413, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f548, [%rd166];\n" " // end inline asm\n" " add.s64 %rd167, %rd414, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f549, [%rd167];\n" " // end inline asm\n" " shl.b64 %rd170, %rd16, 3;\n" " add.s64 %rd171, %rd413, %rd170;\n" " add.s64 %rd168, %rd171, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r320,%r321}, [%rd168];\n" " // end inline asm\n" " add.s64 %rd172, %rd414, %rd170;\n" " add.s64 %rd169, %rd172, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r322,%r323}, [%rd169];\n" " // end inline asm\n" " shr.u32 %r324, %r318, 16;\n" " shr.u32 %r325, %r319, 16;\n" " and.b32 %r326, %r318, 15;\n" " or.b32 %r327, %r326, 1258291200;\n" " mov.b32 %f550, %r327;\n" " mov.f32 %f551, 0fCB000004;\n" " mov.f32 %f552, 0f3F800000;\n" " fma.rn.f32 %f553, %f550, %f552, %f551;\n" " and.b32 %r328, %r320, 255;\n" " cvt.rn.f32.u32 %f554, %r328;\n" " mov.f32 %f555, 0f42000000;\n" " sub.rn.f32 %f556, %f554, %f555;\n" " and.b32 %r329, %r319, 15;\n" " or.b32 %r330, %r329, 1258291200;\n" " mov.b32 %f557, %r330;\n" " fma.rn.f32 %f558, %f557, %f552, %f551;\n" " and.b32 %r331, %r322, 255;\n" " cvt.rn.f32.u32 %f559, %r331;\n" " sub.rn.f32 %f560, %f559, %f555;\n" " mul.rn.f32 %f561, %f556, %f553;\n" " ld.shared.f32 %f562, [%r50];\n" " mov.f32 %f563, 0f00000000;\n" " fma.rn.f32 %f564, %f561, %f562, %f563;\n" " mul.rn.f32 %f565, %f560, %f558;\n" " fma.rn.f32 %f566, %f565, %f562, %f563;\n" " and.b32 %r332, %r318, 240;\n" " or.b32 %r333, %r332, 1258291200;\n" " mov.b32 %f567, %r333;\n" " mov.f32 %f568, 0fC9000040;\n" " mov.f32 %f569, 0f3D800000;\n" " fma.rn.f32 %f570, %f567, %f569, %f568;\n" " bfe.u32 %r334, %r320, 8, 8;\n" " cvt.rn.f32.u32 %f571, %r334;\n" " sub.rn.f32 %f572, %f571, %f555;\n" " and.b32 %r335, %r319, 240;\n" " or.b32 %r336, %r335, 1258291200;\n" " mov.b32 %f573, %r336;\n" " fma.rn.f32 %f574, %f573, %f569, %f568;\n" " bfe.u32 %r337, %r322, 8, 8;\n" " cvt.rn.f32.u32 %f575, %r337;\n" " sub.rn.f32 %f576, %f575, %f555;\n" " mul.rn.f32 %f577, %f572, %f570;\n" " ld.shared.f32 %f578, [%r50+128];\n" " fma.rn.f32 %f579, %f577, %f578, %f564;\n" " mul.rn.f32 %f580, %f576, %f574;\n" " fma.rn.f32 %f581, %f580, %f578, %f566;\n" " and.b32 %r338, %r318, 3840;\n" " or.b32 %r339, %r338, 1258291200;\n" " mov.b32 %f582, %r339;\n" " mov.f32 %f583, 0fC7000400;\n" " mov.f32 %f584, 0f3B800000;\n" " fma.rn.f32 %f585, %f582, %f584, %f583;\n" " bfe.u32 %r340, %r320, 16, 8;\n" " cvt.rn.f32.u32 %f586, %r340;\n" " sub.rn.f32 %f587, %f586, %f555;\n" " and.b32 %r341, %r319, 3840;\n" " or.b32 %r342, %r341, 1258291200;\n" " mov.b32 %f588, %r342;\n" " fma.rn.f32 %f589, %f588, %f584, %f583;\n" " bfe.u32 %r343, %r322, 16, 8;\n" " cvt.rn.f32.u32 %f590, %r343;\n" " sub.rn.f32 %f591, %f590, %f555;\n" " mul.rn.f32 %f592, %f587, %f585;\n" " ld.shared.f32 %f593, [%r50+256];\n" " fma.rn.f32 %f594, %f592, %f593, %f579;\n" " mul.rn.f32 %f595, %f591, %f589;\n" " fma.rn.f32 %f596, %f595, %f593, %f581;\n" " and.b32 %r344, %r318, 61440;\n" " or.b32 %r345, %r344, 1258291200;\n" " mov.b32 %f597, %r345;\n" " mov.f32 %f598, 0fC5004000;\n" " mov.f32 %f599, 0f39800000;\n" " fma.rn.f32 %f600, %f597, %f599, %f598;\n" " shr.u32 %r346, %r320, 24;\n" " cvt.rn.f32.u32 %f601, %r346;\n" " sub.rn.f32 %f602, %f601, %f555;\n" " and.b32 %r347, %r319, 61440;\n" " or.b32 %r348, %r347, 1258291200;\n" " mov.b32 %f603, %r348;\n" " fma.rn.f32 %f604, %f603, %f599, %f598;\n" " shr.u32 %r349, %r322, 24;\n" " cvt.rn.f32.u32 %f605, %r349;\n" " sub.rn.f32 %f606, %f605, %f555;\n" " mul.rn.f32 %f607, %f602, %f600;\n" " ld.shared.f32 %f608, [%r50+384];\n" " fma.rn.f32 %f609, %f607, %f608, %f594;\n" " mul.rn.f32 %f610, %f606, %f604;\n" " fma.rn.f32 %f611, %f610, %f608, %f596;\n" " and.b32 %r350, %r318, 983040;\n" " or.b32 %r351, %r350, 1258291200;\n" " mov.b32 %f612, %r351;\n" " mov.f32 %f613, 0fC3040000;\n" " mov.f32 %f614, 0f37800000;\n" " fma.rn.f32 %f615, %f612, %f614, %f613;\n" " and.b32 %r352, %r321, 255;\n" " cvt.rn.f32.u32 %f616, %r352;\n" " sub.rn.f32 %f617, %f616, %f555;\n" " and.b32 %r353, %r319, 983040;\n" " or.b32 %r354, %r353, 1258291200;\n" " mov.b32 %f618, %r354;\n" " fma.rn.f32 %f619, %f618, %f614, %f613;\n" " and.b32 %r355, %r323, 255;\n" " cvt.rn.f32.u32 %f620, %r355;\n" " sub.rn.f32 %f621, %f620, %f555;\n" " mul.rn.f32 %f622, %f617, %f615;\n" " ld.shared.f32 %f623, [%r50+512];\n" " fma.rn.f32 %f624, %f622, %f623, %f609;\n" " mul.rn.f32 %f625, %f621, %f619;\n" " fma.rn.f32 %f626, %f625, %f623, %f611;\n" " and.b32 %r356, %r324, 240;\n" " or.b32 %r357, %r356, 1258291200;\n" " mov.b32 %f627, %r357;\n" " fma.rn.f32 %f628, %f627, %f569, %f568;\n" " bfe.u32 %r358, %r321, 8, 8;\n" " cvt.rn.f32.u32 %f629, %r358;\n" " sub.rn.f32 %f630, %f629, %f555;\n" " and.b32 %r359, %r325, 240;\n" " or.b32 %r360, %r359, 1258291200;\n" " mov.b32 %f631, %r360;\n" " fma.rn.f32 %f632, %f631, %f569, %f568;\n" " bfe.u32 %r361, %r323, 8, 8;\n" " cvt.rn.f32.u32 %f633, %r361;\n" " sub.rn.f32 %f634, %f633, %f555;\n" " mul.rn.f32 %f635, %f630, %f628;\n" " ld.shared.f32 %f636, [%r50+640];\n" " fma.rn.f32 %f637, %f635, %f636, %f624;\n" " mul.rn.f32 %f638, %f634, %f632;\n" " fma.rn.f32 %f639, %f638, %f636, %f626;\n" " and.b32 %r362, %r324, 3840;\n" " or.b32 %r363, %r362, 1258291200;\n" " mov.b32 %f640, %r363;\n" " fma.rn.f32 %f641, %f640, %f584, %f583;\n" " bfe.u32 %r364, %r321, 16, 8;\n" " cvt.rn.f32.u32 %f642, %r364;\n" " sub.rn.f32 %f643, %f642, %f555;\n" " and.b32 %r365, %r325, 3840;\n" " or.b32 %r366, %r365, 1258291200;\n" " mov.b32 %f644, %r366;\n" " fma.rn.f32 %f645, %f644, %f584, %f583;\n" " bfe.u32 %r367, %r323, 16, 8;\n" " cvt.rn.f32.u32 %f646, %r367;\n" " sub.rn.f32 %f647, %f646, %f555;\n" " mul.rn.f32 %f648, %f643, %f641;\n" " ld.shared.f32 %f649, [%r50+768];\n" " fma.rn.f32 %f650, %f648, %f649, %f637;\n" " mul.rn.f32 %f651, %f647, %f645;\n" " fma.rn.f32 %f652, %f651, %f649, %f639;\n" " and.b32 %r368, %r324, 61440;\n" " or.b32 %r369, %r368, 1258291200;\n" " mov.b32 %f653, %r369;\n" " fma.rn.f32 %f654, %f653, %f599, %f598;\n" " shr.u32 %r370, %r321, 24;\n" " cvt.rn.f32.u32 %f655, %r370;\n" " sub.rn.f32 %f656, %f655, %f555;\n" " and.b32 %r371, %r325, 61440;\n" " or.b32 %r372, %r371, 1258291200;\n" " mov.b32 %f657, %r372;\n" " fma.rn.f32 %f658, %f657, %f599, %f598;\n" " shr.u32 %r373, %r323, 24;\n" " cvt.rn.f32.u32 %f659, %r373;\n" " sub.rn.f32 %f660, %f659, %f555;\n" " mul.rn.f32 %f661, %f656, %f654;\n" " ld.shared.f32 %f662, [%r50+896];\n" " fma.rn.f32 %f663, %f661, %f662, %f650;\n" " mul.rn.f32 %f664, %f660, %f658;\n" " fma.rn.f32 %f665, %f664, %f662, %f652;\n" " fma.rn.f32 %f2017, %f548, %f663, %f2017;\n" " fma.rn.f32 %f2019, %f549, %f665, %f2019;\n" " bra.uni $L__BB0_32;\n" "\n" "$L__BB0_29:\n" " shl.b64 %rd354, %rd15, 2;\n" " add.s64 %rd353, %rd414, %rd354;\n" " add.s64 %rd352, %rd413, %rd354;\n" " // begin inline asm\n" " ld.global.nc.u32 %r242, [%rd352];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r243, [%rd353];\n" " // end inline asm\n" " add.s64 %rd158, %rd413, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r244,%r245}, [%rd158];\n" " // end inline asm\n" " add.s64 %rd159, %rd414, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r246,%r247}, [%rd159];\n" " // end inline asm\n" " add.s64 %rd160, %rd413, 136;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r248,%r249}, [%rd160];\n" " // end inline asm\n" " add.s64 %rd161, %rd414, 136;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r250,%r251}, [%rd161];\n" " // end inline asm\n" " add.s64 %rd162, %rd413, 144;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f381,%f382}, [%rd162];\n" " // end inline asm\n" " add.s64 %rd163, %rd414, 144;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f383,%f384}, [%rd163];\n" " // end inline asm\n" " shr.u32 %r252, %r242, 16;\n" " shr.u32 %r253, %r243, 16;\n" " and.b32 %r254, %r244, 255;\n" " cvt.rn.f32.u32 %f385, %r254;\n" " mul.rn.f32 %f386, %f381, %f385;\n" " and.b32 %r255, %r242, 15;\n" " or.b32 %r256, %r255, 1258291200;\n" " mov.b32 %f387, %r256;\n" " mov.f32 %f388, 0fCB000000;\n" " mov.f32 %f389, 0f3F800000;\n" " fma.rn.f32 %f390, %f387, %f389, %f388;\n" " mul.rn.f32 %f391, %f386, %f390;\n" " and.b32 %r257, %r248, 255;\n" " cvt.rn.f32.u32 %f392, %r257;\n" " mul.rn.f32 %f393, %f382, %f392;\n" " sub.rn.f32 %f394, %f391, %f393;\n" " and.b32 %r258, %r246, 255;\n" " cvt.rn.f32.u32 %f395, %r258;\n" " mul.rn.f32 %f396, %f383, %f395;\n" " and.b32 %r259, %r243, 15;\n" " or.b32 %r260, %r259, 1258291200;\n" " mov.b32 %f397, %r260;\n" " fma.rn.f32 %f398, %f397, %f389, %f388;\n" " mul.rn.f32 %f399, %f396, %f398;\n" " and.b32 %r261, %r250, 255;\n" " cvt.rn.f32.u32 %f400, %r261;\n" " mul.rn.f32 %f401, %f384, %f400;\n" " sub.rn.f32 %f402, %f399, %f401;\n" " ld.shared.f32 %f403, [%r50];\n" " mov.f32 %f404, 0f00000000;\n" " fma.rn.f32 %f405, %f394, %f403, %f404;\n" " fma.rn.f32 %f406, %f402, %f403, %f404;\n" " bfe.u32 %r262, %r244, 8, 8;\n" " cvt.rn.f32.u32 %f407, %r262;\n" " mul.rn.f32 %f408, %f381, %f407;\n" " and.b32 %r263, %r242, 240;\n" " or.b32 %r264, %r263, 1258291200;\n" " mov.b32 %f409, %r264;\n" " mov.f32 %f410, 0fC9000000;\n" " mov.f32 %f411, 0f3D800000;\n" " fma.rn.f32 %f412, %f409, %f411, %f410;\n" " mul.rn.f32 %f413, %f408, %f412;\n" " bfe.u32 %r265, %r248, 8, 8;\n" " cvt.rn.f32.u32 %f414, %r265;\n" " mul.rn.f32 %f415, %f382, %f414;\n" " sub.rn.f32 %f416, %f413, %f415;\n" " bfe.u32 %r266, %r246, 8, 8;\n" " cvt.rn.f32.u32 %f417, %r266;\n" " mul.rn.f32 %f418, %f383, %f417;\n" " and.b32 %r267, %r243, 240;\n" " or.b32 %r268, %r267, 1258291200;\n" " mov.b32 %f419, %r268;\n" " fma.rn.f32 %f420, %f419, %f411, %f410;\n" " mul.rn.f32 %f421, %f418, %f420;\n" " bfe.u32 %r269, %r250, 8, 8;\n" " cvt.rn.f32.u32 %f422, %r269;\n" " mul.rn.f32 %f423, %f384, %f422;\n" " sub.rn.f32 %f424, %f421, %f423;\n" " ld.shared.f32 %f425, [%r50+128];\n" " fma.rn.f32 %f426, %f416, %f425, %f405;\n" " fma.rn.f32 %f427, %f424, %f425, %f406;\n" " bfe.u32 %r270, %r244, 16, 8;\n" " cvt.rn.f32.u32 %f428, %r270;\n" " mul.rn.f32 %f429, %f381, %f428;\n" " and.b32 %r271, %r242, 3840;\n" " or.b32 %r272, %r271, 1258291200;\n" " mov.b32 %f430, %r272;\n" " mov.f32 %f431, 0fC7000000;\n" " mov.f32 %f432, 0f3B800000;\n" " fma.rn.f32 %f433, %f430, %f432, %f431;\n" " mul.rn.f32 %f434, %f429, %f433;\n" " bfe.u32 %r273, %r248, 16, 8;\n" " cvt.rn.f32.u32 %f435, %r273;\n" " mul.rn.f32 %f436, %f382, %f435;\n" " sub.rn.f32 %f437, %f434, %f436;\n" " bfe.u32 %r274, %r246, 16, 8;\n" " cvt.rn.f32.u32 %f438, %r274;\n" " mul.rn.f32 %f439, %f383, %f438;\n" " and.b32 %r275, %r243, 3840;\n" " or.b32 %r276, %r275, 1258291200;\n" " mov.b32 %f440, %r276;\n" " fma.rn.f32 %f441, %f440, %f432, %f431;\n" " mul.rn.f32 %f442, %f439, %f441;\n" " bfe.u32 %r277, %r250, 16, 8;\n" " cvt.rn.f32.u32 %f443, %r277;\n" " mul.rn.f32 %f444, %f384, %f443;\n" " sub.rn.f32 %f445, %f442, %f444;\n" " ld.shared.f32 %f446, [%r50+256];\n" " fma.rn.f32 %f447, %f437, %f446, %f426;\n" " fma.rn.f32 %f448, %f445, %f446, %f427;\n" " shr.u32 %r278, %r244, 24;\n" " cvt.rn.f32.u32 %f449, %r278;\n" " mul.rn.f32 %f450, %f381, %f449;\n" " and.b32 %r279, %r242, 61440;\n" " or.b32 %r280, %r279, 1258291200;\n" " mov.b32 %f451, %r280;\n" " mov.f32 %f452, 0fC5000000;\n" " mov.f32 %f453, 0f39800000;\n" " fma.rn.f32 %f454, %f451, %f453, %f452;\n" " mul.rn.f32 %f455, %f450, %f454;\n" " shr.u32 %r281, %r248, 24;\n" " cvt.rn.f32.u32 %f456, %r281;\n" " mul.rn.f32 %f457, %f382, %f456;\n" " sub.rn.f32 %f458, %f455, %f457;\n" " shr.u32 %r282, %r246, 24;\n" " cvt.rn.f32.u32 %f459, %r282;\n" " mul.rn.f32 %f460, %f383, %f459;\n" " and.b32 %r283, %r243, 61440;\n" " or.b32 %r284, %r283, 1258291200;\n" " mov.b32 %f461, %r284;\n" " fma.rn.f32 %f462, %f461, %f453, %f452;\n" " mul.rn.f32 %f463, %f460, %f462;\n" " shr.u32 %r285, %r250, 24;\n" " cvt.rn.f32.u32 %f464, %r285;\n" " mul.rn.f32 %f465, %f384, %f464;\n" " sub.rn.f32 %f466, %f463, %f465;\n" " ld.shared.f32 %f467, [%r50+384];\n" " fma.rn.f32 %f468, %f458, %f467, %f447;\n" " fma.rn.f32 %f469, %f466, %f467, %f448;\n" " and.b32 %r286, %r245, 255;\n" " cvt.rn.f32.u32 %f470, %r286;\n" " mul.rn.f32 %f471, %f381, %f470;\n" " and.b32 %r287, %r242, 983040;\n" " or.b32 %r288, %r287, 1258291200;\n" " mov.b32 %f472, %r288;\n" " mov.f32 %f473, 0fC3000000;\n" " mov.f32 %f474, 0f37800000;\n" " fma.rn.f32 %f475, %f472, %f474, %f473;\n" " mul.rn.f32 %f476, %f471, %f475;\n" " and.b32 %r289, %r249, 255;\n" " cvt.rn.f32.u32 %f477, %r289;\n" " mul.rn.f32 %f478, %f382, %f477;\n" " sub.rn.f32 %f479, %f476, %f478;\n" " and.b32 %r290, %r247, 255;\n" " cvt.rn.f32.u32 %f480, %r290;\n" " mul.rn.f32 %f481, %f383, %f480;\n" " and.b32 %r291, %r243, 983040;\n" " or.b32 %r292, %r291, 1258291200;\n" " mov.b32 %f482, %r292;\n" " fma.rn.f32 %f483, %f482, %f474, %f473;\n" " mul.rn.f32 %f484, %f481, %f483;\n" " and.b32 %r293, %r251, 255;\n" " cvt.rn.f32.u32 %f485, %r293;\n" " mul.rn.f32 %f486, %f384, %f485;\n" " sub.rn.f32 %f487, %f484, %f486;\n" " ld.shared.f32 %f488, [%r50+512];\n" " fma.rn.f32 %f489, %f479, %f488, %f468;\n" " fma.rn.f32 %f490, %f487, %f488, %f469;\n" " bfe.u32 %r294, %r245, 8, 8;\n" " cvt.rn.f32.u32 %f491, %r294;\n" " mul.rn.f32 %f492, %f381, %f491;\n" " and.b32 %r295, %r252, 240;\n" " or.b32 %r296, %r295, 1258291200;\n" " mov.b32 %f493, %r296;\n" " fma.rn.f32 %f494, %f493, %f411, %f410;\n" " mul.rn.f32 %f495, %f492, %f494;\n" " bfe.u32 %r297, %r249, 8, 8;\n" " cvt.rn.f32.u32 %f496, %r297;\n" " mul.rn.f32 %f497, %f382, %f496;\n" " sub.rn.f32 %f498, %f495, %f497;\n" " bfe.u32 %r298, %r247, 8, 8;\n" " cvt.rn.f32.u32 %f499, %r298;\n" " mul.rn.f32 %f500, %f383, %f499;\n" " and.b32 %r299, %r253, 240;\n" " or.b32 %r300, %r299, 1258291200;\n" " mov.b32 %f501, %r300;\n" " fma.rn.f32 %f502, %f501, %f411, %f410;\n" " mul.rn.f32 %f503, %f500, %f502;\n" " bfe.u32 %r301, %r251, 8, 8;\n" " cvt.rn.f32.u32 %f504, %r301;\n" " mul.rn.f32 %f505, %f384, %f504;\n" " sub.rn.f32 %f506, %f503, %f505;\n" " ld.shared.f32 %f507, [%r50+640];\n" " fma.rn.f32 %f508, %f498, %f507, %f489;\n" " fma.rn.f32 %f509, %f506, %f507, %f490;\n" " bfe.u32 %r302, %r245, 16, 8;\n" " cvt.rn.f32.u32 %f510, %r302;\n" " mul.rn.f32 %f511, %f381, %f510;\n" " and.b32 %r303, %r252, 3840;\n" " or.b32 %r304, %r303, 1258291200;\n" " mov.b32 %f512, %r304;\n" " fma.rn.f32 %f513, %f512, %f432, %f431;\n" " mul.rn.f32 %f514, %f511, %f513;\n" " bfe.u32 %r305, %r249, 16, 8;\n" " cvt.rn.f32.u32 %f515, %r305;\n" " mul.rn.f32 %f516, %f382, %f515;\n" " sub.rn.f32 %f517, %f514, %f516;\n" " bfe.u32 %r306, %r247, 16, 8;\n" " cvt.rn.f32.u32 %f518, %r306;\n" " mul.rn.f32 %f519, %f383, %f518;\n" " and.b32 %r307, %r253, 3840;\n" " or.b32 %r308, %r307, 1258291200;\n" " mov.b32 %f520, %r308;\n" " fma.rn.f32 %f521, %f520, %f432, %f431;\n" " mul.rn.f32 %f522, %f519, %f521;\n" " bfe.u32 %r309, %r251, 16, 8;\n" " cvt.rn.f32.u32 %f523, %r309;\n" " mul.rn.f32 %f524, %f384, %f523;\n" " sub.rn.f32 %f525, %f522, %f524;\n" " ld.shared.f32 %f526, [%r50+768];\n" " fma.rn.f32 %f527, %f517, %f526, %f508;\n" " fma.rn.f32 %f528, %f525, %f526, %f509;\n" " shr.u32 %r310, %r245, 24;\n" " cvt.rn.f32.u32 %f529, %r310;\n" " mul.rn.f32 %f530, %f381, %f529;\n" " and.b32 %r311, %r252, 61440;\n" " or.b32 %r312, %r311, 1258291200;\n" " mov.b32 %f531, %r312;\n" " fma.rn.f32 %f532, %f531, %f453, %f452;\n" " mul.rn.f32 %f533, %f530, %f532;\n" " shr.u32 %r313, %r249, 24;\n" " cvt.rn.f32.u32 %f534, %r313;\n" " mul.rn.f32 %f535, %f382, %f534;\n" " sub.rn.f32 %f536, %f533, %f535;\n" " shr.u32 %r314, %r247, 24;\n" " cvt.rn.f32.u32 %f537, %r314;\n" " mul.rn.f32 %f538, %f383, %f537;\n" " and.b32 %r315, %r253, 61440;\n" " or.b32 %r316, %r315, 1258291200;\n" " mov.b32 %f539, %r316;\n" " fma.rn.f32 %f540, %f539, %f453, %f452;\n" " mul.rn.f32 %f541, %f538, %f540;\n" " shr.u32 %r317, %r251, 24;\n" " cvt.rn.f32.u32 %f542, %r317;\n" " mul.rn.f32 %f543, %f384, %f542;\n" " sub.rn.f32 %f544, %f541, %f543;\n" " ld.shared.f32 %f545, [%r50+896];\n" " fma.rn.f32 %f546, %f536, %f545, %f527;\n" " fma.rn.f32 %f547, %f544, %f545, %f528;\n" " add.rn.f32 %f2017, %f2017, %f546;\n" " add.rn.f32 %f2019, %f2019, %f547;\n" " bra.uni $L__BB0_32;\n" "\n" "$L__BB0_31:\n" " shl.b64 %rd348, %rd15, 2;\n" " add.s64 %rd347, %rd414, %rd348;\n" " add.s64 %rd346, %rd413, %rd348;\n" " // begin inline asm\n" " ld.global.nc.u32 %r374, [%rd346];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r375, [%rd347];\n" " // end inline asm\n" " shl.b64 %rd181, %rd15, 1;\n" " add.s64 %rd182, %rd413, %rd181;\n" " add.s64 %rd175, %rd182, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs1, [%rd175];\n" " // end inline asm\n" " cvt.u32.u16 %r380, %rs1;\n" " add.s64 %rd183, %rd414, %rd181;\n" " add.s64 %rd176, %rd183, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs2, [%rd176];\n" " // end inline asm\n" " cvt.u32.u16 %r381, %rs2;\n" " add.s64 %rd177, %rd413, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f666, [%rd177];\n" " // end inline asm\n" " add.s64 %rd178, %rd414, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f667, [%rd178];\n" " // end inline asm\n" " shl.b64 %rd184, %rd16, 3;\n" " add.s64 %rd185, %rd413, %rd184;\n" " add.s64 %rd179, %rd185, 192;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r376,%r377}, [%rd179];\n" " // end inline asm\n" " add.s64 %rd186, %rd414, %rd184;\n" " add.s64 %rd180, %rd186, 192;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r378,%r379}, [%rd180];\n" " // end inline asm\n" " shr.u32 %r382, %r374, 16;\n" " shr.u32 %r383, %r375, 16;\n" " and.b32 %r384, %r374, 15;\n" " or.b32 %r385, %r384, 1258291200;\n" " mov.b32 %f668, %r385;\n" " mov.f32 %f669, 0fCB000020;\n" " mov.f32 %f670, 0f3F800000;\n" " fma.rn.f32 %f671, %f668, %f670, %f669;\n" " and.b32 %r386, %r380, 3;\n" " or.b32 %r387, %r386, 1258291200;\n" " mov.b32 %f672, %r387;\n" " mov.f32 %f673, 0fCD000000;\n" " mov.f32 %f674, 0f41800000;\n" " fma.rn.f32 %f675, %f672, %f674, %f673;\n" " add.rn.f32 %f676, %f671, %f675;\n" " and.b32 %r388, %r375, 15;\n" " or.b32 %r389, %r388, 1258291200;\n" " mov.b32 %f677, %r389;\n" " fma.rn.f32 %f678, %f677, %f670, %f669;\n" " and.b32 %r390, %r381, 3;\n" " or.b32 %r391, %r390, 1258291200;\n" " mov.b32 %f679, %r391;\n" " fma.rn.f32 %f680, %f679, %f674, %f673;\n" " add.rn.f32 %f681, %f678, %f680;\n" " and.b32 %r392, %r376, 255;\n" " cvt.rn.f32.u32 %f682, %r392;\n" " mov.f32 %f683, 0f43000000;\n" " sub.rn.f32 %f684, %f682, %f683;\n" " and.b32 %r393, %r378, 255;\n" " cvt.rn.f32.u32 %f685, %r393;\n" " sub.rn.f32 %f686, %f685, %f683;\n" " mul.rn.f32 %f687, %f684, %f676;\n" " ld.shared.f32 %f688, [%r50];\n" " mov.f32 %f689, 0f00000000;\n" " fma.rn.f32 %f690, %f687, %f688, %f689;\n" " mul.rn.f32 %f691, %f686, %f681;\n" " fma.rn.f32 %f692, %f691, %f688, %f689;\n" " and.b32 %r394, %r374, 3840;\n" " or.b32 %r395, %r394, 1258291200;\n" " mov.b32 %f693, %r395;\n" " mov.f32 %f694, 0fC7002000;\n" " mov.f32 %f695, 0f3B800000;\n" " fma.rn.f32 %f696, %f693, %f695, %f694;\n" " and.b32 %r396, %r380, 12;\n" " or.b32 %r397, %r396, 1258291200;\n" " mov.b32 %f697, %r397;\n" " mov.f32 %f698, 0fCC000000;\n" " mov.f32 %f699, 0f40800000;\n" " fma.rn.f32 %f700, %f697, %f699, %f698;\n" " add.rn.f32 %f701, %f696, %f700;\n" " and.b32 %r398, %r375, 3840;\n" " or.b32 %r399, %r398, 1258291200;\n" " mov.b32 %f702, %r399;\n" " fma.rn.f32 %f703, %f702, %f695, %f694;\n" " and.b32 %r400, %r381, 12;\n" " or.b32 %r401, %r400, 1258291200;\n" " mov.b32 %f704, %r401;\n" " fma.rn.f32 %f705, %f704, %f699, %f698;\n" " add.rn.f32 %f706, %f703, %f705;\n" " bfe.u32 %r402, %r376, 8, 8;\n" " cvt.rn.f32.u32 %f707, %r402;\n" " sub.rn.f32 %f708, %f707, %f683;\n" " bfe.u32 %r403, %r378, 8, 8;\n" " cvt.rn.f32.u32 %f709, %r403;\n" " sub.rn.f32 %f710, %f709, %f683;\n" " mul.rn.f32 %f711, %f708, %f701;\n" " ld.shared.f32 %f712, [%r50+128];\n" " fma.rn.f32 %f713, %f711, %f712, %f690;\n" " mul.rn.f32 %f714, %f710, %f706;\n" " fma.rn.f32 %f715, %f714, %f712, %f692;\n" " and.b32 %r404, %r374, 240;\n" " or.b32 %r405, %r404, 1258291200;\n" " mov.b32 %f716, %r405;\n" " mov.f32 %f717, 0fC9000200;\n" " mov.f32 %f718, 0f3D800000;\n" " fma.rn.f32 %f719, %f716, %f718, %f717;\n" " and.b32 %r406, %r380, 48;\n" " or.b32 %r407, %r406, 1258291200;\n" " mov.b32 %f720, %r407;\n" " mov.f32 %f721, 0fCB000000;\n" " fma.rn.f32 %f722, %f720, %f670, %f721;\n" " add.rn.f32 %f723, %f719, %f722;\n" " and.b32 %r408, %r375, 240;\n" " or.b32 %r409, %r408, 1258291200;\n" " mov.b32 %f724, %r409;\n" " fma.rn.f32 %f725, %f724, %f718, %f717;\n" " and.b32 %r410, %r381, 48;\n" " or.b32 %r411, %r410, 1258291200;\n" " mov.b32 %f726, %r411;\n" " fma.rn.f32 %f727, %f726, %f670, %f721;\n" " add.rn.f32 %f728, %f725, %f727;\n" " bfe.u32 %r412, %r376, 16, 8;\n" " cvt.rn.f32.u32 %f729, %r412;\n" " sub.rn.f32 %f730, %f729, %f683;\n" " bfe.u32 %r413, %r378, 16, 8;\n" " cvt.rn.f32.u32 %f731, %r413;\n" " sub.rn.f32 %f732, %f731, %f683;\n" " mul.rn.f32 %f733, %f730, %f723;\n" " ld.shared.f32 %f734, [%r50+256];\n" " fma.rn.f32 %f735, %f733, %f734, %f713;\n" " mul.rn.f32 %f736, %f732, %f728;\n" " fma.rn.f32 %f737, %f736, %f734, %f715;\n" " and.b32 %r414, %r374, 61440;\n" " or.b32 %r415, %r414, 1258291200;\n" " mov.b32 %f738, %r415;\n" " mov.f32 %f739, 0fC5020000;\n" " mov.f32 %f740, 0f39800000;\n" " fma.rn.f32 %f741, %f738, %f740, %f739;\n" " and.b32 %r416, %r380, 192;\n" " or.b32 %r417, %r416, 1258291200;\n" " mov.b32 %f742, %r417;\n" " mov.f32 %f743, 0fCA000000;\n" " mov.f32 %f744, 0f3E800000;\n" " fma.rn.f32 %f745, %f742, %f744, %f743;\n" " add.rn.f32 %f746, %f741, %f745;\n" " and.b32 %r418, %r375, 61440;\n" " or.b32 %r419, %r418, 1258291200;\n" " mov.b32 %f747, %r419;\n" " fma.rn.f32 %f748, %f747, %f740, %f739;\n" " and.b32 %r420, %r381, 192;\n" " or.b32 %r421, %r420, 1258291200;\n" " mov.b32 %f749, %r421;\n" " fma.rn.f32 %f750, %f749, %f744, %f743;\n" " add.rn.f32 %f751, %f748, %f750;\n" " shr.u32 %r422, %r376, 24;\n" " cvt.rn.f32.u32 %f752, %r422;\n" " sub.rn.f32 %f753, %f752, %f683;\n" " shr.u32 %r423, %r378, 24;\n" " cvt.rn.f32.u32 %f754, %r423;\n" " sub.rn.f32 %f755, %f754, %f683;\n" " mul.rn.f32 %f756, %f753, %f746;\n" " ld.shared.f32 %f757, [%r50+384];\n" " fma.rn.f32 %f758, %f756, %f757, %f735;\n" " mul.rn.f32 %f759, %f755, %f751;\n" " fma.rn.f32 %f760, %f759, %f757, %f737;\n" " and.b32 %r424, %r374, 983040;\n" " or.b32 %r425, %r424, 1258291200;\n" " mov.b32 %f761, %r425;\n" " mov.f32 %f762, 0fC3200000;\n" " mov.f32 %f763, 0f37800000;\n" " fma.rn.f32 %f764, %f761, %f763, %f762;\n" " and.b32 %r426, %r380, 768;\n" " or.b32 %r427, %r426, 1258291200;\n" " mov.b32 %f765, %r427;\n" " mov.f32 %f766, 0fC9000000;\n" " fma.rn.f32 %f767, %f765, %f718, %f766;\n" " add.rn.f32 %f768, %f764, %f767;\n" " and.b32 %r428, %r375, 983040;\n" " or.b32 %r429, %r428, 1258291200;\n" " mov.b32 %f769, %r429;\n" " fma.rn.f32 %f770, %f769, %f763, %f762;\n" " and.b32 %r430, %r381, 768;\n" " or.b32 %r431, %r430, 1258291200;\n" " mov.b32 %f771, %r431;\n" " fma.rn.f32 %f772, %f771, %f718, %f766;\n" " add.rn.f32 %f773, %f770, %f772;\n" " and.b32 %r432, %r377, 255;\n" " cvt.rn.f32.u32 %f774, %r432;\n" " sub.rn.f32 %f775, %f774, %f683;\n" " and.b32 %r433, %r379, 255;\n" " cvt.rn.f32.u32 %f776, %r433;\n" " sub.rn.f32 %f777, %f776, %f683;\n" " mul.rn.f32 %f778, %f775, %f768;\n" " ld.shared.f32 %f779, [%r50+512];\n" " fma.rn.f32 %f780, %f778, %f779, %f758;\n" " mul.rn.f32 %f781, %f777, %f773;\n" " fma.rn.f32 %f782, %f781, %f779, %f760;\n" " and.b32 %r434, %r382, 3840;\n" " or.b32 %r435, %r434, 1258291200;\n" " mov.b32 %f783, %r435;\n" " fma.rn.f32 %f784, %f783, %f695, %f694;\n" " and.b32 %r436, %r380, 3072;\n" " or.b32 %r437, %r436, 1258291200;\n" " mov.b32 %f785, %r437;\n" " mov.f32 %f786, 0fC8000000;\n" " mov.f32 %f787, 0f3C800000;\n" " fma.rn.f32 %f788, %f785, %f787, %f786;\n" " add.rn.f32 %f789, %f784, %f788;\n" " and.b32 %r438, %r383, 3840;\n" " or.b32 %r439, %r438, 1258291200;\n" " mov.b32 %f790, %r439;\n" " fma.rn.f32 %f791, %f790, %f695, %f694;\n" " and.b32 %r440, %r381, 3072;\n" " or.b32 %r441, %r440, 1258291200;\n" " mov.b32 %f792, %r441;\n" " fma.rn.f32 %f793, %f792, %f787, %f786;\n" " add.rn.f32 %f794, %f791, %f793;\n" " bfe.u32 %r442, %r377, 8, 8;\n" " cvt.rn.f32.u32 %f795, %r442;\n" " sub.rn.f32 %f796, %f795, %f683;\n" " bfe.u32 %r443, %r379, 8, 8;\n" " cvt.rn.f32.u32 %f797, %r443;\n" " sub.rn.f32 %f798, %f797, %f683;\n" " mul.rn.f32 %f799, %f796, %f789;\n" " ld.shared.f32 %f800, [%r50+640];\n" " fma.rn.f32 %f801, %f799, %f800, %f780;\n" " mul.rn.f32 %f802, %f798, %f794;\n" " fma.rn.f32 %f803, %f802, %f800, %f782;\n" " and.b32 %r444, %r382, 240;\n" " or.b32 %r445, %r444, 1258291200;\n" " mov.b32 %f804, %r445;\n" " fma.rn.f32 %f805, %f804, %f718, %f717;\n" " and.b32 %r446, %r380, 12288;\n" " or.b32 %r447, %r446, 1258291200;\n" " mov.b32 %f806, %r447;\n" " mov.f32 %f807, 0fC7000000;\n" " fma.rn.f32 %f808, %f806, %f695, %f807;\n" " add.rn.f32 %f809, %f805, %f808;\n" " and.b32 %r448, %r383, 240;\n" " or.b32 %r449, %r448, 1258291200;\n" " mov.b32 %f810, %r449;\n" " fma.rn.f32 %f811, %f810, %f718, %f717;\n" " and.b32 %r450, %r381, 12288;\n" " or.b32 %r451, %r450, 1258291200;\n" " mov.b32 %f812, %r451;\n" " fma.rn.f32 %f813, %f812, %f695, %f807;\n" " add.rn.f32 %f814, %f811, %f813;\n" " bfe.u32 %r452, %r377, 16, 8;\n" " cvt.rn.f32.u32 %f815, %r452;\n" " sub.rn.f32 %f816, %f815, %f683;\n" " bfe.u32 %r453, %r379, 16, 8;\n" " cvt.rn.f32.u32 %f817, %r453;\n" " sub.rn.f32 %f818, %f817, %f683;\n" " mul.rn.f32 %f819, %f816, %f809;\n" " ld.shared.f32 %f820, [%r50+768];\n" " fma.rn.f32 %f821, %f819, %f820, %f801;\n" " mul.rn.f32 %f822, %f818, %f814;\n" " fma.rn.f32 %f823, %f822, %f820, %f803;\n" " and.b32 %r454, %r382, 61440;\n" " or.b32 %r455, %r454, 1258291200;\n" " mov.b32 %f824, %r455;\n" " fma.rn.f32 %f825, %f824, %f740, %f739;\n" " and.b32 %r456, %r380, 49152;\n" " or.b32 %r457, %r456, 1258291200;\n" " mov.b32 %f826, %r457;\n" " mov.f32 %f827, 0fC6000000;\n" " mov.f32 %f828, 0f3A800000;\n" " fma.rn.f32 %f829, %f826, %f828, %f827;\n" " add.rn.f32 %f830, %f825, %f829;\n" " and.b32 %r458, %r383, 61440;\n" " or.b32 %r459, %r458, 1258291200;\n" " mov.b32 %f831, %r459;\n" " fma.rn.f32 %f832, %f831, %f740, %f739;\n" " and.b32 %r460, %r381, 49152;\n" " or.b32 %r461, %r460, 1258291200;\n" " mov.b32 %f833, %r461;\n" " fma.rn.f32 %f834, %f833, %f828, %f827;\n" " add.rn.f32 %f835, %f832, %f834;\n" " shr.u32 %r462, %r377, 24;\n" " cvt.rn.f32.u32 %f836, %r462;\n" " sub.rn.f32 %f837, %f836, %f683;\n" " shr.u32 %r463, %r379, 24;\n" " cvt.rn.f32.u32 %f838, %r463;\n" " sub.rn.f32 %f839, %f838, %f683;\n" " mul.rn.f32 %f840, %f837, %f830;\n" " ld.shared.f32 %f841, [%r50+896];\n" " fma.rn.f32 %f842, %f840, %f841, %f821;\n" " mul.rn.f32 %f843, %f839, %f835;\n" " fma.rn.f32 %f844, %f843, %f841, %f823;\n" " fma.rn.f32 %f2017, %f666, %f842, %f2017;\n" " fma.rn.f32 %f2019, %f667, %f844, %f2019;\n" "\n" "$L__BB0_32:\n" " @%p56 bra $L__BB0_40;\n" "\n" " @%p26 bra $L__BB0_38;\n" "\n" " setp.eq.s32 %p31, %r29, 12;\n" " @%p31 bra $L__BB0_37;\n" "\n" " setp.ne.s32 %p32, %r29, 13;\n" " @%p32 bra $L__BB0_39;\n" "\n" " shl.b64 %rd363, %rd15, 2;\n" " add.s64 %rd362, %rd394, %rd363;\n" " add.s64 %rd361, %rd393, %rd363;\n" " // begin inline asm\n" " ld.global.nc.u32 %r464, [%rd361];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r465, [%rd362];\n" " // end inline asm\n" " add.s64 %rd190, %rd393, %rd17;\n" " // begin inline asm\n" " ld.global.nc.u8 %r466, [%rd190];\n" " // end inline asm\n" " add.s64 %rd191, %rd394, %rd17;\n" " // begin inline asm\n" " ld.global.nc.u8 %r467, [%rd191];\n" " // end inline asm\n" " add.s64 %rd192, %rd393, 160;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r468,%r469}, [%rd192];\n" " // end inline asm\n" " add.s64 %rd193, %rd394, 160;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r470,%r471}, [%rd193];\n" " // end inline asm\n" " add.s64 %rd194, %rd393, 168;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r472,%r473}, [%rd194];\n" " // end inline asm\n" " add.s64 %rd195, %rd394, 168;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r474,%r475}, [%rd195];\n" " // end inline asm\n" " add.s64 %rd196, %rd393, 176;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f845,%f846}, [%rd196];\n" " // end inline asm\n" " add.s64 %rd197, %rd394, 176;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f847,%f848}, [%rd197];\n" " // end inline asm\n" " shr.u32 %r476, %r464, 16;\n" " shr.u32 %r477, %r465, 16;\n" " and.b32 %r478, %r464, 15;\n" " or.b32 %r479, %r478, 1258291200;\n" " mov.b32 %f849, %r479;\n" " mov.f32 %f850, 0fCB000000;\n" " mov.f32 %f851, 0f3F800000;\n" " fma.rn.f32 %f852, %f849, %f851, %f850;\n" " and.b32 %r480, %r466, 1;\n" " or.b32 %r481, %r480, 1258291200;\n" " mov.b32 %f853, %r481;\n" " mov.f32 %f854, 0fCD000000;\n" " mov.f32 %f855, 0f41800000;\n" " fma.rn.f32 %f856, %f853, %f855, %f854;\n" " add.rn.f32 %f857, %f852, %f856;\n" " and.b32 %r482, %r465, 15;\n" " or.b32 %r483, %r482, 1258291200;\n" " mov.b32 %f858, %r483;\n" " fma.rn.f32 %f859, %f858, %f851, %f850;\n" " and.b32 %r484, %r467, 1;\n" " or.b32 %r485, %r484, 1258291200;\n" " mov.b32 %f860, %r485;\n" " fma.rn.f32 %f861, %f860, %f855, %f854;\n" " add.rn.f32 %f862, %f859, %f861;\n" " and.b32 %r486, %r468, 255;\n" " cvt.rn.f32.u32 %f863, %r486;\n" " mul.rn.f32 %f864, %f845, %f863;\n" " mul.rn.f32 %f865, %f864, %f857;\n" " and.b32 %r487, %r472, 255;\n" " cvt.rn.f32.u32 %f866, %r487;\n" " mul.rn.f32 %f867, %f846, %f866;\n" " sub.rn.f32 %f868, %f865, %f867;\n" " and.b32 %r488, %r470, 255;\n" " cvt.rn.f32.u32 %f869, %r488;\n" " mul.rn.f32 %f870, %f847, %f869;\n" " mul.rn.f32 %f871, %f870, %f862;\n" " and.b32 %r489, %r474, 255;\n" " cvt.rn.f32.u32 %f872, %r489;\n" " mul.rn.f32 %f873, %f848, %f872;\n" " sub.rn.f32 %f874, %f871, %f873;\n" " ld.shared.f32 %f875, [%r50];\n" " mov.f32 %f876, 0f00000000;\n" " fma.rn.f32 %f877, %f868, %f875, %f876;\n" " fma.rn.f32 %f878, %f874, %f875, %f876;\n" " and.b32 %r490, %r464, 240;\n" " or.b32 %r491, %r490, 1258291200;\n" " mov.b32 %f879, %r491;\n" " mov.f32 %f880, 0fC9000000;\n" " mov.f32 %f881, 0f3D800000;\n" " fma.rn.f32 %f882, %f879, %f881, %f880;\n" " and.b32 %r492, %r466, 2;\n" " or.b32 %r493, %r492, 1258291200;\n" " mov.b32 %f883, %r493;\n" " mov.f32 %f884, 0fCC800000;\n" " mov.f32 %f885, 0f41000000;\n" " fma.rn.f32 %f886, %f883, %f885, %f884;\n" " add.rn.f32 %f887, %f882, %f886;\n" " and.b32 %r494, %r465, 240;\n" " or.b32 %r495, %r494, 1258291200;\n" " mov.b32 %f888, %r495;\n" " fma.rn.f32 %f889, %f888, %f881, %f880;\n" " and.b32 %r496, %r467, 2;\n" " or.b32 %r497, %r496, 1258291200;\n" " mov.b32 %f890, %r497;\n" " fma.rn.f32 %f891, %f890, %f885, %f884;\n" " add.rn.f32 %f892, %f889, %f891;\n" " bfe.u32 %r498, %r468, 8, 8;\n" " cvt.rn.f32.u32 %f893, %r498;\n" " mul.rn.f32 %f894, %f845, %f893;\n" " mul.rn.f32 %f895, %f894, %f887;\n" " bfe.u32 %r499, %r472, 8, 8;\n" " cvt.rn.f32.u32 %f896, %r499;\n" " mul.rn.f32 %f897, %f846, %f896;\n" " sub.rn.f32 %f898, %f895, %f897;\n" " bfe.u32 %r500, %r470, 8, 8;\n" " cvt.rn.f32.u32 %f899, %r500;\n" " mul.rn.f32 %f900, %f847, %f899;\n" " mul.rn.f32 %f901, %f900, %f892;\n" " bfe.u32 %r501, %r474, 8, 8;\n" " cvt.rn.f32.u32 %f902, %r501;\n" " mul.rn.f32 %f903, %f848, %f902;\n" " sub.rn.f32 %f904, %f901, %f903;\n" " ld.shared.f32 %f905, [%r50+128];\n" " fma.rn.f32 %f906, %f898, %f905, %f877;\n" " fma.rn.f32 %f907, %f904, %f905, %f878;\n" " and.b32 %r502, %r464, 3840;\n" " or.b32 %r503, %r502, 1258291200;\n" " mov.b32 %f908, %r503;\n" " mov.f32 %f909, 0fC7000000;\n" " mov.f32 %f910, 0f3B800000;\n" " fma.rn.f32 %f911, %f908, %f910, %f909;\n" " and.b32 %r504, %r466, 4;\n" " or.b32 %r505, %r504, 1258291200;\n" " mov.b32 %f912, %r505;\n" " mov.f32 %f913, 0fCC000000;\n" " mov.f32 %f914, 0f40800000;\n" " fma.rn.f32 %f915, %f912, %f914, %f913;\n" " add.rn.f32 %f916, %f911, %f915;\n" " and.b32 %r506, %r465, 3840;\n" " or.b32 %r507, %r506, 1258291200;\n" " mov.b32 %f917, %r507;\n" " fma.rn.f32 %f918, %f917, %f910, %f909;\n" " and.b32 %r508, %r467, 4;\n" " or.b32 %r509, %r508, 1258291200;\n" " mov.b32 %f919, %r509;\n" " fma.rn.f32 %f920, %f919, %f914, %f913;\n" " add.rn.f32 %f921, %f918, %f920;\n" " bfe.u32 %r510, %r468, 16, 8;\n" " cvt.rn.f32.u32 %f922, %r510;\n" " mul.rn.f32 %f923, %f845, %f922;\n" " mul.rn.f32 %f924, %f923, %f916;\n" " bfe.u32 %r511, %r472, 16, 8;\n" " cvt.rn.f32.u32 %f925, %r511;\n" " mul.rn.f32 %f926, %f846, %f925;\n" " sub.rn.f32 %f927, %f924, %f926;\n" " bfe.u32 %r512, %r470, 16, 8;\n" " cvt.rn.f32.u32 %f928, %r512;\n" " mul.rn.f32 %f929, %f847, %f928;\n" " mul.rn.f32 %f930, %f929, %f921;\n" " bfe.u32 %r513, %r474, 16, 8;\n" " cvt.rn.f32.u32 %f931, %r513;\n" " mul.rn.f32 %f932, %f848, %f931;\n" " sub.rn.f32 %f933, %f930, %f932;\n" " ld.shared.f32 %f934, [%r50+256];\n" " fma.rn.f32 %f935, %f927, %f934, %f906;\n" " fma.rn.f32 %f936, %f933, %f934, %f907;\n" " and.b32 %r514, %r464, 61440;\n" " or.b32 %r515, %r514, 1258291200;\n" " mov.b32 %f937, %r515;\n" " mov.f32 %f938, 0fC5000000;\n" " mov.f32 %f939, 0f39800000;\n" " fma.rn.f32 %f940, %f937, %f939, %f938;\n" " and.b32 %r516, %r466, 8;\n" " or.b32 %r517, %r516, 1258291200;\n" " mov.b32 %f941, %r517;\n" " mov.f32 %f942, 0fCB800000;\n" " mov.f32 %f943, 0f40000000;\n" " fma.rn.f32 %f944, %f941, %f943, %f942;\n" " add.rn.f32 %f945, %f940, %f944;\n" " and.b32 %r518, %r465, 61440;\n" " or.b32 %r519, %r518, 1258291200;\n" " mov.b32 %f946, %r519;\n" " fma.rn.f32 %f947, %f946, %f939, %f938;\n" " and.b32 %r520, %r467, 8;\n" " or.b32 %r521, %r520, 1258291200;\n" " mov.b32 %f948, %r521;\n" " fma.rn.f32 %f949, %f948, %f943, %f942;\n" " add.rn.f32 %f950, %f947, %f949;\n" " shr.u32 %r522, %r468, 24;\n" " cvt.rn.f32.u32 %f951, %r522;\n" " mul.rn.f32 %f952, %f845, %f951;\n" " mul.rn.f32 %f953, %f952, %f945;\n" " shr.u32 %r523, %r472, 24;\n" " cvt.rn.f32.u32 %f954, %r523;\n" " mul.rn.f32 %f955, %f846, %f954;\n" " sub.rn.f32 %f956, %f953, %f955;\n" " shr.u32 %r524, %r470, 24;\n" " cvt.rn.f32.u32 %f957, %r524;\n" " mul.rn.f32 %f958, %f847, %f957;\n" " mul.rn.f32 %f959, %f958, %f950;\n" " shr.u32 %r525, %r474, 24;\n" " cvt.rn.f32.u32 %f960, %r525;\n" " mul.rn.f32 %f961, %f848, %f960;\n" " sub.rn.f32 %f962, %f959, %f961;\n" " ld.shared.f32 %f963, [%r50+384];\n" " fma.rn.f32 %f964, %f956, %f963, %f935;\n" " fma.rn.f32 %f965, %f962, %f963, %f936;\n" " and.b32 %r526, %r464, 983040;\n" " or.b32 %r527, %r526, 1258291200;\n" " mov.b32 %f966, %r527;\n" " mov.f32 %f967, 0fC3000000;\n" " mov.f32 %f968, 0f37800000;\n" " fma.rn.f32 %f969, %f966, %f968, %f967;\n" " and.b32 %r528, %r466, 16;\n" " or.b32 %r529, %r528, 1258291200;\n" " mov.b32 %f970, %r529;\n" " fma.rn.f32 %f971, %f970, %f851, %f850;\n" " add.rn.f32 %f972, %f969, %f971;\n" " and.b32 %r530, %r465, 983040;\n" " or.b32 %r531, %r530, 1258291200;\n" " mov.b32 %f973, %r531;\n" " fma.rn.f32 %f974, %f973, %f968, %f967;\n" " and.b32 %r532, %r467, 16;\n" " or.b32 %r533, %r532, 1258291200;\n" " mov.b32 %f975, %r533;\n" " fma.rn.f32 %f976, %f975, %f851, %f850;\n" " add.rn.f32 %f977, %f974, %f976;\n" " and.b32 %r534, %r469, 255;\n" " cvt.rn.f32.u32 %f978, %r534;\n" " mul.rn.f32 %f979, %f845, %f978;\n" " mul.rn.f32 %f980, %f979, %f972;\n" " and.b32 %r535, %r473, 255;\n" " cvt.rn.f32.u32 %f981, %r535;\n" " mul.rn.f32 %f982, %f846, %f981;\n" " sub.rn.f32 %f983, %f980, %f982;\n" " and.b32 %r536, %r471, 255;\n" " cvt.rn.f32.u32 %f984, %r536;\n" " mul.rn.f32 %f985, %f847, %f984;\n" " mul.rn.f32 %f986, %f985, %f977;\n" " and.b32 %r537, %r475, 255;\n" " cvt.rn.f32.u32 %f987, %r537;\n" " mul.rn.f32 %f988, %f848, %f987;\n" " sub.rn.f32 %f989, %f986, %f988;\n" " ld.shared.f32 %f990, [%r50+512];\n" " fma.rn.f32 %f991, %f983, %f990, %f964;\n" " fma.rn.f32 %f992, %f989, %f990, %f965;\n" " and.b32 %r538, %r476, 240;\n" " or.b32 %r539, %r538, 1258291200;\n" " mov.b32 %f993, %r539;\n" " fma.rn.f32 %f994, %f993, %f881, %f880;\n" " and.b32 %r540, %r466, 32;\n" " or.b32 %r541, %r540, 1258291200;\n" " mov.b32 %f995, %r541;\n" " mov.f32 %f996, 0fCA800000;\n" " mov.f32 %f997, 0f3F000000;\n" " fma.rn.f32 %f998, %f995, %f997, %f996;\n" " add.rn.f32 %f999, %f994, %f998;\n" " and.b32 %r542, %r477, 240;\n" " or.b32 %r543, %r542, 1258291200;\n" " mov.b32 %f1000, %r543;\n" " fma.rn.f32 %f1001, %f1000, %f881, %f880;\n" " and.b32 %r544, %r467, 32;\n" " or.b32 %r545, %r544, 1258291200;\n" " mov.b32 %f1002, %r545;\n" " fma.rn.f32 %f1003, %f1002, %f997, %f996;\n" " add.rn.f32 %f1004, %f1001, %f1003;\n" " bfe.u32 %r546, %r469, 8, 8;\n" " cvt.rn.f32.u32 %f1005, %r546;\n" " mul.rn.f32 %f1006, %f845, %f1005;\n" " mul.rn.f32 %f1007, %f1006, %f999;\n" " bfe.u32 %r547, %r473, 8, 8;\n" " cvt.rn.f32.u32 %f1008, %r547;\n" " mul.rn.f32 %f1009, %f846, %f1008;\n" " sub.rn.f32 %f1010, %f1007, %f1009;\n" " bfe.u32 %r548, %r471, 8, 8;\n" " cvt.rn.f32.u32 %f1011, %r548;\n" " mul.rn.f32 %f1012, %f847, %f1011;\n" " mul.rn.f32 %f1013, %f1012, %f1004;\n" " bfe.u32 %r549, %r475, 8, 8;\n" " cvt.rn.f32.u32 %f1014, %r549;\n" " mul.rn.f32 %f1015, %f848, %f1014;\n" " sub.rn.f32 %f1016, %f1013, %f1015;\n" " ld.shared.f32 %f1017, [%r50+640];\n" " fma.rn.f32 %f1018, %f1010, %f1017, %f991;\n" " fma.rn.f32 %f1019, %f1016, %f1017, %f992;\n" " and.b32 %r550, %r476, 3840;\n" " or.b32 %r551, %r550, 1258291200;\n" " mov.b32 %f1020, %r551;\n" " fma.rn.f32 %f1021, %f1020, %f910, %f909;\n" " and.b32 %r552, %r466, 64;\n" " or.b32 %r553, %r552, 1258291200;\n" " mov.b32 %f1022, %r553;\n" " mov.f32 %f1023, 0fCA000000;\n" " mov.f32 %f1024, 0f3E800000;\n" " fma.rn.f32 %f1025, %f1022, %f1024, %f1023;\n" " add.rn.f32 %f1026, %f1021, %f1025;\n" " and.b32 %r554, %r477, 3840;\n" " or.b32 %r555, %r554, 1258291200;\n" " mov.b32 %f1027, %r555;\n" " fma.rn.f32 %f1028, %f1027, %f910, %f909;\n" " and.b32 %r556, %r467, 64;\n" " or.b32 %r557, %r556, 1258291200;\n" " mov.b32 %f1029, %r557;\n" " fma.rn.f32 %f1030, %f1029, %f1024, %f1023;\n" " add.rn.f32 %f1031, %f1028, %f1030;\n" " bfe.u32 %r558, %r469, 16, 8;\n" " cvt.rn.f32.u32 %f1032, %r558;\n" " mul.rn.f32 %f1033, %f845, %f1032;\n" " mul.rn.f32 %f1034, %f1033, %f1026;\n" " bfe.u32 %r559, %r473, 16, 8;\n" " cvt.rn.f32.u32 %f1035, %r559;\n" " mul.rn.f32 %f1036, %f846, %f1035;\n" " sub.rn.f32 %f1037, %f1034, %f1036;\n" " bfe.u32 %r560, %r471, 16, 8;\n" " cvt.rn.f32.u32 %f1038, %r560;\n" " mul.rn.f32 %f1039, %f847, %f1038;\n" " mul.rn.f32 %f1040, %f1039, %f1031;\n" " bfe.u32 %r561, %r475, 16, 8;\n" " cvt.rn.f32.u32 %f1041, %r561;\n" " mul.rn.f32 %f1042, %f848, %f1041;\n" " sub.rn.f32 %f1043, %f1040, %f1042;\n" " ld.shared.f32 %f1044, [%r50+768];\n" " fma.rn.f32 %f1045, %f1037, %f1044, %f1018;\n" " fma.rn.f32 %f1046, %f1043, %f1044, %f1019;\n" " and.b32 %r562, %r476, 61440;\n" " or.b32 %r563, %r562, 1258291200;\n" " mov.b32 %f1047, %r563;\n" " fma.rn.f32 %f1048, %f1047, %f939, %f938;\n" " and.b32 %r564, %r466, 128;\n" " or.b32 %r565, %r564, 1258291200;\n" " mov.b32 %f1049, %r565;\n" " mov.f32 %f1050, 0fC9800000;\n" " mov.f32 %f1051, 0f3E000000;\n" " fma.rn.f32 %f1052, %f1049, %f1051, %f1050;\n" " add.rn.f32 %f1053, %f1048, %f1052;\n" " and.b32 %r566, %r477, 61440;\n" " or.b32 %r567, %r566, 1258291200;\n" " mov.b32 %f1054, %r567;\n" " fma.rn.f32 %f1055, %f1054, %f939, %f938;\n" " and.b32 %r568, %r467, 128;\n" " or.b32 %r569, %r568, 1258291200;\n" " mov.b32 %f1056, %r569;\n" " fma.rn.f32 %f1057, %f1056, %f1051, %f1050;\n" " add.rn.f32 %f1058, %f1055, %f1057;\n" " shr.u32 %r570, %r469, 24;\n" " cvt.rn.f32.u32 %f1059, %r570;\n" " mul.rn.f32 %f1060, %f845, %f1059;\n" " mul.rn.f32 %f1061, %f1060, %f1053;\n" " shr.u32 %r571, %r473, 24;\n" " cvt.rn.f32.u32 %f1062, %r571;\n" " mul.rn.f32 %f1063, %f846, %f1062;\n" " sub.rn.f32 %f1064, %f1061, %f1063;\n" " shr.u32 %r572, %r471, 24;\n" " cvt.rn.f32.u32 %f1065, %r572;\n" " mul.rn.f32 %f1066, %f847, %f1065;\n" " mul.rn.f32 %f1067, %f1066, %f1058;\n" " shr.u32 %r573, %r475, 24;\n" " cvt.rn.f32.u32 %f1068, %r573;\n" " mul.rn.f32 %f1069, %f848, %f1068;\n" " sub.rn.f32 %f1070, %f1067, %f1069;\n" " ld.shared.f32 %f1071, [%r50+896];\n" " fma.rn.f32 %f1072, %f1064, %f1071, %f1045;\n" " fma.rn.f32 %f1073, %f1070, %f1071, %f1046;\n" " add.rn.f32 %f2018, %f2018, %f1072;\n" " add.rn.f32 %f2020, %f2020, %f1073;\n" " bra.uni $L__BB0_40;\n" "\n" "$L__BB0_38:\n" " shl.b64 %rd369, %rd15, 2;\n" " add.s64 %rd368, %rd394, %rd369;\n" " add.s64 %rd367, %rd393, %rd369;\n" " // begin inline asm\n" " ld.global.nc.u32 %r650, [%rd367];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r651, [%rd368];\n" " // end inline asm\n" " add.s64 %rd208, %rd393, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1241, [%rd208];\n" " // end inline asm\n" " add.s64 %rd209, %rd394, 144;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1242, [%rd209];\n" " // end inline asm\n" " shl.b64 %rd212, %rd16, 3;\n" " add.s64 %rd213, %rd393, %rd212;\n" " add.s64 %rd210, %rd213, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r652,%r653}, [%rd210];\n" " // end inline asm\n" " add.s64 %rd214, %rd394, %rd212;\n" " add.s64 %rd211, %rd214, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r654,%r655}, [%rd211];\n" " // end inline asm\n" " shr.u32 %r656, %r650, 16;\n" " shr.u32 %r657, %r651, 16;\n" " and.b32 %r658, %r650, 15;\n" " or.b32 %r659, %r658, 1258291200;\n" " mov.b32 %f1243, %r659;\n" " mov.f32 %f1244, 0fCB000004;\n" " mov.f32 %f1245, 0f3F800000;\n" " fma.rn.f32 %f1246, %f1243, %f1245, %f1244;\n" " and.b32 %r660, %r652, 255;\n" " cvt.rn.f32.u32 %f1247, %r660;\n" " mov.f32 %f1248, 0f42000000;\n" " sub.rn.f32 %f1249, %f1247, %f1248;\n" " and.b32 %r661, %r651, 15;\n" " or.b32 %r662, %r661, 1258291200;\n" " mov.b32 %f1250, %r662;\n" " fma.rn.f32 %f1251, %f1250, %f1245, %f1244;\n" " and.b32 %r663, %r654, 255;\n" " cvt.rn.f32.u32 %f1252, %r663;\n" " sub.rn.f32 %f1253, %f1252, %f1248;\n" " mul.rn.f32 %f1254, %f1249, %f1246;\n" " ld.shared.f32 %f1255, [%r50];\n" " mov.f32 %f1256, 0f00000000;\n" " fma.rn.f32 %f1257, %f1254, %f1255, %f1256;\n" " mul.rn.f32 %f1258, %f1253, %f1251;\n" " fma.rn.f32 %f1259, %f1258, %f1255, %f1256;\n" " and.b32 %r664, %r650, 240;\n" " or.b32 %r665, %r664, 1258291200;\n" " mov.b32 %f1260, %r665;\n" " mov.f32 %f1261, 0fC9000040;\n" " mov.f32 %f1262, 0f3D800000;\n" " fma.rn.f32 %f1263, %f1260, %f1262, %f1261;\n" " bfe.u32 %r666, %r652, 8, 8;\n" " cvt.rn.f32.u32 %f1264, %r666;\n" " sub.rn.f32 %f1265, %f1264, %f1248;\n" " and.b32 %r667, %r651, 240;\n" " or.b32 %r668, %r667, 1258291200;\n" " mov.b32 %f1266, %r668;\n" " fma.rn.f32 %f1267, %f1266, %f1262, %f1261;\n" " bfe.u32 %r669, %r654, 8, 8;\n" " cvt.rn.f32.u32 %f1268, %r669;\n" " sub.rn.f32 %f1269, %f1268, %f1248;\n" " mul.rn.f32 %f1270, %f1265, %f1263;\n" " ld.shared.f32 %f1271, [%r50+128];\n" " fma.rn.f32 %f1272, %f1270, %f1271, %f1257;\n" " mul.rn.f32 %f1273, %f1269, %f1267;\n" " fma.rn.f32 %f1274, %f1273, %f1271, %f1259;\n" " and.b32 %r670, %r650, 3840;\n" " or.b32 %r671, %r670, 1258291200;\n" " mov.b32 %f1275, %r671;\n" " mov.f32 %f1276, 0fC7000400;\n" " mov.f32 %f1277, 0f3B800000;\n" " fma.rn.f32 %f1278, %f1275, %f1277, %f1276;\n" " bfe.u32 %r672, %r652, 16, 8;\n" " cvt.rn.f32.u32 %f1279, %r672;\n" " sub.rn.f32 %f1280, %f1279, %f1248;\n" " and.b32 %r673, %r651, 3840;\n" " or.b32 %r674, %r673, 1258291200;\n" " mov.b32 %f1281, %r674;\n" " fma.rn.f32 %f1282, %f1281, %f1277, %f1276;\n" " bfe.u32 %r675, %r654, 16, 8;\n" " cvt.rn.f32.u32 %f1283, %r675;\n" " sub.rn.f32 %f1284, %f1283, %f1248;\n" " mul.rn.f32 %f1285, %f1280, %f1278;\n" " ld.shared.f32 %f1286, [%r50+256];\n" " fma.rn.f32 %f1287, %f1285, %f1286, %f1272;\n" " mul.rn.f32 %f1288, %f1284, %f1282;\n" " fma.rn.f32 %f1289, %f1288, %f1286, %f1274;\n" " and.b32 %r676, %r650, 61440;\n" " or.b32 %r677, %r676, 1258291200;\n" " mov.b32 %f1290, %r677;\n" " mov.f32 %f1291, 0fC5004000;\n" " mov.f32 %f1292, 0f39800000;\n" " fma.rn.f32 %f1293, %f1290, %f1292, %f1291;\n" " shr.u32 %r678, %r652, 24;\n" " cvt.rn.f32.u32 %f1294, %r678;\n" " sub.rn.f32 %f1295, %f1294, %f1248;\n" " and.b32 %r679, %r651, 61440;\n" " or.b32 %r680, %r679, 1258291200;\n" " mov.b32 %f1296, %r680;\n" " fma.rn.f32 %f1297, %f1296, %f1292, %f1291;\n" " shr.u32 %r681, %r654, 24;\n" " cvt.rn.f32.u32 %f1298, %r681;\n" " sub.rn.f32 %f1299, %f1298, %f1248;\n" " mul.rn.f32 %f1300, %f1295, %f1293;\n" " ld.shared.f32 %f1301, [%r50+384];\n" " fma.rn.f32 %f1302, %f1300, %f1301, %f1287;\n" " mul.rn.f32 %f1303, %f1299, %f1297;\n" " fma.rn.f32 %f1304, %f1303, %f1301, %f1289;\n" " and.b32 %r682, %r650, 983040;\n" " or.b32 %r683, %r682, 1258291200;\n" " mov.b32 %f1305, %r683;\n" " mov.f32 %f1306, 0fC3040000;\n" " mov.f32 %f1307, 0f37800000;\n" " fma.rn.f32 %f1308, %f1305, %f1307, %f1306;\n" " and.b32 %r684, %r653, 255;\n" " cvt.rn.f32.u32 %f1309, %r684;\n" " sub.rn.f32 %f1310, %f1309, %f1248;\n" " and.b32 %r685, %r651, 983040;\n" " or.b32 %r686, %r685, 1258291200;\n" " mov.b32 %f1311, %r686;\n" " fma.rn.f32 %f1312, %f1311, %f1307, %f1306;\n" " and.b32 %r687, %r655, 255;\n" " cvt.rn.f32.u32 %f1313, %r687;\n" " sub.rn.f32 %f1314, %f1313, %f1248;\n" " mul.rn.f32 %f1315, %f1310, %f1308;\n" " ld.shared.f32 %f1316, [%r50+512];\n" " fma.rn.f32 %f1317, %f1315, %f1316, %f1302;\n" " mul.rn.f32 %f1318, %f1314, %f1312;\n" " fma.rn.f32 %f1319, %f1318, %f1316, %f1304;\n" " and.b32 %r688, %r656, 240;\n" " or.b32 %r689, %r688, 1258291200;\n" " mov.b32 %f1320, %r689;\n" " fma.rn.f32 %f1321, %f1320, %f1262, %f1261;\n" " bfe.u32 %r690, %r653, 8, 8;\n" " cvt.rn.f32.u32 %f1322, %r690;\n" " sub.rn.f32 %f1323, %f1322, %f1248;\n" " and.b32 %r691, %r657, 240;\n" " or.b32 %r692, %r691, 1258291200;\n" " mov.b32 %f1324, %r692;\n" " fma.rn.f32 %f1325, %f1324, %f1262, %f1261;\n" " bfe.u32 %r693, %r655, 8, 8;\n" " cvt.rn.f32.u32 %f1326, %r693;\n" " sub.rn.f32 %f1327, %f1326, %f1248;\n" " mul.rn.f32 %f1328, %f1323, %f1321;\n" " ld.shared.f32 %f1329, [%r50+640];\n" " fma.rn.f32 %f1330, %f1328, %f1329, %f1317;\n" " mul.rn.f32 %f1331, %f1327, %f1325;\n" " fma.rn.f32 %f1332, %f1331, %f1329, %f1319;\n" " and.b32 %r694, %r656, 3840;\n" " or.b32 %r695, %r694, 1258291200;\n" " mov.b32 %f1333, %r695;\n" " fma.rn.f32 %f1334, %f1333, %f1277, %f1276;\n" " bfe.u32 %r696, %r653, 16, 8;\n" " cvt.rn.f32.u32 %f1335, %r696;\n" " sub.rn.f32 %f1336, %f1335, %f1248;\n" " and.b32 %r697, %r657, 3840;\n" " or.b32 %r698, %r697, 1258291200;\n" " mov.b32 %f1337, %r698;\n" " fma.rn.f32 %f1338, %f1337, %f1277, %f1276;\n" " bfe.u32 %r699, %r655, 16, 8;\n" " cvt.rn.f32.u32 %f1339, %r699;\n" " sub.rn.f32 %f1340, %f1339, %f1248;\n" " mul.rn.f32 %f1341, %f1336, %f1334;\n" " ld.shared.f32 %f1342, [%r50+768];\n" " fma.rn.f32 %f1343, %f1341, %f1342, %f1330;\n" " mul.rn.f32 %f1344, %f1340, %f1338;\n" " fma.rn.f32 %f1345, %f1344, %f1342, %f1332;\n" " and.b32 %r700, %r656, 61440;\n" " or.b32 %r701, %r700, 1258291200;\n" " mov.b32 %f1346, %r701;\n" " fma.rn.f32 %f1347, %f1346, %f1292, %f1291;\n" " shr.u32 %r702, %r653, 24;\n" " cvt.rn.f32.u32 %f1348, %r702;\n" " sub.rn.f32 %f1349, %f1348, %f1248;\n" " and.b32 %r703, %r657, 61440;\n" " or.b32 %r704, %r703, 1258291200;\n" " mov.b32 %f1350, %r704;\n" " fma.rn.f32 %f1351, %f1350, %f1292, %f1291;\n" " shr.u32 %r705, %r655, 24;\n" " cvt.rn.f32.u32 %f1352, %r705;\n" " sub.rn.f32 %f1353, %f1352, %f1248;\n" " mul.rn.f32 %f1354, %f1349, %f1347;\n" " ld.shared.f32 %f1355, [%r50+896];\n" " fma.rn.f32 %f1356, %f1354, %f1355, %f1343;\n" " mul.rn.f32 %f1357, %f1353, %f1351;\n" " fma.rn.f32 %f1358, %f1357, %f1355, %f1345;\n" " fma.rn.f32 %f2018, %f1241, %f1356, %f2018;\n" " fma.rn.f32 %f2020, %f1242, %f1358, %f2020;\n" " bra.uni $L__BB0_40;\n" "\n" "$L__BB0_37:\n" " shl.b64 %rd366, %rd15, 2;\n" " add.s64 %rd365, %rd394, %rd366;\n" " add.s64 %rd364, %rd393, %rd366;\n" " // begin inline asm\n" " ld.global.nc.u32 %r574, [%rd364];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r575, [%rd365];\n" " // end inline asm\n" " add.s64 %rd200, %rd393, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r576,%r577}, [%rd200];\n" " // end inline asm\n" " add.s64 %rd201, %rd394, 128;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r578,%r579}, [%rd201];\n" " // end inline asm\n" " add.s64 %rd202, %rd393, 136;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r580,%r581}, [%rd202];\n" " // end inline asm\n" " add.s64 %rd203, %rd394, 136;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r582,%r583}, [%rd203];\n" " // end inline asm\n" " add.s64 %rd204, %rd393, 144;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f1074,%f1075}, [%rd204];\n" " // end inline asm\n" " add.s64 %rd205, %rd394, 144;\n" " // begin inline asm\n" " ld.global.nc.v2.f32 {%f1076,%f1077}, [%rd205];\n" " // end inline asm\n" " shr.u32 %r584, %r574, 16;\n" " shr.u32 %r585, %r575, 16;\n" " and.b32 %r586, %r576, 255;\n" " cvt.rn.f32.u32 %f1078, %r586;\n" " mul.rn.f32 %f1079, %f1074, %f1078;\n" " and.b32 %r587, %r574, 15;\n" " or.b32 %r588, %r587, 1258291200;\n" " mov.b32 %f1080, %r588;\n" " mov.f32 %f1081, 0fCB000000;\n" " mov.f32 %f1082, 0f3F800000;\n" " fma.rn.f32 %f1083, %f1080, %f1082, %f1081;\n" " mul.rn.f32 %f1084, %f1079, %f1083;\n" " and.b32 %r589, %r580, 255;\n" " cvt.rn.f32.u32 %f1085, %r589;\n" " mul.rn.f32 %f1086, %f1075, %f1085;\n" " sub.rn.f32 %f1087, %f1084, %f1086;\n" " and.b32 %r590, %r578, 255;\n" " cvt.rn.f32.u32 %f1088, %r590;\n" " mul.rn.f32 %f1089, %f1076, %f1088;\n" " and.b32 %r591, %r575, 15;\n" " or.b32 %r592, %r591, 1258291200;\n" " mov.b32 %f1090, %r592;\n" " fma.rn.f32 %f1091, %f1090, %f1082, %f1081;\n" " mul.rn.f32 %f1092, %f1089, %f1091;\n" " and.b32 %r593, %r582, 255;\n" " cvt.rn.f32.u32 %f1093, %r593;\n" " mul.rn.f32 %f1094, %f1077, %f1093;\n" " sub.rn.f32 %f1095, %f1092, %f1094;\n" " ld.shared.f32 %f1096, [%r50];\n" " mov.f32 %f1097, 0f00000000;\n" " fma.rn.f32 %f1098, %f1087, %f1096, %f1097;\n" " fma.rn.f32 %f1099, %f1095, %f1096, %f1097;\n" " bfe.u32 %r594, %r576, 8, 8;\n" " cvt.rn.f32.u32 %f1100, %r594;\n" " mul.rn.f32 %f1101, %f1074, %f1100;\n" " and.b32 %r595, %r574, 240;\n" " or.b32 %r596, %r595, 1258291200;\n" " mov.b32 %f1102, %r596;\n" " mov.f32 %f1103, 0fC9000000;\n" " mov.f32 %f1104, 0f3D800000;\n" " fma.rn.f32 %f1105, %f1102, %f1104, %f1103;\n" " mul.rn.f32 %f1106, %f1101, %f1105;\n" " bfe.u32 %r597, %r580, 8, 8;\n" " cvt.rn.f32.u32 %f1107, %r597;\n" " mul.rn.f32 %f1108, %f1075, %f1107;\n" " sub.rn.f32 %f1109, %f1106, %f1108;\n" " bfe.u32 %r598, %r578, 8, 8;\n" " cvt.rn.f32.u32 %f1110, %r598;\n" " mul.rn.f32 %f1111, %f1076, %f1110;\n" " and.b32 %r599, %r575, 240;\n" " or.b32 %r600, %r599, 1258291200;\n" " mov.b32 %f1112, %r600;\n" " fma.rn.f32 %f1113, %f1112, %f1104, %f1103;\n" " mul.rn.f32 %f1114, %f1111, %f1113;\n" " bfe.u32 %r601, %r582, 8, 8;\n" " cvt.rn.f32.u32 %f1115, %r601;\n" " mul.rn.f32 %f1116, %f1077, %f1115;\n" " sub.rn.f32 %f1117, %f1114, %f1116;\n" " ld.shared.f32 %f1118, [%r50+128];\n" " fma.rn.f32 %f1119, %f1109, %f1118, %f1098;\n" " fma.rn.f32 %f1120, %f1117, %f1118, %f1099;\n" " bfe.u32 %r602, %r576, 16, 8;\n" " cvt.rn.f32.u32 %f1121, %r602;\n" " mul.rn.f32 %f1122, %f1074, %f1121;\n" " and.b32 %r603, %r574, 3840;\n" " or.b32 %r604, %r603, 1258291200;\n" " mov.b32 %f1123, %r604;\n" " mov.f32 %f1124, 0fC7000000;\n" " mov.f32 %f1125, 0f3B800000;\n" " fma.rn.f32 %f1126, %f1123, %f1125, %f1124;\n" " mul.rn.f32 %f1127, %f1122, %f1126;\n" " bfe.u32 %r605, %r580, 16, 8;\n" " cvt.rn.f32.u32 %f1128, %r605;\n" " mul.rn.f32 %f1129, %f1075, %f1128;\n" " sub.rn.f32 %f1130, %f1127, %f1129;\n" " bfe.u32 %r606, %r578, 16, 8;\n" " cvt.rn.f32.u32 %f1131, %r606;\n" " mul.rn.f32 %f1132, %f1076, %f1131;\n" " and.b32 %r607, %r575, 3840;\n" " or.b32 %r608, %r607, 1258291200;\n" " mov.b32 %f1133, %r608;\n" " fma.rn.f32 %f1134, %f1133, %f1125, %f1124;\n" " mul.rn.f32 %f1135, %f1132, %f1134;\n" " bfe.u32 %r609, %r582, 16, 8;\n" " cvt.rn.f32.u32 %f1136, %r609;\n" " mul.rn.f32 %f1137, %f1077, %f1136;\n" " sub.rn.f32 %f1138, %f1135, %f1137;\n" " ld.shared.f32 %f1139, [%r50+256];\n" " fma.rn.f32 %f1140, %f1130, %f1139, %f1119;\n" " fma.rn.f32 %f1141, %f1138, %f1139, %f1120;\n" " shr.u32 %r610, %r576, 24;\n" " cvt.rn.f32.u32 %f1142, %r610;\n" " mul.rn.f32 %f1143, %f1074, %f1142;\n" " and.b32 %r611, %r574, 61440;\n" " or.b32 %r612, %r611, 1258291200;\n" " mov.b32 %f1144, %r612;\n" " mov.f32 %f1145, 0fC5000000;\n" " mov.f32 %f1146, 0f39800000;\n" " fma.rn.f32 %f1147, %f1144, %f1146, %f1145;\n" " mul.rn.f32 %f1148, %f1143, %f1147;\n" " shr.u32 %r613, %r580, 24;\n" " cvt.rn.f32.u32 %f1149, %r613;\n" " mul.rn.f32 %f1150, %f1075, %f1149;\n" " sub.rn.f32 %f1151, %f1148, %f1150;\n" " shr.u32 %r614, %r578, 24;\n" " cvt.rn.f32.u32 %f1152, %r614;\n" " mul.rn.f32 %f1153, %f1076, %f1152;\n" " and.b32 %r615, %r575, 61440;\n" " or.b32 %r616, %r615, 1258291200;\n" " mov.b32 %f1154, %r616;\n" " fma.rn.f32 %f1155, %f1154, %f1146, %f1145;\n" " mul.rn.f32 %f1156, %f1153, %f1155;\n" " shr.u32 %r617, %r582, 24;\n" " cvt.rn.f32.u32 %f1157, %r617;\n" " mul.rn.f32 %f1158, %f1077, %f1157;\n" " sub.rn.f32 %f1159, %f1156, %f1158;\n" " ld.shared.f32 %f1160, [%r50+384];\n" " fma.rn.f32 %f1161, %f1151, %f1160, %f1140;\n" " fma.rn.f32 %f1162, %f1159, %f1160, %f1141;\n" " and.b32 %r618, %r577, 255;\n" " cvt.rn.f32.u32 %f1163, %r618;\n" " mul.rn.f32 %f1164, %f1074, %f1163;\n" " and.b32 %r619, %r574, 983040;\n" " or.b32 %r620, %r619, 1258291200;\n" " mov.b32 %f1165, %r620;\n" " mov.f32 %f1166, 0fC3000000;\n" " mov.f32 %f1167, 0f37800000;\n" " fma.rn.f32 %f1168, %f1165, %f1167, %f1166;\n" " mul.rn.f32 %f1169, %f1164, %f1168;\n" " and.b32 %r621, %r581, 255;\n" " cvt.rn.f32.u32 %f1170, %r621;\n" " mul.rn.f32 %f1171, %f1075, %f1170;\n" " sub.rn.f32 %f1172, %f1169, %f1171;\n" " and.b32 %r622, %r579, 255;\n" " cvt.rn.f32.u32 %f1173, %r622;\n" " mul.rn.f32 %f1174, %f1076, %f1173;\n" " and.b32 %r623, %r575, 983040;\n" " or.b32 %r624, %r623, 1258291200;\n" " mov.b32 %f1175, %r624;\n" " fma.rn.f32 %f1176, %f1175, %f1167, %f1166;\n" " mul.rn.f32 %f1177, %f1174, %f1176;\n" " and.b32 %r625, %r583, 255;\n" " cvt.rn.f32.u32 %f1178, %r625;\n" " mul.rn.f32 %f1179, %f1077, %f1178;\n" " sub.rn.f32 %f1180, %f1177, %f1179;\n" " ld.shared.f32 %f1181, [%r50+512];\n" " fma.rn.f32 %f1182, %f1172, %f1181, %f1161;\n" " fma.rn.f32 %f1183, %f1180, %f1181, %f1162;\n" " bfe.u32 %r626, %r577, 8, 8;\n" " cvt.rn.f32.u32 %f1184, %r626;\n" " mul.rn.f32 %f1185, %f1074, %f1184;\n" " and.b32 %r627, %r584, 240;\n" " or.b32 %r628, %r627, 1258291200;\n" " mov.b32 %f1186, %r628;\n" " fma.rn.f32 %f1187, %f1186, %f1104, %f1103;\n" " mul.rn.f32 %f1188, %f1185, %f1187;\n" " bfe.u32 %r629, %r581, 8, 8;\n" " cvt.rn.f32.u32 %f1189, %r629;\n" " mul.rn.f32 %f1190, %f1075, %f1189;\n" " sub.rn.f32 %f1191, %f1188, %f1190;\n" " bfe.u32 %r630, %r579, 8, 8;\n" " cvt.rn.f32.u32 %f1192, %r630;\n" " mul.rn.f32 %f1193, %f1076, %f1192;\n" " and.b32 %r631, %r585, 240;\n" " or.b32 %r632, %r631, 1258291200;\n" " mov.b32 %f1194, %r632;\n" " fma.rn.f32 %f1195, %f1194, %f1104, %f1103;\n" " mul.rn.f32 %f1196, %f1193, %f1195;\n" " bfe.u32 %r633, %r583, 8, 8;\n" " cvt.rn.f32.u32 %f1197, %r633;\n" " mul.rn.f32 %f1198, %f1077, %f1197;\n" " sub.rn.f32 %f1199, %f1196, %f1198;\n" " ld.shared.f32 %f1200, [%r50+640];\n" " fma.rn.f32 %f1201, %f1191, %f1200, %f1182;\n" " fma.rn.f32 %f1202, %f1199, %f1200, %f1183;\n" " bfe.u32 %r634, %r577, 16, 8;\n" " cvt.rn.f32.u32 %f1203, %r634;\n" " mul.rn.f32 %f1204, %f1074, %f1203;\n" " and.b32 %r635, %r584, 3840;\n" " or.b32 %r636, %r635, 1258291200;\n" " mov.b32 %f1205, %r636;\n" " fma.rn.f32 %f1206, %f1205, %f1125, %f1124;\n" " mul.rn.f32 %f1207, %f1204, %f1206;\n" " bfe.u32 %r637, %r581, 16, 8;\n" " cvt.rn.f32.u32 %f1208, %r637;\n" " mul.rn.f32 %f1209, %f1075, %f1208;\n" " sub.rn.f32 %f1210, %f1207, %f1209;\n" " bfe.u32 %r638, %r579, 16, 8;\n" " cvt.rn.f32.u32 %f1211, %r638;\n" " mul.rn.f32 %f1212, %f1076, %f1211;\n" " and.b32 %r639, %r585, 3840;\n" " or.b32 %r640, %r639, 1258291200;\n" " mov.b32 %f1213, %r640;\n" " fma.rn.f32 %f1214, %f1213, %f1125, %f1124;\n" " mul.rn.f32 %f1215, %f1212, %f1214;\n" " bfe.u32 %r641, %r583, 16, 8;\n" " cvt.rn.f32.u32 %f1216, %r641;\n" " mul.rn.f32 %f1217, %f1077, %f1216;\n" " sub.rn.f32 %f1218, %f1215, %f1217;\n" " ld.shared.f32 %f1219, [%r50+768];\n" " fma.rn.f32 %f1220, %f1210, %f1219, %f1201;\n" " fma.rn.f32 %f1221, %f1218, %f1219, %f1202;\n" " shr.u32 %r642, %r577, 24;\n" " cvt.rn.f32.u32 %f1222, %r642;\n" " mul.rn.f32 %f1223, %f1074, %f1222;\n" " and.b32 %r643, %r584, 61440;\n" " or.b32 %r644, %r643, 1258291200;\n" " mov.b32 %f1224, %r644;\n" " fma.rn.f32 %f1225, %f1224, %f1146, %f1145;\n" " mul.rn.f32 %f1226, %f1223, %f1225;\n" " shr.u32 %r645, %r581, 24;\n" " cvt.rn.f32.u32 %f1227, %r645;\n" " mul.rn.f32 %f1228, %f1075, %f1227;\n" " sub.rn.f32 %f1229, %f1226, %f1228;\n" " shr.u32 %r646, %r579, 24;\n" " cvt.rn.f32.u32 %f1230, %r646;\n" " mul.rn.f32 %f1231, %f1076, %f1230;\n" " and.b32 %r647, %r585, 61440;\n" " or.b32 %r648, %r647, 1258291200;\n" " mov.b32 %f1232, %r648;\n" " fma.rn.f32 %f1233, %f1232, %f1146, %f1145;\n" " mul.rn.f32 %f1234, %f1231, %f1233;\n" " shr.u32 %r649, %r583, 24;\n" " cvt.rn.f32.u32 %f1235, %r649;\n" " mul.rn.f32 %f1236, %f1077, %f1235;\n" " sub.rn.f32 %f1237, %f1234, %f1236;\n" " ld.shared.f32 %f1238, [%r50+896];\n" " fma.rn.f32 %f1239, %f1229, %f1238, %f1220;\n" " fma.rn.f32 %f1240, %f1237, %f1238, %f1221;\n" " add.rn.f32 %f2018, %f2018, %f1239;\n" " add.rn.f32 %f2020, %f2020, %f1240;\n" " bra.uni $L__BB0_40;\n" "\n" "$L__BB0_39:\n" " shl.b64 %rd360, %rd15, 2;\n" " add.s64 %rd359, %rd394, %rd360;\n" " add.s64 %rd358, %rd393, %rd360;\n" " // begin inline asm\n" " ld.global.nc.u32 %r706, [%rd358];\n" " // end inline asm\n" " // begin inline asm\n" " ld.global.nc.u32 %r707, [%rd359];\n" " // end inline asm\n" " shl.b64 %rd223, %rd15, 1;\n" " add.s64 %rd224, %rd393, %rd223;\n" " add.s64 %rd217, %rd224, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs3, [%rd217];\n" " // end inline asm\n" " cvt.u32.u16 %r712, %rs3;\n" " add.s64 %rd225, %rd394, %rd223;\n" " add.s64 %rd218, %rd225, 128;\n" " // begin inline asm\n" " ld.global.nc.u16 %rs4, [%rd218];\n" " // end inline asm\n" " cvt.u32.u16 %r713, %rs4;\n" " add.s64 %rd219, %rd393, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1359, [%rd219];\n" " // end inline asm\n" " add.s64 %rd220, %rd394, 208;\n" " // begin inline asm\n" " ld.global.nc.f32 %f1360, [%rd220];\n" " // end inline asm\n" " shl.b64 %rd226, %rd16, 3;\n" " add.s64 %rd227, %rd393, %rd226;\n" " add.s64 %rd221, %rd227, 192;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r708,%r709}, [%rd221];\n" " // end inline asm\n" " add.s64 %rd228, %rd394, %rd226;\n" " add.s64 %rd222, %rd228, 192;\n" " // begin inline asm\n" " ld.global.nc.v2.u32 {%r710,%r711}, [%rd222];\n" " // end inline asm\n" " shr.u32 %r714, %r706, 16;\n" " shr.u32 %r715, %r707, 16;\n" " and.b32 %r716, %r706, 15;\n" " or.b32 %r717, %r716, 1258291200;\n" " mov.b32 %f1361, %r717;\n" " mov.f32 %f1362, 0fCB000020;\n" " mov.f32 %f1363, 0f3F800000;\n" " fma.rn.f32 %f1364, %f1361, %f1363, %f1362;\n" " and.b32 %r718, %r712, 3;\n" " or.b32 %r719, %r718, 1258291200;\n" " mov.b32 %f1365, %r719;\n" " mov.f32 %f1366, 0fCD000000;\n" " mov.f32 %f1367, 0f41800000;\n" " fma.rn.f32 %f1368, %f1365, %f1367, %f1366;\n" " add.rn.f32 %f1369, %f1364, %f1368;\n" " and.b32 %r720, %r707, 15;\n" " or.b32 %r721, %r720, 1258291200;\n" " mov.b32 %f1370, %r721;\n" " fma.rn.f32 %f1371, %f1370, %f1363, %f1362;\n" " and.b32 %r722, %r713, 3;\n" " or.b32 %r723, %r722, 1258291200;\n" " mov.b32 %f1372, %r723;\n" " fma.rn.f32 %f1373, %f1372, %f1367, %f1366;\n" " add.rn.f32 %f1374, %f1371, %f1373;\n" " and.b32 %r724, %r708, 255;\n" " cvt.rn.f32.u32 %f1375, %r724;\n" " mov.f32 %f1376, 0f43000000;\n" " sub.rn.f32 %f1377, %f1375, %f1376;\n" " and.b32 %r725, %r710, 255;\n" " cvt.rn.f32.u32 %f1378, %r725;\n" " sub.rn.f32 %f1379, %f1378, %f1376;\n" " mul.rn.f32 %f1380, %f1377, %f1369;\n" " ld.shared.f32 %f1381, [%r50];\n" " mov.f32 %f1382, 0f00000000;\n" " fma.rn.f32 %f1383, %f1380, %f1381, %f1382;\n" " mul.rn.f32 %f1384, %f1379, %f1374;\n" " fma.rn.f32 %f1385, %f1384, %f1381, %f1382;\n" " and.b32 %r726, %r706, 3840;\n" " or.b32 %r727, %r726, 1258291200;\n" " mov.b32 %f1386, %r727;\n" " mov.f32 %f1387, 0fC7002000;\n" " mov.f32 %f1388, 0f3B800000;\n" " fma.rn.f32 %f1389, %f1386, %f1388, %f1387;\n" " and.b32 %r728, %r712, 12;\n" " or.b32 %r729, %r728, 1258291200;\n" " mov.b32 %f1390, %r729;\n" " mov.f32 %f1391, 0fCC000000;\n" " mov.f32 %f1392, 0f40800000;\n" " fma.rn.f32 %f1393, %f1390, %f1392, %f1391;\n" " add.rn.f32 %f1394, %f1389, %f1393;\n" " and.b32 %r730, %r707, 3840;\n" " or.b32 %r731, %r730, 1258291200;\n" " mov.b32 %f1395, %r731;\n" " fma.rn.f32 %f1396, %f1395, %f1388, %f1387;\n" " and.b32 %r732, %r713, 12;\n" " or.b32 %r733, %r732, 1258291200;\n" " mov.b32 %f1397, %r733;\n" " fma.rn.f32 %f1398, %f1397, %f1392, %f1391;\n" " add.rn.f32 %f1399, %f1396, %f1398;\n" " bfe.u32 %r734, %r708, 8, 8;\n" " cvt.rn.f32.u32 %f1400, %r734;\n" " sub.rn.f32 %f1401, %f1400, %f1376;\n" " bfe.u32 %r735, %r710, 8, 8;\n" " cvt.rn.f32.u32 %f1402, %r735;\n" " sub.rn.f32 %f1403, %f1402, %f1376;\n" " mul.rn.f32 %f1404, %f1401, %f1394;\n" " ld.shared.f32 %f1405, [%r50+128];\n" " fma.rn.f32 %f1406, %f1404, %f1405, %f1383;\n" " mul.rn.f32 %f1407, %f1403, %f1399;\n" " fma.rn.f32 %f1408, %f1407, %f1405, %f1385;\n" " and.b32 %r736, %r706, 240;\n" " or.b32 %r737, %r736, 1258291200;\n" " mov.b32 %f1409, %r737;\n" " mov.f32 %f1410, 0fC9000200;\n" " mov.f32 %f1411, 0f3D800000;\n" " fma.rn.f32 %f1412, %f1409, %f1411, %f1410;\n" " and.b32 %r738, %r712, 48;\n" " or.b32 %r739, %r738, 1258291200;\n" " mov.b32 %f1413, %r739;\n" " mov.f32 %f1414, 0fCB000000;\n" " fma.rn.f32 %f1415, %f1413, %f1363, %f1414;\n" " add.rn.f32 %f1416, %f1412, %f1415;\n" " and.b32 %r740, %r707, 240;\n" " or.b32 %r741, %r740, 1258291200;\n" " mov.b32 %f1417, %r741;\n" " fma.rn.f32 %f1418, %f1417, %f1411, %f1410;\n" " and.b32 %r742, %r713, 48;\n" " or.b32 %r743, %r742, 1258291200;\n" " mov.b32 %f1419, %r743;\n" " fma.rn.f32 %f1420, %f1419, %f1363, %f1414;\n" " add.rn.f32 %f1421, %f1418, %f1420;\n" " bfe.u32 %r744, %r708, 16, 8;\n" " cvt.rn.f32.u32 %f1422, %r744;\n" " sub.rn.f32 %f1423, %f1422, %f1376;\n" " bfe.u32 %r745, %r710, 16, 8;\n" " cvt.rn.f32.u32 %f1424, %r745;\n" " sub.rn.f32 %f1425, %f1424, %f1376;\n" " mul.rn.f32 %f1426, %f1423, %f1416;\n" " ld.shared.f32 %f1427, [%r50+256];\n" " fma.rn.f32 %f1428, %f1426, %f1427, %f1406;\n" " mul.rn.f32 %f1429, %f1425, %f1421;\n" " fma.rn.f32 %f1430, %f1429, %f1427, %f1408;\n" " and.b32 %r746, %r706, 61440;\n" " or.b32 %r747, %r746, 1258291200;\n" " mov.b32 %f1431, %r747;\n" " mov.f32 %f1432, 0fC5020000;\n" " mov.f32 %f1433, 0f39800000;\n" " fma.rn.f32 %f1434, %f1431, %f1433, %f1432;\n" " and.b32 %r748, %r712, 192;\n" " or.b32 %r749, %r748, 1258291200;\n" " mov.b32 %f1435, %r749;\n" " mov.f32 %f1436, 0fCA000000;\n" " mov.f32 %f1437, 0f3E800000;\n" " fma.rn.f32 %f1438, %f1435, %f1437, %f1436;\n" " add.rn.f32 %f1439, %f1434, %f1438;\n" " and.b32 %r750, %r707, 61440;\n" " or.b32 %r751, %r750, 1258291200;\n" " mov.b32 %f1440, %r751;\n" " fma.rn.f32 %f1441, %f1440, %f1433, %f1432;\n" " and.b32 %r752, %r713, 192;\n" " or.b32 %r753, %r752, 1258291200;\n" " mov.b32 %f1442, %r753;\n" " fma.rn.f32 %f1443, %f1442, %f1437, %f1436;\n" " add.rn.f32 %f1444, %f1441, %f1443;\n" " shr.u32 %r754, %r708, 24;\n" " cvt.rn.f32.u32 %f1445, %r754;\n" " sub.rn.f32 %f1446, %f1445, %f1376;\n" " shr.u32 %r755, %r710, 24;\n" " cvt.rn.f32.u32 %f1447, %r755;\n" " sub.rn.f32 %f1448, %f1447, %f1376;\n" " mul.rn.f32 %f1449, %f1446, %f1439;\n" " ld.shared.f32 %f1450, [%r50+384];\n" " fma.rn.f32 %f1451, %f1449, %f1450, %f1428;\n" " mul.rn.f32 %f1452, %f1448, %f1444;\n" " fma.rn.f32 %f1453, %f1452, %f1450, %f1430;\n" " and.b32 %r756, %r706, 983040;\n" " or.b32 %r757, %r756, 1258291200;\n" " mov.b32 %f1454, %r757;\n" " mov.f32 %f1455, 0fC3200000;\n" " mov.f32 %f1456, 0f37800000;\n" " fma.rn.f32 %f1457, %f1454, %f1456, %f1455;\n" " and.b32 %r758, %r712, 768;\n" " or.b32 %r759, %r758, 1258291200;\n" " mov.b32 %f1458, %r759;\n" " mov.f32 %f1459, 0fC9000000;\n" " fma.rn.f32 %f1460, %f1458, %f1411, %f1459;\n" " add.rn.f32 %f1461, %f1457, %f1460;\n" " and.b32 %r760, %r707, 983040;\n" " or.b32 %r761, %r760, 1258291200;\n" " mov.b32 %f1462, %r761;\n" " fma.rn.f32 %f1463, %f1462, %f1456, %f1455;\n" " and.b32 %r762, %r713, 768;\n" " or.b32 %r763, %r762, 1258291200;\n" " mov.b32 %f1464, %r763;\n" " fma.rn.f32 %f1465, %f1464, %f1411, %f1459;\n" " add.rn.f32 %f1466, %f1463, %f1465;\n" " and.b32 %r764, %r709, 255;\n" " cvt.rn.f32.u32 %f1467, %r764;\n" " sub.rn.f32 %f1468, %f1467, %f1376;\n" " and.b32 %r765, %r711, 255;\n" " cvt.rn.f32.u32 %f1469, %r765;\n" " sub.rn.f32 %f1470, %f1469, %f1376;\n" " mul.rn.f32 %f1471, %f1468, %f1461;\n" " ld.shared.f32 %f1472, [%r50+512];\n" " fma.rn.f32 %f1473, %f1471, %f1472, %f1451;\n" " mul.rn.f32 %f1474, %f1470, %f1466;\n" " fma.rn.f32 %f1475, %f1474, %f1472, %f1453;\n" " and.b32 %r766, %r714, 3840;\n" " or.b32 %r767, %r766, 1258291200;\n" " mov.b32 %f1476, %r767;\n" " fma.rn.f32 %f1477, %f1476, %f1388, %f1387;\n" " and.b32 %r768, %r712, 3072;\n" " or.b32 %r769, %r768, 1258291200;\n" " mov.b32 %f1478, %r769;\n" " mov.f32 %f1479, 0fC8000000;\n" " mov.f32 %f1480, 0f3C800000;\n" " fma.rn.f32 %f1481, %f1478, %f1480, %f1479;\n" " add.rn.f32 %f1482, %f1477, %f1481;\n" " and.b32 %r770, %r715, 3840;\n" " or.b32 %r771, %r770, 1258291200;\n" " mov.b32 %f1483, %r771;\n" " fma.rn.f32 %f1484, %f1483, %f1388, %f1387;\n" " and.b32 %r772, %r713, 3072;\n" " or.b32 %r773, %r772, 1258291200;\n" " mov.b32 %f1485, %r773;\n" " fma.rn.f32 %f1486, %f1485, %f1480, %f1479;\n" " add.rn.f32 %f1487, %f1484, %f1486;\n" " bfe.u32 %r774, %r709, 8, 8;\n" " cvt.rn.f32.u32 %f1488, %r774;\n" " sub.rn.f32 %f1489, %f1488, %f1376;\n" " bfe.u32 %r775, %r711, 8, 8;\n" " cvt.rn.f32.u32 %f1490, %r775;\n" " sub.rn.f32 %f1491, %f1490, %f1376;\n" " mul.rn.f32 %f1492, %f1489, %f1482;\n" " ld.shared.f32 %f1493, [%r50+640];\n" " fma.rn.f32 %f1494, %f1492, %f1493, %f1473;\n" " mul.rn.f32 %f1495, %f1491, %f1487;\n" " fma.rn.f32 %f1496, %f1495, %f1493, %f1475;\n" " and.b32 %r776, %r714, 240;\n" " or.b32 %r777, %r776, 1258291200;\n" " mov.b32 %f1497, %r777;\n" " fma.rn.f32 %f1498, %f1497, %f1411, %f1410;\n" " and.b32 %r778, %r712, 12288;\n" " or.b32 %r779, %r778, 1258291200;\n" " mov.b32 %f1499, %r779;\n" " mov.f32 %f1500, 0fC7000000;\n" " fma.rn.f32 %f1501, %f1499, %f1388, %f1500;\n" " add.rn.f32 %f1502, %f1498, %f1501;\n" " and.b32 %r780, %r715, 240;\n" " or.b32 %r781, %r780, 1258291200;\n" " mov.b32 %f1503, %r781;\n" " fma.rn.f32 %f1504, %f1503, %f1411, %f1410;\n" " and.b32 %r782, %r713, 12288;\n" " or.b32 %r783, %r782, 1258291200;\n" " mov.b32 %f1505, %r783;\n" " fma.rn.f32 %f1506, %f1505, %f1388, %f1500;\n" " add.rn.f32 %f1507, %f1504, %f1506;\n" " bfe.u32 %r784, %r709, 16, 8;\n" " cvt.rn.f32.u32 %f1508, %r784;\n" " sub.rn.f32 %f1509, %f1508, %f1376;\n" " bfe.u32 %r785, %r711, 16, 8;\n" " cvt.rn.f32.u32 %f1510, %r785;\n" " sub.rn.f32 %f1511, %f1510, %f1376;\n" " mul.rn.f32 %f1512, %f1509, %f1502;\n" " ld.shared.f32 %f1513, [%r50+768];\n" " fma.rn.f32 %f1514, %f1512, %f1513, %f1494;\n" " mul.rn.f32 %f1515, %f1511, %f1507;\n" " fma.rn.f32 %f1516, %f1515, %f1513, %f1496;\n" " and.b32 %r786, %r714, 61440;\n" " or.b32 %r787, %r786, 1258291200;\n" " mov.b32 %f1517, %r787;\n" " fma.rn.f32 %f1518, %f1517, %f1433, %f1432;\n" " and.b32 %r788, %r712, 49152;\n" " or.b32 %r789, %r788, 1258291200;\n" " mov.b32 %f1519, %r789;\n" " mov.f32 %f1520, 0fC6000000;\n" " mov.f32 %f1521, 0f3A800000;\n" " fma.rn.f32 %f1522, %f1519, %f1521, %f1520;\n" " add.rn.f32 %f1523, %f1518, %f1522;\n" " and.b32 %r790, %r715, 61440;\n" " or.b32 %r791, %r790, 1258291200;\n" " mov.b32 %f1524, %r791;\n" " fma.rn.f32 %f1525, %f1524, %f1433, %f1432;\n" " and.b32 %r792, %r713, 49152;\n" " or.b32 %r793, %r792, 1258291200;\n" " mov.b32 %f1526, %r793;\n" " fma.rn.f32 %f1527, %f1526, %f1521, %f1520;\n" " add.rn.f32 %f1528, %f1525, %f1527;\n" " shr.u32 %r794, %r709, 24;\n" " cvt.rn.f32.u32 %f1529, %r794;\n" " sub.rn.f32 %f1530, %f1529, %f1376;\n" " shr.u32 %r795, %r711, 24;\n" " cvt.rn.f32.u32 %f1531, %r795;\n" " sub.rn.f32 %f1532, %f1531, %f1376;\n" " mul.rn.f32 %f1533, %f1530, %f1523;\n" " ld.shared.f32 %f1534, [%r50+896];\n" " fma.rn.f32 %f1535, %f1533, %f1534, %f1514;\n" " mul.rn.f32 %f1536, %f1532, %f1528;\n" " fma.rn.f32 %f1537, %f1536, %f1534, %f1516;\n" " fma.rn.f32 %f2018, %f1359, %f1535, %f2018;\n" " fma.rn.f32 %f2020, %f1360, %f1537, %f2020;\n" "\n" "$L__BB0_40:\n" " @%p56 bra $L__BB0_42;\n" "\n" " add.s64 %rd393, %rd393, %rd13;\n" " add.s64 %rd394, %rd394, %rd13;\n" "\n" "$L__BB0_42:\n" " sub.s32 %r922, %r103, %r955;\n" " min.s32 %r921, %r922, 2048;\n" " shr.s32 %r920, %r921, 8;\n" " add.s64 %rd413, %rd413, %rd13;\n" " add.s64 %rd414, %rd414, %rd13;\n" " add.s32 %r960, %r960, 1024;\n" " add.s32 %r959, %r959, 1;\n" " setp.lt.s32 %p34, %r959, %r920;\n" " @%p34 bra $L__BB0_25;\n" "\n" "$L__BB0_99:\n" " add.s32 %r955, %r955, 2048;\n" " setp.gt.s32 %p69, %r103, %r955;\n" " add.s32 %r954, %r954, 1;\n" " @%p69 bra $L__BB0_13;\n" "\n" "$L__BB0_100:\n" " mov.u32 %r840, 16;\n" " mov.u32 %r849, 31;\n" " // begin inline asm\n" " shfl.down.b32 %f1651, %f2017, %r840, %r849;\n" " // end inline asm\n" " add.f32 %f1654, %f2017, %f1651;\n" " mov.u32 %r842, 8;\n" " // begin inline asm\n" " shfl.down.b32 %f1653, %f1654, %r842, %r849;\n" " // end inline asm\n" " add.f32 %f1656, %f1654, %f1653;\n" " mov.u32 %r844, 4;\n" " // begin inline asm\n" " shfl.down.b32 %f1655, %f1656, %r844, %r849;\n" " // end inline asm\n" " add.f32 %f1658, %f1656, %f1655;\n" " mov.u32 %r846, 2;\n" " // begin inline asm\n" " shfl.down.b32 %f1657, %f1658, %r846, %r849;\n" " // end inline asm\n" " add.f32 %f1660, %f1658, %f1657;\n" " mov.u32 %r848, 1;\n" " // begin inline asm\n" " shfl.down.b32 %f1659, %f1660, %r848, %r849;\n" " // end inline asm\n" " add.f32 %f105, %f1660, %f1659;\n" " // begin inline asm\n" " shfl.down.b32 %f1661, %f2019, %r840, %r849;\n" " // end inline asm\n" " add.f32 %f1664, %f2019, %f1661;\n" " // begin inline asm\n" " shfl.down.b32 %f1663, %f1664, %r842, %r849;\n" " // end inline asm\n" " add.f32 %f1666, %f1664, %f1663;\n" " // begin inline asm\n" " shfl.down.b32 %f1665, %f1666, %r844, %r849;\n" " // end inline asm\n" " add.f32 %f1668, %f1666, %f1665;\n" " // begin inline asm\n" " shfl.down.b32 %f1667, %f1668, %r846, %r849;\n" " // end inline asm\n" " add.f32 %f1670, %f1668, %f1667;\n" " // begin inline asm\n" " shfl.down.b32 %f1669, %f1670, %r848, %r849;\n" " // end inline asm\n" " add.f32 %f114, %f1670, %f1669;\n" " setp.ne.s32 %p70, %r105, 2;\n" " @%p70 bra $L__BB0_102;\n" "\n" " // begin inline asm\n" " shfl.down.b32 %f1671, %f2018, %r840, %r849;\n" " // end inline asm\n" " add.f32 %f1674, %f2018, %f1671;\n" " // begin inline asm\n" " shfl.down.b32 %f1673, %f1674, %r842, %r849;\n" " // end inline asm\n" " add.f32 %f1676, %f1674, %f1673;\n" " // begin inline asm\n" " shfl.down.b32 %f1675, %f1676, %r844, %r849;\n" " // end inline asm\n" " add.f32 %f1678, %f1676, %f1675;\n" " // begin inline asm\n" " shfl.down.b32 %f1677, %f1678, %r846, %r849;\n" " // end inline asm\n" " add.f32 %f1680, %f1678, %f1677;\n" " // begin inline asm\n" " shfl.down.b32 %f1679, %f1680, %r848, %r849;\n" " // end inline asm\n" " add.f32 %f2018, %f1680, %f1679;\n" " // begin inline asm\n" " shfl.down.b32 %f1681, %f2020, %r840, %r849;\n" " // end inline asm\n" " add.f32 %f1684, %f2020, %f1681;\n" " // begin inline asm\n" " shfl.down.b32 %f1683, %f1684, %r842, %r849;\n" " // end inline asm\n" " add.f32 %f1686, %f1684, %f1683;\n" " // begin inline asm\n" " shfl.down.b32 %f1685, %f1686, %r844, %r849;\n" " // end inline asm\n" " add.f32 %f1688, %f1686, %f1685;\n" " // begin inline asm\n" " shfl.down.b32 %f1687, %f1688, %r846, %r849;\n" " // end inline asm\n" " add.f32 %f1690, %f1688, %f1687;\n" " // begin inline asm\n" " shfl.down.b32 %f1689, %f1690, %r848, %r849;\n" " // end inline asm\n" " add.f32 %f2020, %f1690, %f1689;\n" "\n" "$L__BB0_102:\n" " cvt.u32.u64 %r870, %rd4;\n" " setp.ge.s32 %p71, %r870, %r102;\n" " setp.ne.s32 %p72, %r27, 0;\n" " or.pred %p73, %p72, %p71;\n" " @%p73 bra $L__BB0_120;\n" "\n" " @%p70 bra $L__BB0_105;\n" "\n" " neg.f32 %f1691, %f105;\n" " mov.f32 %f1692, 0f3F000000;\n" " mov.f32 %f1693, 0f3BBB989D;\n" " fma.rn.f32 %f1694, %f1691, %f1693, %f1692;\n" " mov.f32 %f1695, 0f3FB8AA3B;\n" " mov.f32 %f1696, 0f437C0000;\n" " cvt.sat.f32.f32 %f1697, %f1694;\n" " mov.f32 %f1698, 0f4B400001;\n" " fma.rm.f32 %f1699, %f1697, %f1696, %f1698;\n" " add.f32 %f1700, %f1699, 0fCB40007F;\n" " neg.f32 %f1701, %f1700;\n" " fma.rn.f32 %f1702, %f1691, %f1695, %f1701;\n" " mov.f32 %f1703, 0f32A57060;\n" " fma.rn.f32 %f1704, %f1691, %f1703, %f1702;\n" " mov.b32 %r871, %f1699;\n" " shl.b32 %r872, %r871, 23;\n" " mov.b32 %f1705, %r872;\n" " ex2.approx.ftz.f32 %f1706, %f1704;\n" " fma.rn.f32 %f1707, %f1706, %f1705, 0f3F800000;\n" " div.rn.f32 %f1708, %f105, %f1707;\n" " mul.f32 %f105, %f2018, %f1708;\n" "\n" "$L__BB0_105:\n" " ld.global.u32 %r873, [%rd3+20];\n" " ld.global.u64 %rd286, [%rd3+-32];\n" " shl.b64 %rd287, %rd4, 2;\n" " add.s64 %rd99, %rd286, %rd287;\n" " setp.gt.s32 %p75, %r873, 2;\n" " @%p75 bra $L__BB0_109;\n" "\n" " setp.eq.s32 %p78, %r873, 1;\n" " @%p78 bra $L__BB0_119;\n" "\n" " setp.eq.s32 %p79, %r873, 2;\n" " @%p79 bra $L__BB0_113;\n" " bra.uni $L__BB0_108;\n" "\n" "$L__BB0_113:\n" " ld.global.u64 %rd100, [%rd3+-16];\n" " ld.global.u64 %rd297, [%rd3+-24];\n" " add.s64 %rd299, %rd297, %rd287;\n" " ld.f32 %f1717, [%rd299];\n" " add.f32 %f2029, %f105, %f1717;\n" " setp.gt.f32 %p80, %f2029, 0f41A00000;\n" " @%p80 bra $L__BB0_118;\n" "\n" " mov.f32 %f1718, 0f3F000000;\n" " mov.f32 %f1719, 0f3BBB989D;\n" " fma.rn.f32 %f1720, %f2029, %f1719, %f1718;\n" " mov.f32 %f1721, 0f3FB8AA3B;\n" " mov.f32 %f1722, 0f437C0000;\n" " cvt.sat.f32.f32 %f1723, %f1720;\n" " mov.f32 %f1724, 0f4B400001;\n" " fma.rm.f32 %f1725, %f1723, %f1722, %f1724;\n" " add.f32 %f1726, %f1725, 0fCB40007F;\n" " neg.f32 %f1727, %f1726;\n" " fma.rn.f32 %f1728, %f2029, %f1721, %f1727;\n" " mov.f32 %f1729, 0f32A57060;\n" " fma.rn.f32 %f1730, %f2029, %f1729, %f1728;\n" " mov.b32 %r874, %f1725;\n" " shl.b32 %r875, %r874, 23;\n" " mov.b32 %f1731, %r875;\n" " ex2.approx.ftz.f32 %f1732, %f1730;\n" " mul.f32 %f107, %f1732, %f1731;\n" " mov.f32 %f1733, 0f3F800000;\n" " add.rz.f32 %f1734, %f107, %f1733;\n" " mov.b32 %r876, %f1734;\n" " add.s32 %r877, %r876, -1061158912;\n" " and.b32 %r878, %r877, -8388608;\n" " mov.b32 %r65, %f107;\n" " sub.s32 %r879, %r65, %r878;\n" " mov.b32 %f1735, %r879;\n" " mov.u32 %r880, 1082130432;\n" " sub.s32 %r881, %r880, %r878;\n" " mov.b32 %f1736, %r881;\n" " mov.f32 %f1737, 0fBF800000;\n" " mov.f32 %f1738, 0f3E800000;\n" " fma.rn.f32 %f1739, %f1738, %f1736, %f1737;\n" " add.f32 %f1740, %f1739, %f1735;\n" " cvt.rn.f32.s32 %f1741, %r878;\n" " mul.f32 %f1742, %f1741, 0f34000000;\n" " mov.f32 %f1743, 0f3DD80012;\n" " mov.f32 %f1744, 0fBD39BF78;\n" " fma.rn.f32 %f1745, %f1744, %f1740, %f1743;\n" " mov.f32 %f1746, 0fBE0778E0;\n" " fma.rn.f32 %f1747, %f1745, %f1740, %f1746;\n" " mov.f32 %f1748, 0f3E146475;\n" " fma.rn.f32 %f1749, %f1747, %f1740, %f1748;\n" " mov.f32 %f1750, 0fBE2A68DD;\n" " fma.rn.f32 %f1751, %f1749, %f1740, %f1750;\n" " mov.f32 %f1752, 0f3E4CAF9E;\n" " fma.rn.f32 %f1753, %f1751, %f1740, %f1752;\n" " mov.f32 %f1754, 0fBE800042;\n" " fma.rn.f32 %f1755, %f1753, %f1740, %f1754;\n" " mov.f32 %f1756, 0f3EAAAAE6;\n" " fma.rn.f32 %f1757, %f1755, %f1740, %f1756;\n" " mov.f32 %f1758, 0fBF000000;\n" " fma.rn.f32 %f1759, %f1757, %f1740, %f1758;\n" " mul.f32 %f1760, %f1740, %f1759;\n" " fma.rn.f32 %f1761, %f1760, %f1740, %f1740;\n" " mov.f32 %f1762, 0f3F317218;\n" " fma.rn.f32 %f2029, %f1742, %f1762, %f1761;\n" " setp.lt.u32 %p81, %r65, 2139095040;\n" " @%p81 bra $L__BB0_118;\n" "\n" " setp.lt.s32 %p82, %r65, -1082130431;\n" " @%p82 bra $L__BB0_117;\n" "\n" " mov.f32 %f1763, 0f7F800000;\n" " fma.rn.f32 %f2029, %f107, %f1763, %f1763;\n" "\n" "$L__BB0_117:\n" " setp.eq.f32 %p83, %f107, 0f00000000;\n" " selp.f32 %f2029, 0f80000000, %f2029, %p83;\n" "\n" "$L__BB0_118:\n" " add.s64 %rd301, %rd100, %rd287;\n" " ld.f32 %f1764, [%rd301];\n" " mul.f32 %f1765, %f2029, %f1764;\n" " st.f32 [%rd99], %f1765;\n" " bra.uni $L__BB0_120;\n" "\n" "$L__BB0_109:\n" " setp.eq.s32 %p76, %r873, 3;\n" " @%p76 bra $L__BB0_112;\n" "\n" " setp.ne.s32 %p77, %r873, 4;\n" " @%p77 bra $L__BB0_108;\n" "\n" " ld.global.u64 %rd288, [%rd3+-16];\n" " ld.global.u64 %rd289, [%rd3+-24];\n" " add.s64 %rd291, %rd289, %rd287;\n" " ld.global.u64 %rd292, [%rd3+-8];\n" " ld.f32 %f1709, [%rd292];\n" " mul.rn.f32 %f1710, %f105, %f1709;\n" " ld.f32 %f1711, [%rd291];\n" " add.rn.f32 %f1712, %f1711, %f1710;\n" " st.f32 [%rd99], %f1712;\n" " add.s64 %rd293, %rd288, %rd287;\n" " ld.f32 %f1713, [%rd293];\n" " add.rn.f32 %f1714, %f1713, %f1712;\n" " st.f32 [%rd293], %f1714;\n" " bra.uni $L__BB0_120;\n" "\n" "$L__BB0_119:\n" " neg.f32 %f1766, %f105;\n" " mov.f32 %f1767, 0f3F000000;\n" " mov.f32 %f1768, 0f3BBB989D;\n" " fma.rn.f32 %f1769, %f1766, %f1768, %f1767;\n" " mov.f32 %f1770, 0f3FB8AA3B;\n" " mov.f32 %f1771, 0f437C0000;\n" " cvt.sat.f32.f32 %f1772, %f1769;\n" " mov.f32 %f1773, 0f4B400001;\n" " fma.rm.f32 %f1774, %f1772, %f1771, %f1773;\n" " add.f32 %f1775, %f1774, 0fCB40007F;\n" " neg.f32 %f1776, %f1775;\n" " fma.rn.f32 %f1777, %f1766, %f1770, %f1776;\n" " mov.f32 %f1778, 0f32A57060;\n" " fma.rn.f32 %f1779, %f1766, %f1778, %f1777;\n" " mov.b32 %r882, %f1774;\n" " shl.b32 %r883, %r882, 23;\n" " mov.b32 %f1780, %r883;\n" " ex2.approx.ftz.f32 %f1781, %f1779;\n" " fma.rn.f32 %f1782, %f1781, %f1780, 0f3F800000;\n" " rcp.rn.f32 %f1783, %f1782;\n" " st.f32 [%rd99], %f1783;\n" " bra.uni $L__BB0_120;\n" "\n" "$L__BB0_112:\n" " ld.global.u64 %rd294, [%rd3+-16];\n" " st.f32 [%rd99], %f105;\n" " add.s64 %rd296, %rd294, %rd287;\n" " ld.f32 %f1715, [%rd296];\n" " add.f32 %f1716, %f105, %f1715;\n" " st.f32 [%rd296], %f1716;\n" " bra.uni $L__BB0_120;\n" "\n" "$L__BB0_108:\n" " st.f32 [%rd99], %f105;\n" "\n" "$L__BB0_120:\n" " or.pred %p86, %p72, %p12;\n" " @%p86 bra $L__BB0_167;\n" "\n" " @%p70 bra $L__BB0_123;\n" "\n" " neg.f32 %f1784, %f114;\n" " mov.f32 %f1785, 0f3F000000;\n" " mov.f32 %f1786, 0f3BBB989D;\n" " fma.rn.f32 %f1787, %f1784, %f1786, %f1785;\n" " mov.f32 %f1788, 0f3FB8AA3B;\n" " mov.f32 %f1789, 0f437C0000;\n" " cvt.sat.f32.f32 %f1790, %f1787;\n" " mov.f32 %f1791, 0f4B400001;\n" " fma.rm.f32 %f1792, %f1790, %f1789, %f1791;\n" " add.f32 %f1793, %f1792, 0fCB40007F;\n" " neg.f32 %f1794, %f1793;\n" " fma.rn.f32 %f1795, %f1784, %f1788, %f1794;\n" " mov.f32 %f1796, 0f32A57060;\n" " fma.rn.f32 %f1797, %f1784, %f1796, %f1795;\n" " mov.b32 %r884, %f1792;\n" " shl.b32 %r885, %r884, 23;\n" " mov.b32 %f1798, %r885;\n" " ex2.approx.ftz.f32 %f1799, %f1797;\n" " fma.rn.f32 %f1800, %f1799, %f1798, 0f3F800000;\n" " div.rn.f32 %f1801, %f114, %f1800;\n" " mul.f32 %f114, %f2020, %f1801;\n" "\n" "$L__BB0_123:\n" " mov.u32 %r929, %ctaid.x;\n" " sub.s32 %r928, %r929, %r106;\n" " shr.u32 %r927, %r26, 4;\n" " and.b32 %r926, %r927, 268435454;\n" " shl.b32 %r925, %r928, 4;\n" " add.s32 %r924, %r925, %r926;\n" " or.b32 %r923, %r924, 1;\n" " ld.global.u32 %r886, [%rd3+20];\n" " cvt.s64.s32 %rd101, %r923;\n" " ld.global.u64 %rd302, [%rd3+-32];\n" " mul.wide.s32 %rd303, %r923, 4;\n" " add.s64 %rd102, %rd302, %rd303;\n" " setp.gt.s32 %p88, %r886, 2;\n" " @%p88 bra $L__BB0_127;\n" "\n" " setp.eq.s32 %p91, %r886, 1;\n" " @%p91 bra $L__BB0_137;\n" "\n" " setp.eq.s32 %p92, %r886, 2;\n" " @%p92 bra $L__BB0_131;\n" " bra.uni $L__BB0_126;\n" "\n" "$L__BB0_131:\n" " ld.global.u64 %rd103, [%rd3+-16];\n" " ld.global.u64 %rd313, [%rd3+-24];\n" " shl.b64 %rd314, %rd101, 2;\n" " add.s64 %rd315, %rd313, %rd314;\n" " ld.f32 %f1810, [%rd315];\n" " add.f32 %f2032, %f114, %f1810;\n" " setp.gt.f32 %p93, %f2032, 0f41A00000;\n" " @%p93 bra $L__BB0_136;\n" "\n" " mov.f32 %f1811, 0f3F000000;\n" " mov.f32 %f1812, 0f3BBB989D;\n" " fma.rn.f32 %f1813, %f2032, %f1812, %f1811;\n" " mov.f32 %f1814, 0f3FB8AA3B;\n" " mov.f32 %f1815, 0f437C0000;\n" " cvt.sat.f32.f32 %f1816, %f1813;\n" " mov.f32 %f1817, 0f4B400001;\n" " fma.rm.f32 %f1818, %f1816, %f1815, %f1817;\n" " add.f32 %f1819, %f1818, 0fCB40007F;\n" " neg.f32 %f1820, %f1819;\n" " fma.rn.f32 %f1821, %f2032, %f1814, %f1820;\n" " mov.f32 %f1822, 0f32A57060;\n" " fma.rn.f32 %f1823, %f2032, %f1822, %f1821;\n" " mov.b32 %r887, %f1818;\n" " shl.b32 %r888, %r887, 23;\n" " mov.b32 %f1824, %r888;\n" " ex2.approx.ftz.f32 %f1825, %f1823;\n" " mul.f32 %f116, %f1825, %f1824;\n" " mov.f32 %f1826, 0f3F800000;\n" " add.rz.f32 %f1827, %f116, %f1826;\n" " mov.b32 %r889, %f1827;\n" " add.s32 %r890, %r889, -1061158912;\n" " and.b32 %r891, %r890, -8388608;\n" " mov.b32 %r66, %f116;\n" " sub.s32 %r892, %r66, %r891;\n" " mov.b32 %f1828, %r892;\n" " mov.u32 %r893, 1082130432;\n" " sub.s32 %r894, %r893, %r891;\n" " mov.b32 %f1829, %r894;\n" " mov.f32 %f1830, 0fBF800000;\n" " mov.f32 %f1831, 0f3E800000;\n" " fma.rn.f32 %f1832, %f1831, %f1829, %f1830;\n" " add.f32 %f1833, %f1832, %f1828;\n" " cvt.rn.f32.s32 %f1834, %r891;\n" " mul.f32 %f1835, %f1834, 0f34000000;\n" " mov.f32 %f1836, 0f3DD80012;\n" " mov.f32 %f1837, 0fBD39BF78;\n" " fma.rn.f32 %f1838, %f1837, %f1833, %f1836;\n" " mov.f32 %f1839, 0fBE0778E0;\n" " fma.rn.f32 %f1840, %f1838, %f1833, %f1839;\n" " mov.f32 %f1841, 0f3E146475;\n" " fma.rn.f32 %f1842, %f1840, %f1833, %f1841;\n" " mov.f32 %f1843, 0fBE2A68DD;\n" " fma.rn.f32 %f1844, %f1842, %f1833, %f1843;\n" " mov.f32 %f1845, 0f3E4CAF9E;\n" " fma.rn.f32 %f1846, %f1844, %f1833, %f1845;\n" " mov.f32 %f1847, 0fBE800042;\n" " fma.rn.f32 %f1848, %f1846, %f1833, %f1847;\n" " mov.f32 %f1849, 0f3EAAAAE6;\n" " fma.rn.f32 %f1850, %f1848, %f1833, %f1849;\n" " mov.f32 %f1851, 0fBF000000;\n" " fma.rn.f32 %f1852, %f1850, %f1833, %f1851;\n" " mul.f32 %f1853, %f1833, %f1852;\n" " fma.rn.f32 %f1854, %f1853, %f1833, %f1833;\n" " mov.f32 %f1855, 0f3F317218;\n" " fma.rn.f32 %f2032, %f1835, %f1855, %f1854;\n" " setp.lt.u32 %p94, %r66, 2139095040;\n" " @%p94 bra $L__BB0_136;\n" "\n" " setp.lt.s32 %p95, %r66, -1082130431;\n" " @%p95 bra $L__BB0_135;\n" "\n" " mov.f32 %f1856, 0f7F800000;\n" " fma.rn.f32 %f2032, %f116, %f1856, %f1856;\n" "\n" "$L__BB0_135:\n" " setp.eq.f32 %p96, %f116, 0f00000000;\n" " selp.f32 %f2032, 0f80000000, %f2032, %p96;\n" "\n" "$L__BB0_136:\n" " add.s64 %rd317, %rd103, %rd314;\n" " ld.f32 %f1857, [%rd317];\n" " mul.f32 %f1858, %f2032, %f1857;\n" " st.f32 [%rd102], %f1858;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_127:\n" " setp.eq.s32 %p89, %r886, 3;\n" " @%p89 bra $L__BB0_130;\n" "\n" " setp.ne.s32 %p90, %r886, 4;\n" " @%p90 bra $L__BB0_126;\n" "\n" " ld.global.u64 %rd304, [%rd3+-16];\n" " ld.global.u64 %rd305, [%rd3+-24];\n" " shl.b64 %rd306, %rd101, 2;\n" " add.s64 %rd307, %rd305, %rd306;\n" " ld.global.u64 %rd308, [%rd3+-8];\n" " ld.f32 %f1802, [%rd308];\n" " mul.rn.f32 %f1803, %f114, %f1802;\n" " ld.f32 %f1804, [%rd307];\n" " add.rn.f32 %f1805, %f1804, %f1803;\n" " st.f32 [%rd102], %f1805;\n" " add.s64 %rd309, %rd304, %rd306;\n" " ld.f32 %f1806, [%rd309];\n" " add.rn.f32 %f1807, %f1806, %f1805;\n" " st.f32 [%rd309], %f1807;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_137:\n" " neg.f32 %f1859, %f114;\n" " mov.f32 %f1860, 0f3F000000;\n" " mov.f32 %f1861, 0f3BBB989D;\n" " fma.rn.f32 %f1862, %f1859, %f1861, %f1860;\n" " mov.f32 %f1863, 0f3FB8AA3B;\n" " mov.f32 %f1864, 0f437C0000;\n" " cvt.sat.f32.f32 %f1865, %f1862;\n" " mov.f32 %f1866, 0f4B400001;\n" " fma.rm.f32 %f1867, %f1865, %f1864, %f1866;\n" " add.f32 %f1868, %f1867, 0fCB40007F;\n" " neg.f32 %f1869, %f1868;\n" " fma.rn.f32 %f1870, %f1859, %f1863, %f1869;\n" " mov.f32 %f1871, 0f32A57060;\n" " fma.rn.f32 %f1872, %f1859, %f1871, %f1870;\n" " mov.b32 %r895, %f1867;\n" " shl.b32 %r896, %r895, 23;\n" " mov.b32 %f1873, %r896;\n" " ex2.approx.ftz.f32 %f1874, %f1872;\n" " fma.rn.f32 %f1875, %f1874, %f1873, 0f3F800000;\n" " rcp.rn.f32 %f1876, %f1875;\n" " st.f32 [%rd102], %f1876;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_130:\n" " ld.global.u64 %rd310, [%rd3+-16];\n" " st.f32 [%rd102], %f114;\n" " shl.b64 %rd311, %rd101, 2;\n" " add.s64 %rd312, %rd310, %rd311;\n" " ld.f32 %f1808, [%rd312];\n" " add.f32 %f1809, %f114, %f1808;\n" " st.f32 [%rd312], %f1809;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_126:\n" " st.f32 [%rd102], %f114;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_156:\n" " setp.eq.s32 %p108, %r909, 3;\n" " @%p108 bra $L__BB0_159;\n" "\n" " setp.ne.s32 %p109, %r909, 4;\n" " @%p109 bra $L__BB0_155;\n" "\n" " ld.global.u64 %rd331, [%rd3+-16];\n" " ld.global.u64 %rd332, [%rd3+-24];\n" " add.s64 %rd334, %rd332, %rd330;\n" " ld.global.u64 %rd335, [%rd3+-8];\n" " ld.f32 %f1897, [%rd335];\n" " mul.rn.f32 %f1898, %f129, %f1897;\n" " ld.f32 %f1899, [%rd334];\n" " add.rn.f32 %f1900, %f1899, %f1898;\n" " st.f32 [%rd116], %f1900;\n" " add.s64 %rd336, %rd331, %rd330;\n" " ld.f32 %f1901, [%rd336];\n" " add.rn.f32 %f1902, %f1901, %f1900;\n" " st.f32 [%rd336], %f1902;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_155:\n" " st.f32 [%rd116], %f129;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_166:\n" " neg.f32 %f1954, %f129;\n" " mov.f32 %f1955, 0f3F000000;\n" " mov.f32 %f1956, 0f3BBB989D;\n" " fma.rn.f32 %f1957, %f1954, %f1956, %f1955;\n" " mov.f32 %f1958, 0f3FB8AA3B;\n" " mov.f32 %f1959, 0f437C0000;\n" " cvt.sat.f32.f32 %f1960, %f1957;\n" " mov.f32 %f1961, 0f4B400001;\n" " fma.rm.f32 %f1962, %f1960, %f1959, %f1961;\n" " add.f32 %f1963, %f1962, 0fCB40007F;\n" " neg.f32 %f1964, %f1963;\n" " fma.rn.f32 %f1965, %f1954, %f1958, %f1964;\n" " mov.f32 %f1966, 0f32A57060;\n" " fma.rn.f32 %f1967, %f1954, %f1966, %f1965;\n" " mov.b32 %r918, %f1962;\n" " shl.b32 %r919, %r918, 23;\n" " mov.b32 %f1968, %r919;\n" " ex2.approx.ftz.f32 %f1969, %f1967;\n" " fma.rn.f32 %f1970, %f1969, %f1968, 0f3F800000;\n" " rcp.rn.f32 %f1971, %f1970;\n" " st.f32 [%rd116], %f1971;\n" " bra.uni $L__BB0_167;\n" "\n" "$L__BB0_159:\n" " ld.global.u64 %rd337, [%rd3+-16];\n" " st.f32 [%rd116], %f129;\n" " add.s64 %rd339, %rd337, %rd330;\n" " ld.f32 %f1903, [%rd339];\n" " add.f32 %f1904, %f129, %f1903;\n" " st.f32 [%rd339], %f1904;\n" "\n" "$L__BB0_167:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_qknorm_rope\n" ".visible .entry fc_attn_qknorm_rope(\n" " .param .u64 fc_attn_qknorm_rope_param_0,\n" " .param .u64 fc_attn_qknorm_rope_param_1,\n" " .param .u64 fc_attn_qknorm_rope_param_2,\n" " .param .u64 fc_attn_qknorm_rope_param_3,\n" " .param .u32 fc_attn_qknorm_rope_param_4,\n" " .param .u32 fc_attn_qknorm_rope_param_5,\n" " .param .u32 fc_attn_qknorm_rope_param_6,\n" " .param .f32 fc_attn_qknorm_rope_param_7,\n" " .param .u32 fc_attn_qknorm_rope_param_8,\n" " .param .u32 fc_attn_qknorm_rope_param_9,\n" " .param .f32 fc_attn_qknorm_rope_param_10\n" ")\n" "{\n" " .local .align 4 .b8 __local_depot1[28];\n" " .reg .b64 %SP;\n" " .reg .b64 %SPL;\n" " .reg .pred %p<73>;\n" " .reg .f32 %f<247>;\n" " .reg .b32 %r<200>;\n" " .reg .f64 %fd<5>;\n" " .reg .b64 %rd<116>;\n" "\n" "\n" " mov.u64 %SPL, __local_depot1;\n" " ld.param.u64 %rd41, [fc_attn_qknorm_rope_param_0];\n" " ld.param.u64 %rd42, [fc_attn_qknorm_rope_param_1];\n" " ld.param.u64 %rd43, [fc_attn_qknorm_rope_param_2];\n" " ld.param.u64 %rd44, [fc_attn_qknorm_rope_param_3];\n" " ld.param.u32 %r64, [fc_attn_qknorm_rope_param_4];\n" " ld.param.u32 %r68, [fc_attn_qknorm_rope_param_5];\n" " ld.param.u32 %r65, [fc_attn_qknorm_rope_param_6];\n" " ld.param.f32 %f62, [fc_attn_qknorm_rope_param_7];\n" " ld.param.u32 %r66, [fc_attn_qknorm_rope_param_8];\n" " ld.param.f32 %f63, [fc_attn_qknorm_rope_param_10];\n" " add.u64 %rd1, %SPL, 0;\n" " add.s32 %r69, %r68, %r64;\n" " mov.u32 %r1, %ctaid.x;\n" " setp.ge.s32 %p2, %r1, %r69;\n" " @%p2 bra $L__BB1_70;\n" "\n" " cvta.to.global.u64 %rd46, %rd41;\n" " mul.wide.s32 %rd47, %r65, %r1;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd49, %rd46, %rd48;\n" " sub.s32 %r70, %r1, %r64;\n" " mul.wide.s32 %rd50, %r65, %r70;\n" " cvta.to.global.u64 %rd51, %rd42;\n" " shl.b64 %rd52, %rd50, 2;\n" " add.s64 %rd53, %rd51, %rd52;\n" " setp.lt.s32 %p3, %r1, %r64;\n" " selp.b64 %rd2, %rd49, %rd53, %p3;\n" " selp.b64 %rd3, %rd43, %rd44, %p3;\n" " cvta.to.global.u64 %rd4, %rd3;\n" " mov.u32 %r2, %ntid.x;\n" " mov.u32 %r3, %tid.x;\n" " setp.ge.s32 %p4, %r3, %r65;\n" " mov.f32 %f232, 0f00000000;\n" " @%p4 bra $L__BB1_8;\n" "\n" " not.b32 %r71, %r3;\n" " add.s32 %r72, %r71, %r65;\n" " div.u32 %r4, %r72, %r2;\n" " add.s32 %r73, %r4, 1;\n" " and.b32 %r184, %r73, 3;\n" " setp.eq.s32 %p5, %r184, 0;\n" " mov.f32 %f232, 0f00000000;\n" " mov.u32 %r185, %r3;\n" " @%p5 bra $L__BB1_5;\n" "\n" " mul.wide.s32 %rd54, %r3, 4;\n" " add.s64 %rd107, %rd2, %rd54;\n" " mul.wide.s32 %rd6, %r2, 4;\n" " mov.f32 %f232, 0f00000000;\n" " mov.u32 %r185, %r3;\n" "\n" "$L__BB1_4:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f68, [%rd107];\n" " fma.rn.f32 %f232, %f68, %f68, %f232;\n" " add.s32 %r185, %r185, %r2;\n" " add.s64 %rd107, %rd107, %rd6;\n" " add.s32 %r184, %r184, -1;\n" " setp.ne.s32 %p6, %r184, 0;\n" " @%p6 bra $L__BB1_4;\n" "\n" "$L__BB1_5:\n" " setp.lt.u32 %p7, %r4, 3;\n" " @%p7 bra $L__BB1_8;\n" "\n" " mul.wide.s32 %rd9, %r2, 4;\n" "\n" "$L__BB1_7:\n" " mul.wide.s32 %rd55, %r185, 4;\n" " add.s64 %rd56, %rd2, %rd55;\n" " ld.global.f32 %f69, [%rd56];\n" " fma.rn.f32 %f70, %f69, %f69, %f232;\n" " add.s64 %rd57, %rd56, %rd9;\n" " ld.global.f32 %f71, [%rd57];\n" " fma.rn.f32 %f72, %f71, %f71, %f70;\n" " add.s32 %r74, %r185, %r2;\n" " add.s32 %r75, %r74, %r2;\n" " add.s64 %rd58, %rd57, %rd9;\n" " ld.global.f32 %f73, [%rd58];\n" " fma.rn.f32 %f74, %f73, %f73, %f72;\n" " add.s32 %r76, %r75, %r2;\n" " add.s64 %rd59, %rd58, %rd9;\n" " ld.global.f32 %f75, [%rd59];\n" " fma.rn.f32 %f232, %f75, %f75, %f74;\n" " add.s32 %r185, %r76, %r2;\n" " setp.lt.s32 %p8, %r185, %r65;\n" " @%p8 bra $L__BB1_7;\n" "\n" "$L__BB1_8:\n" " shl.b32 %r77, %r3, 2;\n" " mov.u32 %r78, sdata;\n" " add.s32 %r13, %r78, %r77;\n" " st.shared.f32 [%r13], %f232;\n" " bar.sync 0;\n" " setp.lt.s32 %p9, %r2, 2;\n" " @%p9 bra $L__BB1_13;\n" "\n" " shr.u32 %r79, %r2, 31;\n" " add.s32 %r80, %r2, %r79;\n" " shr.s32 %r187, %r80, 1;\n" "\n" "$L__BB1_10:\n" " setp.ge.s32 %p10, %r3, %r187;\n" " @%p10 bra $L__BB1_12;\n" "\n" " shl.b32 %r81, %r187, 2;\n" " add.s32 %r82, %r13, %r81;\n" " ld.shared.f32 %f76, [%r13];\n" " ld.shared.f32 %f77, [%r82];\n" " add.f32 %f78, %f77, %f76;\n" " st.shared.f32 [%r13], %f78;\n" "\n" "$L__BB1_12:\n" " bar.sync 0;\n" " shr.s32 %r16, %r187, 1;\n" " setp.gt.s32 %p11, %r187, 1;\n" " mov.u32 %r187, %r16;\n" " @%p11 bra $L__BB1_10;\n" "\n" "$L__BB1_13:\n" " cvt.rn.f32.s32 %f79, %r65;\n" " ld.shared.f32 %f80, [sdata];\n" " div.rn.f32 %f81, %f80, %f79;\n" " add.f32 %f82, %f81, %f62;\n" " sqrt.rn.f32 %f83, %f82;\n" " rcp.rn.f32 %f8, %f83;\n" " @%p4 bra $L__BB1_30;\n" "\n" " not.b32 %r83, %r3;\n" " add.s32 %r84, %r83, %r65;\n" " div.u32 %r17, %r84, %r2;\n" " add.s32 %r85, %r17, 1;\n" " and.b32 %r189, %r85, 3;\n" " setp.eq.s32 %p13, %r189, 0;\n" " mov.u32 %r190, %r3;\n" " @%p13 bra $L__BB1_19;\n" "\n" " mul.wide.s32 %rd60, %r3, 4;\n" " add.s64 %rd109, %rd4, %rd60;\n" " mul.wide.s32 %rd11, %r2, 4;\n" " add.s64 %rd108, %rd2, %rd60;\n" " setp.eq.s64 %p14, %rd3, 0;\n" " mov.u32 %r190, %r3;\n" "\n" "$L__BB1_16:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f84, [%rd108];\n" " mul.f32 %f233, %f8, %f84;\n" " @%p14 bra $L__BB1_18;\n" "\n" " ld.global.f32 %f85, [%rd109];\n" " mul.f32 %f233, %f233, %f85;\n" "\n" "$L__BB1_18:\n" " st.global.f32 [%rd108], %f233;\n" " add.s32 %r190, %r190, %r2;\n" " add.s64 %rd109, %rd109, %rd11;\n" " add.s64 %rd108, %rd108, %rd11;\n" " add.s32 %r189, %r189, -1;\n" " setp.ne.s32 %p15, %r189, 0;\n" " @%p15 bra $L__BB1_16;\n" "\n" "$L__BB1_19:\n" " setp.lt.u32 %p16, %r17, 3;\n" " @%p16 bra $L__BB1_30;\n" "\n" " mul.wide.s32 %rd17, %r2, 4;\n" " setp.eq.s64 %p17, %rd3, 0;\n" "\n" "$L__BB1_21:\n" " cvt.s64.s32 %rd18, %r190;\n" " mul.wide.s32 %rd61, %r190, 4;\n" " add.s64 %rd19, %rd2, %rd61;\n" " ld.global.f32 %f86, [%rd19];\n" " mul.f32 %f234, %f8, %f86;\n" " @%p17 bra $L__BB1_23;\n" "\n" " shl.b64 %rd62, %rd18, 2;\n" " add.s64 %rd63, %rd4, %rd62;\n" " ld.global.f32 %f87, [%rd63];\n" " mul.f32 %f234, %f234, %f87;\n" "\n" "$L__BB1_23:\n" " st.global.f32 [%rd19], %f234;\n" " cvt.u32.u64 %r86, %rd18;\n" " add.s32 %r87, %r86, %r2;\n" " cvt.s64.s32 %rd20, %r87;\n" " add.s64 %rd22, %rd19, %rd17;\n" " ld.global.f32 %f88, [%rd22];\n" " mul.f32 %f235, %f8, %f88;\n" " @%p17 bra $L__BB1_25;\n" "\n" " shl.b64 %rd64, %rd20, 2;\n" " add.s64 %rd65, %rd4, %rd64;\n" " ld.global.f32 %f89, [%rd65];\n" " mul.f32 %f235, %f235, %f89;\n" "\n" "$L__BB1_25:\n" " st.global.f32 [%rd22], %f235;\n" " cvt.u32.u64 %r88, %rd20;\n" " add.s32 %r89, %r88, %r2;\n" " cvt.s64.s32 %rd23, %r89;\n" " add.s64 %rd25, %rd22, %rd17;\n" " ld.global.f32 %f90, [%rd25];\n" " mul.f32 %f236, %f8, %f90;\n" " @%p17 bra $L__BB1_27;\n" "\n" " shl.b64 %rd66, %rd23, 2;\n" " add.s64 %rd67, %rd4, %rd66;\n" " ld.global.f32 %f91, [%rd67];\n" " mul.f32 %f236, %f236, %f91;\n" "\n" "$L__BB1_27:\n" " st.global.f32 [%rd25], %f236;\n" " cvt.u32.u64 %r90, %rd23;\n" " add.s32 %r91, %r90, %r2;\n" " cvt.s64.s32 %rd26, %r91;\n" " add.s64 %rd27, %rd25, %rd17;\n" " ld.global.f32 %f92, [%rd27];\n" " mul.f32 %f237, %f8, %f92;\n" " @%p17 bra $L__BB1_29;\n" "\n" " shl.b64 %rd68, %rd26, 2;\n" " add.s64 %rd69, %rd4, %rd68;\n" " ld.global.f32 %f93, [%rd69];\n" " mul.f32 %f237, %f237, %f93;\n" "\n" "$L__BB1_29:\n" " st.global.f32 [%rd27], %f237;\n" " cvt.u32.u64 %r92, %rd26;\n" " add.s32 %r190, %r92, %r2;\n" " setp.lt.s32 %p21, %r190, %r65;\n" " @%p21 bra $L__BB1_21;\n" "\n" "$L__BB1_30:\n" " bar.sync 0;\n" " setp.lt.s32 %p22, %r66, %r65;\n" " setp.gt.s32 %p23, %r66, 0;\n" " and.pred %p24, %p23, %p22;\n" " selp.b32 %r26, %r66, %r65, %p24;\n" " shr.u32 %r93, %r26, 31;\n" " add.s32 %r94, %r26, %r93;\n" " shr.s32 %r27, %r94, 1;\n" " setp.ge.s32 %p25, %r3, %r27;\n" " @%p25 bra $L__BB1_70;\n" "\n" " shl.b32 %r95, %r3, 1;\n" " cvt.rn.f32.s32 %f95, %r95;\n" " cvt.rn.f32.s32 %f96, %r26;\n" " div.rn.f32 %f24, %f95, %f96;\n" " mul.f32 %f97, %f24, 0f3F000000;\n" " cvt.rzi.f32.f32 %f98, %f97;\n" " add.f32 %f99, %f98, %f98;\n" " sub.f32 %f100, %f24, %f99;\n" " abs.f32 %f25, %f100;\n" " abs.f32 %f26, %f63;\n" " setp.lt.f32 %p26, %f26, 0f00800000;\n" " mul.f32 %f101, %f26, 0f4B800000;\n" " selp.f32 %f102, %f101, %f26, %p26;\n" " selp.f32 %f103, 0fC3170000, 0fC2FE0000, %p26;\n" " mov.b32 %r96, %f102;\n" " and.b32 %r97, %r96, 8388607;\n" " or.b32 %r98, %r97, 1065353216;\n" " mov.b32 %f104, %r98;\n" " shr.u32 %r99, %r96, 23;\n" " cvt.rn.f32.u32 %f105, %r99;\n" " add.f32 %f106, %f103, %f105;\n" " setp.gt.f32 %p27, %f104, 0f3FB504F3;\n" " mul.f32 %f107, %f104, 0f3F000000;\n" " add.f32 %f108, %f106, 0f3F800000;\n" " selp.f32 %f109, %f108, %f106, %p27;\n" " selp.f32 %f110, %f107, %f104, %p27;\n" " add.f32 %f111, %f110, 0fBF800000;\n" " add.f32 %f112, %f110, 0f3F800000;\n" " rcp.approx.ftz.f32 %f113, %f112;\n" " add.f32 %f114, %f111, %f111;\n" " mul.f32 %f115, %f114, %f113;\n" " mul.f32 %f116, %f115, %f115;\n" " mov.f32 %f117, 0f3C4CAF63;\n" " mov.f32 %f118, 0f3B18F0FE;\n" " fma.rn.f32 %f119, %f118, %f116, %f117;\n" " mov.f32 %f120, 0f3DAAAABD;\n" " fma.rn.f32 %f121, %f119, %f116, %f120;\n" " mul.rn.f32 %f122, %f121, %f116;\n" " mul.rn.f32 %f123, %f122, %f115;\n" " sub.f32 %f124, %f111, %f115;\n" " add.f32 %f125, %f124, %f124;\n" " neg.f32 %f126, %f115;\n" " fma.rn.f32 %f127, %f126, %f111, %f125;\n" " mul.rn.f32 %f128, %f113, %f127;\n" " add.f32 %f129, %f123, %f115;\n" " sub.f32 %f130, %f115, %f129;\n" " add.f32 %f131, %f123, %f130;\n" " add.f32 %f132, %f128, %f131;\n" " add.f32 %f133, %f129, %f132;\n" " sub.f32 %f134, %f129, %f133;\n" " add.f32 %f135, %f132, %f134;\n" " mov.f32 %f136, 0f3F317200;\n" " mul.rn.f32 %f137, %f109, %f136;\n" " mov.f32 %f138, 0f35BFBE8E;\n" " mul.rn.f32 %f139, %f109, %f138;\n" " add.f32 %f140, %f137, %f133;\n" " sub.f32 %f141, %f137, %f140;\n" " add.f32 %f142, %f133, %f141;\n" " add.f32 %f143, %f135, %f142;\n" " add.f32 %f144, %f139, %f143;\n" " add.f32 %f145, %f140, %f144;\n" " sub.f32 %f146, %f140, %f145;\n" " add.f32 %f147, %f144, %f146;\n" " abs.f32 %f27, %f24;\n" " setp.gt.f32 %p28, %f27, 0f77F684DF;\n" " mul.f32 %f148, %f24, 0f39000000;\n" " selp.f32 %f149, %f148, %f24, %p28;\n" " mul.rn.f32 %f150, %f149, %f145;\n" " neg.f32 %f151, %f150;\n" " fma.rn.f32 %f152, %f149, %f145, %f151;\n" " fma.rn.f32 %f153, %f149, %f147, %f152;\n" " mov.f32 %f154, 0f00000000;\n" " fma.rn.f32 %f155, %f154, %f145, %f153;\n" " add.rn.f32 %f156, %f150, %f155;\n" " neg.f32 %f157, %f156;\n" " add.rn.f32 %f158, %f150, %f157;\n" " add.rn.f32 %f159, %f158, %f155;\n" " mov.b32 %r100, %f156;\n" " setp.eq.s32 %p29, %r100, 1118925336;\n" " add.s32 %r101, %r100, -1;\n" " mov.b32 %f160, %r101;\n" " add.f32 %f161, %f159, 0f37000000;\n" " selp.f32 %f28, %f161, %f159, %p29;\n" " selp.f32 %f162, %f160, %f156, %p29;\n" " mov.f32 %f163, 0f3FB8AA3B;\n" " mul.rn.f32 %f164, %f162, %f163;\n" " cvt.rzi.f32.f32 %f165, %f164;\n" " abs.f32 %f166, %f165;\n" " setp.gt.f32 %p30, %f166, 0f42FC0000;\n" " mov.b32 %r102, %f165;\n" " and.b32 %r103, %r102, -2147483648;\n" " or.b32 %r104, %r103, 1123811328;\n" " mov.b32 %f167, %r104;\n" " selp.f32 %f168, %f167, %f165, %p30;\n" " mov.f32 %f169, 0fBF317218;\n" " fma.rn.f32 %f170, %f168, %f169, %f162;\n" " mov.f32 %f171, 0f3102E308;\n" " fma.rn.f32 %f172, %f168, %f171, %f170;\n" " mul.f32 %f173, %f172, 0f3FB8AA3B;\n" " add.f32 %f174, %f168, 0f4B40007F;\n" " mov.b32 %r105, %f174;\n" " shl.b32 %r106, %r105, 23;\n" " mov.b32 %f175, %r106;\n" " ex2.approx.ftz.f32 %f176, %f173;\n" " mul.f32 %f29, %f176, %f175;\n" " setp.eq.f32 %p31, %f29, 0f7F800000;\n" " mov.f32 %f238, 0f7F800000;\n" " @%p31 bra $L__BB1_33;\n" "\n" " fma.rn.f32 %f238, %f29, %f28, %f29;\n" "\n" "$L__BB1_33:\n" " setp.lt.f32 %p32, %f63, 0f00000000;\n" " setp.eq.f32 %p33, %f25, 0f3F800000;\n" " and.pred %p1, %p32, %p33;\n" " setp.eq.f32 %p34, %f63, 0f00000000;\n" " @%p34 bra $L__BB1_37;\n" " bra.uni $L__BB1_34;\n" "\n" "$L__BB1_37:\n" " add.f32 %f180, %f63, %f63;\n" " mov.b32 %r109, %f180;\n" " selp.b32 %r110, %r109, 0, %p33;\n" " or.b32 %r111, %r110, 2139095040;\n" " setp.lt.f32 %p38, %f24, 0f00000000;\n" " selp.b32 %r112, %r111, %r110, %p38;\n" " mov.b32 %f240, %r112;\n" " bra.uni $L__BB1_38;\n" "\n" "$L__BB1_34:\n" " mov.b32 %r107, %f238;\n" " xor.b32 %r108, %r107, -2147483648;\n" " mov.b32 %f177, %r108;\n" " selp.f32 %f240, %f177, %f238, %p1;\n" " setp.geu.f32 %p35, %f63, 0f00000000;\n" " @%p35 bra $L__BB1_38;\n" "\n" " cvt.rzi.f32.f32 %f178, %f24;\n" " setp.eq.f32 %p36, %f178, %f24;\n" " @%p36 bra $L__BB1_38;\n" "\n" " mov.f32 %f240, 0f7FFFFFFF;\n" "\n" "$L__BB1_38:\n" " add.f32 %f181, %f26, %f27;\n" " mov.b32 %r113, %f181;\n" " setp.lt.s32 %p39, %r113, 2139095040;\n" " @%p39 bra $L__BB1_45;\n" "\n" " setp.gtu.f32 %p40, %f26, 0f7F800000;\n" " setp.gtu.f32 %p41, %f27, 0f7F800000;\n" " or.pred %p42, %p40, %p41;\n" " @%p42 bra $L__BB1_44;\n" " bra.uni $L__BB1_40;\n" "\n" "$L__BB1_44:\n" " add.f32 %f240, %f24, %f63;\n" " bra.uni $L__BB1_45;\n" "\n" "$L__BB1_40:\n" " setp.eq.f32 %p43, %f27, 0f7F800000;\n" " @%p43 bra $L__BB1_43;\n" " bra.uni $L__BB1_41;\n" "\n" "$L__BB1_43:\n" " setp.gt.f32 %p46, %f26, 0f3F800000;\n" " selp.b32 %r117, 2139095040, 0, %p46;\n" " xor.b32 %r118, %r117, 2139095040;\n" " setp.lt.f32 %p47, %f24, 0f00000000;\n" " selp.b32 %r119, %r118, %r117, %p47;\n" " mov.b32 %f182, %r119;\n" " setp.eq.f32 %p48, %f63, 0fBF800000;\n" " selp.f32 %f240, 0f3F800000, %f182, %p48;\n" " bra.uni $L__BB1_45;\n" "\n" "$L__BB1_41:\n" " setp.neu.f32 %p44, %f26, 0f7F800000;\n" " @%p44 bra $L__BB1_45;\n" "\n" " setp.ge.f32 %p45, %f24, 0f00000000;\n" " selp.b32 %r114, 2139095040, 0, %p45;\n" " or.b32 %r115, %r114, -2147483648;\n" " selp.b32 %r116, %r115, %r114, %p1;\n" " mov.b32 %f240, %r116;\n" "\n" "$L__BB1_45:\n" " ld.param.u32 %r178, [fc_attn_qknorm_rope_param_9];\n" " setp.eq.f32 %p49, %f24, 0f00000000;\n" " setp.eq.f32 %p50, %f63, 0f3F800000;\n" " or.pred %p51, %p50, %p49;\n" " rcp.rn.f32 %f183, %f240;\n" " selp.f32 %f184, 0f3F800000, %f183, %p51;\n" " cvt.rn.f32.s32 %f185, %r178;\n" " mul.f32 %f39, %f184, %f185;\n" " mul.f32 %f186, %f39, 0f3F22F983;\n" " cvt.rni.s32.f32 %r199, %f186;\n" " cvt.rn.f32.s32 %f187, %r199;\n" " mov.f32 %f188, 0fBFC90FDA;\n" " fma.rn.f32 %f189, %f187, %f188, %f39;\n" " mov.f32 %f190, 0fB3A22168;\n" " fma.rn.f32 %f191, %f187, %f190, %f189;\n" " mov.f32 %f192, 0fA7C234C5;\n" " fma.rn.f32 %f244, %f187, %f192, %f191;\n" " abs.f32 %f41, %f39;\n" " setp.leu.f32 %p52, %f41, 0f47CE4780;\n" " add.s64 %rd28, %rd1, 24;\n" " mov.u32 %r195, %r199;\n" " mov.f32 %f241, %f244;\n" " @%p52 bra $L__BB1_53;\n" "\n" " setp.eq.f32 %p53, %f41, 0f7F800000;\n" " @%p53 bra $L__BB1_52;\n" " bra.uni $L__BB1_47;\n" "\n" "$L__BB1_52:\n" " mul.rn.f32 %f241, %f39, %f154;\n" " mov.u32 %r195, %r199;\n" " bra.uni $L__BB1_53;\n" "\n" "$L__BB1_47:\n" " mov.b32 %r29, %f39;\n" " bfe.u32 %r121, %r29, 23, 8;\n" " add.s32 %r30, %r121, -128;\n" " shl.b32 %r122, %r29, 8;\n" " or.b32 %r31, %r122, -2147483648;\n" " shr.u32 %r32, %r30, 5;\n" " mov.u64 %rd112, 0;\n" " mov.u32 %r192, 0;\n" " mov.u64 %rd110, __cudart_i2opi_f;\n" " mov.u64 %rd111, %rd1;\n" "\n" "$L__BB1_48:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r123, [%rd110];\n" " mad.wide.u32 %rd72, %r123, %r31, %rd112;\n" " shr.u64 %rd112, %rd72, 32;\n" " st.local.u32 [%rd111], %rd72;\n" " add.s64 %rd111, %rd111, 4;\n" " add.s64 %rd110, %rd110, 4;\n" " add.s32 %r192, %r192, 1;\n" " setp.ne.s32 %p54, %r192, 6;\n" " @%p54 bra $L__BB1_48;\n" "\n" " st.local.u32 [%rd28], %rd112;\n" " mov.u32 %r124, 4;\n" " sub.s32 %r35, %r124, %r32;\n" " mov.u32 %r125, 6;\n" " sub.s32 %r126, %r125, %r32;\n" " mul.wide.s32 %rd73, %r126, 4;\n" " add.s64 %rd74, %rd1, %rd73;\n" " ld.local.u32 %r193, [%rd74];\n" " ld.local.u32 %r194, [%rd74+-4];\n" " and.b32 %r38, %r30, 31;\n" " setp.eq.s32 %p55, %r38, 0;\n" " @%p55 bra $L__BB1_51;\n" "\n" " mov.u32 %r127, 32;\n" " sub.s32 %r128, %r127, %r38;\n" " shr.u32 %r129, %r194, %r128;\n" " shl.b32 %r130, %r193, %r38;\n" " add.s32 %r193, %r129, %r130;\n" " mul.wide.s32 %rd75, %r35, 4;\n" " add.s64 %rd76, %rd1, %rd75;\n" " ld.local.u32 %r131, [%rd76];\n" " shr.u32 %r132, %r131, %r128;\n" " shl.b32 %r133, %r194, %r38;\n" " add.s32 %r194, %r132, %r133;\n" "\n" "$L__BB1_51:\n" " and.b32 %r134, %r29, -2147483648;\n" " shr.u32 %r135, %r194, 30;\n" " shl.b32 %r136, %r193, 2;\n" " or.b32 %r137, %r135, %r136;\n" " shr.u32 %r138, %r137, 31;\n" " shr.u32 %r139, %r193, 30;\n" " add.s32 %r140, %r138, %r139;\n" " neg.s32 %r141, %r140;\n" " setp.eq.s32 %p56, %r134, 0;\n" " selp.b32 %r195, %r140, %r141, %p56;\n" " setp.ne.s32 %p57, %r138, 0;\n" " xor.b32 %r142, %r134, -2147483648;\n" " selp.b32 %r143, %r142, %r134, %p57;\n" " selp.b32 %r144, -1, 0, %p57;\n" " xor.b32 %r145, %r137, %r144;\n" " shl.b32 %r146, %r194, 2;\n" " xor.b32 %r147, %r146, %r144;\n" " cvt.u64.u32 %rd77, %r145;\n" " shl.b64 %rd78, %rd77, 32;\n" " cvt.u64.u32 %rd79, %r147;\n" " or.b64 %rd80, %rd78, %rd79;\n" " cvt.rn.f64.s64 %fd1, %rd80;\n" " mul.f64 %fd2, %fd1, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f193, %fd2;\n" " setp.eq.s32 %p58, %r143, 0;\n" " neg.f32 %f194, %f193;\n" " selp.f32 %f241, %f193, %f194, %p58;\n" "\n" "$L__BB1_53:\n" " add.s32 %r45, %r195, 1;\n" " and.b32 %r46, %r45, 1;\n" " setp.eq.s32 %p59, %r46, 0;\n" " selp.f32 %f45, %f241, 0f3F800000, %p59;\n" " mul.rn.f32 %f46, %f241, %f241;\n" " mov.f32 %f242, 0fB94D4153;\n" " @%p59 bra $L__BB1_55;\n" "\n" " mov.f32 %f197, 0fBAB607ED;\n" " mov.f32 %f198, 0f37CBAC00;\n" " fma.rn.f32 %f242, %f198, %f46, %f197;\n" "\n" "$L__BB1_55:\n" " selp.f32 %f199, 0f3C0885E4, 0f3D2AAABB, %p59;\n" " fma.rn.f32 %f200, %f242, %f46, %f199;\n" " selp.f32 %f201, 0fBE2AAAA8, 0fBEFFFFFF, %p59;\n" " fma.rn.f32 %f202, %f200, %f46, %f201;\n" " fma.rn.f32 %f204, %f46, %f45, %f154;\n" " fma.rn.f32 %f243, %f202, %f204, %f45;\n" " and.b32 %r148, %r45, 2;\n" " setp.eq.s32 %p61, %r148, 0;\n" " @%p61 bra $L__BB1_57;\n" "\n" " mov.f32 %f206, 0fBF800000;\n" " fma.rn.f32 %f243, %f243, %f206, %f154;\n" "\n" "$L__BB1_57:\n" " @%p52 bra $L__BB1_65;\n" "\n" " setp.eq.f32 %p63, %f41, 0f7F800000;\n" " @%p63 bra $L__BB1_64;\n" " bra.uni $L__BB1_59;\n" "\n" "$L__BB1_64:\n" " mul.rn.f32 %f244, %f39, %f154;\n" " bra.uni $L__BB1_65;\n" "\n" "$L__BB1_59:\n" " mov.b32 %r47, %f39;\n" " bfe.u32 %r150, %r47, 23, 8;\n" " add.s32 %r48, %r150, -128;\n" " shl.b32 %r151, %r47, 8;\n" " or.b32 %r49, %r151, -2147483648;\n" " shr.u32 %r50, %r48, 5;\n" " mov.u64 %rd115, 0;\n" " mov.u32 %r196, 0;\n" " mov.u64 %rd113, __cudart_i2opi_f;\n" " mov.u64 %rd114, %rd1;\n" "\n" "$L__BB1_60:\n" " .pragma \"nounroll\";\n" " ld.global.nc.u32 %r152, [%rd113];\n" " mad.wide.u32 %rd83, %r152, %r49, %rd115;\n" " shr.u64 %rd115, %rd83, 32;\n" " st.local.u32 [%rd114], %rd83;\n" " add.s64 %rd114, %rd114, 4;\n" " add.s64 %rd113, %rd113, 4;\n" " add.s32 %r196, %r196, 1;\n" " setp.ne.s32 %p64, %r196, 6;\n" " @%p64 bra $L__BB1_60;\n" "\n" " st.local.u32 [%rd28], %rd115;\n" " mov.u32 %r153, 4;\n" " sub.s32 %r53, %r153, %r50;\n" " mov.u32 %r154, 6;\n" " sub.s32 %r155, %r154, %r50;\n" " mul.wide.s32 %rd84, %r155, 4;\n" " add.s64 %rd85, %rd1, %rd84;\n" " ld.local.u32 %r197, [%rd85];\n" " ld.local.u32 %r198, [%rd85+-4];\n" " and.b32 %r56, %r48, 31;\n" " setp.eq.s32 %p65, %r56, 0;\n" " @%p65 bra $L__BB1_63;\n" "\n" " mov.u32 %r156, 32;\n" " sub.s32 %r157, %r156, %r56;\n" " shr.u32 %r158, %r198, %r157;\n" " shl.b32 %r159, %r197, %r56;\n" " add.s32 %r197, %r158, %r159;\n" " mul.wide.s32 %rd86, %r53, 4;\n" " add.s64 %rd87, %rd1, %rd86;\n" " ld.local.u32 %r160, [%rd87];\n" " shr.u32 %r161, %r160, %r157;\n" " shl.b32 %r162, %r198, %r56;\n" " add.s32 %r198, %r161, %r162;\n" "\n" "$L__BB1_63:\n" " and.b32 %r163, %r47, -2147483648;\n" " shr.u32 %r164, %r198, 30;\n" " shl.b32 %r165, %r197, 2;\n" " or.b32 %r166, %r164, %r165;\n" " shr.u32 %r167, %r166, 31;\n" " shr.u32 %r168, %r197, 30;\n" " add.s32 %r169, %r167, %r168;\n" " neg.s32 %r170, %r169;\n" " setp.eq.s32 %p66, %r163, 0;\n" " selp.b32 %r199, %r169, %r170, %p66;\n" " setp.ne.s32 %p67, %r167, 0;\n" " xor.b32 %r171, %r163, -2147483648;\n" " selp.b32 %r172, %r171, %r163, %p67;\n" " selp.b32 %r173, -1, 0, %p67;\n" " xor.b32 %r174, %r166, %r173;\n" " shl.b32 %r175, %r198, 2;\n" " xor.b32 %r176, %r175, %r173;\n" " cvt.u64.u32 %rd88, %r174;\n" " shl.b64 %rd89, %rd88, 32;\n" " cvt.u64.u32 %rd90, %r176;\n" " or.b64 %rd91, %rd89, %rd90;\n" " cvt.rn.f64.s64 %fd3, %rd91;\n" " mul.f64 %fd4, %fd3, 0d3BF921FB54442D19;\n" " cvt.rn.f32.f64 %f207, %fd4;\n" " setp.eq.s32 %p68, %r172, 0;\n" " neg.f32 %f208, %f207;\n" " selp.f32 %f244, %f207, %f208, %p68;\n" "\n" "$L__BB1_65:\n" " and.b32 %r63, %r199, 1;\n" " setp.eq.s32 %p69, %r63, 0;\n" " selp.f32 %f55, %f244, 0f3F800000, %p69;\n" " mul.rn.f32 %f56, %f244, %f244;\n" " mov.f32 %f245, 0fB94D4153;\n" " @%p69 bra $L__BB1_67;\n" "\n" " mov.f32 %f211, 0fBAB607ED;\n" " mov.f32 %f212, 0f37CBAC00;\n" " fma.rn.f32 %f245, %f212, %f56, %f211;\n" "\n" "$L__BB1_67:\n" " selp.f32 %f213, 0f3C0885E4, 0f3D2AAABB, %p69;\n" " fma.rn.f32 %f214, %f245, %f56, %f213;\n" " selp.f32 %f215, 0fBE2AAAA8, 0fBEFFFFFF, %p69;\n" " fma.rn.f32 %f216, %f214, %f56, %f215;\n" " fma.rn.f32 %f218, %f56, %f55, %f154;\n" " fma.rn.f32 %f246, %f216, %f218, %f55;\n" " and.b32 %r177, %r199, 2;\n" " setp.eq.s32 %p71, %r177, 0;\n" " @%p71 bra $L__BB1_69;\n" "\n" " mov.f32 %f220, 0fBF800000;\n" " fma.rn.f32 %f246, %f246, %f220, %f154;\n" "\n" "$L__BB1_69:\n" " ld.param.u32 %r182, [fc_attn_qknorm_rope_param_6];\n" " mov.u32 %r181, %ctaid.x;\n" " ld.param.u32 %r180, [fc_attn_qknorm_rope_param_4];\n" " ld.param.u64 %rd106, [fc_attn_qknorm_rope_param_0];\n" " mul.wide.s32 %rd105, %r182, %r181;\n" " shl.b64 %rd104, %rd105, 2;\n" " cvta.to.global.u64 %rd103, %rd106;\n" " ld.param.u64 %rd102, [fc_attn_qknorm_rope_param_1];\n" " sub.s32 %r179, %r181, %r180;\n" " mul.wide.s32 %rd101, %r182, %r179;\n" " shl.b64 %rd100, %rd101, 2;\n" " cvta.to.global.u64 %rd99, %rd102;\n" " setp.lt.s32 %p72, %r181, %r180;\n" " add.s64 %rd98, %rd99, %rd100;\n" " add.s64 %rd97, %rd103, %rd104;\n" " selp.b64 %rd96, %rd97, %rd98, %p72;\n" " mul.wide.s32 %rd92, %r3, 4;\n" " add.s64 %rd93, %rd96, %rd92;\n" " mul.wide.s32 %rd94, %r27, 4;\n" " add.s64 %rd95, %rd93, %rd94;\n" " ld.global.f32 %f221, [%rd93];\n" " mul.rn.f32 %f222, %f221, %f243;\n" " neg.f32 %f223, %f246;\n" " ld.global.f32 %f224, [%rd95];\n" " fma.rn.f32 %f225, %f224, %f223, %f222;\n" " st.global.f32 [%rd93], %f225;\n" " mul.rn.f32 %f226, %f221, %f246;\n" " fma.rn.f32 %f227, %f224, %f243, %f226;\n" " st.global.f32 [%rd95], %f227;\n" "\n" "$L__BB1_70:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_TOPK_LOGITS[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_topk_partial_argmax\n" ".extern .shared .align 16 .b8 sh[];\n" "\n" ".visible .entry fc_topk_partial_argmax(\n" " .param .u64 fc_topk_partial_argmax_param_0,\n" " .param .u32 fc_topk_partial_argmax_param_1,\n" " .param .u64 fc_topk_partial_argmax_param_2,\n" " .param .u64 fc_topk_partial_argmax_param_3\n" ")\n" "{\n" " .reg .pred %p<20>;\n" " .reg .f32 %f<28>;\n" " .reg .b32 %r<70>;\n" " .reg .b64 %rd<22>;\n" "\n" "\n" " ld.param.u64 %rd9, [fc_topk_partial_argmax_param_0];\n" " ld.param.u32 %r32, [fc_topk_partial_argmax_param_1];\n" " ld.param.u64 %rd7, [fc_topk_partial_argmax_param_2];\n" " ld.param.u64 %rd8, [fc_topk_partial_argmax_param_3];\n" " cvta.to.global.u64 %rd1, %rd9;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r34, %nctaid.x;\n" " mul.lo.s32 %r2, %r34, %r1;\n" " mov.u32 %r3, %ctaid.x;\n" " mov.u32 %r4, %tid.x;\n" " mad.lo.s32 %r63, %r3, %r1, %r4;\n" " setp.ge.s32 %p1, %r63, %r32;\n" " mov.u32 %r68, 2147483647;\n" " mov.f32 %f27, 0fFF800000;\n" " @%p1 bra $L__BB0_7;\n" "\n" " add.s32 %r37, %r2, %r32;\n" " add.s32 %r38, %r63, %r2;\n" " not.b32 %r39, %r38;\n" " add.s32 %r40, %r37, %r39;\n" " div.u32 %r41, %r40, %r2;\n" " add.s32 %r6, %r41, 1;\n" " and.b32 %r67, %r6, 3;\n" " setp.lt.u32 %p2, %r41, 3;\n" " mov.f32 %f27, 0fFF800000;\n" " mov.u32 %r68, 2147483647;\n" " @%p2 bra $L__BB0_4;\n" "\n" " sub.s32 %r61, %r6, %r67;\n" " mul.wide.s32 %rd2, %r2, 4;\n" " mov.f32 %f27, 0fFF800000;\n" " mov.u32 %r68, 2147483647;\n" "\n" "$L__BB0_3:\n" " mul.wide.s32 %rd10, %r63, 4;\n" " add.s64 %rd11, %rd1, %rd10;\n" " ld.global.f32 %f13, [%rd11];\n" " setp.gt.f32 %p3, %f13, %f27;\n" " selp.f32 %f14, %f13, %f27, %p3;\n" " selp.b32 %r43, %r63, %r68, %p3;\n" " add.s64 %rd12, %rd11, %rd2;\n" " ld.global.f32 %f15, [%rd12];\n" " setp.gt.f32 %p4, %f15, %f14;\n" " selp.f32 %f16, %f15, %f14, %p4;\n" " add.s32 %r44, %r63, %r2;\n" " selp.b32 %r45, %r44, %r43, %p4;\n" " add.s32 %r46, %r44, %r2;\n" " add.s64 %rd13, %rd12, %rd2;\n" " ld.global.f32 %f17, [%rd13];\n" " setp.gt.f32 %p5, %f17, %f16;\n" " selp.f32 %f18, %f17, %f16, %p5;\n" " selp.b32 %r47, %r46, %r45, %p5;\n" " add.s32 %r48, %r46, %r2;\n" " add.s64 %rd14, %rd13, %rd2;\n" " ld.global.f32 %f19, [%rd14];\n" " setp.gt.f32 %p6, %f19, %f18;\n" " selp.f32 %f27, %f19, %f18, %p6;\n" " selp.b32 %r68, %r48, %r47, %p6;\n" " add.s32 %r63, %r48, %r2;\n" " add.s32 %r61, %r61, -4;\n" " setp.ne.s32 %p7, %r61, 0;\n" " @%p7 bra $L__BB0_3;\n" "\n" "$L__BB0_4:\n" " setp.eq.s32 %p8, %r67, 0;\n" " @%p8 bra $L__BB0_7;\n" "\n" " mul.wide.s32 %rd15, %r63, 4;\n" " add.s64 %rd21, %rd1, %rd15;\n" " mul.wide.s32 %rd4, %r2, 4;\n" "\n" "$L__BB0_6:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f20, [%rd21];\n" " setp.gt.f32 %p9, %f20, %f27;\n" " selp.f32 %f27, %f20, %f27, %p9;\n" " selp.b32 %r68, %r63, %r68, %p9;\n" " add.s32 %r63, %r63, %r2;\n" " add.s64 %rd21, %rd21, %rd4;\n" " add.s32 %r67, %r67, -1;\n" " setp.ne.s32 %p10, %r67, 0;\n" " @%p10 bra $L__BB0_6;\n" "\n" "$L__BB0_7:\n" " shl.b32 %r49, %r1, 2;\n" " mov.u32 %r50, sh;\n" " add.s32 %r25, %r50, %r49;\n" " shl.b32 %r51, %r4, 2;\n" " add.s32 %r26, %r50, %r51;\n" " st.shared.f32 [%r26], %f27;\n" " add.s32 %r27, %r25, %r51;\n" " st.shared.u32 [%r27], %r68;\n" " bar.sync 0;\n" " shr.u32 %r69, %r1, 1;\n" " setp.eq.s32 %p11, %r69, 0;\n" " @%p11 bra $L__BB0_12;\n" "\n" "$L__BB0_8:\n" " mov.u32 %r29, %r69;\n" " setp.ge.s32 %p12, %r4, %r29;\n" " @%p12 bra $L__BB0_11;\n" "\n" " shl.b32 %r52, %r29, 2;\n" " add.s32 %r53, %r26, %r52;\n" " add.s32 %r30, %r53, %r49;\n" " ld.shared.f32 %f21, [%r26];\n" " ld.shared.f32 %f8, [%r53];\n" " setp.leu.f32 %p13, %f8, %f21;\n" " setp.neu.f32 %p14, %f8, %f21;\n" " ld.shared.u32 %r55, [%r27];\n" " ld.shared.u32 %r56, [%r30];\n" " setp.ge.s32 %p15, %r56, %r55;\n" " or.pred %p16, %p14, %p15;\n" " and.pred %p17, %p13, %p16;\n" " @%p17 bra $L__BB0_11;\n" "\n" " st.shared.f32 [%r26], %f8;\n" " ld.shared.u32 %r57, [%r30];\n" " st.shared.u32 [%r27], %r57;\n" "\n" "$L__BB0_11:\n" " bar.sync 0;\n" " shr.s32 %r69, %r29, 1;\n" " setp.gt.s32 %p18, %r29, 1;\n" " @%p18 bra $L__BB0_8;\n" "\n" "$L__BB0_12:\n" " setp.ne.s32 %p19, %r4, 0;\n" " @%p19 bra $L__BB0_14;\n" "\n" " ld.shared.f32 %f22, [sh];\n" " cvta.to.global.u64 %rd16, %rd7;\n" " mul.wide.u32 %rd17, %r3, 4;\n" " add.s64 %rd18, %rd16, %rd17;\n" " st.global.f32 [%rd18], %f22;\n" " ld.shared.u32 %r58, [%r25];\n" " cvta.to.global.u64 %rd19, %rd8;\n" " add.s64 %rd20, %rd19, %rd17;\n" " st.global.u32 [%rd20], %r58;\n" "\n" "$L__BB0_14:\n" " ret;\n" "\n" "}\n" " // .globl fc_topk_finish\n" ".visible .entry fc_topk_finish(\n" " .param .u64 fc_topk_finish_param_0,\n" " .param .u64 fc_topk_finish_param_1,\n" " .param .u64 fc_topk_finish_param_2,\n" " .param .u32 fc_topk_finish_param_3,\n" " .param .u64 fc_topk_finish_param_4,\n" " .param .u64 fc_topk_finish_param_5,\n" " .param .u32 fc_topk_finish_param_6,\n" " .param .u32 fc_topk_finish_param_7\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<5>;\n" " .reg .b32 %r<32>;\n" " .reg .b64 %rd<19>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_topk_finish_param_0];\n" " ld.param.u64 %rd2, [fc_topk_finish_param_1];\n" " ld.param.u64 %rd3, [fc_topk_finish_param_2];\n" " ld.param.u32 %r15, [fc_topk_finish_param_3];\n" " ld.param.u64 %rd4, [fc_topk_finish_param_4];\n" " ld.param.u64 %rd5, [fc_topk_finish_param_5];\n" " ld.param.u32 %r13, [fc_topk_finish_param_6];\n" " ld.param.u32 %r14, [fc_topk_finish_param_7];\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r2, %tid.x;\n" " setp.lt.s32 %p1, %r2, %r15;\n" " shl.b32 %r16, %r2, 2;\n" " mov.u32 %r17, sh;\n" " add.s32 %r3, %r17, %r16;\n" " @%p1 bra $L__BB1_2;\n" " bra.uni $L__BB1_1;\n" "\n" "$L__BB1_2:\n" " cvta.to.global.u64 %rd6, %rd2;\n" " mul.wide.s32 %rd7, %r2, 4;\n" " add.s64 %rd8, %rd6, %rd7;\n" " ld.global.f32 %f2, [%rd8];\n" " st.shared.f32 [%r3], %f2;\n" " cvta.to.global.u64 %rd9, %rd3;\n" " add.s64 %rd10, %rd9, %rd7;\n" " ld.global.u32 %r30, [%rd10];\n" " bra.uni $L__BB1_3;\n" "\n" "$L__BB1_1:\n" " mov.u32 %r19, -8388608;\n" " st.shared.u32 [%r3], %r19;\n" " mov.u32 %r30, 2147483647;\n" "\n" "$L__BB1_3:\n" " shl.b32 %r20, %r1, 2;\n" " add.s32 %r6, %r17, %r20;\n" " add.s32 %r7, %r6, %r16;\n" " st.shared.u32 [%r7], %r30;\n" " bar.sync 0;\n" " shr.u32 %r31, %r1, 1;\n" " setp.eq.s32 %p2, %r31, 0;\n" " @%p2 bra $L__BB1_8;\n" "\n" "$L__BB1_4:\n" " mov.u32 %r9, %r31;\n" " setp.ge.s32 %p3, %r2, %r9;\n" " @%p3 bra $L__BB1_7;\n" "\n" " shl.b32 %r23, %r9, 2;\n" " add.s32 %r24, %r3, %r23;\n" " add.s32 %r10, %r24, %r20;\n" " ld.shared.f32 %f3, [%r3];\n" " ld.shared.f32 %f1, [%r24];\n" " setp.leu.f32 %p4, %f1, %f3;\n" " setp.neu.f32 %p5, %f1, %f3;\n" " ld.shared.u32 %r26, [%r7];\n" " ld.shared.u32 %r27, [%r10];\n" " setp.ge.s32 %p6, %r27, %r26;\n" " or.pred %p7, %p5, %p6;\n" " and.pred %p8, %p4, %p7;\n" " @%p8 bra $L__BB1_7;\n" "\n" " st.shared.f32 [%r3], %f1;\n" " ld.shared.u32 %r28, [%r10];\n" " st.shared.u32 [%r7], %r28;\n" "\n" "$L__BB1_7:\n" " bar.sync 0;\n" " shr.s32 %r31, %r9, 1;\n" " setp.gt.s32 %p9, %r9, 1;\n" " @%p9 bra $L__BB1_4;\n" "\n" "$L__BB1_8:\n" " setp.ne.s32 %p10, %r2, 0;\n" " @%p10 bra $L__BB1_11;\n" "\n" " ld.shared.f32 %f4, [sh];\n" " cvta.to.global.u64 %rd11, %rd4;\n" " mul.wide.s32 %rd12, %r13, 4;\n" " add.s64 %rd13, %rd11, %rd12;\n" " st.global.f32 [%rd13], %f4;\n" " ld.shared.u32 %r12, [%r6];\n" " cvta.to.global.u64 %rd14, %rd5;\n" " add.s64 %rd15, %rd14, %rd12;\n" " st.global.u32 [%rd15], %r12;\n" " setp.eq.s32 %p11, %r14, 0;\n" " @%p11 bra $L__BB1_11;\n" "\n" " cvta.to.global.u64 %rd16, %rd1;\n" " mul.wide.s32 %rd17, %r12, 4;\n" " add.s64 %rd18, %rd16, %rd17;\n" " mov.u32 %r29, -8388608;\n" " st.global.u32 [%rd18], %r29;\n" "\n" "$L__BB1_11:\n" " ret;\n" "\n" "}\n" " // .globl fc_topk_restore\n" ".visible .entry fc_topk_restore(\n" " .param .u64 fc_topk_restore_param_0,\n" " .param .u64 fc_topk_restore_param_1,\n" " .param .u64 fc_topk_restore_param_2,\n" " .param .u32 fc_topk_restore_param_3\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<2>;\n" " .reg .b32 %r<4>;\n" " .reg .b64 %rd<12>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_topk_restore_param_0];\n" " ld.param.u64 %rd2, [fc_topk_restore_param_1];\n" " ld.param.u64 %rd3, [fc_topk_restore_param_2];\n" " ld.param.u32 %r2, [fc_topk_restore_param_3];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r2;\n" " @%p1 bra $L__BB2_2;\n" "\n" " cvta.to.global.u64 %rd4, %rd2;\n" " mul.wide.s32 %rd5, %r1, 4;\n" " add.s64 %rd6, %rd4, %rd5;\n" " ld.global.f32 %f1, [%rd6];\n" " cvta.to.global.u64 %rd7, %rd3;\n" " add.s64 %rd8, %rd7, %rd5;\n" " ld.global.u32 %r3, [%rd8];\n" " cvta.to.global.u64 %rd9, %rd1;\n" " mul.wide.s32 %rd10, %r3, 4;\n" " add.s64 %rd11, %rd9, %rd10;\n" " st.global.f32 [%rd11], %f1;\n" "\n" "$L__BB2_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_topk_partial_argmax_slots\n" ".visible .entry fc_topk_partial_argmax_slots(\n" " .param .u64 fc_topk_partial_argmax_slots_param_0,\n" " .param .u32 fc_topk_partial_argmax_slots_param_1,\n" " .param .u64 fc_topk_partial_argmax_slots_param_2,\n" " .param .u64 fc_topk_partial_argmax_slots_param_3\n" ")\n" "{\n" " .reg .pred %p<20>;\n" " .reg .f32 %f<28>;\n" " .reg .b32 %r<71>;\n" " .reg .b64 %rd<33>;\n" "\n" "\n" " ld.param.u64 %rd11, [fc_topk_partial_argmax_slots_param_0];\n" " ld.param.u32 %r33, [fc_topk_partial_argmax_slots_param_1];\n" " ld.param.u64 %rd9, [fc_topk_partial_argmax_slots_param_2];\n" " ld.param.u64 %rd10, [fc_topk_partial_argmax_slots_param_3];\n" " cvta.to.global.u64 %rd1, %rd11;\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r35, %ctaid.y;\n" " cvt.u64.u32 %rd2, %r35;\n" " cvt.s64.s32 %rd12, %r33;\n" " mul.lo.s64 %rd3, %rd12, %rd2;\n" " mov.u32 %r2, %nctaid.x;\n" " mul.lo.s32 %r3, %r2, %r1;\n" " mov.u32 %r4, %ctaid.x;\n" " mov.u32 %r5, %tid.x;\n" " mad.lo.s32 %r64, %r4, %r1, %r5;\n" " setp.ge.s32 %p1, %r64, %r33;\n" " mov.u32 %r69, 2147483647;\n" " mov.f32 %f27, 0fFF800000;\n" " @%p1 bra $L__BB3_7;\n" "\n" " add.s32 %r38, %r3, %r33;\n" " add.s32 %r39, %r64, %r3;\n" " not.b32 %r40, %r39;\n" " add.s32 %r41, %r38, %r40;\n" " div.u32 %r42, %r41, %r3;\n" " add.s32 %r7, %r42, 1;\n" " and.b32 %r68, %r7, 3;\n" " setp.lt.u32 %p2, %r42, 3;\n" " mov.f32 %f27, 0fFF800000;\n" " mov.u32 %r69, 2147483647;\n" " @%p2 bra $L__BB3_4;\n" "\n" " sub.s32 %r62, %r7, %r68;\n" " mul.wide.s32 %rd4, %r3, 4;\n" " mov.f32 %f27, 0fFF800000;\n" " mov.u32 %r69, 2147483647;\n" "\n" "$L__BB3_3:\n" " cvt.s64.s32 %rd13, %r64;\n" " add.s64 %rd14, %rd3, %rd13;\n" " shl.b64 %rd15, %rd14, 2;\n" " add.s64 %rd16, %rd1, %rd15;\n" " ld.global.f32 %f13, [%rd16];\n" " setp.gt.f32 %p3, %f13, %f27;\n" " selp.f32 %f14, %f13, %f27, %p3;\n" " selp.b32 %r44, %r64, %r69, %p3;\n" " add.s64 %rd17, %rd16, %rd4;\n" " ld.global.f32 %f15, [%rd17];\n" " setp.gt.f32 %p4, %f15, %f14;\n" " selp.f32 %f16, %f15, %f14, %p4;\n" " add.s32 %r45, %r64, %r3;\n" " selp.b32 %r46, %r45, %r44, %p4;\n" " add.s32 %r47, %r45, %r3;\n" " add.s64 %rd18, %rd17, %rd4;\n" " ld.global.f32 %f17, [%rd18];\n" " setp.gt.f32 %p5, %f17, %f16;\n" " selp.f32 %f18, %f17, %f16, %p5;\n" " selp.b32 %r48, %r47, %r46, %p5;\n" " add.s32 %r49, %r47, %r3;\n" " add.s64 %rd19, %rd18, %rd4;\n" " ld.global.f32 %f19, [%rd19];\n" " setp.gt.f32 %p6, %f19, %f18;\n" " selp.f32 %f27, %f19, %f18, %p6;\n" " selp.b32 %r69, %r49, %r48, %p6;\n" " add.s32 %r64, %r49, %r3;\n" " add.s32 %r62, %r62, -4;\n" " setp.ne.s32 %p7, %r62, 0;\n" " @%p7 bra $L__BB3_3;\n" "\n" "$L__BB3_4:\n" " setp.eq.s32 %p8, %r68, 0;\n" " @%p8 bra $L__BB3_7;\n" "\n" " cvt.s64.s32 %rd20, %r64;\n" " add.s64 %rd21, %rd3, %rd20;\n" " shl.b64 %rd22, %rd21, 2;\n" " add.s64 %rd32, %rd1, %rd22;\n" " mul.wide.s32 %rd6, %r3, 4;\n" "\n" "$L__BB3_6:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f20, [%rd32];\n" " setp.gt.f32 %p9, %f20, %f27;\n" " selp.f32 %f27, %f20, %f27, %p9;\n" " selp.b32 %r69, %r64, %r69, %p9;\n" " add.s32 %r64, %r64, %r3;\n" " add.s64 %rd32, %rd32, %rd6;\n" " add.s32 %r68, %r68, -1;\n" " setp.ne.s32 %p10, %r68, 0;\n" " @%p10 bra $L__BB3_6;\n" "\n" "$L__BB3_7:\n" " shl.b32 %r50, %r1, 2;\n" " mov.u32 %r51, sh;\n" " add.s32 %r26, %r51, %r50;\n" " shl.b32 %r52, %r5, 2;\n" " add.s32 %r27, %r51, %r52;\n" " st.shared.f32 [%r27], %f27;\n" " add.s32 %r28, %r26, %r52;\n" " st.shared.u32 [%r28], %r69;\n" " bar.sync 0;\n" " shr.u32 %r70, %r1, 1;\n" " setp.eq.s32 %p11, %r70, 0;\n" " @%p11 bra $L__BB3_12;\n" "\n" "$L__BB3_8:\n" " mov.u32 %r30, %r70;\n" " setp.ge.s32 %p12, %r5, %r30;\n" " @%p12 bra $L__BB3_11;\n" "\n" " shl.b32 %r53, %r30, 2;\n" " add.s32 %r54, %r27, %r53;\n" " add.s32 %r31, %r54, %r50;\n" " ld.shared.f32 %f21, [%r27];\n" " ld.shared.f32 %f8, [%r54];\n" " setp.leu.f32 %p13, %f8, %f21;\n" " setp.neu.f32 %p14, %f8, %f21;\n" " ld.shared.u32 %r56, [%r28];\n" " ld.shared.u32 %r57, [%r31];\n" " setp.ge.s32 %p15, %r57, %r56;\n" " or.pred %p16, %p14, %p15;\n" " and.pred %p17, %p13, %p16;\n" " @%p17 bra $L__BB3_11;\n" "\n" " st.shared.f32 [%r27], %f8;\n" " ld.shared.u32 %r58, [%r31];\n" " st.shared.u32 [%r28], %r58;\n" "\n" "$L__BB3_11:\n" " bar.sync 0;\n" " shr.s32 %r70, %r30, 1;\n" " setp.gt.s32 %p18, %r30, 1;\n" " @%p18 bra $L__BB3_8;\n" "\n" "$L__BB3_12:\n" " setp.ne.s32 %p19, %r5, 0;\n" " @%p19 bra $L__BB3_14;\n" "\n" " ld.shared.f32 %f22, [sh];\n" " cvt.u64.u32 %rd23, %r2;\n" " mul.lo.s64 %rd24, %rd2, %rd23;\n" " cvt.u64.u32 %rd25, %r4;\n" " add.s64 %rd26, %rd24, %rd25;\n" " cvta.to.global.u64 %rd27, %rd9;\n" " shl.b64 %rd28, %rd26, 2;\n" " add.s64 %rd29, %rd27, %rd28;\n" " st.global.f32 [%rd29], %f22;\n" " ld.shared.u32 %r59, [%r26];\n" " cvta.to.global.u64 %rd30, %rd10;\n" " add.s64 %rd31, %rd30, %rd28;\n" " st.global.u32 [%rd31], %r59;\n" "\n" "$L__BB3_14:\n" " ret;\n" "\n" "}\n" " // .globl fc_topk_finish_slots\n" ".visible .entry fc_topk_finish_slots(\n" " .param .u64 fc_topk_finish_slots_param_0,\n" " .param .u64 fc_topk_finish_slots_param_1,\n" " .param .u64 fc_topk_finish_slots_param_2,\n" " .param .u32 fc_topk_finish_slots_param_3,\n" " .param .u32 fc_topk_finish_slots_param_4,\n" " .param .u64 fc_topk_finish_slots_param_5,\n" " .param .u64 fc_topk_finish_slots_param_6,\n" " .param .u32 fc_topk_finish_slots_param_7,\n" " .param .u32 fc_topk_finish_slots_param_8,\n" " .param .u32 fc_topk_finish_slots_param_9\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<5>;\n" " .reg .b32 %r<35>;\n" " .reg .b64 %rd<32>;\n" "\n" "\n" " ld.param.u64 %rd2, [fc_topk_finish_slots_param_0];\n" " ld.param.u64 %rd3, [fc_topk_finish_slots_param_1];\n" " ld.param.u64 %rd4, [fc_topk_finish_slots_param_2];\n" " ld.param.u32 %r13, [fc_topk_finish_slots_param_3];\n" " ld.param.u32 %r14, [fc_topk_finish_slots_param_4];\n" " ld.param.u64 %rd5, [fc_topk_finish_slots_param_5];\n" " ld.param.u64 %rd6, [fc_topk_finish_slots_param_6];\n" " ld.param.u32 %r15, [fc_topk_finish_slots_param_7];\n" " ld.param.u32 %r16, [fc_topk_finish_slots_param_8];\n" " ld.param.u32 %r17, [fc_topk_finish_slots_param_9];\n" " mov.u32 %r1, %ntid.x;\n" " mov.u32 %r18, %ctaid.y;\n" " cvt.s64.s32 %rd1, %r18;\n" " mov.u32 %r2, %tid.x;\n" " setp.lt.s32 %p1, %r2, %r14;\n" " shl.b32 %r19, %r2, 2;\n" " mov.u32 %r20, sh;\n" " add.s32 %r3, %r20, %r19;\n" " @%p1 bra $L__BB4_2;\n" " bra.uni $L__BB4_1;\n" "\n" "$L__BB4_2:\n" " cvta.to.global.u64 %rd7, %rd3;\n" " cvt.s64.s32 %rd8, %r2;\n" " cvt.s64.s32 %rd9, %r14;\n" " mul.lo.s64 %rd10, %rd9, %rd1;\n" " add.s64 %rd11, %rd10, %rd8;\n" " shl.b64 %rd12, %rd11, 2;\n" " add.s64 %rd13, %rd7, %rd12;\n" " ld.global.f32 %f2, [%rd13];\n" " st.shared.f32 [%r3], %f2;\n" " cvta.to.global.u64 %rd14, %rd4;\n" " add.s64 %rd15, %rd14, %rd12;\n" " ld.global.u32 %r33, [%rd15];\n" " bra.uni $L__BB4_3;\n" "\n" "$L__BB4_1:\n" " mov.u32 %r22, -8388608;\n" " st.shared.u32 [%r3], %r22;\n" " mov.u32 %r33, 2147483647;\n" "\n" "$L__BB4_3:\n" " shl.b32 %r23, %r1, 2;\n" " add.s32 %r6, %r20, %r23;\n" " add.s32 %r7, %r6, %r19;\n" " st.shared.u32 [%r7], %r33;\n" " bar.sync 0;\n" " shr.u32 %r34, %r1, 1;\n" " setp.eq.s32 %p2, %r34, 0;\n" " @%p2 bra $L__BB4_8;\n" "\n" "$L__BB4_4:\n" " mov.u32 %r9, %r34;\n" " setp.ge.s32 %p3, %r2, %r9;\n" " @%p3 bra $L__BB4_7;\n" "\n" " shl.b32 %r26, %r9, 2;\n" " add.s32 %r27, %r3, %r26;\n" " add.s32 %r10, %r27, %r23;\n" " ld.shared.f32 %f3, [%r3];\n" " ld.shared.f32 %f1, [%r27];\n" " setp.leu.f32 %p4, %f1, %f3;\n" " setp.neu.f32 %p5, %f1, %f3;\n" " ld.shared.u32 %r29, [%r7];\n" " ld.shared.u32 %r30, [%r10];\n" " setp.ge.s32 %p6, %r30, %r29;\n" " or.pred %p7, %p5, %p6;\n" " and.pred %p8, %p4, %p7;\n" " @%p8 bra $L__BB4_7;\n" "\n" " st.shared.f32 [%r3], %f1;\n" " ld.shared.u32 %r31, [%r10];\n" " st.shared.u32 [%r7], %r31;\n" "\n" "$L__BB4_7:\n" " bar.sync 0;\n" " shr.s32 %r34, %r9, 1;\n" " setp.gt.s32 %p9, %r9, 1;\n" " @%p9 bra $L__BB4_4;\n" "\n" "$L__BB4_8:\n" " setp.ne.s32 %p10, %r2, 0;\n" " @%p10 bra $L__BB4_11;\n" "\n" " ld.shared.f32 %f4, [sh];\n" " cvt.s64.s32 %rd16, %r15;\n" " mul.lo.s64 %rd17, %rd16, %rd1;\n" " cvt.s64.s32 %rd18, %r16;\n" " add.s64 %rd19, %rd17, %rd18;\n" " cvta.to.global.u64 %rd20, %rd5;\n" " shl.b64 %rd21, %rd19, 2;\n" " add.s64 %rd22, %rd20, %rd21;\n" " st.global.f32 [%rd22], %f4;\n" " ld.shared.u32 %r12, [%r6];\n" " cvta.to.global.u64 %rd23, %rd6;\n" " add.s64 %rd24, %rd23, %rd21;\n" " st.global.u32 [%rd24], %r12;\n" " setp.eq.s32 %p11, %r17, 0;\n" " @%p11 bra $L__BB4_11;\n" "\n" " cvt.s64.s32 %rd25, %r13;\n" " mul.lo.s64 %rd26, %rd25, %rd1;\n" " cvt.s64.s32 %rd27, %r12;\n" " add.s64 %rd28, %rd26, %rd27;\n" " cvta.to.global.u64 %rd29, %rd2;\n" " shl.b64 %rd30, %rd28, 2;\n" " add.s64 %rd31, %rd29, %rd30;\n" " mov.u32 %r32, -8388608;\n" " st.global.u32 [%rd31], %r32;\n" "\n" "$L__BB4_11:\n" " ret;\n" "\n" "}\n" " // .globl fc_topk_restore_slots\n" ".visible .entry fc_topk_restore_slots(\n" " .param .u64 fc_topk_restore_slots_param_0,\n" " .param .u64 fc_topk_restore_slots_param_1,\n" " .param .u64 fc_topk_restore_slots_param_2,\n" " .param .u32 fc_topk_restore_slots_param_3,\n" " .param .u32 fc_topk_restore_slots_param_4\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<2>;\n" " .reg .b32 %r<5>;\n" " .reg .b64 %rd<18>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_topk_restore_slots_param_0];\n" " ld.param.u64 %rd2, [fc_topk_restore_slots_param_1];\n" " ld.param.u64 %rd3, [fc_topk_restore_slots_param_2];\n" " ld.param.u32 %r2, [fc_topk_restore_slots_param_3];\n" " ld.param.u32 %r3, [fc_topk_restore_slots_param_4];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r3;\n" " @%p1 bra $L__BB5_2;\n" "\n" " mov.u32 %r4, %ctaid.x;\n" " mul.wide.s32 %rd4, %r3, %r4;\n" " cvt.s64.s32 %rd5, %r1;\n" " add.s64 %rd6, %rd4, %rd5;\n" " cvta.to.global.u64 %rd7, %rd2;\n" " shl.b64 %rd8, %rd6, 2;\n" " add.s64 %rd9, %rd7, %rd8;\n" " ld.global.f32 %f1, [%rd9];\n" " mul.wide.s32 %rd10, %r2, %r4;\n" " cvta.to.global.u64 %rd11, %rd3;\n" " add.s64 %rd12, %rd11, %rd8;\n" " ld.global.s32 %rd13, [%rd12];\n" " add.s64 %rd14, %rd10, %rd13;\n" " cvta.to.global.u64 %rd15, %rd1;\n" " shl.b64 %rd16, %rd14, 2;\n" " add.s64 %rd17, %rd15, %rd16;\n" " st.global.f32 [%rd17], %f1;\n" "\n" "$L__BB5_2:\n" " ret;\n" "\n" "}\n" "\n" ; static const char FC_PTX_ATTENTION[] = "//\n" "// Generated by NVIDIA NVVM Compiler\n" "//\n" "// Compiler Build ID: CL-30521435\n" "// Cuda compilation tools, release 11.4, V11.4.152\n" "// Based on NVVM 7.0.1\n" "//\n" "\n" ".version 7.4\n" ".target sm_37\n" ".address_size 64\n" "\n" " // .globl fc_attn_qz_split\n" "// _ZZ20fc_attn_kv_append_q4E3s_k has been demoted\n" "// _ZZ20fc_attn_kv_append_q4E3s_v has been demoted\n" "// _ZZ26fc_attn_kv_append_slots_q4E3s_k has been demoted\n" "// _ZZ26fc_attn_kv_append_slots_q4E3s_v has been demoted\n" "// _ZZ24fc_attn_kv_append_q4_devE3s_k has been demoted\n" "// _ZZ24fc_attn_kv_append_q4_devE3s_v has been demoted\n" ".extern .shared .align 16 .b8 shb[];\n" ".extern .shared .align 16 .b8 sh[];\n" ".extern .shared .align 16 .b8 shk[];\n" ".extern .shared .align 16 .b8 shv[];\n" "\n" ".visible .entry fc_attn_qz_split(\n" " .param .u64 fc_attn_qz_split_param_0,\n" " .param .u64 fc_attn_qz_split_param_1,\n" " .param .u64 fc_attn_qz_split_param_2,\n" " .param .u32 fc_attn_qz_split_param_3,\n" " .param .u32 fc_attn_qz_split_param_4\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<3>;\n" " .reg .b32 %r<10>;\n" " .reg .b64 %rd<18>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_attn_qz_split_param_0];\n" " ld.param.u64 %rd2, [fc_attn_qz_split_param_1];\n" " ld.param.u64 %rd3, [fc_attn_qz_split_param_2];\n" " ld.param.u32 %r2, [fc_attn_qz_split_param_3];\n" " ld.param.u32 %r3, [fc_attn_qz_split_param_4];\n" " mov.u32 %r4, %ntid.x;\n" " mov.u32 %r5, %ctaid.x;\n" " mov.u32 %r6, %tid.x;\n" " mad.lo.s32 %r1, %r5, %r4, %r6;\n" " setp.ge.s32 %p1, %r1, %r3;\n" " @%p1 bra $L__BB0_2;\n" "\n" " cvta.to.global.u64 %rd4, %rd2;\n" " div.s32 %r7, %r1, %r2;\n" " mul.wide.s32 %rd5, %r2, %r7;\n" " shl.b64 %rd6, %rd5, 1;\n" " mul.lo.s32 %r8, %r7, %r2;\n" " sub.s32 %r9, %r1, %r8;\n" " cvt.s64.s32 %rd7, %r9;\n" " add.s64 %rd8, %rd6, %rd7;\n" " cvta.to.global.u64 %rd9, %rd1;\n" " shl.b64 %rd10, %rd8, 2;\n" " add.s64 %rd11, %rd9, %rd10;\n" " ld.global.f32 %f1, [%rd11];\n" " mul.wide.s32 %rd12, %r1, 4;\n" " add.s64 %rd13, %rd4, %rd12;\n" " st.global.f32 [%rd13], %f1;\n" " mul.wide.s32 %rd14, %r2, 4;\n" " add.s64 %rd15, %rd11, %rd14;\n" " ld.global.f32 %f2, [%rd15];\n" " cvta.to.global.u64 %rd16, %rd3;\n" " add.s64 %rd17, %rd16, %rd12;\n" " st.global.f32 [%rd17], %f2;\n" "\n" "$L__BB0_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_kv_append\n" ".visible .entry fc_attn_kv_append(\n" " .param .u64 fc_attn_kv_append_param_0,\n" " .param .u64 fc_attn_kv_append_param_1,\n" " .param .u64 fc_attn_kv_append_param_2,\n" " .param .u64 fc_attn_kv_append_param_3,\n" " .param .u32 fc_attn_kv_append_param_4,\n" " .param .u32 fc_attn_kv_append_param_5,\n" " .param .u32 fc_attn_kv_append_param_6,\n" " .param .u32 fc_attn_kv_append_param_7\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<3>;\n" " .reg .b32 %r<10>;\n" " .reg .b64 %rd<27>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_attn_kv_append_param_0];\n" " ld.param.u64 %rd2, [fc_attn_kv_append_param_1];\n" " ld.param.u64 %rd3, [fc_attn_kv_append_param_2];\n" " ld.param.u64 %rd4, [fc_attn_kv_append_param_3];\n" " ld.param.u32 %r2, [fc_attn_kv_append_param_4];\n" " ld.param.u32 %r3, [fc_attn_kv_append_param_5];\n" " ld.param.u32 %r4, [fc_attn_kv_append_param_6];\n" " ld.param.u32 %r5, [fc_attn_kv_append_param_7];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r3;\n" " @%p1 bra $L__BB1_2;\n" "\n" " mov.u32 %r6, %ctaid.x;\n" " mov.u32 %r7, %ctaid.y;\n" " add.s32 %r8, %r6, %r5;\n" " rem.s32 %r9, %r8, %r4;\n" " mul.wide.s32 %rd5, %r2, %r6;\n" " cvt.s64.s32 %rd6, %r7;\n" " add.s64 %rd7, %rd5, %rd6;\n" " cvt.s64.s32 %rd8, %r3;\n" " mul.lo.s64 %rd9, %rd7, %rd8;\n" " cvt.s64.s32 %rd10, %r1;\n" " add.s64 %rd11, %rd9, %rd10;\n" " mul.wide.s32 %rd12, %r4, %r7;\n" " cvt.s64.s32 %rd13, %r9;\n" " add.s64 %rd14, %rd12, %rd13;\n" " mul.lo.s64 %rd15, %rd14, %rd8;\n" " add.s64 %rd16, %rd15, %rd10;\n" " cvta.to.global.u64 %rd17, %rd1;\n" " shl.b64 %rd18, %rd11, 2;\n" " add.s64 %rd19, %rd17, %rd18;\n" " ld.global.f32 %f1, [%rd19];\n" " cvta.to.global.u64 %rd20, %rd3;\n" " shl.b64 %rd21, %rd16, 2;\n" " add.s64 %rd22, %rd20, %rd21;\n" " st.global.f32 [%rd22], %f1;\n" " cvta.to.global.u64 %rd23, %rd2;\n" " add.s64 %rd24, %rd23, %rd18;\n" " ld.global.f32 %f2, [%rd24];\n" " cvta.to.global.u64 %rd25, %rd4;\n" " add.s64 %rd26, %rd25, %rd21;\n" " st.global.f32 [%rd26], %f2;\n" "\n" "$L__BB1_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_prefill\n" ".visible .entry fc_attn_flash_prefill(\n" " .param .u64 fc_attn_flash_prefill_param_0,\n" " .param .u64 fc_attn_flash_prefill_param_1,\n" " .param .u64 fc_attn_flash_prefill_param_2,\n" " .param .u64 fc_attn_flash_prefill_param_3,\n" " .param .u32 fc_attn_flash_prefill_param_4,\n" " .param .u32 fc_attn_flash_prefill_param_5,\n" " .param .u32 fc_attn_flash_prefill_param_6,\n" " .param .u32 fc_attn_flash_prefill_param_7,\n" " .param .u32 fc_attn_flash_prefill_param_8,\n" " .param .u32 fc_attn_flash_prefill_param_9,\n" " .param .f32 fc_attn_flash_prefill_param_10\n" ")\n" "{\n" " .reg .pred %p<163>;\n" " .reg .f32 %f<637>;\n" " .reg .b32 %r<545>;\n" " .reg .b64 %rd<115>;\n" "\n" "\n" " ld.param.u64 %rd26, [fc_attn_flash_prefill_param_0];\n" " ld.param.u64 %rd24, [fc_attn_flash_prefill_param_1];\n" " ld.param.u64 %rd27, [fc_attn_flash_prefill_param_3];\n" " ld.param.u32 %r101, [fc_attn_flash_prefill_param_4];\n" " ld.param.u32 %r102, [fc_attn_flash_prefill_param_5];\n" " ld.param.u32 %r103, [fc_attn_flash_prefill_param_6];\n" " ld.param.u32 %r104, [fc_attn_flash_prefill_param_7];\n" " ld.param.u32 %r106, [fc_attn_flash_prefill_param_9];\n" " ld.param.f32 %f179, [fc_attn_flash_prefill_param_10];\n" " cvta.to.global.u64 %rd1, %rd26;\n" " cvta.to.global.u64 %rd2, %rd27;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %ctaid.x;\n" " add.s32 %r3, %r2, %r104;\n" " mul.lo.s32 %r4, %r102, %r101;\n" " add.s32 %r5, %r102, 16;\n" " mul.wide.s32 %rd3, %r5, 16;\n" " shl.b32 %r107, %r101, 4;\n" " cvt.s64.s32 %rd28, %r107;\n" " add.s64 %rd4, %rd3, %rd28;\n" " add.s64 %rd5, %rd4, %rd28;\n" " cvt.s64.s32 %rd6, %r101;\n" " add.s64 %rd7, %rd5, %rd6;\n" " mov.u32 %r108, %nctaid.y;\n" " cvt.u64.u32 %rd29, %r108;\n" " cvt.s64.s32 %rd30, %r2;\n" " mul.lo.s64 %rd31, %rd30, %rd29;\n" " mov.u32 %r109, %ctaid.y;\n" " cvt.s64.s32 %rd8, %r109;\n" " add.s64 %rd32, %rd31, %rd8;\n" " cvt.s64.s32 %rd9, %r102;\n" " mul.wide.s32 %rd33, %r102, %r101;\n" " mul.lo.s64 %rd10, %rd33, %rd32;\n" " setp.lt.s32 %p2, %r101, 1;\n" " @%p2 bra $L__BB2_7;\n" "\n" " cvt.s64.s32 %rd11, %r1;\n" " and.b32 %r528, %r101, 3;\n" " add.s32 %r111, %r101, -1;\n" " setp.lt.u32 %p3, %r111, 3;\n" " mov.u32 %r526, 0;\n" " @%p3 bra $L__BB2_4;\n" "\n" " sub.s32 %r524, %r101, %r528;\n" " add.s64 %rd34, %rd10, %rd11;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd114, %rd1, %rd35;\n" " shl.b64 %rd13, %rd9, 2;\n" " shl.b32 %r113, %r1, 2;\n" " mov.u32 %r114, sh;\n" " add.s32 %r525, %r114, %r113;\n" " shl.b32 %r9, %r102, 2;\n" " mov.u32 %r526, 0;\n" "\n" "$L__BB2_3:\n" " ld.global.f32 %f180, [%rd114];\n" " st.shared.f32 [%r525], %f180;\n" " add.s64 %rd36, %rd114, %rd13;\n" " ld.global.f32 %f181, [%rd36];\n" " add.s32 %r115, %r525, %r9;\n" " st.shared.f32 [%r115], %f181;\n" " add.s64 %rd37, %rd36, %rd13;\n" " ld.global.f32 %f182, [%rd37];\n" " add.s32 %r116, %r115, %r9;\n" " st.shared.f32 [%r116], %f182;\n" " add.s64 %rd38, %rd37, %rd13;\n" " add.s64 %rd114, %rd38, %rd13;\n" " ld.global.f32 %f183, [%rd38];\n" " add.s32 %r117, %r116, %r9;\n" " add.s32 %r525, %r117, %r9;\n" " st.shared.f32 [%r117], %f183;\n" " add.s32 %r526, %r526, 4;\n" " add.s32 %r524, %r524, -4;\n" " setp.ne.s32 %p4, %r524, 0;\n" " @%p4 bra $L__BB2_3;\n" "\n" "$L__BB2_4:\n" " setp.eq.s32 %p5, %r528, 0;\n" " @%p5 bra $L__BB2_7;\n" "\n" " add.s64 %rd16, %rd10, %rd11;\n" " mov.u32 %r120, sh;\n" "\n" "$L__BB2_6:\n" " .pragma \"nounroll\";\n" " cvt.s64.s32 %rd39, %r526;\n" " mul.lo.s64 %rd40, %rd39, %rd9;\n" " add.s64 %rd41, %rd16, %rd40;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd43, %rd1, %rd42;\n" " ld.global.f32 %f184, [%rd43];\n" " mad.lo.s32 %r118, %r526, %r102, %r1;\n" " shl.b32 %r119, %r118, 2;\n" " add.s32 %r121, %r120, %r119;\n" " st.shared.f32 [%r121], %f184;\n" " add.s32 %r526, %r526, 1;\n" " add.s32 %r528, %r528, -1;\n" " setp.ne.s32 %p6, %r528, 0;\n" " @%p6 bra $L__BB2_6;\n" "\n" "$L__BB2_7:\n" " setp.ge.s32 %p7, %r1, %r101;\n" " add.s32 %r122, %r4, %r1;\n" " cvt.u32.u64 %r123, %rd5;\n" " add.s32 %r124, %r122, %r123;\n" " shl.b32 %r125, %r124, 2;\n" " mov.u32 %r126, sh;\n" " add.s32 %r21, %r126, %r125;\n" " cvt.u32.u64 %r127, %rd7;\n" " add.s32 %r128, %r122, %r127;\n" " shl.b32 %r129, %r128, 2;\n" " add.s32 %r22, %r126, %r129;\n" " @%p7 bra $L__BB2_9;\n" "\n" " mov.u32 %r130, -8388608;\n" " st.shared.u32 [%r21], %r130;\n" " mov.u32 %r131, 0;\n" " st.shared.u32 [%r22], %r131;\n" "\n" "$L__BB2_9:\n" " bar.sync 0;\n" " mul.lo.s64 %rd44, %rd9, %rd8;\n" " cvt.s64.s32 %rd45, %r103;\n" " mul.lo.s64 %rd17, %rd44, %rd45;\n" " shr.s32 %r132, %r1, 31;\n" " shr.u32 %r133, %r132, 28;\n" " add.s32 %r134, %r1, %r133;\n" " shr.s32 %r24, %r134, 4;\n" " and.b32 %r135, %r134, -16;\n" " sub.s32 %r25, %r1, %r135;\n" " cvt.s64.s32 %rd18, %r1;\n" " setp.lt.s32 %p8, %r3, 0;\n" " mov.f32 %f605, 0f00000000;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" " @%p8 bra $L__BB2_151;\n" "\n" " shr.s32 %r138, %r102, 31;\n" " shr.u32 %r139, %r138, 28;\n" " add.s32 %r140, %r102, %r139;\n" " shr.s32 %r141, %r140, 4;\n" " sub.s32 %r142, %r3, %r106;\n" " add.s32 %r143, %r142, 1;\n" " mov.u32 %r529, 0;\n" " max.s32 %r26, %r143, 0;\n" " add.s32 %r144, %r4, %r24;\n" " cvt.u32.u64 %r145, %rd3;\n" " add.s32 %r146, %r144, %r145;\n" " mad.lo.s32 %r27, %r5, %r24, %r4;\n" " shl.b32 %r147, %r1, 4;\n" " add.s32 %r148, %r147, %r145;\n" " add.s32 %r28, %r4, %r148;\n" " add.s64 %rd19, %rd17, %rd18;\n" " add.s32 %r149, %r127, %r101;\n" " add.s32 %r150, %r4, %r149;\n" " add.s32 %r152, %r122, %r149;\n" " shl.b32 %r153, %r152, 2;\n" " add.s32 %r29, %r126, %r153;\n" " cvt.u64.u32 %rd46, %r1;\n" " add.s64 %rd20, %rd4, %rd46;\n" " mov.u32 %r155, -2;\n" " sub.s32 %r156, %r155, %r2;\n" " sub.s32 %r30, %r156, %r104;\n" " bfe.u32 %r32, %r140, 4, 1;\n" " sub.s32 %r33, %r141, %r32;\n" " shl.b32 %r34, %r102, 1;\n" " mul.lo.s32 %r35, %r102, 3;\n" " shl.b32 %r36, %r102, 2;\n" " mul.lo.s32 %r37, %r102, 5;\n" " mul.lo.s32 %r38, %r102, 6;\n" " mul.lo.s32 %r39, %r102, 7;\n" " shl.b32 %r157, %r146, 2;\n" " add.s32 %r40, %r126, %r157;\n" " shl.b32 %r158, %r150, 2;\n" " add.s32 %r41, %r126, %r158;\n" " shl.b32 %r159, %r122, 2;\n" " add.s32 %r42, %r126, %r159;\n" " mad.lo.s32 %r161, %r101, %r5, %r1;\n" " add.s32 %r162, %r161, %r145;\n" " shl.b32 %r163, %r162, 2;\n" " add.s32 %r164, %r126, %r163;\n" " add.s32 %r45, %r164, -1024;\n" " cvta.to.global.u64 %rd21, %rd24;\n" " mov.f32 %f605, 0f00000000;\n" " setp.lt.s32 %p10, %r106, 1;\n" " setp.ge.s32 %p18, %r1, %r102;\n" " setp.lt.s32 %p21, %r102, 16;\n" " setp.eq.s32 %p51, %r25, 0;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" " mov.u32 %r530, %r529;\n" "\n" "$L__BB2_11:\n" " ld.param.u32 %r423, [fc_attn_flash_prefill_param_8];\n" " add.s32 %r422, %r3, 1;\n" " shl.b32 %r165, %r529, 4;\n" " add.s32 %r166, %r30, %r165;\n" " max.s32 %r167, %r166, -17;\n" " not.b32 %r168, %r167;\n" " max.s32 %r51, %r168, 1;\n" " sub.s32 %r52, %r422, %r530;\n" " min.s32 %r53, %r52, 16;\n" " setp.lt.s32 %p9, %r530, %r423;\n" " or.pred %p11, %p10, %p9;\n" " add.s32 %r169, %r53, %r530;\n" " setp.gt.s32 %p12, %r169, %r26;\n" " setp.le.s32 %p13, %r530, %r3;\n" " and.pred %p14, %p13, %p12;\n" " or.pred %p15, %p11, %p14;\n" " not.pred %p16, %p15;\n" " @%p16 bra $L__BB2_150;\n" "\n" " setp.lt.s32 %p17, %r52, 1;\n" " @%p17 bra $L__BB2_18;\n" "\n" " mov.u32 %r531, 0;\n" "\n" "$L__BB2_14:\n" " @%p18 bra $L__BB2_17;\n" "\n" " add.s32 %r428, %r102, 16;\n" " mov.u32 %r533, %tid.x;\n" " add.s32 %r171, %r531, %r530;\n" " cvt.s64.s32 %rd47, %r171;\n" " mul.lo.s64 %rd48, %rd47, %rd9;\n" " add.s64 %rd22, %rd48, %rd17;\n" " mul.lo.s32 %r172, %r531, %r428;\n" " shl.b32 %r173, %r172, 2;\n" " add.s32 %r532, %r42, %r173;\n" "\n" "$L__BB2_16:\n" " mov.u32 %r431, %ntid.x;\n" " mov.u32 %r430, %ntid.x;\n" " shl.b32 %r429, %r430, 2;\n" " cvt.s64.s32 %rd49, %r533;\n" " add.s64 %rd50, %rd22, %rd49;\n" " shl.b64 %rd51, %rd50, 2;\n" " add.s64 %rd52, %rd21, %rd51;\n" " ld.global.f32 %f201, [%rd52];\n" " st.shared.f32 [%r532], %f201;\n" " add.s32 %r532, %r532, %r429;\n" " add.s32 %r533, %r533, %r430;\n" " setp.lt.s32 %p19, %r533, %r102;\n" " @%p19 bra $L__BB2_16;\n" "\n" "$L__BB2_17:\n" " add.s32 %r531, %r531, 1;\n" " setp.lt.s32 %p20, %r531, %r53;\n" " @%p20 bra $L__BB2_14;\n" "\n" "$L__BB2_18:\n" " bar.sync 0;\n" " setp.ge.s32 %p22, %r24, %r53;\n" " or.pred %p23, %p21, %p22;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p23 bra $L__BB2_71;\n" "\n" " and.b32 %r424, %r102, -16;\n" " setp.eq.s32 %p24, %r424, 16;\n" " mov.u32 %r536, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p24 bra $L__BB2_54;\n" "\n" " mov.u32 %r536, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " mov.u32 %r535, %r33;\n" "\n" "$L__BB2_21:\n" " shl.b32 %r176, %r536, 4;\n" " add.s32 %r63, %r176, %r25;\n" " add.s32 %r177, %r27, %r63;\n" " shl.b32 %r178, %r177, 2;\n" " add.s32 %r64, %r126, %r178;\n" " ld.shared.f32 %f17, [%r64];\n" " @%p2 bra $L__BB2_23;\n" "\n" " shl.b32 %r517, %r536, 4;\n" " add.s32 %r516, %r517, %r25;\n" " shl.b32 %r515, %r516, 2;\n" " add.s32 %r514, %r126, %r515;\n" " ld.shared.f32 %f226, [%r514];\n" " fma.rn.f32 %f536, %f17, %f226, %f536;\n" "\n" "$L__BB2_23:\n" " setp.lt.s32 %p26, %r101, 2;\n" " @%p26 bra $L__BB2_25;\n" "\n" " shl.b32 %r522, %r536, 4;\n" " add.s32 %r521, %r522, %r25;\n" " add.s32 %r520, %r521, %r102;\n" " shl.b32 %r519, %r520, 2;\n" " add.s32 %r518, %r126, %r519;\n" " ld.shared.f32 %f227, [%r518];\n" " fma.rn.f32 %f537, %f17, %f227, %f537;\n" "\n" "$L__BB2_25:\n" " setp.lt.s32 %p27, %r101, 3;\n" " @%p27 bra $L__BB2_27;\n" "\n" " shl.b32 %r463, %r536, 4;\n" " add.s32 %r462, %r463, %r25;\n" " add.s32 %r461, %r34, %r462;\n" " shl.b32 %r460, %r461, 2;\n" " add.s32 %r459, %r126, %r460;\n" " ld.shared.f32 %f228, [%r459];\n" " fma.rn.f32 %f538, %f17, %f228, %f538;\n" "\n" "$L__BB2_27:\n" " setp.lt.s32 %p28, %r101, 4;\n" " @%p28 bra $L__BB2_29;\n" "\n" " shl.b32 %r473, %r536, 4;\n" " add.s32 %r472, %r473, %r25;\n" " add.s32 %r471, %r35, %r472;\n" " shl.b32 %r470, %r471, 2;\n" " add.s32 %r469, %r126, %r470;\n" " ld.shared.f32 %f229, [%r469];\n" " fma.rn.f32 %f539, %f17, %f229, %f539;\n" "\n" "$L__BB2_29:\n" " setp.lt.s32 %p29, %r101, 5;\n" " @%p29 bra $L__BB2_31;\n" "\n" " shl.b32 %r483, %r536, 4;\n" " add.s32 %r482, %r483, %r25;\n" " add.s32 %r481, %r36, %r482;\n" " shl.b32 %r480, %r481, 2;\n" " add.s32 %r479, %r126, %r480;\n" " ld.shared.f32 %f230, [%r479];\n" " fma.rn.f32 %f540, %f17, %f230, %f540;\n" "\n" "$L__BB2_31:\n" " setp.lt.s32 %p30, %r101, 6;\n" " @%p30 bra $L__BB2_33;\n" "\n" " shl.b32 %r493, %r536, 4;\n" " add.s32 %r492, %r493, %r25;\n" " add.s32 %r491, %r37, %r492;\n" " shl.b32 %r490, %r491, 2;\n" " add.s32 %r489, %r126, %r490;\n" " ld.shared.f32 %f231, [%r489];\n" " fma.rn.f32 %f541, %f17, %f231, %f541;\n" "\n" "$L__BB2_33:\n" " setp.lt.s32 %p31, %r101, 7;\n" " @%p31 bra $L__BB2_35;\n" "\n" " shl.b32 %r503, %r536, 4;\n" " add.s32 %r502, %r503, %r25;\n" " add.s32 %r501, %r38, %r502;\n" " shl.b32 %r500, %r501, 2;\n" " add.s32 %r499, %r126, %r500;\n" " ld.shared.f32 %f232, [%r499];\n" " fma.rn.f32 %f542, %f17, %f232, %f542;\n" "\n" "$L__BB2_35:\n" " setp.lt.s32 %p32, %r101, 8;\n" " @%p32 bra $L__BB2_37;\n" "\n" " shl.b32 %r513, %r536, 4;\n" " add.s32 %r512, %r513, %r25;\n" " add.s32 %r511, %r39, %r512;\n" " shl.b32 %r510, %r511, 2;\n" " add.s32 %r509, %r126, %r510;\n" " ld.shared.f32 %f233, [%r509];\n" " fma.rn.f32 %f543, %f17, %f233, %f543;\n" "\n" "$L__BB2_37:\n" " shl.b32 %r444, %r536, 4;\n" " add.s32 %r443, %r444, %r25;\n" " add.s32 %r442, %r27, %r443;\n" " shl.b32 %r441, %r442, 2;\n" " add.s32 %r440, %r126, %r441;\n" " ld.shared.f32 %f34, [%r440+64];\n" " @%p2 bra $L__BB2_39;\n" "\n" " shl.b32 %r458, %r536, 4;\n" " add.s32 %r457, %r458, %r25;\n" " shl.b32 %r456, %r457, 2;\n" " add.s32 %r455, %r126, %r456;\n" " ld.shared.f32 %f234, [%r455+64];\n" " fma.rn.f32 %f536, %f34, %f234, %f536;\n" "\n" "$L__BB2_39:\n" " setp.lt.s32 %p156, %r101, 2;\n" " @%p156 bra $L__BB2_41;\n" "\n" " shl.b32 %r454, %r536, 4;\n" " add.s32 %r453, %r454, %r25;\n" " add.s32 %r452, %r453, %r102;\n" " shl.b32 %r451, %r452, 2;\n" " add.s32 %r450, %r126, %r451;\n" " ld.shared.f32 %f235, [%r450+64];\n" " fma.rn.f32 %f537, %f34, %f235, %f537;\n" "\n" "$L__BB2_41:\n" " setp.lt.s32 %p157, %r101, 3;\n" " @%p157 bra $L__BB2_43;\n" "\n" " shl.b32 %r449, %r536, 4;\n" " add.s32 %r448, %r449, %r25;\n" " add.s32 %r447, %r34, %r448;\n" " shl.b32 %r446, %r447, 2;\n" " add.s32 %r445, %r126, %r446;\n" " ld.shared.f32 %f236, [%r445+64];\n" " fma.rn.f32 %f538, %f34, %f236, %f538;\n" "\n" "$L__BB2_43:\n" " setp.lt.s32 %p158, %r101, 4;\n" " @%p158 bra $L__BB2_45;\n" "\n" " shl.b32 %r468, %r536, 4;\n" " add.s32 %r467, %r468, %r25;\n" " add.s32 %r466, %r35, %r467;\n" " shl.b32 %r465, %r466, 2;\n" " add.s32 %r464, %r126, %r465;\n" " ld.shared.f32 %f237, [%r464+64];\n" " fma.rn.f32 %f539, %f34, %f237, %f539;\n" "\n" "$L__BB2_45:\n" " setp.lt.s32 %p159, %r101, 5;\n" " @%p159 bra $L__BB2_47;\n" "\n" " shl.b32 %r478, %r536, 4;\n" " add.s32 %r477, %r478, %r25;\n" " add.s32 %r476, %r36, %r477;\n" " shl.b32 %r475, %r476, 2;\n" " add.s32 %r474, %r126, %r475;\n" " ld.shared.f32 %f238, [%r474+64];\n" " fma.rn.f32 %f540, %f34, %f238, %f540;\n" "\n" "$L__BB2_47:\n" " setp.lt.s32 %p160, %r101, 6;\n" " @%p160 bra $L__BB2_49;\n" "\n" " shl.b32 %r488, %r536, 4;\n" " add.s32 %r487, %r488, %r25;\n" " add.s32 %r486, %r37, %r487;\n" " shl.b32 %r485, %r486, 2;\n" " add.s32 %r484, %r126, %r485;\n" " ld.shared.f32 %f239, [%r484+64];\n" " fma.rn.f32 %f541, %f34, %f239, %f541;\n" "\n" "$L__BB2_49:\n" " setp.lt.s32 %p161, %r101, 7;\n" " @%p161 bra $L__BB2_51;\n" "\n" " shl.b32 %r498, %r536, 4;\n" " add.s32 %r497, %r498, %r25;\n" " add.s32 %r496, %r38, %r497;\n" " shl.b32 %r495, %r496, 2;\n" " add.s32 %r494, %r126, %r495;\n" " ld.shared.f32 %f240, [%r494+64];\n" " fma.rn.f32 %f542, %f34, %f240, %f542;\n" "\n" "$L__BB2_51:\n" " setp.lt.s32 %p162, %r101, 8;\n" " @%p162 bra $L__BB2_53;\n" "\n" " shl.b32 %r508, %r536, 4;\n" " add.s32 %r507, %r508, %r25;\n" " add.s32 %r506, %r39, %r507;\n" " shl.b32 %r505, %r506, 2;\n" " add.s32 %r504, %r126, %r505;\n" " ld.shared.f32 %f241, [%r504+64];\n" " fma.rn.f32 %f543, %f34, %f241, %f543;\n" "\n" "$L__BB2_53:\n" " add.s32 %r536, %r536, 2;\n" " add.s32 %r535, %r535, -2;\n" " setp.ne.s32 %p41, %r535, 0;\n" " @%p41 bra $L__BB2_21;\n" "\n" "$L__BB2_54:\n" " setp.eq.s32 %p42, %r32, 0;\n" " @%p42 bra $L__BB2_71;\n" "\n" " shl.b32 %r202, %r536, 4;\n" " add.s32 %r76, %r202, %r25;\n" " add.s32 %r203, %r27, %r76;\n" " shl.b32 %r204, %r203, 2;\n" " add.s32 %r206, %r126, %r204;\n" " ld.shared.f32 %f59, [%r206];\n" " @%p2 bra $L__BB2_57;\n" "\n" " shl.b32 %r207, %r76, 2;\n" " add.s32 %r209, %r126, %r207;\n" " ld.shared.f32 %f242, [%r209];\n" " fma.rn.f32 %f536, %f59, %f242, %f536;\n" "\n" "$L__BB2_57:\n" " setp.lt.s32 %p44, %r101, 2;\n" " @%p44 bra $L__BB2_59;\n" "\n" " add.s32 %r210, %r76, %r102;\n" " shl.b32 %r211, %r210, 2;\n" " add.s32 %r213, %r126, %r211;\n" " ld.shared.f32 %f243, [%r213];\n" " fma.rn.f32 %f537, %f59, %f243, %f537;\n" "\n" "$L__BB2_59:\n" " setp.lt.s32 %p45, %r101, 3;\n" " @%p45 bra $L__BB2_61;\n" "\n" " add.s32 %r214, %r34, %r76;\n" " shl.b32 %r215, %r214, 2;\n" " add.s32 %r217, %r126, %r215;\n" " ld.shared.f32 %f244, [%r217];\n" " fma.rn.f32 %f538, %f59, %f244, %f538;\n" "\n" "$L__BB2_61:\n" " setp.lt.s32 %p46, %r101, 4;\n" " @%p46 bra $L__BB2_63;\n" "\n" " add.s32 %r218, %r35, %r76;\n" " shl.b32 %r219, %r218, 2;\n" " add.s32 %r221, %r126, %r219;\n" " ld.shared.f32 %f245, [%r221];\n" " fma.rn.f32 %f539, %f59, %f245, %f539;\n" "\n" "$L__BB2_63:\n" " setp.lt.s32 %p47, %r101, 5;\n" " @%p47 bra $L__BB2_65;\n" "\n" " add.s32 %r222, %r36, %r76;\n" " shl.b32 %r223, %r222, 2;\n" " add.s32 %r225, %r126, %r223;\n" " ld.shared.f32 %f246, [%r225];\n" " fma.rn.f32 %f540, %f59, %f246, %f540;\n" "\n" "$L__BB2_65:\n" " setp.lt.s32 %p48, %r101, 6;\n" " @%p48 bra $L__BB2_67;\n" "\n" " add.s32 %r226, %r37, %r76;\n" " shl.b32 %r227, %r226, 2;\n" " add.s32 %r229, %r126, %r227;\n" " ld.shared.f32 %f247, [%r229];\n" " fma.rn.f32 %f541, %f59, %f247, %f541;\n" "\n" "$L__BB2_67:\n" " setp.lt.s32 %p49, %r101, 7;\n" " @%p49 bra $L__BB2_69;\n" "\n" " add.s32 %r230, %r38, %r76;\n" " shl.b32 %r231, %r230, 2;\n" " add.s32 %r233, %r126, %r231;\n" " ld.shared.f32 %f248, [%r233];\n" " fma.rn.f32 %f542, %f59, %f248, %f542;\n" "\n" "$L__BB2_69:\n" " setp.lt.s32 %p50, %r101, 8;\n" " @%p50 bra $L__BB2_71;\n" "\n" " add.s32 %r234, %r39, %r76;\n" " shl.b32 %r235, %r234, 2;\n" " add.s32 %r237, %r126, %r235;\n" " ld.shared.f32 %f249, [%r237];\n" " fma.rn.f32 %f543, %f59, %f249, %f543;\n" "\n" "$L__BB2_71:\n" " setp.lt.s32 %p52, %r24, %r53;\n" " and.pred %p1, %p51, %p52;\n" " @%p2 bra $L__BB2_74;\n" "\n" " mov.b32 %r238, %f536;\n" " mov.u32 %r239, 2;\n" " mov.u32 %r240, 4127;\n" " mov.u32 %r241, 8;\n" " mov.u32 %r242, -1;\n" " shfl.sync.down.b32 %r243|%p54, %r238, %r241, %r240, %r242;\n" " mov.b32 %f250, %r243;\n" " add.f32 %f251, %f536, %f250;\n" " mov.b32 %r244, %f251;\n" " mov.u32 %r245, 4;\n" " shfl.sync.down.b32 %r246|%p55, %r244, %r245, %r240, %r242;\n" " mov.b32 %f252, %r246;\n" " add.f32 %f253, %f251, %f252;\n" " mov.b32 %r247, %f253;\n" " shfl.sync.down.b32 %r248|%p56, %r247, %r239, %r240, %r242;\n" " mov.b32 %f254, %r248;\n" " add.f32 %f255, %f253, %f254;\n" " mov.b32 %r249, %f255;\n" " mov.u32 %r250, 1;\n" " shfl.sync.down.b32 %r251|%p57, %r249, %r250, %r240, %r242;\n" " mov.b32 %f256, %r251;\n" " add.f32 %f83, %f255, %f256;\n" " not.pred %p58, %p1;\n" " @%p58 bra $L__BB2_74;\n" "\n" " mul.f32 %f257, %f83, %f179;\n" " st.shared.f32 [%r40], %f257;\n" "\n" "$L__BB2_74:\n" " setp.lt.s32 %p59, %r101, 2;\n" " @%p59 bra $L__BB2_77;\n" "\n" " mov.b32 %r252, %f537;\n" " mov.u32 %r253, 2;\n" " mov.u32 %r254, 4127;\n" " mov.u32 %r255, 8;\n" " mov.u32 %r256, -1;\n" " shfl.sync.down.b32 %r257|%p60, %r252, %r255, %r254, %r256;\n" " mov.b32 %f258, %r257;\n" " add.f32 %f259, %f537, %f258;\n" " mov.b32 %r258, %f259;\n" " mov.u32 %r259, 4;\n" " shfl.sync.down.b32 %r260|%p61, %r258, %r259, %r254, %r256;\n" " mov.b32 %f260, %r260;\n" " add.f32 %f261, %f259, %f260;\n" " mov.b32 %r261, %f261;\n" " shfl.sync.down.b32 %r262|%p62, %r261, %r253, %r254, %r256;\n" " mov.b32 %f262, %r262;\n" " add.f32 %f263, %f261, %f262;\n" " mov.b32 %r263, %f263;\n" " mov.u32 %r264, 1;\n" " shfl.sync.down.b32 %r265|%p63, %r263, %r264, %r254, %r256;\n" " mov.b32 %f264, %r265;\n" " add.f32 %f84, %f263, %f264;\n" " not.pred %p64, %p1;\n" " @%p64 bra $L__BB2_77;\n" "\n" " mul.f32 %f265, %f84, %f179;\n" " st.shared.f32 [%r40+64], %f265;\n" "\n" "$L__BB2_77:\n" " setp.lt.s32 %p65, %r101, 3;\n" " @%p65 bra $L__BB2_80;\n" "\n" " mov.b32 %r266, %f538;\n" " mov.u32 %r267, 2;\n" " mov.u32 %r268, 4127;\n" " mov.u32 %r269, 8;\n" " mov.u32 %r270, -1;\n" " shfl.sync.down.b32 %r271|%p66, %r266, %r269, %r268, %r270;\n" " mov.b32 %f266, %r271;\n" " add.f32 %f267, %f538, %f266;\n" " mov.b32 %r272, %f267;\n" " mov.u32 %r273, 4;\n" " shfl.sync.down.b32 %r274|%p67, %r272, %r273, %r268, %r270;\n" " mov.b32 %f268, %r274;\n" " add.f32 %f269, %f267, %f268;\n" " mov.b32 %r275, %f269;\n" " shfl.sync.down.b32 %r276|%p68, %r275, %r267, %r268, %r270;\n" " mov.b32 %f270, %r276;\n" " add.f32 %f271, %f269, %f270;\n" " mov.b32 %r277, %f271;\n" " mov.u32 %r278, 1;\n" " shfl.sync.down.b32 %r279|%p69, %r277, %r278, %r268, %r270;\n" " mov.b32 %f272, %r279;\n" " add.f32 %f85, %f271, %f272;\n" " not.pred %p70, %p1;\n" " @%p70 bra $L__BB2_80;\n" "\n" " mul.f32 %f273, %f85, %f179;\n" " st.shared.f32 [%r40+128], %f273;\n" "\n" "$L__BB2_80:\n" " setp.lt.s32 %p71, %r101, 4;\n" " @%p71 bra $L__BB2_83;\n" "\n" " mov.b32 %r280, %f539;\n" " mov.u32 %r281, 2;\n" " mov.u32 %r282, 4127;\n" " mov.u32 %r283, 8;\n" " mov.u32 %r284, -1;\n" " shfl.sync.down.b32 %r285|%p72, %r280, %r283, %r282, %r284;\n" " mov.b32 %f274, %r285;\n" " add.f32 %f275, %f539, %f274;\n" " mov.b32 %r286, %f275;\n" " mov.u32 %r287, 4;\n" " shfl.sync.down.b32 %r288|%p73, %r286, %r287, %r282, %r284;\n" " mov.b32 %f276, %r288;\n" " add.f32 %f277, %f275, %f276;\n" " mov.b32 %r289, %f277;\n" " shfl.sync.down.b32 %r290|%p74, %r289, %r281, %r282, %r284;\n" " mov.b32 %f278, %r290;\n" " add.f32 %f279, %f277, %f278;\n" " mov.b32 %r291, %f279;\n" " mov.u32 %r292, 1;\n" " shfl.sync.down.b32 %r293|%p75, %r291, %r292, %r282, %r284;\n" " mov.b32 %f280, %r293;\n" " add.f32 %f86, %f279, %f280;\n" " not.pred %p76, %p1;\n" " @%p76 bra $L__BB2_83;\n" "\n" " mul.f32 %f281, %f86, %f179;\n" " st.shared.f32 [%r40+192], %f281;\n" "\n" "$L__BB2_83:\n" " setp.lt.s32 %p77, %r101, 5;\n" " @%p77 bra $L__BB2_86;\n" "\n" " mov.b32 %r294, %f540;\n" " mov.u32 %r295, 2;\n" " mov.u32 %r296, 4127;\n" " mov.u32 %r297, 8;\n" " mov.u32 %r298, -1;\n" " shfl.sync.down.b32 %r299|%p78, %r294, %r297, %r296, %r298;\n" " mov.b32 %f282, %r299;\n" " add.f32 %f283, %f540, %f282;\n" " mov.b32 %r300, %f283;\n" " mov.u32 %r301, 4;\n" " shfl.sync.down.b32 %r302|%p79, %r300, %r301, %r296, %r298;\n" " mov.b32 %f284, %r302;\n" " add.f32 %f285, %f283, %f284;\n" " mov.b32 %r303, %f285;\n" " shfl.sync.down.b32 %r304|%p80, %r303, %r295, %r296, %r298;\n" " mov.b32 %f286, %r304;\n" " add.f32 %f287, %f285, %f286;\n" " mov.b32 %r305, %f287;\n" " mov.u32 %r306, 1;\n" " shfl.sync.down.b32 %r307|%p81, %r305, %r306, %r296, %r298;\n" " mov.b32 %f288, %r307;\n" " add.f32 %f87, %f287, %f288;\n" " not.pred %p82, %p1;\n" " @%p82 bra $L__BB2_86;\n" "\n" " mul.f32 %f289, %f87, %f179;\n" " st.shared.f32 [%r40+256], %f289;\n" "\n" "$L__BB2_86:\n" " setp.lt.s32 %p83, %r101, 6;\n" " @%p83 bra $L__BB2_89;\n" "\n" " mov.b32 %r308, %f541;\n" " mov.u32 %r309, 2;\n" " mov.u32 %r310, 4127;\n" " mov.u32 %r311, 8;\n" " mov.u32 %r312, -1;\n" " shfl.sync.down.b32 %r313|%p84, %r308, %r311, %r310, %r312;\n" " mov.b32 %f290, %r313;\n" " add.f32 %f291, %f541, %f290;\n" " mov.b32 %r314, %f291;\n" " mov.u32 %r315, 4;\n" " shfl.sync.down.b32 %r316|%p85, %r314, %r315, %r310, %r312;\n" " mov.b32 %f292, %r316;\n" " add.f32 %f293, %f291, %f292;\n" " mov.b32 %r317, %f293;\n" " shfl.sync.down.b32 %r318|%p86, %r317, %r309, %r310, %r312;\n" " mov.b32 %f294, %r318;\n" " add.f32 %f295, %f293, %f294;\n" " mov.b32 %r319, %f295;\n" " mov.u32 %r320, 1;\n" " shfl.sync.down.b32 %r321|%p87, %r319, %r320, %r310, %r312;\n" " mov.b32 %f296, %r321;\n" " add.f32 %f88, %f295, %f296;\n" " not.pred %p88, %p1;\n" " @%p88 bra $L__BB2_89;\n" "\n" " mul.f32 %f297, %f88, %f179;\n" " st.shared.f32 [%r40+320], %f297;\n" "\n" "$L__BB2_89:\n" " setp.lt.s32 %p89, %r101, 7;\n" " @%p89 bra $L__BB2_92;\n" "\n" " mov.b32 %r322, %f542;\n" " mov.u32 %r323, 2;\n" " mov.u32 %r324, 4127;\n" " mov.u32 %r325, 8;\n" " mov.u32 %r326, -1;\n" " shfl.sync.down.b32 %r327|%p90, %r322, %r325, %r324, %r326;\n" " mov.b32 %f298, %r327;\n" " add.f32 %f299, %f542, %f298;\n" " mov.b32 %r328, %f299;\n" " mov.u32 %r329, 4;\n" " shfl.sync.down.b32 %r330|%p91, %r328, %r329, %r324, %r326;\n" " mov.b32 %f300, %r330;\n" " add.f32 %f301, %f299, %f300;\n" " mov.b32 %r331, %f301;\n" " shfl.sync.down.b32 %r332|%p92, %r331, %r323, %r324, %r326;\n" " mov.b32 %f302, %r332;\n" " add.f32 %f303, %f301, %f302;\n" " mov.b32 %r333, %f303;\n" " mov.u32 %r334, 1;\n" " shfl.sync.down.b32 %r335|%p93, %r333, %r334, %r324, %r326;\n" " mov.b32 %f304, %r335;\n" " add.f32 %f89, %f303, %f304;\n" " not.pred %p94, %p1;\n" " @%p94 bra $L__BB2_92;\n" "\n" " mul.f32 %f305, %f89, %f179;\n" " st.shared.f32 [%r40+384], %f305;\n" "\n" "$L__BB2_92:\n" " setp.lt.s32 %p95, %r101, 8;\n" " @%p95 bra $L__BB2_95;\n" "\n" " mov.b32 %r336, %f543;\n" " mov.u32 %r337, 2;\n" " mov.u32 %r338, 4127;\n" " mov.u32 %r339, 8;\n" " mov.u32 %r340, -1;\n" " shfl.sync.down.b32 %r341|%p96, %r336, %r339, %r338, %r340;\n" " mov.b32 %f306, %r341;\n" " add.f32 %f307, %f543, %f306;\n" " mov.b32 %r342, %f307;\n" " mov.u32 %r343, 4;\n" " shfl.sync.down.b32 %r344|%p97, %r342, %r343, %r338, %r340;\n" " mov.b32 %f308, %r344;\n" " add.f32 %f309, %f307, %f308;\n" " mov.b32 %r345, %f309;\n" " shfl.sync.down.b32 %r346|%p98, %r345, %r337, %r338, %r340;\n" " mov.b32 %f310, %r346;\n" " add.f32 %f311, %f309, %f310;\n" " mov.b32 %r347, %f311;\n" " mov.u32 %r348, 1;\n" " shfl.sync.down.b32 %r349|%p99, %r347, %r348, %r338, %r340;\n" " mov.b32 %f312, %r349;\n" " add.f32 %f90, %f311, %f312;\n" " not.pred %p100, %p1;\n" " @%p100 bra $L__BB2_95;\n" "\n" " mul.f32 %f313, %f90, %f179;\n" " st.shared.f32 [%r40+448], %f313;\n" "\n" "$L__BB2_95:\n" " bar.sync 0;\n" " @%p7 bra $L__BB2_113;\n" "\n" " ld.shared.f32 %f91, [%r21];\n" " mov.f32 %f578, %f91;\n" " @%p17 bra $L__BB2_104;\n" "\n" " and.b32 %r77, %r51, 3;\n" " add.s32 %r351, %r51, -1;\n" " setp.lt.u32 %p103, %r351, 3;\n" " mov.u32 %r539, 0;\n" " mov.f32 %f578, %f91;\n" " @%p103 bra $L__BB2_100;\n" "\n" " sub.s32 %r538, %r51, %r77;\n" " mov.u32 %r539, 0;\n" " mov.f32 %f578, %f91;\n" "\n" "$L__BB2_99:\n" " add.s32 %r353, %r28, %r539;\n" " shl.b32 %r354, %r353, 2;\n" " add.s32 %r356, %r126, %r354;\n" " ld.shared.f32 %f315, [%r356];\n" " max.f32 %f316, %f578, %f315;\n" " ld.shared.f32 %f317, [%r356+4];\n" " max.f32 %f318, %f316, %f317;\n" " ld.shared.f32 %f319, [%r356+8];\n" " max.f32 %f320, %f318, %f319;\n" " ld.shared.f32 %f321, [%r356+12];\n" " max.f32 %f578, %f320, %f321;\n" " add.s32 %r539, %r539, 4;\n" " add.s32 %r538, %r538, -4;\n" " setp.ne.s32 %p104, %r538, 0;\n" " @%p104 bra $L__BB2_99;\n" "\n" "$L__BB2_100:\n" " setp.eq.s32 %p105, %r77, 0;\n" " @%p105 bra $L__BB2_104;\n" "\n" " add.s32 %r357, %r28, %r539;\n" " shl.b32 %r358, %r357, 2;\n" " add.s32 %r84, %r126, %r358;\n" " ld.shared.f32 %f322, [%r84];\n" " max.f32 %f578, %f578, %f322;\n" " setp.eq.s32 %p106, %r77, 1;\n" " @%p106 bra $L__BB2_104;\n" "\n" " ld.shared.f32 %f323, [%r84+4];\n" " max.f32 %f578, %f578, %f323;\n" " setp.eq.s32 %p107, %r77, 2;\n" " @%p107 bra $L__BB2_104;\n" "\n" " ld.shared.f32 %f324, [%r84+8];\n" " max.f32 %f578, %f578, %f324;\n" "\n" "$L__BB2_104:\n" " mov.f32 %f582, 0f00000000;\n" " @%p17 bra $L__BB2_112;\n" "\n" " and.b32 %r85, %r51, 3;\n" " add.s32 %r361, %r51, -1;\n" " setp.lt.u32 %p109, %r361, 3;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r543, 0;\n" " @%p109 bra $L__BB2_108;\n" "\n" " sub.s32 %r541, %r51, %r85;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r543, 0;\n" " mov.u32 %r542, %r45;\n" "\n" "$L__BB2_107:\n" " shl.b32 %r426, %r101, 2;\n" " add.s32 %r425, %r426, 1024;\n" " add.s32 %r363, %r28, %r543;\n" " shl.b32 %r364, %r363, 2;\n" " add.s32 %r366, %r126, %r364;\n" " ld.shared.f32 %f329, [%r366];\n" " sub.f32 %f330, %f329, %f578;\n" " mov.f32 %f331, 0f3F000000;\n" " mov.f32 %f332, 0f3BBB989D;\n" " fma.rn.f32 %f333, %f330, %f332, %f331;\n" " mov.f32 %f334, 0f3FB8AA3B;\n" " mov.f32 %f335, 0f437C0000;\n" " cvt.sat.f32.f32 %f336, %f333;\n" " mov.f32 %f337, 0f4B400001;\n" " fma.rm.f32 %f338, %f336, %f335, %f337;\n" " add.f32 %f339, %f338, 0fCB40007F;\n" " neg.f32 %f340, %f339;\n" " fma.rn.f32 %f341, %f330, %f334, %f340;\n" " mov.f32 %f342, 0f32A57060;\n" " fma.rn.f32 %f343, %f330, %f342, %f341;\n" " mov.b32 %r367, %f338;\n" " shl.b32 %r368, %r367, 23;\n" " mov.b32 %f344, %r368;\n" " ex2.approx.ftz.f32 %f345, %f343;\n" " mul.f32 %f346, %f345, %f344;\n" " st.shared.f32 [%r542+1024], %f346;\n" " add.f32 %f347, %f582, %f346;\n" " ld.shared.f32 %f348, [%r366+4];\n" " sub.f32 %f349, %f348, %f578;\n" " fma.rn.f32 %f350, %f349, %f332, %f331;\n" " cvt.sat.f32.f32 %f351, %f350;\n" " fma.rm.f32 %f352, %f351, %f335, %f337;\n" " add.f32 %f353, %f352, 0fCB40007F;\n" " neg.f32 %f354, %f353;\n" " fma.rn.f32 %f355, %f349, %f334, %f354;\n" " fma.rn.f32 %f356, %f349, %f342, %f355;\n" " mov.b32 %r369, %f352;\n" " shl.b32 %r370, %r369, 23;\n" " mov.b32 %f357, %r370;\n" " ex2.approx.ftz.f32 %f358, %f356;\n" " mul.f32 %f359, %f358, %f357;\n" " add.s32 %r371, %r542, %r426;\n" " st.shared.f32 [%r371+1024], %f359;\n" " add.f32 %f360, %f347, %f359;\n" " ld.shared.f32 %f361, [%r366+8];\n" " sub.f32 %f362, %f361, %f578;\n" " fma.rn.f32 %f363, %f362, %f332, %f331;\n" " cvt.sat.f32.f32 %f364, %f363;\n" " fma.rm.f32 %f365, %f364, %f335, %f337;\n" " add.f32 %f366, %f365, 0fCB40007F;\n" " neg.f32 %f367, %f366;\n" " fma.rn.f32 %f368, %f362, %f334, %f367;\n" " fma.rn.f32 %f369, %f362, %f342, %f368;\n" " mov.b32 %r372, %f365;\n" " shl.b32 %r373, %r372, 23;\n" " mov.b32 %f370, %r373;\n" " ex2.approx.ftz.f32 %f371, %f369;\n" " mul.f32 %f372, %f371, %f370;\n" " add.s32 %r374, %r371, %r426;\n" " add.s32 %r375, %r371, %r425;\n" " st.shared.f32 [%r375], %f372;\n" " add.f32 %f373, %f360, %f372;\n" " ld.shared.f32 %f374, [%r366+12];\n" " sub.f32 %f375, %f374, %f578;\n" " fma.rn.f32 %f376, %f375, %f332, %f331;\n" " cvt.sat.f32.f32 %f377, %f376;\n" " fma.rm.f32 %f378, %f377, %f335, %f337;\n" " add.f32 %f379, %f378, 0fCB40007F;\n" " neg.f32 %f380, %f379;\n" " fma.rn.f32 %f381, %f375, %f334, %f380;\n" " fma.rn.f32 %f382, %f375, %f342, %f381;\n" " mov.b32 %r376, %f378;\n" " shl.b32 %r377, %r376, 23;\n" " mov.b32 %f383, %r377;\n" " ex2.approx.ftz.f32 %f384, %f382;\n" " mul.f32 %f385, %f384, %f383;\n" " add.s32 %r378, %r374, %r426;\n" " add.s32 %r379, %r374, %r425;\n" " add.s32 %r542, %r378, %r426;\n" " st.shared.f32 [%r379], %f385;\n" " add.f32 %f582, %f373, %f385;\n" " add.s32 %r543, %r543, 4;\n" " add.s32 %r541, %r541, -4;\n" " setp.ne.s32 %p110, %r541, 0;\n" " @%p110 bra $L__BB2_107;\n" "\n" "$L__BB2_108:\n" " setp.eq.s32 %p111, %r85, 0;\n" " @%p111 bra $L__BB2_112;\n" "\n" " add.s32 %r380, %r28, %r543;\n" " shl.b32 %r381, %r380, 2;\n" " add.s32 %r94, %r126, %r381;\n" " ld.shared.f32 %f386, [%r94];\n" " sub.f32 %f387, %f386, %f578;\n" " mov.f32 %f388, 0f3F000000;\n" " mov.f32 %f389, 0f3BBB989D;\n" " fma.rn.f32 %f390, %f387, %f389, %f388;\n" " mov.f32 %f391, 0f3FB8AA3B;\n" " mov.f32 %f392, 0f437C0000;\n" " cvt.sat.f32.f32 %f393, %f390;\n" " mov.f32 %f394, 0f4B400001;\n" " fma.rm.f32 %f395, %f393, %f392, %f394;\n" " add.f32 %f396, %f395, 0fCB40007F;\n" " neg.f32 %f397, %f396;\n" " fma.rn.f32 %f398, %f387, %f391, %f397;\n" " mov.f32 %f399, 0f32A57060;\n" " fma.rn.f32 %f400, %f387, %f399, %f398;\n" " mov.b32 %r383, %f395;\n" " shl.b32 %r384, %r383, 23;\n" " mov.b32 %f401, %r384;\n" " ex2.approx.ftz.f32 %f402, %f400;\n" " mul.f32 %f403, %f402, %f401;\n" " cvt.u64.u32 %rd53, %r543;\n" " mul.lo.s64 %rd54, %rd53, %rd6;\n" " add.s64 %rd55, %rd20, %rd54;\n" " cvt.u32.u64 %r385, %rd55;\n" " add.s32 %r386, %r4, %r385;\n" " shl.b32 %r387, %r386, 2;\n" " add.s32 %r388, %r126, %r387;\n" " st.shared.f32 [%r388], %f403;\n" " add.f32 %f582, %f582, %f403;\n" " setp.eq.s32 %p112, %r85, 1;\n" " @%p112 bra $L__BB2_112;\n" "\n" " mov.f32 %f513, 0f32A57060;\n" " mov.f32 %f512, 0f3FB8AA3B;\n" " mov.f32 %f511, 0f4B400001;\n" " mov.f32 %f510, 0f437C0000;\n" " mov.f32 %f509, 0f3F000000;\n" " mov.f32 %f508, 0f3BBB989D;\n" " add.s32 %r418, %r28, %r543;\n" " shl.b32 %r417, %r418, 2;\n" " add.s32 %r416, %r126, %r417;\n" " ld.shared.f32 %f404, [%r416+4];\n" " sub.f32 %f405, %f404, %f578;\n" " fma.rn.f32 %f408, %f405, %f508, %f509;\n" " cvt.sat.f32.f32 %f411, %f408;\n" " fma.rm.f32 %f413, %f411, %f510, %f511;\n" " add.f32 %f414, %f413, 0fCB40007F;\n" " neg.f32 %f415, %f414;\n" " fma.rn.f32 %f416, %f405, %f512, %f415;\n" " fma.rn.f32 %f418, %f405, %f513, %f416;\n" " mov.b32 %r389, %f413;\n" " shl.b32 %r390, %r389, 23;\n" " mov.b32 %f419, %r390;\n" " ex2.approx.ftz.f32 %f420, %f418;\n" " mul.f32 %f421, %f420, %f419;\n" " add.s32 %r391, %r543, 1;\n" " cvt.u64.u32 %rd56, %r391;\n" " mul.lo.s64 %rd57, %rd56, %rd6;\n" " add.s64 %rd58, %rd20, %rd57;\n" " cvt.u32.u64 %r392, %rd58;\n" " add.s32 %r393, %r4, %r392;\n" " shl.b32 %r394, %r393, 2;\n" " add.s32 %r396, %r126, %r394;\n" " st.shared.f32 [%r396], %f421;\n" " add.f32 %f582, %f582, %f421;\n" " setp.eq.s32 %p113, %r85, 2;\n" " @%p113 bra $L__BB2_112;\n" "\n" " mov.f32 %f519, 0f32A57060;\n" " mov.f32 %f518, 0f3FB8AA3B;\n" " mov.f32 %f517, 0f4B400001;\n" " mov.f32 %f516, 0f437C0000;\n" " mov.f32 %f515, 0f3F000000;\n" " mov.f32 %f514, 0f3BBB989D;\n" " add.s32 %r421, %r28, %r543;\n" " shl.b32 %r420, %r421, 2;\n" " add.s32 %r419, %r126, %r420;\n" " ld.shared.f32 %f422, [%r419+8];\n" " sub.f32 %f423, %f422, %f578;\n" " fma.rn.f32 %f426, %f423, %f514, %f515;\n" " cvt.sat.f32.f32 %f429, %f426;\n" " fma.rm.f32 %f431, %f429, %f516, %f517;\n" " add.f32 %f432, %f431, 0fCB40007F;\n" " neg.f32 %f433, %f432;\n" " fma.rn.f32 %f434, %f423, %f518, %f433;\n" " fma.rn.f32 %f436, %f423, %f519, %f434;\n" " mov.b32 %r397, %f431;\n" " shl.b32 %r398, %r397, 23;\n" " mov.b32 %f437, %r398;\n" " ex2.approx.ftz.f32 %f438, %f436;\n" " mul.f32 %f439, %f438, %f437;\n" " add.s32 %r399, %r543, 2;\n" " cvt.u64.u32 %rd59, %r399;\n" " mul.lo.s64 %rd60, %rd59, %rd6;\n" " add.s64 %rd61, %rd20, %rd60;\n" " cvt.u32.u64 %r400, %rd61;\n" " add.s32 %r401, %r4, %r400;\n" " shl.b32 %r402, %r401, 2;\n" " add.s32 %r404, %r126, %r402;\n" " st.shared.f32 [%r404], %f439;\n" " add.f32 %f582, %f582, %f439;\n" "\n" "$L__BB2_112:\n" " sub.f32 %f440, %f91, %f578;\n" " mov.f32 %f441, 0f3F000000;\n" " mov.f32 %f442, 0f3BBB989D;\n" " fma.rn.f32 %f443, %f440, %f442, %f441;\n" " mov.f32 %f444, 0f3FB8AA3B;\n" " mov.f32 %f445, 0f437C0000;\n" " cvt.sat.f32.f32 %f446, %f443;\n" " mov.f32 %f447, 0f4B400001;\n" " fma.rm.f32 %f448, %f446, %f445, %f447;\n" " add.f32 %f449, %f448, 0fCB40007F;\n" " neg.f32 %f450, %f449;\n" " fma.rn.f32 %f451, %f440, %f444, %f450;\n" " mov.f32 %f452, 0f32A57060;\n" " fma.rn.f32 %f453, %f440, %f452, %f451;\n" " ex2.approx.ftz.f32 %f454, %f453;\n" " mov.b32 %r405, %f448;\n" " shl.b32 %r406, %r405, 23;\n" " mov.b32 %f455, %r406;\n" " mul.f32 %f456, %f454, %f455;\n" " ld.shared.f32 %f457, [%r22];\n" " fma.rn.f32 %f458, %f456, %f457, %f582;\n" " st.shared.f32 [%r22], %f458;\n" " st.shared.f32 [%r21], %f578;\n" " st.shared.f32 [%r29], %f456;\n" "\n" "$L__BB2_113:\n" " bar.sync 0;\n" " @%p17 bra $L__BB2_149;\n" "\n" " mov.u32 %r544, 0;\n" "\n" "$L__BB2_115:\n" " ld.param.u64 %rd113, [fc_attn_flash_prefill_param_2];\n" " add.s32 %r408, %r544, %r530;\n" " cvt.s64.s32 %rd63, %r408;\n" " mul.lo.s64 %rd64, %rd63, %rd9;\n" " add.s64 %rd65, %rd19, %rd64;\n" " shl.b64 %rd66, %rd65, 2;\n" " add.s64 %rd62, %rd113, %rd66;\n" " // begin inline asm\n" " ld.global.nc.f32 %f459, [%rd62];\n" " // end inline asm\n" " cvt.u64.u32 %rd67, %r544;\n" " mul.lo.s64 %rd68, %rd67, %rd6;\n" " add.s64 %rd69, %rd68, %rd4;\n" " cvt.u32.u64 %r409, %rd69;\n" " add.s32 %r410, %r4, %r409;\n" " shl.b32 %r411, %r410, 2;\n" " add.s32 %r96, %r126, %r411;\n" " setp.eq.s32 %p115, %r544, 0;\n" " @%p115 bra $L__BB2_132;\n" "\n" " @%p2 bra $L__BB2_118;\n" "\n" " ld.shared.f32 %f460, [%r96];\n" " fma.rn.f32 %f612, %f459, %f460, %f612;\n" "\n" "$L__BB2_118:\n" " setp.lt.s32 %p142, %r101, 2;\n" " @%p142 bra $L__BB2_120;\n" "\n" " ld.shared.f32 %f461, [%r96+4];\n" " fma.rn.f32 %f611, %f459, %f461, %f611;\n" "\n" "$L__BB2_120:\n" " setp.lt.s32 %p143, %r101, 3;\n" " @%p143 bra $L__BB2_122;\n" "\n" " ld.shared.f32 %f462, [%r96+8];\n" " fma.rn.f32 %f610, %f459, %f462, %f610;\n" "\n" "$L__BB2_122:\n" " setp.lt.s32 %p144, %r101, 4;\n" " @%p144 bra $L__BB2_124;\n" "\n" " ld.shared.f32 %f463, [%r96+12];\n" " fma.rn.f32 %f609, %f459, %f463, %f609;\n" "\n" "$L__BB2_124:\n" " setp.lt.s32 %p145, %r101, 5;\n" " @%p145 bra $L__BB2_126;\n" "\n" " ld.shared.f32 %f464, [%r96+16];\n" " fma.rn.f32 %f608, %f459, %f464, %f608;\n" "\n" "$L__BB2_126:\n" " setp.lt.s32 %p146, %r101, 6;\n" " @%p146 bra $L__BB2_128;\n" "\n" " ld.shared.f32 %f465, [%r96+20];\n" " fma.rn.f32 %f607, %f459, %f465, %f607;\n" "\n" "$L__BB2_128:\n" " setp.lt.s32 %p147, %r101, 7;\n" " @%p147 bra $L__BB2_130;\n" "\n" " ld.shared.f32 %f466, [%r96+24];\n" " fma.rn.f32 %f606, %f459, %f466, %f606;\n" "\n" "$L__BB2_130:\n" " setp.lt.s32 %p148, %r101, 8;\n" " @%p148 bra $L__BB2_148;\n" "\n" " ld.shared.f32 %f467, [%r96+28];\n" " fma.rn.f32 %f605, %f459, %f467, %f605;\n" " bra.uni $L__BB2_148;\n" "\n" "$L__BB2_132:\n" " @%p2 bra $L__BB2_134;\n" "\n" " ld.shared.f32 %f468, [%r41];\n" " ld.shared.f32 %f469, [%r96];\n" " mul.f32 %f470, %f459, %f469;\n" " fma.rn.f32 %f612, %f468, %f612, %f470;\n" "\n" "$L__BB2_134:\n" " setp.lt.s32 %p149, %r101, 2;\n" " @%p149 bra $L__BB2_136;\n" "\n" " ld.shared.f32 %f471, [%r41+4];\n" " ld.shared.f32 %f472, [%r96+4];\n" " mul.f32 %f473, %f459, %f472;\n" " fma.rn.f32 %f611, %f471, %f611, %f473;\n" "\n" "$L__BB2_136:\n" " setp.lt.s32 %p150, %r101, 3;\n" " @%p150 bra $L__BB2_138;\n" "\n" " ld.shared.f32 %f474, [%r41+8];\n" " ld.shared.f32 %f475, [%r96+8];\n" " mul.f32 %f476, %f459, %f475;\n" " fma.rn.f32 %f610, %f474, %f610, %f476;\n" "\n" "$L__BB2_138:\n" " setp.lt.s32 %p151, %r101, 4;\n" " @%p151 bra $L__BB2_140;\n" "\n" " ld.shared.f32 %f477, [%r41+12];\n" " ld.shared.f32 %f478, [%r96+12];\n" " mul.f32 %f479, %f459, %f478;\n" " fma.rn.f32 %f609, %f477, %f609, %f479;\n" "\n" "$L__BB2_140:\n" " setp.lt.s32 %p152, %r101, 5;\n" " @%p152 bra $L__BB2_142;\n" "\n" " ld.shared.f32 %f480, [%r41+16];\n" " ld.shared.f32 %f481, [%r96+16];\n" " mul.f32 %f482, %f459, %f481;\n" " fma.rn.f32 %f608, %f480, %f608, %f482;\n" "\n" "$L__BB2_142:\n" " setp.lt.s32 %p153, %r101, 6;\n" " @%p153 bra $L__BB2_144;\n" "\n" " ld.shared.f32 %f483, [%r41+20];\n" " ld.shared.f32 %f484, [%r96+20];\n" " mul.f32 %f485, %f459, %f484;\n" " fma.rn.f32 %f607, %f483, %f607, %f485;\n" "\n" "$L__BB2_144:\n" " setp.lt.s32 %p154, %r101, 7;\n" " @%p154 bra $L__BB2_146;\n" "\n" " ld.shared.f32 %f486, [%r41+24];\n" " ld.shared.f32 %f487, [%r96+24];\n" " mul.f32 %f488, %f459, %f487;\n" " fma.rn.f32 %f606, %f486, %f606, %f488;\n" "\n" "$L__BB2_146:\n" " setp.lt.s32 %p155, %r101, 8;\n" " @%p155 bra $L__BB2_148;\n" "\n" " ld.shared.f32 %f489, [%r41+28];\n" " ld.shared.f32 %f490, [%r96+28];\n" " mul.f32 %f491, %f459, %f490;\n" " fma.rn.f32 %f605, %f489, %f605, %f491;\n" "\n" "$L__BB2_148:\n" " add.s32 %r544, %r544, 1;\n" " setp.lt.s32 %p132, %r544, %r53;\n" " @%p132 bra $L__BB2_115;\n" "\n" "$L__BB2_149:\n" " bar.sync 0;\n" "\n" "$L__BB2_150:\n" " add.s32 %r530, %r530, 16;\n" " setp.ge.s32 %p133, %r3, %r530;\n" " add.s32 %r529, %r529, 1;\n" " @%p133 bra $L__BB2_11;\n" "\n" "$L__BB2_151:\n" " ld.param.u32 %r439, [fc_attn_flash_prefill_param_5];\n" " add.s32 %r438, %r439, 16;\n" " mul.wide.s32 %rd112, %r438, 16;\n" " shl.b32 %r437, %r101, 4;\n" " cvt.s64.s32 %rd111, %r437;\n" " add.s64 %rd110, %rd112, %rd111;\n" " cvt.s64.s32 %rd109, %r101;\n" " add.s64 %rd108, %rd110, %rd111;\n" " add.s64 %rd107, %rd108, %rd109;\n" " cvt.u32.u64 %r436, %rd107;\n" " mov.u32 %r435, %tid.x;\n" " cvt.s64.s32 %rd106, %r435;\n" " mov.u32 %r434, %ctaid.x;\n" " mov.u32 %r433, %nctaid.y;\n" " cvt.u64.u32 %rd105, %r433;\n" " cvt.s64.s32 %rd104, %r434;\n" " mov.u32 %r432, %ctaid.y;\n" " cvt.s64.s32 %rd103, %r432;\n" " mul.lo.s64 %rd102, %rd104, %rd105;\n" " add.s64 %rd101, %rd102, %rd103;\n" " mul.wide.s32 %rd100, %r439, %r101;\n" " mul.lo.s64 %rd99, %rd100, %rd101;\n" " add.s64 %rd23, %rd99, %rd106;\n" " add.s32 %r413, %r4, %r436;\n" " shl.b32 %r414, %r413, 2;\n" " add.s32 %r100, %r126, %r414;\n" " @%p2 bra $L__BB2_153;\n" "\n" " ld.shared.f32 %f492, [%r100];\n" " div.rn.f32 %f493, %f612, %f492;\n" " shl.b64 %rd70, %rd23, 2;\n" " add.s64 %rd71, %rd2, %rd70;\n" " st.global.f32 [%rd71], %f493;\n" "\n" "$L__BB2_153:\n" " setp.lt.s32 %p135, %r101, 2;\n" " @%p135 bra $L__BB2_155;\n" "\n" " ld.shared.f32 %f494, [%r100+4];\n" " div.rn.f32 %f495, %f611, %f494;\n" " add.s64 %rd72, %rd23, %rd9;\n" " shl.b64 %rd73, %rd72, 2;\n" " add.s64 %rd74, %rd2, %rd73;\n" " st.global.f32 [%rd74], %f495;\n" "\n" "$L__BB2_155:\n" " setp.lt.s32 %p136, %r101, 3;\n" " @%p136 bra $L__BB2_157;\n" "\n" " ld.shared.f32 %f496, [%r100+8];\n" " div.rn.f32 %f497, %f610, %f496;\n" " shl.b64 %rd75, %rd9, 1;\n" " add.s64 %rd76, %rd23, %rd75;\n" " shl.b64 %rd77, %rd76, 2;\n" " add.s64 %rd78, %rd2, %rd77;\n" " st.global.f32 [%rd78], %f497;\n" "\n" "$L__BB2_157:\n" " setp.lt.s32 %p137, %r101, 4;\n" " @%p137 bra $L__BB2_159;\n" "\n" " ld.shared.f32 %f498, [%r100+12];\n" " div.rn.f32 %f499, %f609, %f498;\n" " mul.lo.s64 %rd79, %rd9, 3;\n" " add.s64 %rd80, %rd23, %rd79;\n" " shl.b64 %rd81, %rd80, 2;\n" " add.s64 %rd82, %rd2, %rd81;\n" " st.global.f32 [%rd82], %f499;\n" "\n" "$L__BB2_159:\n" " setp.lt.s32 %p138, %r101, 5;\n" " @%p138 bra $L__BB2_161;\n" "\n" " ld.shared.f32 %f500, [%r100+16];\n" " div.rn.f32 %f501, %f608, %f500;\n" " shl.b64 %rd83, %rd9, 2;\n" " add.s64 %rd84, %rd23, %rd83;\n" " shl.b64 %rd85, %rd84, 2;\n" " add.s64 %rd86, %rd2, %rd85;\n" " st.global.f32 [%rd86], %f501;\n" "\n" "$L__BB2_161:\n" " setp.lt.s32 %p139, %r101, 6;\n" " @%p139 bra $L__BB2_163;\n" "\n" " ld.shared.f32 %f502, [%r100+20];\n" " div.rn.f32 %f503, %f607, %f502;\n" " mul.lo.s64 %rd87, %rd9, 5;\n" " add.s64 %rd88, %rd23, %rd87;\n" " shl.b64 %rd89, %rd88, 2;\n" " add.s64 %rd90, %rd2, %rd89;\n" " st.global.f32 [%rd90], %f503;\n" "\n" "$L__BB2_163:\n" " setp.lt.s32 %p140, %r101, 7;\n" " @%p140 bra $L__BB2_165;\n" "\n" " ld.shared.f32 %f504, [%r100+24];\n" " div.rn.f32 %f505, %f606, %f504;\n" " mul.lo.s64 %rd91, %rd9, 6;\n" " add.s64 %rd92, %rd23, %rd91;\n" " shl.b64 %rd93, %rd92, 2;\n" " add.s64 %rd94, %rd2, %rd93;\n" " st.global.f32 [%rd94], %f505;\n" "\n" "$L__BB2_165:\n" " setp.lt.s32 %p141, %r101, 8;\n" " @%p141 bra $L__BB2_167;\n" "\n" " ld.shared.f32 %f506, [%r100+28];\n" " div.rn.f32 %f507, %f605, %f506;\n" " mul.lo.s64 %rd95, %rd9, 7;\n" " add.s64 %rd96, %rd23, %rd95;\n" " shl.b64 %rd97, %rd96, 2;\n" " add.s64 %rd98, %rd2, %rd97;\n" " st.global.f32 [%rd98], %f507;\n" "\n" "$L__BB2_167:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_kv_append_q4\n" ".visible .entry fc_attn_kv_append_q4(\n" " .param .u64 fc_attn_kv_append_q4_param_0,\n" " .param .u64 fc_attn_kv_append_q4_param_1,\n" " .param .u64 fc_attn_kv_append_q4_param_2,\n" " .param .u64 fc_attn_kv_append_q4_param_3,\n" " .param .u64 fc_attn_kv_append_q4_param_4,\n" " .param .u64 fc_attn_kv_append_q4_param_5,\n" " .param .u64 fc_attn_kv_append_q4_param_6,\n" " .param .u64 fc_attn_kv_append_q4_param_7,\n" " .param .u32 fc_attn_kv_append_q4_param_8,\n" " .param .u32 fc_attn_kv_append_q4_param_9,\n" " .param .u32 fc_attn_kv_append_q4_param_10,\n" " .param .u32 fc_attn_kv_append_q4_param_11,\n" " .param .u32 fc_attn_kv_append_q4_param_12,\n" " .param .u32 fc_attn_kv_append_q4_param_13\n" ")\n" "{\n" " .reg .pred %p<10>;\n" " .reg .b16 %rs<7>;\n" " .reg .f32 %f<223>;\n" " .reg .b32 %r<66>;\n" " .reg .b64 %rd<55>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ20fc_attn_kv_append_q4E3s_k[4096];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ20fc_attn_kv_append_q4E3s_v[4096];\n" "\n" " ld.param.u64 %rd8, [fc_attn_kv_append_q4_param_0];\n" " ld.param.u64 %rd9, [fc_attn_kv_append_q4_param_1];\n" " ld.param.u64 %rd10, [fc_attn_kv_append_q4_param_2];\n" " ld.param.u64 %rd11, [fc_attn_kv_append_q4_param_3];\n" " ld.param.u64 %rd12, [fc_attn_kv_append_q4_param_4];\n" " ld.param.u64 %rd13, [fc_attn_kv_append_q4_param_5];\n" " ld.param.u64 %rd14, [fc_attn_kv_append_q4_param_6];\n" " ld.param.u64 %rd15, [fc_attn_kv_append_q4_param_7];\n" " ld.param.u32 %r6, [fc_attn_kv_append_q4_param_8];\n" " ld.param.u32 %r7, [fc_attn_kv_append_q4_param_9];\n" " ld.param.u32 %r8, [fc_attn_kv_append_q4_param_10];\n" " ld.param.u32 %r9, [fc_attn_kv_append_q4_param_11];\n" " ld.param.u32 %r10, [fc_attn_kv_append_q4_param_12];\n" " ld.param.u32 %r11, [fc_attn_kv_append_q4_param_13];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r7;\n" " @%p1 bra $L__BB3_17;\n" "\n" " mov.u32 %r12, %ctaid.x;\n" " mov.u32 %r13, %ctaid.y;\n" " shr.s32 %r14, %r7, 31;\n" " shr.u32 %r15, %r14, 27;\n" " add.s32 %r16, %r7, %r15;\n" " shr.s32 %r17, %r16, 5;\n" " mul.lo.s32 %r18, %r17, 18;\n" " mul.wide.s32 %rd16, %r6, %r12;\n" " cvt.s64.s32 %rd1, %r13;\n" " add.s64 %rd17, %rd16, %rd1;\n" " cvt.s64.s32 %rd2, %r7;\n" " mul.lo.s64 %rd18, %rd17, %rd2;\n" " cvt.s64.s32 %rd3, %r1;\n" " add.s64 %rd19, %rd18, %rd3;\n" " mul.wide.s32 %rd20, %r8, %r13;\n" " add.s32 %r19, %r12, %r9;\n" " cvt.s64.s32 %rd4, %r19;\n" " add.s64 %rd21, %rd20, %rd4;\n" " cvt.s64.s32 %rd22, %r18;\n" " mul.lo.s64 %rd23, %rd21, %rd22;\n" " cvta.to.global.u64 %rd24, %rd8;\n" " shl.b64 %rd25, %rd19, 2;\n" " add.s64 %rd26, %rd24, %rd25;\n" " ld.global.f32 %f1, [%rd26];\n" " cvta.to.global.u64 %rd27, %rd9;\n" " add.s64 %rd28, %rd27, %rd25;\n" " ld.global.f32 %f2, [%rd28];\n" " shl.b32 %r20, %r1, 2;\n" " mov.u32 %r21, _ZZ20fc_attn_kv_append_q4E3s_k;\n" " add.s32 %r22, %r21, %r20;\n" " st.shared.f32 [%r22], %f1;\n" " mov.u32 %r23, _ZZ20fc_attn_kv_append_q4E3s_v;\n" " add.s32 %r24, %r23, %r20;\n" " st.shared.f32 [%r24], %f2;\n" " bar.sync 0;\n" " shr.s32 %r25, %r1, 31;\n" " shr.u32 %r26, %r25, 27;\n" " add.s32 %r27, %r1, %r26;\n" " shr.s32 %r28, %r27, 5;\n" " and.b32 %r2, %r27, -32;\n" " mul.lo.s32 %r29, %r28, 18;\n" " cvt.s64.s32 %rd29, %r29;\n" " add.s64 %rd5, %rd23, %rd29;\n" " shl.b32 %r30, %r28, 7;\n" " add.s32 %r3, %r21, %r30;\n" " ld.shared.f32 %f10, [%r3];\n" " abs.f32 %f11, %f10;\n" " mov.f32 %f221, 0f00000000;\n" " max.f32 %f12, %f221, %f11;\n" " ld.shared.f32 %f13, [%r3+4];\n" " abs.f32 %f14, %f13;\n" " max.f32 %f15, %f12, %f14;\n" " ld.shared.f32 %f16, [%r3+8];\n" " abs.f32 %f17, %f16;\n" " max.f32 %f18, %f15, %f17;\n" " ld.shared.f32 %f19, [%r3+12];\n" " abs.f32 %f20, %f19;\n" " max.f32 %f21, %f18, %f20;\n" " ld.shared.f32 %f22, [%r3+16];\n" " abs.f32 %f23, %f22;\n" " max.f32 %f24, %f21, %f23;\n" " ld.shared.f32 %f25, [%r3+20];\n" " abs.f32 %f26, %f25;\n" " max.f32 %f27, %f24, %f26;\n" " ld.shared.f32 %f28, [%r3+24];\n" " abs.f32 %f29, %f28;\n" " max.f32 %f30, %f27, %f29;\n" " ld.shared.f32 %f31, [%r3+28];\n" " abs.f32 %f32, %f31;\n" " max.f32 %f33, %f30, %f32;\n" " ld.shared.f32 %f34, [%r3+32];\n" " abs.f32 %f35, %f34;\n" " max.f32 %f36, %f33, %f35;\n" " ld.shared.f32 %f37, [%r3+36];\n" " abs.f32 %f38, %f37;\n" " max.f32 %f39, %f36, %f38;\n" " ld.shared.f32 %f40, [%r3+40];\n" " abs.f32 %f41, %f40;\n" " max.f32 %f42, %f39, %f41;\n" " ld.shared.f32 %f43, [%r3+44];\n" " abs.f32 %f44, %f43;\n" " max.f32 %f45, %f42, %f44;\n" " ld.shared.f32 %f46, [%r3+48];\n" " abs.f32 %f47, %f46;\n" " max.f32 %f48, %f45, %f47;\n" " ld.shared.f32 %f49, [%r3+52];\n" " abs.f32 %f50, %f49;\n" " max.f32 %f51, %f48, %f50;\n" " ld.shared.f32 %f52, [%r3+56];\n" " abs.f32 %f53, %f52;\n" " max.f32 %f54, %f51, %f53;\n" " ld.shared.f32 %f55, [%r3+60];\n" " abs.f32 %f56, %f55;\n" " max.f32 %f57, %f54, %f56;\n" " ld.shared.f32 %f58, [%r3+64];\n" " abs.f32 %f59, %f58;\n" " max.f32 %f60, %f57, %f59;\n" " ld.shared.f32 %f61, [%r3+68];\n" " abs.f32 %f62, %f61;\n" " max.f32 %f63, %f60, %f62;\n" " ld.shared.f32 %f64, [%r3+72];\n" " abs.f32 %f65, %f64;\n" " max.f32 %f66, %f63, %f65;\n" " ld.shared.f32 %f67, [%r3+76];\n" " abs.f32 %f68, %f67;\n" " max.f32 %f69, %f66, %f68;\n" " ld.shared.f32 %f70, [%r3+80];\n" " abs.f32 %f71, %f70;\n" " max.f32 %f72, %f69, %f71;\n" " ld.shared.f32 %f73, [%r3+84];\n" " abs.f32 %f74, %f73;\n" " max.f32 %f75, %f72, %f74;\n" " ld.shared.f32 %f76, [%r3+88];\n" " abs.f32 %f77, %f76;\n" " max.f32 %f78, %f75, %f77;\n" " ld.shared.f32 %f79, [%r3+92];\n" " abs.f32 %f80, %f79;\n" " max.f32 %f81, %f78, %f80;\n" " ld.shared.f32 %f82, [%r3+96];\n" " abs.f32 %f83, %f82;\n" " max.f32 %f84, %f81, %f83;\n" " ld.shared.f32 %f85, [%r3+100];\n" " abs.f32 %f86, %f85;\n" " max.f32 %f87, %f84, %f86;\n" " ld.shared.f32 %f88, [%r3+104];\n" " abs.f32 %f89, %f88;\n" " max.f32 %f90, %f87, %f89;\n" " ld.shared.f32 %f91, [%r3+108];\n" " abs.f32 %f92, %f91;\n" " max.f32 %f93, %f90, %f92;\n" " ld.shared.f32 %f94, [%r3+112];\n" " abs.f32 %f95, %f94;\n" " max.f32 %f96, %f93, %f95;\n" " ld.shared.f32 %f97, [%r3+116];\n" " abs.f32 %f98, %f97;\n" " max.f32 %f99, %f96, %f98;\n" " ld.shared.f32 %f100, [%r3+120];\n" " abs.f32 %f101, %f100;\n" " max.f32 %f102, %f99, %f101;\n" " ld.shared.f32 %f103, [%r3+124];\n" " abs.f32 %f104, %f103;\n" " max.f32 %f105, %f102, %f104;\n" " and.b32 %r4, %r1, 31;\n" " mul.f32 %f3, %f105, 0fBE000000;\n" " setp.eq.f32 %p2, %f3, 0f00000000;\n" " @%p2 bra $L__BB3_3;\n" "\n" " rcp.rn.f32 %f221, %f3;\n" "\n" "$L__BB3_3:\n" " cvta.to.global.u64 %rd30, %rd10;\n" " add.s64 %rd6, %rd30, %rd5;\n" " setp.ne.s32 %p3, %r4, 0;\n" " @%p3 bra $L__BB3_5;\n" "\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs1, %f3;}\n" "\n" " // end inline asm\n" " st.global.u16 [%rd6], %rs1;\n" "\n" "$L__BB3_5:\n" " setp.gt.u32 %p4, %r4, 15;\n" " @%p4 bra $L__BB3_7;\n" "\n" " shl.b32 %r31, %r4, 2;\n" " add.s32 %r32, %r3, %r31;\n" " ld.shared.f32 %f107, [%r32];\n" " mul.f32 %f108, %f221, %f107;\n" " cvt.rni.f32.f32 %f109, %f108;\n" " cvt.rzi.s32.f32 %r33, %f109;\n" " add.s32 %r34, %r33, 8;\n" " ld.shared.f32 %f110, [%r32+64];\n" " mul.f32 %f111, %f221, %f110;\n" " cvt.rni.f32.f32 %f112, %f111;\n" " cvt.rzi.s32.f32 %r35, %f112;\n" " add.s32 %r36, %r35, 8;\n" " min.s32 %r37, %r34, 15;\n" " max.s32 %r38, %r37, 0;\n" " min.s32 %r39, %r36, 15;\n" " max.s32 %r40, %r39, 0;\n" " shl.b32 %r41, %r40, 4;\n" " or.b32 %r42, %r41, %r38;\n" " add.s32 %r43, %r4, 2;\n" " cvt.u64.u32 %rd31, %r43;\n" " add.s64 %rd32, %rd6, %rd31;\n" " st.global.u8 [%rd32], %r42;\n" "\n" "$L__BB3_7:\n" " shl.b32 %r44, %r2, 2;\n" " add.s32 %r5, %r23, %r44;\n" " ld.shared.f32 %f114, [%r5];\n" " abs.f32 %f115, %f114;\n" " mov.f32 %f222, 0f00000000;\n" " max.f32 %f116, %f222, %f115;\n" " ld.shared.f32 %f117, [%r5+4];\n" " abs.f32 %f118, %f117;\n" " max.f32 %f119, %f116, %f118;\n" " ld.shared.f32 %f120, [%r5+8];\n" " abs.f32 %f121, %f120;\n" " max.f32 %f122, %f119, %f121;\n" " ld.shared.f32 %f123, [%r5+12];\n" " abs.f32 %f124, %f123;\n" " max.f32 %f125, %f122, %f124;\n" " ld.shared.f32 %f126, [%r5+16];\n" " abs.f32 %f127, %f126;\n" " max.f32 %f128, %f125, %f127;\n" " ld.shared.f32 %f129, [%r5+20];\n" " abs.f32 %f130, %f129;\n" " max.f32 %f131, %f128, %f130;\n" " ld.shared.f32 %f132, [%r5+24];\n" " abs.f32 %f133, %f132;\n" " max.f32 %f134, %f131, %f133;\n" " ld.shared.f32 %f135, [%r5+28];\n" " abs.f32 %f136, %f135;\n" " max.f32 %f137, %f134, %f136;\n" " ld.shared.f32 %f138, [%r5+32];\n" " abs.f32 %f139, %f138;\n" " max.f32 %f140, %f137, %f139;\n" " ld.shared.f32 %f141, [%r5+36];\n" " abs.f32 %f142, %f141;\n" " max.f32 %f143, %f140, %f142;\n" " ld.shared.f32 %f144, [%r5+40];\n" " abs.f32 %f145, %f144;\n" " max.f32 %f146, %f143, %f145;\n" " ld.shared.f32 %f147, [%r5+44];\n" " abs.f32 %f148, %f147;\n" " max.f32 %f149, %f146, %f148;\n" " ld.shared.f32 %f150, [%r5+48];\n" " abs.f32 %f151, %f150;\n" " max.f32 %f152, %f149, %f151;\n" " ld.shared.f32 %f153, [%r5+52];\n" " abs.f32 %f154, %f153;\n" " max.f32 %f155, %f152, %f154;\n" " ld.shared.f32 %f156, [%r5+56];\n" " abs.f32 %f157, %f156;\n" " max.f32 %f158, %f155, %f157;\n" " ld.shared.f32 %f159, [%r5+60];\n" " abs.f32 %f160, %f159;\n" " max.f32 %f161, %f158, %f160;\n" " ld.shared.f32 %f162, [%r5+64];\n" " abs.f32 %f163, %f162;\n" " max.f32 %f164, %f161, %f163;\n" " ld.shared.f32 %f165, [%r5+68];\n" " abs.f32 %f166, %f165;\n" " max.f32 %f167, %f164, %f166;\n" " ld.shared.f32 %f168, [%r5+72];\n" " abs.f32 %f169, %f168;\n" " max.f32 %f170, %f167, %f169;\n" " ld.shared.f32 %f171, [%r5+76];\n" " abs.f32 %f172, %f171;\n" " max.f32 %f173, %f170, %f172;\n" " ld.shared.f32 %f174, [%r5+80];\n" " abs.f32 %f175, %f174;\n" " max.f32 %f176, %f173, %f175;\n" " ld.shared.f32 %f177, [%r5+84];\n" " abs.f32 %f178, %f177;\n" " max.f32 %f179, %f176, %f178;\n" " ld.shared.f32 %f180, [%r5+88];\n" " abs.f32 %f181, %f180;\n" " max.f32 %f182, %f179, %f181;\n" " ld.shared.f32 %f183, [%r5+92];\n" " abs.f32 %f184, %f183;\n" " max.f32 %f185, %f182, %f184;\n" " ld.shared.f32 %f186, [%r5+96];\n" " abs.f32 %f187, %f186;\n" " max.f32 %f188, %f185, %f187;\n" " ld.shared.f32 %f189, [%r5+100];\n" " abs.f32 %f190, %f189;\n" " max.f32 %f191, %f188, %f190;\n" " ld.shared.f32 %f192, [%r5+104];\n" " abs.f32 %f193, %f192;\n" " max.f32 %f194, %f191, %f193;\n" " ld.shared.f32 %f195, [%r5+108];\n" " abs.f32 %f196, %f195;\n" " max.f32 %f197, %f194, %f196;\n" " ld.shared.f32 %f198, [%r5+112];\n" " abs.f32 %f199, %f198;\n" " max.f32 %f200, %f197, %f199;\n" " ld.shared.f32 %f201, [%r5+116];\n" " abs.f32 %f202, %f201;\n" " max.f32 %f203, %f200, %f202;\n" " ld.shared.f32 %f204, [%r5+120];\n" " abs.f32 %f205, %f204;\n" " max.f32 %f206, %f203, %f205;\n" " ld.shared.f32 %f207, [%r5+124];\n" " abs.f32 %f208, %f207;\n" " max.f32 %f209, %f206, %f208;\n" " mul.f32 %f6, %f209, 0fBE000000;\n" " setp.eq.f32 %p5, %f6, 0f00000000;\n" " @%p5 bra $L__BB3_9;\n" "\n" " rcp.rn.f32 %f222, %f6;\n" "\n" "$L__BB3_9:\n" " cvta.to.global.u64 %rd33, %rd11;\n" " add.s64 %rd7, %rd33, %rd5;\n" " @%p3 bra $L__BB3_11;\n" "\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs2, %f6;}\n" "\n" " // end inline asm\n" " st.global.u16 [%rd7], %rs2;\n" "\n" "$L__BB3_11:\n" " @%p4 bra $L__BB3_13;\n" "\n" " shl.b32 %r46, %r4, 2;\n" " add.s32 %r47, %r5, %r46;\n" " ld.shared.f32 %f211, [%r47];\n" " mul.f32 %f212, %f222, %f211;\n" " cvt.rni.f32.f32 %f213, %f212;\n" " cvt.rzi.s32.f32 %r48, %f213;\n" " add.s32 %r49, %r48, 8;\n" " ld.shared.f32 %f214, [%r47+64];\n" " mul.f32 %f215, %f222, %f214;\n" " cvt.rni.f32.f32 %f216, %f215;\n" " cvt.rzi.s32.f32 %r50, %f216;\n" " add.s32 %r51, %r50, 8;\n" " min.s32 %r52, %r49, 15;\n" " max.s32 %r53, %r52, 0;\n" " min.s32 %r54, %r51, 15;\n" " max.s32 %r55, %r54, 0;\n" " shl.b32 %r56, %r55, 4;\n" " or.b32 %r57, %r56, %r53;\n" " add.s32 %r58, %r4, 2;\n" " cvt.u64.u32 %rd34, %r58;\n" " add.s64 %rd35, %rd7, %rd34;\n" " st.global.u8 [%rd35], %r57;\n" "\n" "$L__BB3_13:\n" " setp.lt.s32 %p8, %r11, 1;\n" " @%p8 bra $L__BB3_15;\n" "\n" " cvt.u32.u64 %r59, %rd4;\n" " shr.s32 %r60, %r59, 31;\n" " shr.u32 %r61, %r60, 24;\n" " add.s32 %r62, %r59, %r61;\n" " and.b32 %r63, %r62, -256;\n" " sub.s32 %r64, %r59, %r63;\n" " cvt.s64.s32 %rd36, %r64;\n" " shl.b64 %rd37, %rd1, 8;\n" " add.s64 %rd38, %rd37, %rd36;\n" " mul.lo.s64 %rd39, %rd38, %rd2;\n" " add.s64 %rd40, %rd39, %rd3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs3, %f1;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd41, %rd14;\n" " shl.b64 %rd42, %rd40, 1;\n" " add.s64 %rd43, %rd41, %rd42;\n" " st.global.u16 [%rd43], %rs3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs4, %f2;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd44, %rd15;\n" " add.s64 %rd45, %rd44, %rd42;\n" " st.global.u16 [%rd45], %rs4;\n" "\n" "$L__BB3_15:\n" " cvt.u32.u64 %r65, %rd4;\n" " setp.ge.s32 %p9, %r65, %r10;\n" " @%p9 bra $L__BB3_17;\n" "\n" " shl.b64 %rd46, %rd1, 3;\n" " add.s64 %rd47, %rd46, %rd4;\n" " mul.lo.s64 %rd48, %rd47, %rd2;\n" " add.s64 %rd49, %rd48, %rd3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs5, %f1;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd50, %rd12;\n" " shl.b64 %rd51, %rd49, 1;\n" " add.s64 %rd52, %rd50, %rd51;\n" " st.global.u16 [%rd52], %rs5;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs6, %f2;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd53, %rd13;\n" " add.s64 %rd54, %rd53, %rd51;\n" " st.global.u16 [%rd54], %rs6;\n" "\n" "$L__BB3_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_prefill_q4\n" ".visible .entry fc_attn_flash_prefill_q4(\n" " .param .u64 fc_attn_flash_prefill_q4_param_0,\n" " .param .u64 fc_attn_flash_prefill_q4_param_1,\n" " .param .u64 fc_attn_flash_prefill_q4_param_2,\n" " .param .u64 fc_attn_flash_prefill_q4_param_3,\n" " .param .u64 fc_attn_flash_prefill_q4_param_4,\n" " .param .u64 fc_attn_flash_prefill_q4_param_5,\n" " .param .u64 fc_attn_flash_prefill_q4_param_6,\n" " .param .u64 fc_attn_flash_prefill_q4_param_7,\n" " .param .u32 fc_attn_flash_prefill_q4_param_8,\n" " .param .u32 fc_attn_flash_prefill_q4_param_9,\n" " .param .u32 fc_attn_flash_prefill_q4_param_10,\n" " .param .u32 fc_attn_flash_prefill_q4_param_11,\n" " .param .u32 fc_attn_flash_prefill_q4_param_12,\n" " .param .u32 fc_attn_flash_prefill_q4_param_13,\n" " .param .u32 fc_attn_flash_prefill_q4_param_14,\n" " .param .u32 fc_attn_flash_prefill_q4_param_15,\n" " .param .f32 fc_attn_flash_prefill_q4_param_16\n" ")\n" "{\n" " .reg .pred %p<191>;\n" " .reg .b16 %rs<11>;\n" " .reg .f32 %f<767>;\n" " .reg .b32 %r<663>;\n" " .reg .b64 %rd<194>;\n" "\n" "\n" " ld.param.u64 %rd49, [fc_attn_flash_prefill_q4_param_0];\n" " ld.param.u64 %rd50, [fc_attn_flash_prefill_q4_param_1];\n" " ld.param.u64 %rd51, [fc_attn_flash_prefill_q4_param_2];\n" " ld.param.u64 %rd52, [fc_attn_flash_prefill_q4_param_3];\n" " ld.param.u64 %rd53, [fc_attn_flash_prefill_q4_param_4];\n" " ld.param.u64 %rd47, [fc_attn_flash_prefill_q4_param_5];\n" " ld.param.u64 %rd48, [fc_attn_flash_prefill_q4_param_6];\n" " ld.param.u64 %rd54, [fc_attn_flash_prefill_q4_param_7];\n" " ld.param.u32 %r120, [fc_attn_flash_prefill_q4_param_8];\n" " ld.param.u32 %r121, [fc_attn_flash_prefill_q4_param_9];\n" " ld.param.u32 %r122, [fc_attn_flash_prefill_q4_param_10];\n" " ld.param.u32 %r123, [fc_attn_flash_prefill_q4_param_11];\n" " ld.param.u32 %r124, [fc_attn_flash_prefill_q4_param_12];\n" " ld.param.u32 %r125, [fc_attn_flash_prefill_q4_param_13];\n" " ld.param.u32 %r127, [fc_attn_flash_prefill_q4_param_15];\n" " ld.param.f32 %f231, [fc_attn_flash_prefill_q4_param_16];\n" " cvta.to.global.u64 %rd1, %rd49;\n" " cvta.to.global.u64 %rd2, %rd54;\n" " cvta.to.global.u64 %rd3, %rd52;\n" " cvta.to.global.u64 %rd4, %rd50;\n" " cvta.to.global.u64 %rd5, %rd53;\n" " cvta.to.global.u64 %rd6, %rd51;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %ctaid.x;\n" " add.s32 %r3, %r2, %r123;\n" " mov.u32 %r128, %nctaid.x;\n" " add.s32 %r129, %r128, %r123;\n" " add.s32 %r4, %r129, -257;\n" " mul.lo.s32 %r5, %r121, %r120;\n" " add.s32 %r130, %r121, 16;\n" " cvt.s64.s32 %rd7, %r130;\n" " mul.wide.s32 %rd8, %r130, 16;\n" " shl.b32 %r131, %r120, 4;\n" " cvt.s64.s32 %rd55, %r131;\n" " add.s64 %rd9, %rd8, %rd55;\n" " add.s64 %rd10, %rd9, %rd55;\n" " cvt.s64.s32 %rd11, %r120;\n" " add.s64 %rd12, %rd10, %rd11;\n" " mov.u32 %r132, %nctaid.y;\n" " cvt.u64.u32 %rd56, %r132;\n" " cvt.s64.s32 %rd57, %r2;\n" " mul.lo.s64 %rd58, %rd57, %rd56;\n" " mov.u32 %r133, %ctaid.y;\n" " cvt.s64.s32 %rd13, %r133;\n" " add.s64 %rd59, %rd58, %rd13;\n" " cvt.s64.s32 %rd14, %r121;\n" " mul.wide.s32 %rd60, %r121, %r120;\n" " mul.lo.s64 %rd15, %rd60, %rd59;\n" " setp.lt.s32 %p2, %r120, 1;\n" " @%p2 bra $L__BB4_7;\n" "\n" " cvt.s64.s32 %rd16, %r1;\n" " and.b32 %r641, %r120, 3;\n" " add.s32 %r135, %r120, -1;\n" " setp.lt.u32 %p3, %r135, 3;\n" " mov.u32 %r639, 0;\n" " @%p3 bra $L__BB4_4;\n" "\n" " sub.s32 %r637, %r120, %r641;\n" " add.s64 %rd61, %rd15, %rd16;\n" " shl.b64 %rd62, %rd61, 2;\n" " add.s64 %rd192, %rd1, %rd62;\n" " shl.b64 %rd18, %rd14, 2;\n" " shl.b32 %r137, %r1, 2;\n" " mov.u32 %r138, sh;\n" " add.s32 %r638, %r138, %r137;\n" " shl.b32 %r9, %r121, 2;\n" " mov.u32 %r639, 0;\n" "\n" "$L__BB4_3:\n" " ld.global.f32 %f232, [%rd192];\n" " st.shared.f32 [%r638], %f232;\n" " add.s64 %rd63, %rd192, %rd18;\n" " ld.global.f32 %f233, [%rd63];\n" " add.s32 %r139, %r638, %r9;\n" " st.shared.f32 [%r139], %f233;\n" " add.s64 %rd64, %rd63, %rd18;\n" " ld.global.f32 %f234, [%rd64];\n" " add.s32 %r140, %r139, %r9;\n" " st.shared.f32 [%r140], %f234;\n" " add.s64 %rd65, %rd64, %rd18;\n" " add.s64 %rd192, %rd65, %rd18;\n" " ld.global.f32 %f235, [%rd65];\n" " add.s32 %r141, %r140, %r9;\n" " add.s32 %r638, %r141, %r9;\n" " st.shared.f32 [%r141], %f235;\n" " add.s32 %r639, %r639, 4;\n" " add.s32 %r637, %r637, -4;\n" " setp.ne.s32 %p4, %r637, 0;\n" " @%p4 bra $L__BB4_3;\n" "\n" "$L__BB4_4:\n" " setp.eq.s32 %p5, %r641, 0;\n" " @%p5 bra $L__BB4_7;\n" "\n" " mad.lo.s32 %r142, %r639, %r121, %r1;\n" " shl.b32 %r143, %r142, 2;\n" " mov.u32 %r144, sh;\n" " add.s32 %r640, %r144, %r143;\n" " shl.b32 %r18, %r121, 2;\n" " cvt.s64.s32 %rd193, %r639;\n" " add.s64 %rd22, %rd15, %rd16;\n" "\n" "$L__BB4_6:\n" " .pragma \"nounroll\";\n" " mul.lo.s64 %rd66, %rd193, %rd14;\n" " add.s64 %rd67, %rd22, %rd66;\n" " shl.b64 %rd68, %rd67, 2;\n" " add.s64 %rd69, %rd1, %rd68;\n" " ld.global.f32 %f236, [%rd69];\n" " st.shared.f32 [%r640], %f236;\n" " add.s32 %r640, %r640, %r18;\n" " add.s64 %rd193, %rd193, 1;\n" " add.s32 %r641, %r641, -1;\n" " setp.ne.s32 %p6, %r641, 0;\n" " @%p6 bra $L__BB4_6;\n" "\n" "$L__BB4_7:\n" " setp.ge.s32 %p7, %r1, %r120;\n" " add.s32 %r145, %r5, %r1;\n" " cvt.u32.u64 %r146, %rd10;\n" " add.s32 %r147, %r145, %r146;\n" " shl.b32 %r148, %r147, 2;\n" " mov.u32 %r149, sh;\n" " add.s32 %r23, %r149, %r148;\n" " cvt.u32.u64 %r150, %rd12;\n" " add.s32 %r151, %r145, %r150;\n" " shl.b32 %r152, %r151, 2;\n" " add.s32 %r24, %r149, %r152;\n" " @%p7 bra $L__BB4_9;\n" "\n" " mov.u32 %r153, -8388608;\n" " st.shared.u32 [%r23], %r153;\n" " mov.u32 %r154, 0;\n" " st.shared.u32 [%r24], %r154;\n" "\n" "$L__BB4_9:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd70, %r122;\n" " mul.lo.s64 %rd71, %rd70, %rd13;\n" " shr.s32 %r155, %r121, 31;\n" " shr.u32 %r156, %r155, 27;\n" " add.s32 %r157, %r121, %r156;\n" " shr.s32 %r158, %r157, 5;\n" " mul.lo.s32 %r159, %r158, 18;\n" " cvt.s64.s32 %rd25, %r159;\n" " mul.lo.s64 %rd26, %rd71, %rd25;\n" " mul.lo.s64 %rd72, %rd13, %rd14;\n" " shl.b64 %rd27, %rd72, 8;\n" " shl.b64 %rd28, %rd72, 3;\n" " shr.s32 %r160, %r1, 31;\n" " shr.u32 %r161, %r160, 28;\n" " add.s32 %r162, %r1, %r161;\n" " shr.s32 %r26, %r162, 4;\n" " and.b32 %r163, %r162, -16;\n" " sub.s32 %r27, %r1, %r163;\n" " cvt.s64.s32 %rd29, %r1;\n" " setp.lt.s32 %p8, %r3, 0;\n" " mov.f32 %f704, 0f00000000;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " @%p8 bra $L__BB4_213;\n" "\n" " shr.u32 %r167, %r155, 28;\n" " add.s32 %r168, %r121, %r167;\n" " shr.s32 %r169, %r168, 4;\n" " cvt.u32.u64 %r170, %rd7;\n" " sub.s32 %r171, %r3, %r127;\n" " add.s32 %r172, %r171, 1;\n" " mov.u32 %r642, 0;\n" " max.s32 %r28, %r172, 0;\n" " sub.s32 %r29, %r3, %r125;\n" " mov.u32 %r30, %ntid.x;\n" " add.s32 %r173, %r5, %r26;\n" " cvt.u32.u64 %r174, %rd8;\n" " add.s32 %r175, %r173, %r174;\n" " mad.lo.s32 %r31, %r170, %r26, %r5;\n" " shl.b32 %r176, %r1, 4;\n" " add.s32 %r177, %r176, %r174;\n" " add.s32 %r32, %r5, %r177;\n" " add.s64 %rd30, %rd28, %rd29;\n" " add.s32 %r178, %r150, %r120;\n" " add.s32 %r179, %r5, %r178;\n" " add.s64 %rd31, %rd27, %rd29;\n" " shr.u32 %r181, %r160, 27;\n" " add.s32 %r182, %r1, %r181;\n" " shr.s32 %r183, %r182, 5;\n" " mul.lo.s32 %r184, %r183, 18;\n" " cvt.s64.s32 %rd32, %r184;\n" " and.b32 %r185, %r1, 31;\n" " setp.lt.u32 %p9, %r185, 16;\n" " add.s32 %r186, %r185, -16;\n" " selp.b32 %r187, %r185, %r186, %p9;\n" " selp.b32 %r33, 0, 4, %p9;\n" " add.s32 %r188, %r187, 2;\n" " cvt.s64.s32 %rd73, %r188;\n" " add.s64 %rd33, %rd73, %rd32;\n" " add.s32 %r190, %r145, %r178;\n" " shl.b32 %r191, %r190, 2;\n" " add.s32 %r34, %r149, %r191;\n" " cvt.u64.u32 %rd74, %r1;\n" " add.s64 %rd34, %rd9, %rd74;\n" " mov.u32 %r193, -2;\n" " sub.s32 %r194, %r193, %r2;\n" " sub.s32 %r35, %r194, %r123;\n" " bfe.u32 %r37, %r168, 4, 1;\n" " sub.s32 %r38, %r169, %r37;\n" " shl.b32 %r39, %r121, 1;\n" " mul.lo.s32 %r40, %r121, 3;\n" " shl.b32 %r41, %r121, 2;\n" " mul.lo.s32 %r42, %r121, 5;\n" " mul.lo.s32 %r43, %r121, 6;\n" " mul.lo.s32 %r44, %r121, 7;\n" " shl.b32 %r195, %r175, 2;\n" " add.s32 %r45, %r149, %r195;\n" " shl.b32 %r196, %r179, 2;\n" " add.s32 %r46, %r149, %r196;\n" " mad.lo.s32 %r197, %r120, %r170, %r1;\n" " add.s32 %r198, %r197, %r174;\n" " shl.b32 %r199, %r198, 2;\n" " add.s32 %r200, %r149, %r199;\n" " add.s32 %r47, %r200, -1024;\n" " cvta.to.global.u64 %rd35, %rd48;\n" " cvta.to.global.u64 %rd36, %rd47;\n" " mov.f32 %f704, 0f00000000;\n" " setp.lt.s32 %p11, %r127, 1;\n" " setp.gt.s32 %p19, %r125, 0;\n" " setp.ge.s32 %p36, %r1, %r121;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " mov.u32 %r643, %r642;\n" "\n" "$L__BB4_11:\n" " ld.param.u32 %r538, [fc_attn_flash_prefill_q4_param_14];\n" " add.s32 %r537, %r3, 1;\n" " shl.b32 %r201, %r642, 4;\n" " add.s32 %r202, %r35, %r201;\n" " max.s32 %r203, %r202, -17;\n" " not.b32 %r204, %r203;\n" " max.s32 %r53, %r204, 1;\n" " sub.s32 %r54, %r537, %r643;\n" " min.s32 %r55, %r54, 16;\n" " setp.lt.s32 %p10, %r643, %r538;\n" " or.pred %p12, %p11, %p10;\n" " add.s32 %r205, %r55, %r643;\n" " setp.gt.s32 %p13, %r205, %r28;\n" " setp.le.s32 %p14, %r643, %r3;\n" " and.pred %p15, %p14, %p13;\n" " or.pred %p16, %p12, %p15;\n" " not.pred %p17, %p16;\n" " @%p17 bra $L__BB4_212;\n" "\n" " setp.lt.s32 %p18, %r54, 1;\n" " @%p18 bra $L__BB4_36;\n" "\n" " @%p19 bra $L__BB4_23;\n" " bra.uni $L__BB4_14;\n" "\n" "$L__BB4_23:\n" " mov.u32 %r647, 0;\n" "\n" "$L__BB4_24:\n" " cvt.s64.s32 %rd87, %r647;\n" " mul.lo.s64 %rd41, %rd87, %rd7;\n" " add.s32 %r65, %r647, %r643;\n" " setp.lt.s32 %p27, %r65, %r124;\n" " @%p27 bra $L__BB4_32;\n" " bra.uni $L__BB4_25;\n" "\n" "$L__BB4_32:\n" " @%p36 bra $L__BB4_35;\n" "\n" " cvt.u32.u64 %r264, %rd41;\n" " add.s32 %r72, %r5, %r264;\n" " cvt.s64.s32 %rd101, %r65;\n" " mul.lo.s64 %rd102, %rd101, %rd14;\n" " add.s64 %rd44, %rd102, %rd28;\n" " mov.u32 %r650, %r1;\n" "\n" "$L__BB4_34:\n" " cvt.s64.s32 %rd103, %r650;\n" " add.s64 %rd104, %rd44, %rd103;\n" " shl.b64 %rd105, %rd104, 1;\n" " add.s64 %rd106, %rd3, %rd105;\n" " ld.global.u16 %rs5, [%rd106];\n" " // begin inline asm\n" " { cvt.f32.f16 %f261, %rs5;}\n" "\n" " // end inline asm\n" " add.s32 %r265, %r72, %r650;\n" " shl.b32 %r266, %r265, 2;\n" " add.s32 %r268, %r149, %r266;\n" " st.shared.f32 [%r268], %f261;\n" " add.s32 %r650, %r650, %r30;\n" " setp.lt.s32 %p37, %r650, %r121;\n" " @%p37 bra $L__BB4_34;\n" " bra.uni $L__BB4_35;\n" "\n" "$L__BB4_25:\n" " setp.gt.s32 %p28, %r65, %r29;\n" " setp.gt.s32 %p29, %r65, %r4;\n" " and.pred %p30, %p29, %p28;\n" " @%p30 bra $L__BB4_29;\n" " bra.uni $L__BB4_26;\n" "\n" "$L__BB4_29:\n" " @%p36 bra $L__BB4_35;\n" "\n" " cvt.u32.u64 %r258, %rd41;\n" " add.s32 %r69, %r5, %r258;\n" " and.b32 %r259, %r65, 255;\n" " cvt.u64.u32 %rd95, %r259;\n" " mul.lo.s64 %rd96, %rd95, %rd14;\n" " add.s64 %rd43, %rd96, %rd27;\n" " mov.u32 %r649, %r1;\n" "\n" "$L__BB4_31:\n" " cvt.s64.s32 %rd97, %r649;\n" " add.s64 %rd98, %rd43, %rd97;\n" " shl.b64 %rd99, %rd98, 1;\n" " add.s64 %rd100, %rd36, %rd99;\n" " ld.global.u16 %rs4, [%rd100];\n" " // begin inline asm\n" " { cvt.f32.f16 %f260, %rs4;}\n" "\n" " // end inline asm\n" " add.s32 %r260, %r69, %r649;\n" " shl.b32 %r261, %r260, 2;\n" " add.s32 %r263, %r149, %r261;\n" " st.shared.f32 [%r263], %f260;\n" " add.s32 %r649, %r649, %r30;\n" " setp.lt.s32 %p35, %r649, %r121;\n" " @%p35 bra $L__BB4_31;\n" " bra.uni $L__BB4_35;\n" "\n" "$L__BB4_26:\n" " @%p36 bra $L__BB4_35;\n" "\n" " cvt.u32.u64 %r236, %rd41;\n" " add.s32 %r66, %r5, %r236;\n" " cvt.s64.s32 %rd88, %r65;\n" " mul.lo.s64 %rd89, %rd88, %rd25;\n" " add.s64 %rd42, %rd89, %rd26;\n" " mov.u32 %r648, %r1;\n" "\n" "$L__BB4_28:\n" " shr.s32 %r237, %r648, 31;\n" " shr.u32 %r238, %r237, 27;\n" " add.s32 %r239, %r648, %r238;\n" " shr.s32 %r240, %r239, 5;\n" " mul.lo.s32 %r241, %r240, 18;\n" " cvt.s64.s32 %rd90, %r241;\n" " add.s64 %rd91, %rd42, %rd90;\n" " add.s64 %rd92, %rd4, %rd91;\n" " ld.global.u8 %r242, [%rd92];\n" " ld.global.u8 %r243, [%rd92+1];\n" " prmt.b32 %r244, %r243, %r242, 30212;\n" " cvt.u16.u32 %rs3, %r244;\n" " // begin inline asm\n" " { cvt.f32.f16 %f257, %rs3;}\n" "\n" " // end inline asm\n" " and.b32 %r245, %r648, 31;\n" " setp.lt.u32 %p32, %r245, 16;\n" " add.s32 %r246, %r245, -16;\n" " selp.b32 %r247, %r245, %r246, %p32;\n" " selp.b32 %r248, 0, 4, %p32;\n" " add.s32 %r249, %r247, 2;\n" " cvt.s64.s32 %rd93, %r249;\n" " add.s64 %rd94, %rd92, %rd93;\n" " ld.global.u8 %r250, [%rd94];\n" " shr.u32 %r251, %r250, %r248;\n" " and.b32 %r252, %r251, 15;\n" " add.s32 %r253, %r252, -8;\n" " cvt.rn.f32.s32 %f258, %r253;\n" " mul.f32 %f259, %f257, %f258;\n" " add.s32 %r254, %r66, %r648;\n" " shl.b32 %r255, %r254, 2;\n" " add.s32 %r257, %r149, %r255;\n" " st.shared.f32 [%r257], %f259;\n" " add.s32 %r648, %r648, %r30;\n" " setp.lt.s32 %p33, %r648, %r121;\n" " @%p33 bra $L__BB4_28;\n" "\n" "$L__BB4_35:\n" " add.s32 %r647, %r647, 1;\n" " setp.lt.s32 %p38, %r647, %r55;\n" " @%p38 bra $L__BB4_24;\n" " bra.uni $L__BB4_36;\n" "\n" "$L__BB4_14:\n" " mov.u32 %r644, 0;\n" "\n" "$L__BB4_15:\n" " cvt.s64.s32 %rd75, %r644;\n" " mul.lo.s64 %rd37, %rd75, %rd7;\n" " add.s32 %r207, %r644, %r643;\n" " setp.lt.s32 %p20, %r207, %r124;\n" " cvt.s64.s32 %rd38, %r207;\n" " @%p20 bra $L__BB4_19;\n" " bra.uni $L__BB4_16;\n" "\n" "$L__BB4_19:\n" " @%p36 bra $L__BB4_22;\n" "\n" " cvt.u32.u64 %r230, %rd37;\n" " add.s32 %r60, %r5, %r230;\n" " mul.lo.s64 %rd82, %rd38, %rd14;\n" " add.s64 %rd40, %rd82, %rd28;\n" " mov.u32 %r646, %r1;\n" "\n" "$L__BB4_21:\n" " cvt.s64.s32 %rd83, %r646;\n" " add.s64 %rd84, %rd40, %rd83;\n" " shl.b64 %rd85, %rd84, 1;\n" " add.s64 %rd86, %rd3, %rd85;\n" " ld.global.u16 %rs2, [%rd86];\n" " // begin inline asm\n" " { cvt.f32.f16 %f256, %rs2;}\n" "\n" " // end inline asm\n" " add.s32 %r231, %r60, %r646;\n" " shl.b32 %r232, %r231, 2;\n" " add.s32 %r234, %r149, %r232;\n" " st.shared.f32 [%r234], %f256;\n" " add.s32 %r646, %r646, %r30;\n" " setp.lt.s32 %p25, %r646, %r121;\n" " @%p25 bra $L__BB4_21;\n" " bra.uni $L__BB4_22;\n" "\n" "$L__BB4_16:\n" " @%p36 bra $L__BB4_22;\n" "\n" " cvt.u32.u64 %r208, %rd37;\n" " add.s32 %r57, %r5, %r208;\n" " mul.lo.s64 %rd76, %rd38, %rd25;\n" " add.s64 %rd39, %rd76, %rd26;\n" " mov.u32 %r645, %r1;\n" "\n" "$L__BB4_18:\n" " shr.s32 %r209, %r645, 31;\n" " shr.u32 %r210, %r209, 27;\n" " add.s32 %r211, %r645, %r210;\n" " shr.s32 %r212, %r211, 5;\n" " mul.lo.s32 %r213, %r212, 18;\n" " cvt.s64.s32 %rd77, %r213;\n" " add.s64 %rd78, %rd39, %rd77;\n" " add.s64 %rd79, %rd4, %rd78;\n" " ld.global.u8 %r214, [%rd79];\n" " ld.global.u8 %r215, [%rd79+1];\n" " prmt.b32 %r216, %r215, %r214, 30212;\n" " cvt.u16.u32 %rs1, %r216;\n" " // begin inline asm\n" " { cvt.f32.f16 %f253, %rs1;}\n" "\n" " // end inline asm\n" " and.b32 %r217, %r645, 31;\n" " setp.lt.u32 %p22, %r217, 16;\n" " add.s32 %r218, %r217, -16;\n" " selp.b32 %r219, %r217, %r218, %p22;\n" " selp.b32 %r220, 0, 4, %p22;\n" " add.s32 %r221, %r219, 2;\n" " cvt.s64.s32 %rd80, %r221;\n" " add.s64 %rd81, %rd79, %rd80;\n" " ld.global.u8 %r222, [%rd81];\n" " shr.u32 %r223, %r222, %r220;\n" " and.b32 %r224, %r223, 15;\n" " add.s32 %r225, %r224, -8;\n" " cvt.rn.f32.s32 %f254, %r225;\n" " mul.f32 %f255, %f253, %f254;\n" " add.s32 %r226, %r57, %r645;\n" " shl.b32 %r227, %r226, 2;\n" " add.s32 %r229, %r149, %r227;\n" " st.shared.f32 [%r229], %f255;\n" " add.s32 %r645, %r645, %r30;\n" " setp.lt.s32 %p23, %r645, %r121;\n" " @%p23 bra $L__BB4_18;\n" "\n" "$L__BB4_22:\n" " add.s32 %r644, %r644, 1;\n" " setp.lt.s32 %p26, %r644, %r55;\n" " @%p26 bra $L__BB4_15;\n" "\n" "$L__BB4_36:\n" " setp.lt.s32 %p39, %r121, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p40, %r26, %r55;\n" " or.pred %p41, %p39, %p40;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p41 bra $L__BB4_89;\n" "\n" " and.b32 %r539, %r121, -16;\n" " setp.eq.s32 %p42, %r539, 16;\n" " mov.u32 %r653, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p42 bra $L__BB4_72;\n" "\n" " mov.u32 %r653, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " mov.u32 %r652, %r38;\n" "\n" "$L__BB4_39:\n" " shl.b32 %r271, %r653, 4;\n" " add.s32 %r78, %r271, %r27;\n" " add.s32 %r272, %r31, %r78;\n" " shl.b32 %r273, %r272, 2;\n" " add.s32 %r79, %r149, %r273;\n" " ld.shared.f32 %f17, [%r79];\n" " @%p2 bra $L__BB4_41;\n" "\n" " shl.b32 %r626, %r653, 4;\n" " add.s32 %r625, %r626, %r27;\n" " shl.b32 %r624, %r625, 2;\n" " add.s32 %r623, %r149, %r624;\n" " ld.shared.f32 %f286, [%r623];\n" " fma.rn.f32 %f634, %f17, %f286, %f634;\n" "\n" "$L__BB4_41:\n" " setp.lt.s32 %p44, %r120, 2;\n" " @%p44 bra $L__BB4_43;\n" "\n" " shl.b32 %r631, %r653, 4;\n" " add.s32 %r630, %r631, %r27;\n" " add.s32 %r629, %r630, %r121;\n" " shl.b32 %r628, %r629, 2;\n" " add.s32 %r627, %r149, %r628;\n" " ld.shared.f32 %f287, [%r627];\n" " fma.rn.f32 %f635, %f17, %f287, %f635;\n" "\n" "$L__BB4_43:\n" " setp.lt.s32 %p45, %r120, 3;\n" " @%p45 bra $L__BB4_45;\n" "\n" " shl.b32 %r572, %r653, 4;\n" " add.s32 %r571, %r572, %r27;\n" " add.s32 %r570, %r39, %r571;\n" " shl.b32 %r569, %r570, 2;\n" " add.s32 %r568, %r149, %r569;\n" " ld.shared.f32 %f288, [%r568];\n" " fma.rn.f32 %f636, %f17, %f288, %f636;\n" "\n" "$L__BB4_45:\n" " setp.lt.s32 %p46, %r120, 4;\n" " @%p46 bra $L__BB4_47;\n" "\n" " shl.b32 %r582, %r653, 4;\n" " add.s32 %r581, %r582, %r27;\n" " add.s32 %r580, %r40, %r581;\n" " shl.b32 %r579, %r580, 2;\n" " add.s32 %r578, %r149, %r579;\n" " ld.shared.f32 %f289, [%r578];\n" " fma.rn.f32 %f637, %f17, %f289, %f637;\n" "\n" "$L__BB4_47:\n" " setp.lt.s32 %p47, %r120, 5;\n" " @%p47 bra $L__BB4_49;\n" "\n" " shl.b32 %r592, %r653, 4;\n" " add.s32 %r591, %r592, %r27;\n" " add.s32 %r590, %r41, %r591;\n" " shl.b32 %r589, %r590, 2;\n" " add.s32 %r588, %r149, %r589;\n" " ld.shared.f32 %f290, [%r588];\n" " fma.rn.f32 %f638, %f17, %f290, %f638;\n" "\n" "$L__BB4_49:\n" " setp.lt.s32 %p48, %r120, 6;\n" " @%p48 bra $L__BB4_51;\n" "\n" " shl.b32 %r602, %r653, 4;\n" " add.s32 %r601, %r602, %r27;\n" " add.s32 %r600, %r42, %r601;\n" " shl.b32 %r599, %r600, 2;\n" " add.s32 %r598, %r149, %r599;\n" " ld.shared.f32 %f291, [%r598];\n" " fma.rn.f32 %f639, %f17, %f291, %f639;\n" "\n" "$L__BB4_51:\n" " setp.lt.s32 %p49, %r120, 7;\n" " @%p49 bra $L__BB4_53;\n" "\n" " shl.b32 %r612, %r653, 4;\n" " add.s32 %r611, %r612, %r27;\n" " add.s32 %r610, %r43, %r611;\n" " shl.b32 %r609, %r610, 2;\n" " add.s32 %r608, %r149, %r609;\n" " ld.shared.f32 %f292, [%r608];\n" " fma.rn.f32 %f640, %f17, %f292, %f640;\n" "\n" "$L__BB4_53:\n" " setp.lt.s32 %p50, %r120, 8;\n" " @%p50 bra $L__BB4_55;\n" "\n" " shl.b32 %r622, %r653, 4;\n" " add.s32 %r621, %r622, %r27;\n" " add.s32 %r620, %r44, %r621;\n" " shl.b32 %r619, %r620, 2;\n" " add.s32 %r618, %r149, %r619;\n" " ld.shared.f32 %f293, [%r618];\n" " fma.rn.f32 %f641, %f17, %f293, %f641;\n" "\n" "$L__BB4_55:\n" " shl.b32 %r553, %r653, 4;\n" " add.s32 %r552, %r553, %r27;\n" " add.s32 %r551, %r31, %r552;\n" " shl.b32 %r550, %r551, 2;\n" " add.s32 %r549, %r149, %r550;\n" " ld.shared.f32 %f34, [%r549+64];\n" " @%p2 bra $L__BB4_57;\n" "\n" " shl.b32 %r567, %r653, 4;\n" " add.s32 %r566, %r567, %r27;\n" " shl.b32 %r565, %r566, 2;\n" " add.s32 %r564, %r149, %r565;\n" " ld.shared.f32 %f294, [%r564+64];\n" " fma.rn.f32 %f634, %f34, %f294, %f634;\n" "\n" "$L__BB4_57:\n" " setp.lt.s32 %p184, %r120, 2;\n" " @%p184 bra $L__BB4_59;\n" "\n" " shl.b32 %r563, %r653, 4;\n" " add.s32 %r562, %r563, %r27;\n" " add.s32 %r561, %r562, %r121;\n" " shl.b32 %r560, %r561, 2;\n" " add.s32 %r559, %r149, %r560;\n" " ld.shared.f32 %f295, [%r559+64];\n" " fma.rn.f32 %f635, %f34, %f295, %f635;\n" "\n" "$L__BB4_59:\n" " setp.lt.s32 %p185, %r120, 3;\n" " @%p185 bra $L__BB4_61;\n" "\n" " shl.b32 %r558, %r653, 4;\n" " add.s32 %r557, %r558, %r27;\n" " add.s32 %r556, %r39, %r557;\n" " shl.b32 %r555, %r556, 2;\n" " add.s32 %r554, %r149, %r555;\n" " ld.shared.f32 %f296, [%r554+64];\n" " fma.rn.f32 %f636, %f34, %f296, %f636;\n" "\n" "$L__BB4_61:\n" " setp.lt.s32 %p186, %r120, 4;\n" " @%p186 bra $L__BB4_63;\n" "\n" " shl.b32 %r577, %r653, 4;\n" " add.s32 %r576, %r577, %r27;\n" " add.s32 %r575, %r40, %r576;\n" " shl.b32 %r574, %r575, 2;\n" " add.s32 %r573, %r149, %r574;\n" " ld.shared.f32 %f297, [%r573+64];\n" " fma.rn.f32 %f637, %f34, %f297, %f637;\n" "\n" "$L__BB4_63:\n" " setp.lt.s32 %p187, %r120, 5;\n" " @%p187 bra $L__BB4_65;\n" "\n" " shl.b32 %r587, %r653, 4;\n" " add.s32 %r586, %r587, %r27;\n" " add.s32 %r585, %r41, %r586;\n" " shl.b32 %r584, %r585, 2;\n" " add.s32 %r583, %r149, %r584;\n" " ld.shared.f32 %f298, [%r583+64];\n" " fma.rn.f32 %f638, %f34, %f298, %f638;\n" "\n" "$L__BB4_65:\n" " setp.lt.s32 %p188, %r120, 6;\n" " @%p188 bra $L__BB4_67;\n" "\n" " shl.b32 %r597, %r653, 4;\n" " add.s32 %r596, %r597, %r27;\n" " add.s32 %r595, %r42, %r596;\n" " shl.b32 %r594, %r595, 2;\n" " add.s32 %r593, %r149, %r594;\n" " ld.shared.f32 %f299, [%r593+64];\n" " fma.rn.f32 %f639, %f34, %f299, %f639;\n" "\n" "$L__BB4_67:\n" " setp.lt.s32 %p189, %r120, 7;\n" " @%p189 bra $L__BB4_69;\n" "\n" " shl.b32 %r607, %r653, 4;\n" " add.s32 %r606, %r607, %r27;\n" " add.s32 %r605, %r43, %r606;\n" " shl.b32 %r604, %r605, 2;\n" " add.s32 %r603, %r149, %r604;\n" " ld.shared.f32 %f300, [%r603+64];\n" " fma.rn.f32 %f640, %f34, %f300, %f640;\n" "\n" "$L__BB4_69:\n" " setp.lt.s32 %p190, %r120, 8;\n" " @%p190 bra $L__BB4_71;\n" "\n" " shl.b32 %r617, %r653, 4;\n" " add.s32 %r616, %r617, %r27;\n" " add.s32 %r615, %r44, %r616;\n" " shl.b32 %r614, %r615, 2;\n" " add.s32 %r613, %r149, %r614;\n" " ld.shared.f32 %f301, [%r613+64];\n" " fma.rn.f32 %f641, %f34, %f301, %f641;\n" "\n" "$L__BB4_71:\n" " add.s32 %r653, %r653, 2;\n" " add.s32 %r652, %r652, -2;\n" " setp.ne.s32 %p59, %r652, 0;\n" " @%p59 bra $L__BB4_39;\n" "\n" "$L__BB4_72:\n" " setp.eq.s32 %p60, %r37, 0;\n" " @%p60 bra $L__BB4_89;\n" "\n" " shl.b32 %r297, %r653, 4;\n" " add.s32 %r91, %r297, %r27;\n" " add.s32 %r298, %r31, %r91;\n" " shl.b32 %r299, %r298, 2;\n" " add.s32 %r301, %r149, %r299;\n" " ld.shared.f32 %f59, [%r301];\n" " @%p2 bra $L__BB4_75;\n" "\n" " shl.b32 %r302, %r91, 2;\n" " add.s32 %r304, %r149, %r302;\n" " ld.shared.f32 %f302, [%r304];\n" " fma.rn.f32 %f634, %f59, %f302, %f634;\n" "\n" "$L__BB4_75:\n" " setp.lt.s32 %p62, %r120, 2;\n" " @%p62 bra $L__BB4_77;\n" "\n" " add.s32 %r305, %r91, %r121;\n" " shl.b32 %r306, %r305, 2;\n" " add.s32 %r308, %r149, %r306;\n" " ld.shared.f32 %f303, [%r308];\n" " fma.rn.f32 %f635, %f59, %f303, %f635;\n" "\n" "$L__BB4_77:\n" " setp.lt.s32 %p63, %r120, 3;\n" " @%p63 bra $L__BB4_79;\n" "\n" " add.s32 %r309, %r39, %r91;\n" " shl.b32 %r310, %r309, 2;\n" " add.s32 %r312, %r149, %r310;\n" " ld.shared.f32 %f304, [%r312];\n" " fma.rn.f32 %f636, %f59, %f304, %f636;\n" "\n" "$L__BB4_79:\n" " setp.lt.s32 %p64, %r120, 4;\n" " @%p64 bra $L__BB4_81;\n" "\n" " add.s32 %r313, %r40, %r91;\n" " shl.b32 %r314, %r313, 2;\n" " add.s32 %r316, %r149, %r314;\n" " ld.shared.f32 %f305, [%r316];\n" " fma.rn.f32 %f637, %f59, %f305, %f637;\n" "\n" "$L__BB4_81:\n" " setp.lt.s32 %p65, %r120, 5;\n" " @%p65 bra $L__BB4_83;\n" "\n" " add.s32 %r317, %r41, %r91;\n" " shl.b32 %r318, %r317, 2;\n" " add.s32 %r320, %r149, %r318;\n" " ld.shared.f32 %f306, [%r320];\n" " fma.rn.f32 %f638, %f59, %f306, %f638;\n" "\n" "$L__BB4_83:\n" " setp.lt.s32 %p66, %r120, 6;\n" " @%p66 bra $L__BB4_85;\n" "\n" " add.s32 %r321, %r42, %r91;\n" " shl.b32 %r322, %r321, 2;\n" " add.s32 %r324, %r149, %r322;\n" " ld.shared.f32 %f307, [%r324];\n" " fma.rn.f32 %f639, %f59, %f307, %f639;\n" "\n" "$L__BB4_85:\n" " setp.lt.s32 %p67, %r120, 7;\n" " @%p67 bra $L__BB4_87;\n" "\n" " add.s32 %r325, %r43, %r91;\n" " shl.b32 %r326, %r325, 2;\n" " add.s32 %r328, %r149, %r326;\n" " ld.shared.f32 %f308, [%r328];\n" " fma.rn.f32 %f640, %f59, %f308, %f640;\n" "\n" "$L__BB4_87:\n" " setp.lt.s32 %p68, %r120, 8;\n" " @%p68 bra $L__BB4_89;\n" "\n" " add.s32 %r329, %r44, %r91;\n" " shl.b32 %r330, %r329, 2;\n" " add.s32 %r332, %r149, %r330;\n" " ld.shared.f32 %f309, [%r332];\n" " fma.rn.f32 %f641, %f59, %f309, %f641;\n" "\n" "$L__BB4_89:\n" " setp.eq.s32 %p69, %r27, 0;\n" " setp.lt.s32 %p70, %r26, %r55;\n" " and.pred %p1, %p69, %p70;\n" " @%p2 bra $L__BB4_92;\n" "\n" " mov.b32 %r333, %f634;\n" " mov.u32 %r334, 2;\n" " mov.u32 %r335, 4127;\n" " mov.u32 %r336, 8;\n" " mov.u32 %r337, -1;\n" " shfl.sync.down.b32 %r338|%p72, %r333, %r336, %r335, %r337;\n" " mov.b32 %f310, %r338;\n" " add.f32 %f311, %f634, %f310;\n" " mov.b32 %r339, %f311;\n" " mov.u32 %r340, 4;\n" " shfl.sync.down.b32 %r341|%p73, %r339, %r340, %r335, %r337;\n" " mov.b32 %f312, %r341;\n" " add.f32 %f313, %f311, %f312;\n" " mov.b32 %r342, %f313;\n" " shfl.sync.down.b32 %r343|%p74, %r342, %r334, %r335, %r337;\n" " mov.b32 %f314, %r343;\n" " add.f32 %f315, %f313, %f314;\n" " mov.b32 %r344, %f315;\n" " mov.u32 %r345, 1;\n" " shfl.sync.down.b32 %r346|%p75, %r344, %r345, %r335, %r337;\n" " mov.b32 %f316, %r346;\n" " add.f32 %f83, %f315, %f316;\n" " not.pred %p76, %p1;\n" " @%p76 bra $L__BB4_92;\n" "\n" " mul.f32 %f317, %f83, %f231;\n" " st.shared.f32 [%r45], %f317;\n" "\n" "$L__BB4_92:\n" " setp.lt.s32 %p77, %r120, 2;\n" " @%p77 bra $L__BB4_95;\n" "\n" " mov.b32 %r347, %f635;\n" " mov.u32 %r348, 2;\n" " mov.u32 %r349, 4127;\n" " mov.u32 %r350, 8;\n" " mov.u32 %r351, -1;\n" " shfl.sync.down.b32 %r352|%p78, %r347, %r350, %r349, %r351;\n" " mov.b32 %f318, %r352;\n" " add.f32 %f319, %f635, %f318;\n" " mov.b32 %r353, %f319;\n" " mov.u32 %r354, 4;\n" " shfl.sync.down.b32 %r355|%p79, %r353, %r354, %r349, %r351;\n" " mov.b32 %f320, %r355;\n" " add.f32 %f321, %f319, %f320;\n" " mov.b32 %r356, %f321;\n" " shfl.sync.down.b32 %r357|%p80, %r356, %r348, %r349, %r351;\n" " mov.b32 %f322, %r357;\n" " add.f32 %f323, %f321, %f322;\n" " mov.b32 %r358, %f323;\n" " mov.u32 %r359, 1;\n" " shfl.sync.down.b32 %r360|%p81, %r358, %r359, %r349, %r351;\n" " mov.b32 %f324, %r360;\n" " add.f32 %f84, %f323, %f324;\n" " not.pred %p82, %p1;\n" " @%p82 bra $L__BB4_95;\n" "\n" " mul.f32 %f325, %f84, %f231;\n" " st.shared.f32 [%r45+64], %f325;\n" "\n" "$L__BB4_95:\n" " setp.lt.s32 %p83, %r120, 3;\n" " @%p83 bra $L__BB4_98;\n" "\n" " mov.b32 %r361, %f636;\n" " mov.u32 %r362, 2;\n" " mov.u32 %r363, 4127;\n" " mov.u32 %r364, 8;\n" " mov.u32 %r365, -1;\n" " shfl.sync.down.b32 %r366|%p84, %r361, %r364, %r363, %r365;\n" " mov.b32 %f326, %r366;\n" " add.f32 %f327, %f636, %f326;\n" " mov.b32 %r367, %f327;\n" " mov.u32 %r368, 4;\n" " shfl.sync.down.b32 %r369|%p85, %r367, %r368, %r363, %r365;\n" " mov.b32 %f328, %r369;\n" " add.f32 %f329, %f327, %f328;\n" " mov.b32 %r370, %f329;\n" " shfl.sync.down.b32 %r371|%p86, %r370, %r362, %r363, %r365;\n" " mov.b32 %f330, %r371;\n" " add.f32 %f331, %f329, %f330;\n" " mov.b32 %r372, %f331;\n" " mov.u32 %r373, 1;\n" " shfl.sync.down.b32 %r374|%p87, %r372, %r373, %r363, %r365;\n" " mov.b32 %f332, %r374;\n" " add.f32 %f85, %f331, %f332;\n" " not.pred %p88, %p1;\n" " @%p88 bra $L__BB4_98;\n" "\n" " mul.f32 %f333, %f85, %f231;\n" " st.shared.f32 [%r45+128], %f333;\n" "\n" "$L__BB4_98:\n" " setp.lt.s32 %p89, %r120, 4;\n" " @%p89 bra $L__BB4_101;\n" "\n" " mov.b32 %r375, %f637;\n" " mov.u32 %r376, 2;\n" " mov.u32 %r377, 4127;\n" " mov.u32 %r378, 8;\n" " mov.u32 %r379, -1;\n" " shfl.sync.down.b32 %r380|%p90, %r375, %r378, %r377, %r379;\n" " mov.b32 %f334, %r380;\n" " add.f32 %f335, %f637, %f334;\n" " mov.b32 %r381, %f335;\n" " mov.u32 %r382, 4;\n" " shfl.sync.down.b32 %r383|%p91, %r381, %r382, %r377, %r379;\n" " mov.b32 %f336, %r383;\n" " add.f32 %f337, %f335, %f336;\n" " mov.b32 %r384, %f337;\n" " shfl.sync.down.b32 %r385|%p92, %r384, %r376, %r377, %r379;\n" " mov.b32 %f338, %r385;\n" " add.f32 %f339, %f337, %f338;\n" " mov.b32 %r386, %f339;\n" " mov.u32 %r387, 1;\n" " shfl.sync.down.b32 %r388|%p93, %r386, %r387, %r377, %r379;\n" " mov.b32 %f340, %r388;\n" " add.f32 %f86, %f339, %f340;\n" " not.pred %p94, %p1;\n" " @%p94 bra $L__BB4_101;\n" "\n" " mul.f32 %f341, %f86, %f231;\n" " st.shared.f32 [%r45+192], %f341;\n" "\n" "$L__BB4_101:\n" " setp.lt.s32 %p95, %r120, 5;\n" " @%p95 bra $L__BB4_104;\n" "\n" " mov.b32 %r389, %f638;\n" " mov.u32 %r390, 2;\n" " mov.u32 %r391, 4127;\n" " mov.u32 %r392, 8;\n" " mov.u32 %r393, -1;\n" " shfl.sync.down.b32 %r394|%p96, %r389, %r392, %r391, %r393;\n" " mov.b32 %f342, %r394;\n" " add.f32 %f343, %f638, %f342;\n" " mov.b32 %r395, %f343;\n" " mov.u32 %r396, 4;\n" " shfl.sync.down.b32 %r397|%p97, %r395, %r396, %r391, %r393;\n" " mov.b32 %f344, %r397;\n" " add.f32 %f345, %f343, %f344;\n" " mov.b32 %r398, %f345;\n" " shfl.sync.down.b32 %r399|%p98, %r398, %r390, %r391, %r393;\n" " mov.b32 %f346, %r399;\n" " add.f32 %f347, %f345, %f346;\n" " mov.b32 %r400, %f347;\n" " mov.u32 %r401, 1;\n" " shfl.sync.down.b32 %r402|%p99, %r400, %r401, %r391, %r393;\n" " mov.b32 %f348, %r402;\n" " add.f32 %f87, %f347, %f348;\n" " not.pred %p100, %p1;\n" " @%p100 bra $L__BB4_104;\n" "\n" " mul.f32 %f349, %f87, %f231;\n" " st.shared.f32 [%r45+256], %f349;\n" "\n" "$L__BB4_104:\n" " setp.lt.s32 %p101, %r120, 6;\n" " @%p101 bra $L__BB4_107;\n" "\n" " mov.b32 %r403, %f639;\n" " mov.u32 %r404, 2;\n" " mov.u32 %r405, 4127;\n" " mov.u32 %r406, 8;\n" " mov.u32 %r407, -1;\n" " shfl.sync.down.b32 %r408|%p102, %r403, %r406, %r405, %r407;\n" " mov.b32 %f350, %r408;\n" " add.f32 %f351, %f639, %f350;\n" " mov.b32 %r409, %f351;\n" " mov.u32 %r410, 4;\n" " shfl.sync.down.b32 %r411|%p103, %r409, %r410, %r405, %r407;\n" " mov.b32 %f352, %r411;\n" " add.f32 %f353, %f351, %f352;\n" " mov.b32 %r412, %f353;\n" " shfl.sync.down.b32 %r413|%p104, %r412, %r404, %r405, %r407;\n" " mov.b32 %f354, %r413;\n" " add.f32 %f355, %f353, %f354;\n" " mov.b32 %r414, %f355;\n" " mov.u32 %r415, 1;\n" " shfl.sync.down.b32 %r416|%p105, %r414, %r415, %r405, %r407;\n" " mov.b32 %f356, %r416;\n" " add.f32 %f88, %f355, %f356;\n" " not.pred %p106, %p1;\n" " @%p106 bra $L__BB4_107;\n" "\n" " mul.f32 %f357, %f88, %f231;\n" " st.shared.f32 [%r45+320], %f357;\n" "\n" "$L__BB4_107:\n" " setp.lt.s32 %p107, %r120, 7;\n" " @%p107 bra $L__BB4_110;\n" "\n" " mov.b32 %r417, %f640;\n" " mov.u32 %r418, 2;\n" " mov.u32 %r419, 4127;\n" " mov.u32 %r420, 8;\n" " mov.u32 %r421, -1;\n" " shfl.sync.down.b32 %r422|%p108, %r417, %r420, %r419, %r421;\n" " mov.b32 %f358, %r422;\n" " add.f32 %f359, %f640, %f358;\n" " mov.b32 %r423, %f359;\n" " mov.u32 %r424, 4;\n" " shfl.sync.down.b32 %r425|%p109, %r423, %r424, %r419, %r421;\n" " mov.b32 %f360, %r425;\n" " add.f32 %f361, %f359, %f360;\n" " mov.b32 %r426, %f361;\n" " shfl.sync.down.b32 %r427|%p110, %r426, %r418, %r419, %r421;\n" " mov.b32 %f362, %r427;\n" " add.f32 %f363, %f361, %f362;\n" " mov.b32 %r428, %f363;\n" " mov.u32 %r429, 1;\n" " shfl.sync.down.b32 %r430|%p111, %r428, %r429, %r419, %r421;\n" " mov.b32 %f364, %r430;\n" " add.f32 %f89, %f363, %f364;\n" " not.pred %p112, %p1;\n" " @%p112 bra $L__BB4_110;\n" "\n" " mul.f32 %f365, %f89, %f231;\n" " st.shared.f32 [%r45+384], %f365;\n" "\n" "$L__BB4_110:\n" " setp.lt.s32 %p113, %r120, 8;\n" " @%p113 bra $L__BB4_113;\n" "\n" " mov.b32 %r431, %f641;\n" " mov.u32 %r432, 2;\n" " mov.u32 %r433, 4127;\n" " mov.u32 %r434, 8;\n" " mov.u32 %r435, -1;\n" " shfl.sync.down.b32 %r436|%p114, %r431, %r434, %r433, %r435;\n" " mov.b32 %f366, %r436;\n" " add.f32 %f367, %f641, %f366;\n" " mov.b32 %r437, %f367;\n" " mov.u32 %r438, 4;\n" " shfl.sync.down.b32 %r439|%p115, %r437, %r438, %r433, %r435;\n" " mov.b32 %f368, %r439;\n" " add.f32 %f369, %f367, %f368;\n" " mov.b32 %r440, %f369;\n" " shfl.sync.down.b32 %r441|%p116, %r440, %r432, %r433, %r435;\n" " mov.b32 %f370, %r441;\n" " add.f32 %f371, %f369, %f370;\n" " mov.b32 %r442, %f371;\n" " mov.u32 %r443, 1;\n" " shfl.sync.down.b32 %r444|%p117, %r442, %r443, %r433, %r435;\n" " mov.b32 %f372, %r444;\n" " add.f32 %f90, %f371, %f372;\n" " not.pred %p118, %p1;\n" " @%p118 bra $L__BB4_113;\n" "\n" " mul.f32 %f373, %f90, %f231;\n" " st.shared.f32 [%r45+448], %f373;\n" "\n" "$L__BB4_113:\n" " bar.sync 0;\n" " @%p7 bra $L__BB4_131;\n" "\n" " ld.shared.f32 %f91, [%r23];\n" " mov.f32 %f676, %f91;\n" " @%p18 bra $L__BB4_122;\n" "\n" " and.b32 %r92, %r53, 3;\n" " add.s32 %r446, %r53, -1;\n" " setp.lt.u32 %p121, %r446, 3;\n" " mov.u32 %r656, 0;\n" " mov.f32 %f676, %f91;\n" " @%p121 bra $L__BB4_118;\n" "\n" " sub.s32 %r655, %r53, %r92;\n" " mov.u32 %r656, 0;\n" " mov.f32 %f676, %f91;\n" "\n" "$L__BB4_117:\n" " add.s32 %r448, %r32, %r656;\n" " shl.b32 %r449, %r448, 2;\n" " add.s32 %r451, %r149, %r449;\n" " ld.shared.f32 %f375, [%r451];\n" " max.f32 %f376, %f676, %f375;\n" " ld.shared.f32 %f377, [%r451+4];\n" " max.f32 %f378, %f376, %f377;\n" " ld.shared.f32 %f379, [%r451+8];\n" " max.f32 %f380, %f378, %f379;\n" " ld.shared.f32 %f381, [%r451+12];\n" " max.f32 %f676, %f380, %f381;\n" " add.s32 %r656, %r656, 4;\n" " add.s32 %r655, %r655, -4;\n" " setp.ne.s32 %p122, %r655, 0;\n" " @%p122 bra $L__BB4_117;\n" "\n" "$L__BB4_118:\n" " setp.eq.s32 %p123, %r92, 0;\n" " @%p123 bra $L__BB4_122;\n" "\n" " add.s32 %r452, %r32, %r656;\n" " shl.b32 %r453, %r452, 2;\n" " add.s32 %r99, %r149, %r453;\n" " ld.shared.f32 %f382, [%r99];\n" " max.f32 %f676, %f676, %f382;\n" " setp.eq.s32 %p124, %r92, 1;\n" " @%p124 bra $L__BB4_122;\n" "\n" " ld.shared.f32 %f383, [%r99+4];\n" " max.f32 %f676, %f676, %f383;\n" " setp.eq.s32 %p125, %r92, 2;\n" " @%p125 bra $L__BB4_122;\n" "\n" " ld.shared.f32 %f384, [%r99+8];\n" " max.f32 %f676, %f676, %f384;\n" "\n" "$L__BB4_122:\n" " mov.f32 %f680, 0f00000000;\n" " @%p18 bra $L__BB4_130;\n" "\n" " and.b32 %r100, %r53, 3;\n" " add.s32 %r456, %r53, -1;\n" " setp.lt.u32 %p127, %r456, 3;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r660, 0;\n" " @%p127 bra $L__BB4_126;\n" "\n" " sub.s32 %r658, %r53, %r100;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r660, 0;\n" " mov.u32 %r659, %r47;\n" "\n" "$L__BB4_125:\n" " shl.b32 %r541, %r120, 2;\n" " add.s32 %r540, %r541, 1024;\n" " add.s32 %r458, %r32, %r660;\n" " shl.b32 %r459, %r458, 2;\n" " add.s32 %r461, %r149, %r459;\n" " ld.shared.f32 %f389, [%r461];\n" " sub.f32 %f390, %f389, %f676;\n" " mov.f32 %f391, 0f3F000000;\n" " mov.f32 %f392, 0f3BBB989D;\n" " fma.rn.f32 %f393, %f390, %f392, %f391;\n" " mov.f32 %f394, 0f3FB8AA3B;\n" " mov.f32 %f395, 0f437C0000;\n" " cvt.sat.f32.f32 %f396, %f393;\n" " mov.f32 %f397, 0f4B400001;\n" " fma.rm.f32 %f398, %f396, %f395, %f397;\n" " add.f32 %f399, %f398, 0fCB40007F;\n" " neg.f32 %f400, %f399;\n" " fma.rn.f32 %f401, %f390, %f394, %f400;\n" " mov.f32 %f402, 0f32A57060;\n" " fma.rn.f32 %f403, %f390, %f402, %f401;\n" " mov.b32 %r462, %f398;\n" " shl.b32 %r463, %r462, 23;\n" " mov.b32 %f404, %r463;\n" " ex2.approx.ftz.f32 %f405, %f403;\n" " mul.f32 %f406, %f405, %f404;\n" " st.shared.f32 [%r659+1024], %f406;\n" " add.f32 %f407, %f680, %f406;\n" " ld.shared.f32 %f408, [%r461+4];\n" " sub.f32 %f409, %f408, %f676;\n" " fma.rn.f32 %f410, %f409, %f392, %f391;\n" " cvt.sat.f32.f32 %f411, %f410;\n" " fma.rm.f32 %f412, %f411, %f395, %f397;\n" " add.f32 %f413, %f412, 0fCB40007F;\n" " neg.f32 %f414, %f413;\n" " fma.rn.f32 %f415, %f409, %f394, %f414;\n" " fma.rn.f32 %f416, %f409, %f402, %f415;\n" " mov.b32 %r464, %f412;\n" " shl.b32 %r465, %r464, 23;\n" " mov.b32 %f417, %r465;\n" " ex2.approx.ftz.f32 %f418, %f416;\n" " mul.f32 %f419, %f418, %f417;\n" " add.s32 %r466, %r659, %r541;\n" " st.shared.f32 [%r466+1024], %f419;\n" " add.f32 %f420, %f407, %f419;\n" " ld.shared.f32 %f421, [%r461+8];\n" " sub.f32 %f422, %f421, %f676;\n" " fma.rn.f32 %f423, %f422, %f392, %f391;\n" " cvt.sat.f32.f32 %f424, %f423;\n" " fma.rm.f32 %f425, %f424, %f395, %f397;\n" " add.f32 %f426, %f425, 0fCB40007F;\n" " neg.f32 %f427, %f426;\n" " fma.rn.f32 %f428, %f422, %f394, %f427;\n" " fma.rn.f32 %f429, %f422, %f402, %f428;\n" " mov.b32 %r467, %f425;\n" " shl.b32 %r468, %r467, 23;\n" " mov.b32 %f430, %r468;\n" " ex2.approx.ftz.f32 %f431, %f429;\n" " mul.f32 %f432, %f431, %f430;\n" " add.s32 %r469, %r466, %r541;\n" " add.s32 %r470, %r466, %r540;\n" " st.shared.f32 [%r470], %f432;\n" " add.f32 %f433, %f420, %f432;\n" " ld.shared.f32 %f434, [%r461+12];\n" " sub.f32 %f435, %f434, %f676;\n" " fma.rn.f32 %f436, %f435, %f392, %f391;\n" " cvt.sat.f32.f32 %f437, %f436;\n" " fma.rm.f32 %f438, %f437, %f395, %f397;\n" " add.f32 %f439, %f438, 0fCB40007F;\n" " neg.f32 %f440, %f439;\n" " fma.rn.f32 %f441, %f435, %f394, %f440;\n" " fma.rn.f32 %f442, %f435, %f402, %f441;\n" " mov.b32 %r471, %f438;\n" " shl.b32 %r472, %r471, 23;\n" " mov.b32 %f443, %r472;\n" " ex2.approx.ftz.f32 %f444, %f442;\n" " mul.f32 %f445, %f444, %f443;\n" " add.s32 %r473, %r469, %r541;\n" " add.s32 %r474, %r469, %r540;\n" " add.s32 %r659, %r473, %r541;\n" " st.shared.f32 [%r474], %f445;\n" " add.f32 %f680, %f433, %f445;\n" " add.s32 %r660, %r660, 4;\n" " add.s32 %r658, %r658, -4;\n" " setp.ne.s32 %p128, %r658, 0;\n" " @%p128 bra $L__BB4_125;\n" "\n" "$L__BB4_126:\n" " setp.eq.s32 %p129, %r100, 0;\n" " @%p129 bra $L__BB4_130;\n" "\n" " add.s32 %r475, %r32, %r660;\n" " shl.b32 %r476, %r475, 2;\n" " add.s32 %r109, %r149, %r476;\n" " ld.shared.f32 %f446, [%r109];\n" " sub.f32 %f447, %f446, %f676;\n" " mov.f32 %f448, 0f3F000000;\n" " mov.f32 %f449, 0f3BBB989D;\n" " fma.rn.f32 %f450, %f447, %f449, %f448;\n" " mov.f32 %f451, 0f3FB8AA3B;\n" " mov.f32 %f452, 0f437C0000;\n" " cvt.sat.f32.f32 %f453, %f450;\n" " mov.f32 %f454, 0f4B400001;\n" " fma.rm.f32 %f455, %f453, %f452, %f454;\n" " add.f32 %f456, %f455, 0fCB40007F;\n" " neg.f32 %f457, %f456;\n" " fma.rn.f32 %f458, %f447, %f451, %f457;\n" " mov.f32 %f459, 0f32A57060;\n" " fma.rn.f32 %f460, %f447, %f459, %f458;\n" " mov.b32 %r478, %f455;\n" " shl.b32 %r479, %r478, 23;\n" " mov.b32 %f461, %r479;\n" " ex2.approx.ftz.f32 %f462, %f460;\n" " mul.f32 %f463, %f462, %f461;\n" " cvt.u64.u32 %rd107, %r660;\n" " mul.lo.s64 %rd108, %rd107, %rd11;\n" " add.s64 %rd109, %rd34, %rd108;\n" " cvt.u32.u64 %r480, %rd109;\n" " add.s32 %r481, %r5, %r480;\n" " shl.b32 %r482, %r481, 2;\n" " add.s32 %r483, %r149, %r482;\n" " st.shared.f32 [%r483], %f463;\n" " add.f32 %f680, %f680, %f463;\n" " setp.eq.s32 %p130, %r100, 1;\n" " @%p130 bra $L__BB4_130;\n" "\n" " mov.f32 %f611, 0f32A57060;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f609, 0f4B400001;\n" " mov.f32 %f608, 0f437C0000;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f606, 0f3BBB989D;\n" " add.s32 %r533, %r32, %r660;\n" " shl.b32 %r532, %r533, 2;\n" " add.s32 %r531, %r149, %r532;\n" " ld.shared.f32 %f464, [%r531+4];\n" " sub.f32 %f465, %f464, %f676;\n" " fma.rn.f32 %f468, %f465, %f606, %f607;\n" " cvt.sat.f32.f32 %f471, %f468;\n" " fma.rm.f32 %f473, %f471, %f608, %f609;\n" " add.f32 %f474, %f473, 0fCB40007F;\n" " neg.f32 %f475, %f474;\n" " fma.rn.f32 %f476, %f465, %f610, %f475;\n" " fma.rn.f32 %f478, %f465, %f611, %f476;\n" " mov.b32 %r484, %f473;\n" " shl.b32 %r485, %r484, 23;\n" " mov.b32 %f479, %r485;\n" " ex2.approx.ftz.f32 %f480, %f478;\n" " mul.f32 %f481, %f480, %f479;\n" " add.s32 %r486, %r660, 1;\n" " cvt.u64.u32 %rd110, %r486;\n" " mul.lo.s64 %rd111, %rd110, %rd11;\n" " add.s64 %rd112, %rd34, %rd111;\n" " cvt.u32.u64 %r487, %rd112;\n" " add.s32 %r488, %r5, %r487;\n" " shl.b32 %r489, %r488, 2;\n" " add.s32 %r491, %r149, %r489;\n" " st.shared.f32 [%r491], %f481;\n" " add.f32 %f680, %f680, %f481;\n" " setp.eq.s32 %p131, %r100, 2;\n" " @%p131 bra $L__BB4_130;\n" "\n" " mov.f32 %f617, 0f32A57060;\n" " mov.f32 %f616, 0f3FB8AA3B;\n" " mov.f32 %f615, 0f4B400001;\n" " mov.f32 %f614, 0f437C0000;\n" " mov.f32 %f613, 0f3F000000;\n" " mov.f32 %f612, 0f3BBB989D;\n" " add.s32 %r536, %r32, %r660;\n" " shl.b32 %r535, %r536, 2;\n" " add.s32 %r534, %r149, %r535;\n" " ld.shared.f32 %f482, [%r534+8];\n" " sub.f32 %f483, %f482, %f676;\n" " fma.rn.f32 %f486, %f483, %f612, %f613;\n" " cvt.sat.f32.f32 %f489, %f486;\n" " fma.rm.f32 %f491, %f489, %f614, %f615;\n" " add.f32 %f492, %f491, 0fCB40007F;\n" " neg.f32 %f493, %f492;\n" " fma.rn.f32 %f494, %f483, %f616, %f493;\n" " fma.rn.f32 %f496, %f483, %f617, %f494;\n" " mov.b32 %r492, %f491;\n" " shl.b32 %r493, %r492, 23;\n" " mov.b32 %f497, %r493;\n" " ex2.approx.ftz.f32 %f498, %f496;\n" " mul.f32 %f499, %f498, %f497;\n" " add.s32 %r494, %r660, 2;\n" " cvt.u64.u32 %rd113, %r494;\n" " mul.lo.s64 %rd114, %rd113, %rd11;\n" " add.s64 %rd115, %rd34, %rd114;\n" " cvt.u32.u64 %r495, %rd115;\n" " add.s32 %r496, %r5, %r495;\n" " shl.b32 %r497, %r496, 2;\n" " add.s32 %r499, %r149, %r497;\n" " st.shared.f32 [%r499], %f499;\n" " add.f32 %f680, %f680, %f499;\n" "\n" "$L__BB4_130:\n" " sub.f32 %f500, %f91, %f676;\n" " mov.f32 %f501, 0f3F000000;\n" " mov.f32 %f502, 0f3BBB989D;\n" " fma.rn.f32 %f503, %f500, %f502, %f501;\n" " mov.f32 %f504, 0f3FB8AA3B;\n" " mov.f32 %f505, 0f437C0000;\n" " cvt.sat.f32.f32 %f506, %f503;\n" " mov.f32 %f507, 0f4B400001;\n" " fma.rm.f32 %f508, %f506, %f505, %f507;\n" " add.f32 %f509, %f508, 0fCB40007F;\n" " neg.f32 %f510, %f509;\n" " fma.rn.f32 %f511, %f500, %f504, %f510;\n" " mov.f32 %f512, 0f32A57060;\n" " fma.rn.f32 %f513, %f500, %f512, %f511;\n" " ex2.approx.ftz.f32 %f514, %f513;\n" " mov.b32 %r500, %f508;\n" " shl.b32 %r501, %r500, 23;\n" " mov.b32 %f515, %r501;\n" " mul.f32 %f516, %f514, %f515;\n" " ld.shared.f32 %f517, [%r24];\n" " fma.rn.f32 %f518, %f516, %f517, %f680;\n" " st.shared.f32 [%r24], %f518;\n" " st.shared.f32 [%r23], %f676;\n" " st.shared.f32 [%r34], %f516;\n" "\n" "$L__BB4_131:\n" " bar.sync 0;\n" " @%p18 bra $L__BB4_211;\n" "\n" " @%p19 bra $L__BB4_171;\n" " bra.uni $L__BB4_133;\n" "\n" "$L__BB4_171:\n" " mov.u32 %r662, 0;\n" "\n" "$L__BB4_172:\n" " add.s32 %r114, %r662, %r643;\n" " setp.lt.s32 %p153, %r114, %r124;\n" " @%p153 bra $L__BB4_176;\n" " bra.uni $L__BB4_173;\n" "\n" "$L__BB4_176:\n" " add.s32 %r635, %r662, %r643;\n" " cvt.s64.s32 %rd141, %r635;\n" " mul.lo.s64 %rd142, %rd141, %rd14;\n" " add.s64 %rd143, %rd30, %rd142;\n" " shl.b64 %rd144, %rd143, 1;\n" " add.s64 %rd145, %rd5, %rd144;\n" " ld.global.u16 %rs10, [%rd145];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs10;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB4_177;\n" "\n" "$L__BB4_173:\n" " add.s32 %r632, %r662, %r643;\n" " setp.gt.s32 %p154, %r632, %r29;\n" " setp.gt.s32 %p155, %r632, %r4;\n" " and.pred %p156, %p155, %p154;\n" " @%p156 bra $L__BB4_175;\n" " bra.uni $L__BB4_174;\n" "\n" "$L__BB4_175:\n" " add.s32 %r634, %r662, %r643;\n" " and.b32 %r523, %r634, 255;\n" " cvt.u64.u32 %rd136, %r523;\n" " mul.lo.s64 %rd137, %rd136, %rd14;\n" " add.s64 %rd138, %rd31, %rd137;\n" " shl.b64 %rd139, %rd138, 1;\n" " add.s64 %rd140, %rd35, %rd139;\n" " ld.global.u16 %rs9, [%rd140];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs9;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB4_177;\n" "\n" "$L__BB4_174:\n" " add.s32 %r633, %r662, %r643;\n" " cvt.s64.s32 %rd129, %r633;\n" " mul.lo.s64 %rd130, %rd129, %rd25;\n" " add.s64 %rd131, %rd130, %rd26;\n" " add.s64 %rd132, %rd131, %rd32;\n" " add.s64 %rd133, %rd6, %rd132;\n" " ld.global.u8 %r516, [%rd133];\n" " ld.global.u8 %r517, [%rd133+1];\n" " prmt.b32 %r518, %r517, %r516, 30212;\n" " cvt.u16.u32 %rs8, %r518;\n" " // begin inline asm\n" " { cvt.f32.f16 %f554, %rs8;}\n" "\n" " // end inline asm\n" " add.s64 %rd134, %rd33, %rd131;\n" " add.s64 %rd135, %rd6, %rd134;\n" " ld.global.u8 %r519, [%rd135];\n" " shr.u32 %r520, %r519, %r33;\n" " and.b32 %r521, %r520, 15;\n" " add.s32 %r522, %r521, -8;\n" " cvt.rn.f32.s32 %f555, %r522;\n" " mul.f32 %f720, %f554, %f555;\n" "\n" "$L__BB4_177:\n" " cvt.u64.u32 %rd146, %r662;\n" " mul.lo.s64 %rd147, %rd146, %rd11;\n" " add.s64 %rd148, %rd147, %rd9;\n" " cvt.u32.u64 %r524, %rd148;\n" " add.s32 %r525, %r5, %r524;\n" " shl.b32 %r526, %r525, 2;\n" " add.s32 %r115, %r149, %r526;\n" " setp.eq.s32 %p157, %r662, 0;\n" " @%p157 bra $L__BB4_194;\n" "\n" " @%p2 bra $L__BB4_180;\n" "\n" " ld.shared.f32 %f558, [%r115];\n" " fma.rn.f32 %f711, %f720, %f558, %f711;\n" "\n" "$L__BB4_180:\n" " @%p77 bra $L__BB4_182;\n" "\n" " ld.shared.f32 %f559, [%r115+4];\n" " fma.rn.f32 %f710, %f720, %f559, %f710;\n" "\n" "$L__BB4_182:\n" " @%p83 bra $L__BB4_184;\n" "\n" " ld.shared.f32 %f560, [%r115+8];\n" " fma.rn.f32 %f709, %f720, %f560, %f709;\n" "\n" "$L__BB4_184:\n" " @%p89 bra $L__BB4_186;\n" "\n" " ld.shared.f32 %f561, [%r115+12];\n" " fma.rn.f32 %f708, %f720, %f561, %f708;\n" "\n" "$L__BB4_186:\n" " @%p95 bra $L__BB4_188;\n" "\n" " ld.shared.f32 %f562, [%r115+16];\n" " fma.rn.f32 %f707, %f720, %f562, %f707;\n" "\n" "$L__BB4_188:\n" " @%p101 bra $L__BB4_190;\n" "\n" " ld.shared.f32 %f563, [%r115+20];\n" " fma.rn.f32 %f706, %f720, %f563, %f706;\n" "\n" "$L__BB4_190:\n" " @%p107 bra $L__BB4_192;\n" "\n" " ld.shared.f32 %f564, [%r115+24];\n" " fma.rn.f32 %f705, %f720, %f564, %f705;\n" "\n" "$L__BB4_192:\n" " @%p113 bra $L__BB4_210;\n" "\n" " ld.shared.f32 %f565, [%r115+28];\n" " fma.rn.f32 %f704, %f720, %f565, %f704;\n" " bra.uni $L__BB4_210;\n" "\n" "$L__BB4_194:\n" " @%p2 bra $L__BB4_196;\n" "\n" " ld.shared.f32 %f566, [%r46];\n" " ld.shared.f32 %f567, [%r115];\n" " mul.f32 %f568, %f720, %f567;\n" " fma.rn.f32 %f711, %f566, %f711, %f568;\n" "\n" "$L__BB4_196:\n" " @%p77 bra $L__BB4_198;\n" "\n" " ld.shared.f32 %f569, [%r46+4];\n" " ld.shared.f32 %f570, [%r115+4];\n" " mul.f32 %f571, %f720, %f570;\n" " fma.rn.f32 %f710, %f569, %f710, %f571;\n" "\n" "$L__BB4_198:\n" " @%p83 bra $L__BB4_200;\n" "\n" " ld.shared.f32 %f572, [%r46+8];\n" " ld.shared.f32 %f573, [%r115+8];\n" " mul.f32 %f574, %f720, %f573;\n" " fma.rn.f32 %f709, %f572, %f709, %f574;\n" "\n" "$L__BB4_200:\n" " @%p89 bra $L__BB4_202;\n" "\n" " ld.shared.f32 %f575, [%r46+12];\n" " ld.shared.f32 %f576, [%r115+12];\n" " mul.f32 %f577, %f720, %f576;\n" " fma.rn.f32 %f708, %f575, %f708, %f577;\n" "\n" "$L__BB4_202:\n" " @%p95 bra $L__BB4_204;\n" "\n" " ld.shared.f32 %f578, [%r46+16];\n" " ld.shared.f32 %f579, [%r115+16];\n" " mul.f32 %f580, %f720, %f579;\n" " fma.rn.f32 %f707, %f578, %f707, %f580;\n" "\n" "$L__BB4_204:\n" " @%p101 bra $L__BB4_206;\n" "\n" " ld.shared.f32 %f581, [%r46+20];\n" " ld.shared.f32 %f582, [%r115+20];\n" " mul.f32 %f583, %f720, %f582;\n" " fma.rn.f32 %f706, %f581, %f706, %f583;\n" "\n" "$L__BB4_206:\n" " @%p107 bra $L__BB4_208;\n" "\n" " ld.shared.f32 %f584, [%r46+24];\n" " ld.shared.f32 %f585, [%r115+24];\n" " mul.f32 %f586, %f720, %f585;\n" " fma.rn.f32 %f705, %f584, %f705, %f586;\n" "\n" "$L__BB4_208:\n" " @%p113 bra $L__BB4_210;\n" "\n" " ld.shared.f32 %f587, [%r46+28];\n" " ld.shared.f32 %f588, [%r115+28];\n" " mul.f32 %f589, %f720, %f588;\n" " fma.rn.f32 %f704, %f587, %f704, %f589;\n" "\n" "$L__BB4_210:\n" " add.s32 %r662, %r662, 1;\n" " setp.lt.s32 %p174, %r662, %r55;\n" " @%p174 bra $L__BB4_172;\n" " bra.uni $L__BB4_211;\n" "\n" "$L__BB4_133:\n" " mov.u32 %r661, 0;\n" "\n" "$L__BB4_134:\n" " add.s32 %r503, %r661, %r643;\n" " setp.lt.s32 %p134, %r503, %r124;\n" " cvt.s64.s32 %rd45, %r503;\n" " @%p134 bra $L__BB4_136;\n" " bra.uni $L__BB4_135;\n" "\n" "$L__BB4_136:\n" " mul.lo.s64 %rd122, %rd45, %rd14;\n" " add.s64 %rd123, %rd30, %rd122;\n" " shl.b64 %rd124, %rd123, 1;\n" " add.s64 %rd125, %rd5, %rd124;\n" " ld.global.u16 %rs7, [%rd125];\n" " // begin inline asm\n" " { cvt.f32.f16 %f689, %rs7;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB4_137;\n" "\n" "$L__BB4_135:\n" " mul.lo.s64 %rd116, %rd45, %rd25;\n" " add.s64 %rd117, %rd116, %rd26;\n" " add.s64 %rd118, %rd117, %rd32;\n" " add.s64 %rd119, %rd6, %rd118;\n" " ld.global.u8 %r504, [%rd119];\n" " ld.global.u8 %r505, [%rd119+1];\n" " prmt.b32 %r506, %r505, %r504, 30212;\n" " cvt.u16.u32 %rs6, %r506;\n" " // begin inline asm\n" " { cvt.f32.f16 %f519, %rs6;}\n" "\n" " // end inline asm\n" " add.s64 %rd120, %rd33, %rd117;\n" " add.s64 %rd121, %rd6, %rd120;\n" " ld.global.u8 %r507, [%rd121];\n" " shr.u32 %r508, %r507, %r33;\n" " and.b32 %r509, %r508, 15;\n" " add.s32 %r510, %r509, -8;\n" " cvt.rn.f32.s32 %f520, %r510;\n" " mul.f32 %f689, %f519, %f520;\n" "\n" "$L__BB4_137:\n" " cvt.u64.u32 %rd126, %r661;\n" " mul.lo.s64 %rd127, %rd126, %rd11;\n" " add.s64 %rd128, %rd127, %rd9;\n" " cvt.u32.u64 %r511, %rd128;\n" " add.s32 %r512, %r5, %r511;\n" " shl.b32 %r513, %r512, 2;\n" " add.s32 %r111, %r149, %r513;\n" " setp.eq.s32 %p135, %r661, 0;\n" " @%p135 bra $L__BB4_154;\n" "\n" " @%p2 bra $L__BB4_140;\n" "\n" " ld.shared.f32 %f522, [%r111];\n" " fma.rn.f32 %f711, %f689, %f522, %f711;\n" "\n" "$L__BB4_140:\n" " @%p77 bra $L__BB4_142;\n" "\n" " ld.shared.f32 %f523, [%r111+4];\n" " fma.rn.f32 %f710, %f689, %f523, %f710;\n" "\n" "$L__BB4_142:\n" " @%p83 bra $L__BB4_144;\n" "\n" " ld.shared.f32 %f524, [%r111+8];\n" " fma.rn.f32 %f709, %f689, %f524, %f709;\n" "\n" "$L__BB4_144:\n" " @%p89 bra $L__BB4_146;\n" "\n" " ld.shared.f32 %f525, [%r111+12];\n" " fma.rn.f32 %f708, %f689, %f525, %f708;\n" "\n" "$L__BB4_146:\n" " @%p95 bra $L__BB4_148;\n" "\n" " ld.shared.f32 %f526, [%r111+16];\n" " fma.rn.f32 %f707, %f689, %f526, %f707;\n" "\n" "$L__BB4_148:\n" " @%p101 bra $L__BB4_150;\n" "\n" " ld.shared.f32 %f527, [%r111+20];\n" " fma.rn.f32 %f706, %f689, %f527, %f706;\n" "\n" "$L__BB4_150:\n" " @%p107 bra $L__BB4_152;\n" "\n" " ld.shared.f32 %f528, [%r111+24];\n" " fma.rn.f32 %f705, %f689, %f528, %f705;\n" "\n" "$L__BB4_152:\n" " @%p113 bra $L__BB4_170;\n" "\n" " ld.shared.f32 %f529, [%r111+28];\n" " fma.rn.f32 %f704, %f689, %f529, %f704;\n" " bra.uni $L__BB4_170;\n" "\n" "$L__BB4_154:\n" " @%p2 bra $L__BB4_156;\n" "\n" " ld.shared.f32 %f530, [%r46];\n" " ld.shared.f32 %f531, [%r111];\n" " mul.f32 %f532, %f689, %f531;\n" " fma.rn.f32 %f711, %f530, %f711, %f532;\n" "\n" "$L__BB4_156:\n" " @%p77 bra $L__BB4_158;\n" "\n" " ld.shared.f32 %f533, [%r46+4];\n" " ld.shared.f32 %f534, [%r111+4];\n" " mul.f32 %f535, %f689, %f534;\n" " fma.rn.f32 %f710, %f533, %f710, %f535;\n" "\n" "$L__BB4_158:\n" " @%p83 bra $L__BB4_160;\n" "\n" " ld.shared.f32 %f536, [%r46+8];\n" " ld.shared.f32 %f537, [%r111+8];\n" " mul.f32 %f538, %f689, %f537;\n" " fma.rn.f32 %f709, %f536, %f709, %f538;\n" "\n" "$L__BB4_160:\n" " @%p89 bra $L__BB4_162;\n" "\n" " ld.shared.f32 %f539, [%r46+12];\n" " ld.shared.f32 %f540, [%r111+12];\n" " mul.f32 %f541, %f689, %f540;\n" " fma.rn.f32 %f708, %f539, %f708, %f541;\n" "\n" "$L__BB4_162:\n" " @%p95 bra $L__BB4_164;\n" "\n" " ld.shared.f32 %f542, [%r46+16];\n" " ld.shared.f32 %f543, [%r111+16];\n" " mul.f32 %f544, %f689, %f543;\n" " fma.rn.f32 %f707, %f542, %f707, %f544;\n" "\n" "$L__BB4_164:\n" " @%p101 bra $L__BB4_166;\n" "\n" " ld.shared.f32 %f545, [%r46+20];\n" " ld.shared.f32 %f546, [%r111+20];\n" " mul.f32 %f547, %f689, %f546;\n" " fma.rn.f32 %f706, %f545, %f706, %f547;\n" "\n" "$L__BB4_166:\n" " @%p107 bra $L__BB4_168;\n" "\n" " ld.shared.f32 %f548, [%r46+24];\n" " ld.shared.f32 %f549, [%r111+24];\n" " mul.f32 %f550, %f689, %f549;\n" " fma.rn.f32 %f705, %f548, %f705, %f550;\n" "\n" "$L__BB4_168:\n" " @%p113 bra $L__BB4_170;\n" "\n" " ld.shared.f32 %f551, [%r46+28];\n" " ld.shared.f32 %f552, [%r111+28];\n" " mul.f32 %f553, %f689, %f552;\n" " fma.rn.f32 %f704, %f551, %f704, %f553;\n" "\n" "$L__BB4_170:\n" " add.s32 %r661, %r661, 1;\n" " setp.lt.s32 %p152, %r661, %r55;\n" " @%p152 bra $L__BB4_134;\n" "\n" "$L__BB4_211:\n" " bar.sync 0;\n" "\n" "$L__BB4_212:\n" " add.s32 %r643, %r643, 16;\n" " setp.ge.s32 %p175, %r3, %r643;\n" " add.s32 %r642, %r642, 1;\n" " @%p175 bra $L__BB4_11;\n" "\n" "$L__BB4_213:\n" " ld.param.u32 %r548, [fc_attn_flash_prefill_q4_param_9];\n" " add.s32 %r547, %r548, 16;\n" " mul.wide.s32 %rd191, %r547, 16;\n" " shl.b32 %r546, %r120, 4;\n" " cvt.s64.s32 %rd190, %r546;\n" " add.s64 %rd189, %rd191, %rd190;\n" " cvt.s64.s32 %rd188, %r120;\n" " add.s64 %rd187, %rd189, %rd190;\n" " add.s64 %rd186, %rd187, %rd188;\n" " cvt.u32.u64 %r545, %rd186;\n" " cvt.s64.s32 %rd185, %r1;\n" " mov.u32 %r544, %ctaid.x;\n" " mov.u32 %r543, %nctaid.y;\n" " cvt.u64.u32 %rd184, %r543;\n" " cvt.s64.s32 %rd183, %r544;\n" " mov.u32 %r542, %ctaid.y;\n" " cvt.s64.s32 %rd182, %r542;\n" " mul.lo.s64 %rd181, %rd183, %rd184;\n" " add.s64 %rd180, %rd181, %rd182;\n" " mul.wide.s32 %rd179, %r548, %r120;\n" " mul.lo.s64 %rd178, %rd179, %rd180;\n" " add.s64 %rd46, %rd178, %rd185;\n" " add.s32 %r528, %r5, %r545;\n" " shl.b32 %r529, %r528, 2;\n" " add.s32 %r119, %r149, %r529;\n" " @%p2 bra $L__BB4_215;\n" "\n" " ld.shared.f32 %f590, [%r119];\n" " div.rn.f32 %f591, %f711, %f590;\n" " shl.b64 %rd149, %rd46, 2;\n" " add.s64 %rd150, %rd2, %rd149;\n" " st.global.f32 [%rd150], %f591;\n" "\n" "$L__BB4_215:\n" " setp.lt.s32 %p177, %r120, 2;\n" " @%p177 bra $L__BB4_217;\n" "\n" " ld.shared.f32 %f592, [%r119+4];\n" " div.rn.f32 %f593, %f710, %f592;\n" " add.s64 %rd151, %rd46, %rd14;\n" " shl.b64 %rd152, %rd151, 2;\n" " add.s64 %rd153, %rd2, %rd152;\n" " st.global.f32 [%rd153], %f593;\n" "\n" "$L__BB4_217:\n" " setp.lt.s32 %p178, %r120, 3;\n" " @%p178 bra $L__BB4_219;\n" "\n" " ld.shared.f32 %f594, [%r119+8];\n" " div.rn.f32 %f595, %f709, %f594;\n" " shl.b64 %rd154, %rd14, 1;\n" " add.s64 %rd155, %rd46, %rd154;\n" " shl.b64 %rd156, %rd155, 2;\n" " add.s64 %rd157, %rd2, %rd156;\n" " st.global.f32 [%rd157], %f595;\n" "\n" "$L__BB4_219:\n" " setp.lt.s32 %p179, %r120, 4;\n" " @%p179 bra $L__BB4_221;\n" "\n" " ld.shared.f32 %f596, [%r119+12];\n" " div.rn.f32 %f597, %f708, %f596;\n" " mul.lo.s64 %rd158, %rd14, 3;\n" " add.s64 %rd159, %rd46, %rd158;\n" " shl.b64 %rd160, %rd159, 2;\n" " add.s64 %rd161, %rd2, %rd160;\n" " st.global.f32 [%rd161], %f597;\n" "\n" "$L__BB4_221:\n" " setp.lt.s32 %p180, %r120, 5;\n" " @%p180 bra $L__BB4_223;\n" "\n" " ld.shared.f32 %f598, [%r119+16];\n" " div.rn.f32 %f599, %f707, %f598;\n" " shl.b64 %rd162, %rd14, 2;\n" " add.s64 %rd163, %rd46, %rd162;\n" " shl.b64 %rd164, %rd163, 2;\n" " add.s64 %rd165, %rd2, %rd164;\n" " st.global.f32 [%rd165], %f599;\n" "\n" "$L__BB4_223:\n" " setp.lt.s32 %p181, %r120, 6;\n" " @%p181 bra $L__BB4_225;\n" "\n" " ld.shared.f32 %f600, [%r119+20];\n" " div.rn.f32 %f601, %f706, %f600;\n" " mul.lo.s64 %rd166, %rd14, 5;\n" " add.s64 %rd167, %rd46, %rd166;\n" " shl.b64 %rd168, %rd167, 2;\n" " add.s64 %rd169, %rd2, %rd168;\n" " st.global.f32 [%rd169], %f601;\n" "\n" "$L__BB4_225:\n" " setp.lt.s32 %p182, %r120, 7;\n" " @%p182 bra $L__BB4_227;\n" "\n" " ld.shared.f32 %f602, [%r119+24];\n" " div.rn.f32 %f603, %f705, %f602;\n" " mul.lo.s64 %rd170, %rd14, 6;\n" " add.s64 %rd171, %rd46, %rd170;\n" " shl.b64 %rd172, %rd171, 2;\n" " add.s64 %rd173, %rd2, %rd172;\n" " st.global.f32 [%rd173], %f603;\n" "\n" "$L__BB4_227:\n" " setp.lt.s32 %p183, %r120, 8;\n" " @%p183 bra $L__BB4_229;\n" "\n" " ld.shared.f32 %f604, [%r119+28];\n" " div.rn.f32 %f605, %f704, %f604;\n" " mul.lo.s64 %rd174, %rd14, 7;\n" " add.s64 %rd175, %rd46, %rd174;\n" " shl.b64 %rd176, %rd175, 2;\n" " add.s64 %rd177, %rd2, %rd176;\n" " st.global.f32 [%rd177], %f605;\n" "\n" "$L__BB4_229:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_prefill_q4_kr\n" ".visible .entry fc_attn_flash_prefill_q4_kr(\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_0,\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_1,\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_2,\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_3,\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_4,\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_5,\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_6,\n" " .param .u64 fc_attn_flash_prefill_q4_kr_param_7,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_8,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_9,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_10,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_11,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_12,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_13,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_14,\n" " .param .u32 fc_attn_flash_prefill_q4_kr_param_15,\n" " .param .f32 fc_attn_flash_prefill_q4_kr_param_16\n" ")\n" "{\n" " .reg .pred %p<334>;\n" " .reg .b16 %rs<46>;\n" " .reg .f32 %f<1646>;\n" " .reg .b32 %r<581>;\n" " .reg .b64 %rd<307>;\n" "\n" "\n" " ld.param.u64 %rd25, [fc_attn_flash_prefill_q4_kr_param_0];\n" " ld.param.u64 %rd20, [fc_attn_flash_prefill_q4_kr_param_1];\n" " ld.param.u64 %rd21, [fc_attn_flash_prefill_q4_kr_param_2];\n" " ld.param.u64 %rd26, [fc_attn_flash_prefill_q4_kr_param_3];\n" " ld.param.u64 %rd22, [fc_attn_flash_prefill_q4_kr_param_4];\n" " ld.param.u64 %rd27, [fc_attn_flash_prefill_q4_kr_param_5];\n" " ld.param.u64 %rd23, [fc_attn_flash_prefill_q4_kr_param_6];\n" " ld.param.u64 %rd24, [fc_attn_flash_prefill_q4_kr_param_7];\n" " ld.param.u32 %r83, [fc_attn_flash_prefill_q4_kr_param_8];\n" " ld.param.u32 %r84, [fc_attn_flash_prefill_q4_kr_param_9];\n" " ld.param.u32 %r85, [fc_attn_flash_prefill_q4_kr_param_10];\n" " ld.param.u32 %r86, [fc_attn_flash_prefill_q4_kr_param_11];\n" " ld.param.u32 %r87, [fc_attn_flash_prefill_q4_kr_param_12];\n" " ld.param.u32 %r88, [fc_attn_flash_prefill_q4_kr_param_13];\n" " ld.param.u32 %r90, [fc_attn_flash_prefill_q4_kr_param_15];\n" " ld.param.f32 %f673, [fc_attn_flash_prefill_q4_kr_param_16];\n" " cvta.to.global.u64 %rd1, %rd25;\n" " cvta.to.global.u64 %rd2, %rd26;\n" " cvta.to.global.u64 %rd3, %rd27;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r91, %ctaid.x;\n" " add.s32 %r2, %r91, %r86;\n" " mul.lo.s32 %r3, %r84, %r83;\n" " shl.b32 %r4, %r83, 4;\n" " cvt.s64.s32 %rd4, %r83;\n" " mov.u32 %r92, %nctaid.y;\n" " cvt.u64.u32 %rd28, %r92;\n" " cvt.s64.s32 %rd29, %r91;\n" " mul.lo.s64 %rd30, %rd29, %rd28;\n" " mov.u32 %r93, %ctaid.y;\n" " cvt.s64.s32 %rd31, %r93;\n" " add.s64 %rd32, %rd30, %rd31;\n" " cvt.s64.s32 %rd5, %r84;\n" " mul.wide.s32 %rd33, %r84, %r83;\n" " mul.lo.s64 %rd6, %rd33, %rd32;\n" " setp.lt.s32 %p2, %r83, 1;\n" " @%p2 bra $L__BB5_7;\n" "\n" " cvt.s64.s32 %rd7, %r1;\n" " and.b32 %r569, %r83, 3;\n" " add.s32 %r95, %r83, -1;\n" " setp.lt.u32 %p3, %r95, 3;\n" " mov.u32 %r567, 0;\n" " @%p3 bra $L__BB5_4;\n" "\n" " sub.s32 %r565, %r83, %r569;\n" " add.s64 %rd34, %rd6, %rd7;\n" " shl.b64 %rd35, %rd34, 2;\n" " add.s64 %rd306, %rd1, %rd35;\n" " shl.b64 %rd9, %rd5, 2;\n" " shl.b32 %r97, %r1, 2;\n" " mov.u32 %r98, shk;\n" " add.s32 %r566, %r98, %r97;\n" " shl.b32 %r8, %r84, 2;\n" " mov.u32 %r567, 0;\n" "\n" "$L__BB5_3:\n" " ld.global.f32 %f674, [%rd306];\n" " st.shared.f32 [%r566], %f674;\n" " add.s64 %rd36, %rd306, %rd9;\n" " ld.global.f32 %f675, [%rd36];\n" " add.s32 %r99, %r566, %r8;\n" " st.shared.f32 [%r99], %f675;\n" " add.s64 %rd37, %rd36, %rd9;\n" " ld.global.f32 %f676, [%rd37];\n" " add.s32 %r100, %r99, %r8;\n" " st.shared.f32 [%r100], %f676;\n" " add.s64 %rd38, %rd37, %rd9;\n" " add.s64 %rd306, %rd38, %rd9;\n" " ld.global.f32 %f677, [%rd38];\n" " add.s32 %r101, %r100, %r8;\n" " add.s32 %r566, %r101, %r8;\n" " st.shared.f32 [%r101], %f677;\n" " add.s32 %r567, %r567, 4;\n" " add.s32 %r565, %r565, -4;\n" " setp.ne.s32 %p4, %r565, 0;\n" " @%p4 bra $L__BB5_3;\n" "\n" "$L__BB5_4:\n" " setp.eq.s32 %p5, %r569, 0;\n" " add.s64 %rd12, %rd6, %rd7;\n" " @%p5 bra $L__BB5_7;\n" "\n" " mov.u32 %r104, shk;\n" "\n" "$L__BB5_6:\n" " .pragma \"nounroll\";\n" " cvt.s64.s32 %rd39, %r567;\n" " mul.lo.s64 %rd40, %rd39, %rd5;\n" " add.s64 %rd41, %rd12, %rd40;\n" " shl.b64 %rd42, %rd41, 2;\n" " add.s64 %rd43, %rd1, %rd42;\n" " ld.global.f32 %f678, [%rd43];\n" " mad.lo.s32 %r102, %r567, %r84, %r1;\n" " shl.b32 %r103, %r102, 2;\n" " add.s32 %r105, %r104, %r103;\n" " st.shared.f32 [%r105], %f678;\n" " add.s32 %r567, %r567, 1;\n" " add.s32 %r569, %r569, -1;\n" " setp.ne.s32 %p6, %r569, 0;\n" " @%p6 bra $L__BB5_6;\n" "\n" "$L__BB5_7:\n" " mul.wide.s32 %rd44, %r4, 2;\n" " setp.ge.s32 %p7, %r1, %r83;\n" " add.s32 %r106, %r3, %r1;\n" " cvt.u32.u64 %r107, %rd44;\n" " add.s32 %r108, %r106, %r107;\n" " shl.b32 %r109, %r108, 2;\n" " mov.u32 %r110, shk;\n" " add.s32 %r20, %r110, %r109;\n" " add.s64 %rd45, %rd44, %rd4;\n" " cvt.u32.u64 %r21, %rd45;\n" " add.s32 %r111, %r106, %r21;\n" " shl.b32 %r112, %r111, 2;\n" " add.s32 %r22, %r110, %r112;\n" " @%p7 bra $L__BB5_9;\n" "\n" " mov.u32 %r113, -8388608;\n" " st.shared.u32 [%r20], %r113;\n" " mov.u32 %r114, 0;\n" " st.shared.u32 [%r22], %r114;\n" "\n" "$L__BB5_9:\n" " bar.sync 0;\n" " shr.s32 %r115, %r84, 31;\n" " shr.u32 %r116, %r115, 27;\n" " add.s32 %r117, %r84, %r116;\n" " shr.s32 %r118, %r117, 5;\n" " mul.lo.s32 %r119, %r118, 18;\n" " cvt.s64.s32 %rd13, %r119;\n" " shr.s32 %r120, %r1, 31;\n" " shr.u32 %r121, %r120, 28;\n" " add.s32 %r122, %r1, %r121;\n" " shr.s32 %r23, %r122, 4;\n" " and.b32 %r123, %r122, -16;\n" " sub.s32 %r24, %r1, %r123;\n" " setp.lt.s32 %p8, %r2, 0;\n" " mov.f32 %f1567, 0f00000000;\n" " mov.f32 %f1568, %f1567;\n" " mov.f32 %f1569, %f1567;\n" " mov.f32 %f1570, %f1567;\n" " mov.f32 %f1571, %f1567;\n" " mov.f32 %f1572, %f1567;\n" " mov.f32 %f1573, %f1567;\n" " mov.f32 %f1574, %f1567;\n" " @%p8 bra $L__BB5_500;\n" "\n" " add.s32 %r126, %r21, %r83;\n" " add.s32 %r127, %r3, %r126;\n" " shr.u32 %r129, %r120, 27;\n" " add.s32 %r130, %r1, %r129;\n" " shr.s32 %r131, %r130, 5;\n" " mul.lo.s32 %r25, %r131, 18;\n" " shl.b32 %r132, %r24, 2;\n" " add.s32 %r26, %r110, %r132;\n" " shl.b32 %r134, %r84, 2;\n" " add.s32 %r27, %r26, %r134;\n" " add.s32 %r28, %r27, %r134;\n" " add.s32 %r29, %r28, %r134;\n" " add.s32 %r30, %r29, %r134;\n" " add.s32 %r31, %r30, %r134;\n" " add.s32 %r32, %r31, %r134;\n" " add.s32 %r33, %r32, %r134;\n" " shl.b32 %r135, %r127, 2;\n" " add.s32 %r34, %r110, %r135;\n" " mov.u32 %r570, 0;\n" " mov.f32 %f1567, 0f00000000;\n" " setp.lt.s32 %p10, %r90, 1;\n" " setp.lt.s32 %p56, %r84, 16;\n" " cvt.s64.s32 %rd127, %r24;\n" " setp.lt.s32 %p57, %r84, 32;\n" " setp.lt.s32 %p58, %r84, 48;\n" " cvta.to.global.u64 %rd51, %rd20;\n" " cvta.to.global.u64 %rd237, %rd23;\n" " mov.f32 %f1568, %f1567;\n" " mov.f32 %f1569, %f1567;\n" " mov.f32 %f1570, %f1567;\n" " mov.f32 %f1571, %f1567;\n" " mov.f32 %f1572, %f1567;\n" " mov.f32 %f1573, %f1567;\n" " mov.f32 %f1574, %f1567;\n" " mov.u32 %r571, %r570;\n" "\n" "$L__BB5_11:\n" " ld.param.u32 %r550, [fc_attn_flash_prefill_q4_kr_param_14];\n" " mov.u32 %r549, %ctaid.x;\n" " ld.param.u32 %r548, [fc_attn_flash_prefill_q4_kr_param_15];\n" " mov.u32 %r137, -2;\n" " sub.s32 %r138, %r137, %r549;\n" " sub.s32 %r139, %r138, %r86;\n" " shl.b32 %r140, %r570, 4;\n" " add.s32 %r141, %r139, %r140;\n" " max.s32 %r142, %r141, -17;\n" " not.b32 %r143, %r142;\n" " max.s32 %r37, %r143, 1;\n" " add.s32 %r145, %r2, 1;\n" " sub.s32 %r38, %r145, %r571;\n" " min.s32 %r39, %r38, 16;\n" " setp.lt.s32 %p9, %r571, %r550;\n" " or.pred %p11, %p10, %p9;\n" " add.s32 %r146, %r39, %r571;\n" " sub.s32 %r147, %r2, %r548;\n" " add.s32 %r148, %r147, 1;\n" " max.s32 %r149, %r148, 0;\n" " setp.gt.s32 %p12, %r146, %r149;\n" " setp.le.s32 %p13, %r571, %r2;\n" " and.pred %p14, %p13, %p12;\n" " or.pred %p15, %p11, %p14;\n" " not.pred %p16, %p15;\n" " @%p16 bra $L__BB5_499;\n" "\n" " setp.ge.s32 %p17, %r23, %r39;\n" " @%p17 bra $L__BB5_103;\n" "\n" " add.s32 %r40, %r571, %r23;\n" " setp.lt.s32 %p18, %r40, %r87;\n" " @%p18 bra $L__BB5_71;\n" " bra.uni $L__BB5_14;\n" "\n" "$L__BB5_71:\n" " cvt.s64.s32 %rd122, %r40;\n" " mul.lo.s64 %rd123, %rd122, %rd5;\n" " mul.lo.s64 %rd125, %rd31, %rd5;\n" " shl.b64 %rd126, %rd125, 3;\n" " add.s64 %rd17, %rd123, %rd126;\n" " @%p56 bra $L__BB5_73;\n" "\n" " add.s64 %rd128, %rd17, %rd127;\n" " shl.b64 %rd129, %rd128, 1;\n" " add.s64 %rd130, %rd2, %rd129;\n" " ld.global.u16 %rs25, [%rd130];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1295, %rs25;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_73:\n" " @%p57 bra $L__BB5_75;\n" "\n" " add.s32 %r236, %r24, 16;\n" " cvt.s64.s32 %rd131, %r236;\n" " add.s64 %rd132, %rd17, %rd131;\n" " shl.b64 %rd133, %rd132, 1;\n" " add.s64 %rd134, %rd2, %rd133;\n" " ld.global.u16 %rs26, [%rd134];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1294, %rs26;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_75:\n" " @%p58 bra $L__BB5_77;\n" "\n" " add.s32 %r237, %r24, 32;\n" " cvt.s64.s32 %rd135, %r237;\n" " add.s64 %rd136, %rd17, %rd135;\n" " shl.b64 %rd137, %rd136, 1;\n" " add.s64 %rd138, %rd2, %rd137;\n" " ld.global.u16 %rs27, [%rd138];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1293, %rs27;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_77:\n" " setp.lt.s32 %p59, %r84, 64;\n" " @%p59 bra $L__BB5_79;\n" "\n" " add.s32 %r238, %r24, 48;\n" " cvt.s64.s32 %rd139, %r238;\n" " add.s64 %rd140, %rd17, %rd139;\n" " shl.b64 %rd141, %rd140, 1;\n" " add.s64 %rd142, %rd2, %rd141;\n" " ld.global.u16 %rs28, [%rd142];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1292, %rs28;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_79:\n" " setp.lt.s32 %p60, %r84, 80;\n" " @%p60 bra $L__BB5_81;\n" "\n" " add.s32 %r239, %r24, 64;\n" " cvt.s64.s32 %rd143, %r239;\n" " add.s64 %rd144, %rd17, %rd143;\n" " shl.b64 %rd145, %rd144, 1;\n" " add.s64 %rd146, %rd2, %rd145;\n" " ld.global.u16 %rs29, [%rd146];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1291, %rs29;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_81:\n" " setp.lt.s32 %p61, %r84, 96;\n" " @%p61 bra $L__BB5_83;\n" "\n" " add.s32 %r240, %r24, 80;\n" " cvt.s64.s32 %rd147, %r240;\n" " add.s64 %rd148, %rd17, %rd147;\n" " shl.b64 %rd149, %rd148, 1;\n" " add.s64 %rd150, %rd2, %rd149;\n" " ld.global.u16 %rs30, [%rd150];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1290, %rs30;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_83:\n" " setp.lt.s32 %p62, %r84, 112;\n" " @%p62 bra $L__BB5_85;\n" "\n" " add.s32 %r241, %r24, 96;\n" " cvt.s64.s32 %rd151, %r241;\n" " add.s64 %rd152, %rd17, %rd151;\n" " shl.b64 %rd153, %rd152, 1;\n" " add.s64 %rd154, %rd2, %rd153;\n" " ld.global.u16 %rs31, [%rd154];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1289, %rs31;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_85:\n" " setp.lt.s32 %p63, %r84, 128;\n" " @%p63 bra $L__BB5_87;\n" "\n" " add.s32 %r242, %r24, 112;\n" " cvt.s64.s32 %rd155, %r242;\n" " add.s64 %rd156, %rd17, %rd155;\n" " shl.b64 %rd157, %rd156, 1;\n" " add.s64 %rd158, %rd2, %rd157;\n" " ld.global.u16 %rs32, [%rd158];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1288, %rs32;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_87:\n" " setp.lt.s32 %p64, %r84, 144;\n" " @%p64 bra $L__BB5_89;\n" "\n" " add.s32 %r243, %r24, 128;\n" " cvt.s64.s32 %rd159, %r243;\n" " add.s64 %rd160, %rd17, %rd159;\n" " shl.b64 %rd161, %rd160, 1;\n" " add.s64 %rd162, %rd2, %rd161;\n" " ld.global.u16 %rs33, [%rd162];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1287, %rs33;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_89:\n" " setp.lt.s32 %p65, %r84, 160;\n" " @%p65 bra $L__BB5_91;\n" "\n" " add.s32 %r244, %r24, 144;\n" " cvt.s64.s32 %rd163, %r244;\n" " add.s64 %rd164, %rd17, %rd163;\n" " shl.b64 %rd165, %rd164, 1;\n" " add.s64 %rd166, %rd2, %rd165;\n" " ld.global.u16 %rs34, [%rd166];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1286, %rs34;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_91:\n" " setp.lt.s32 %p66, %r84, 176;\n" " @%p66 bra $L__BB5_93;\n" "\n" " add.s32 %r245, %r24, 160;\n" " cvt.s64.s32 %rd167, %r245;\n" " add.s64 %rd168, %rd17, %rd167;\n" " shl.b64 %rd169, %rd168, 1;\n" " add.s64 %rd170, %rd2, %rd169;\n" " ld.global.u16 %rs35, [%rd170];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1285, %rs35;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_93:\n" " setp.lt.s32 %p67, %r84, 192;\n" " @%p67 bra $L__BB5_95;\n" "\n" " add.s32 %r246, %r24, 176;\n" " cvt.s64.s32 %rd171, %r246;\n" " add.s64 %rd172, %rd17, %rd171;\n" " shl.b64 %rd173, %rd172, 1;\n" " add.s64 %rd174, %rd2, %rd173;\n" " ld.global.u16 %rs36, [%rd174];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1284, %rs36;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_95:\n" " setp.lt.s32 %p68, %r84, 208;\n" " @%p68 bra $L__BB5_97;\n" "\n" " add.s32 %r247, %r24, 192;\n" " cvt.s64.s32 %rd175, %r247;\n" " add.s64 %rd176, %rd17, %rd175;\n" " shl.b64 %rd177, %rd176, 1;\n" " add.s64 %rd178, %rd2, %rd177;\n" " ld.global.u16 %rs37, [%rd178];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1283, %rs37;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_97:\n" " setp.lt.s32 %p69, %r84, 224;\n" " @%p69 bra $L__BB5_99;\n" "\n" " add.s32 %r248, %r24, 208;\n" " cvt.s64.s32 %rd179, %r248;\n" " add.s64 %rd180, %rd17, %rd179;\n" " shl.b64 %rd181, %rd180, 1;\n" " add.s64 %rd182, %rd2, %rd181;\n" " ld.global.u16 %rs38, [%rd182];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1282, %rs38;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_99:\n" " setp.lt.s32 %p70, %r84, 240;\n" " @%p70 bra $L__BB5_101;\n" "\n" " add.s32 %r249, %r24, 224;\n" " cvt.s64.s32 %rd183, %r249;\n" " add.s64 %rd184, %rd17, %rd183;\n" " shl.b64 %rd185, %rd184, 1;\n" " add.s64 %rd186, %rd2, %rd185;\n" " ld.global.u16 %rs39, [%rd186];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1281, %rs39;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_101:\n" " setp.lt.s32 %p71, %r84, 256;\n" " @%p71 bra $L__BB5_103;\n" "\n" " add.s32 %r250, %r24, 240;\n" " cvt.s64.s32 %rd187, %r250;\n" " add.s64 %rd188, %rd17, %rd187;\n" " shl.b64 %rd189, %rd188, 1;\n" " add.s64 %rd190, %rd2, %rd189;\n" " ld.global.u16 %rs40, [%rd190];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1280, %rs40;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB5_103;\n" "\n" "$L__BB5_14:\n" " setp.gt.s32 %p19, %r88, 0;\n" " sub.s32 %r152, %r2, %r88;\n" " setp.gt.s32 %p20, %r40, %r152;\n" " and.pred %p21, %p19, %p20;\n" " mov.u32 %r153, %nctaid.x;\n" " add.s32 %r154, %r153, %r86;\n" " add.s32 %r155, %r154, -257;\n" " setp.gt.s32 %p22, %r40, %r155;\n" " and.pred %p23, %p22, %p21;\n" " @%p23 bra $L__BB5_39;\n" " bra.uni $L__BB5_15;\n" "\n" "$L__BB5_39:\n" " shr.s32 %r214, %r40, 31;\n" " shr.u32 %r215, %r214, 24;\n" " add.s32 %r216, %r40, %r215;\n" " and.b32 %r217, %r216, -256;\n" " sub.s32 %r218, %r40, %r217;\n" " cvt.s64.s32 %rd53, %r218;\n" " mul.lo.s64 %rd54, %rd53, %rd5;\n" " mul.lo.s64 %rd56, %rd31, %rd5;\n" " shl.b64 %rd57, %rd56, 8;\n" " add.s64 %rd16, %rd54, %rd57;\n" " @%p56 bra $L__BB5_41;\n" "\n" " add.s64 %rd59, %rd16, %rd127;\n" " shl.b64 %rd60, %rd59, 1;\n" " add.s64 %rd61, %rd3, %rd60;\n" " ld.global.u16 %rs9, [%rd61];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1295, %rs9;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_41:\n" " @%p57 bra $L__BB5_43;\n" "\n" " add.s32 %r220, %r24, 16;\n" " cvt.s64.s32 %rd62, %r220;\n" " add.s64 %rd63, %rd16, %rd62;\n" " shl.b64 %rd64, %rd63, 1;\n" " add.s64 %rd65, %rd3, %rd64;\n" " ld.global.u16 %rs10, [%rd65];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1294, %rs10;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_43:\n" " @%p58 bra $L__BB5_45;\n" "\n" " add.s32 %r221, %r24, 32;\n" " cvt.s64.s32 %rd66, %r221;\n" " add.s64 %rd67, %rd16, %rd66;\n" " shl.b64 %rd68, %rd67, 1;\n" " add.s64 %rd69, %rd3, %rd68;\n" " ld.global.u16 %rs11, [%rd69];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1293, %rs11;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_45:\n" " setp.lt.s32 %p43, %r84, 64;\n" " @%p43 bra $L__BB5_47;\n" "\n" " add.s32 %r222, %r24, 48;\n" " cvt.s64.s32 %rd70, %r222;\n" " add.s64 %rd71, %rd16, %rd70;\n" " shl.b64 %rd72, %rd71, 1;\n" " add.s64 %rd73, %rd3, %rd72;\n" " ld.global.u16 %rs12, [%rd73];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1292, %rs12;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_47:\n" " setp.lt.s32 %p44, %r84, 80;\n" " @%p44 bra $L__BB5_49;\n" "\n" " add.s32 %r223, %r24, 64;\n" " cvt.s64.s32 %rd74, %r223;\n" " add.s64 %rd75, %rd16, %rd74;\n" " shl.b64 %rd76, %rd75, 1;\n" " add.s64 %rd77, %rd3, %rd76;\n" " ld.global.u16 %rs13, [%rd77];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1291, %rs13;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_49:\n" " setp.lt.s32 %p45, %r84, 96;\n" " @%p45 bra $L__BB5_51;\n" "\n" " add.s32 %r224, %r24, 80;\n" " cvt.s64.s32 %rd78, %r224;\n" " add.s64 %rd79, %rd16, %rd78;\n" " shl.b64 %rd80, %rd79, 1;\n" " add.s64 %rd81, %rd3, %rd80;\n" " ld.global.u16 %rs14, [%rd81];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1290, %rs14;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_51:\n" " setp.lt.s32 %p46, %r84, 112;\n" " @%p46 bra $L__BB5_53;\n" "\n" " add.s32 %r225, %r24, 96;\n" " cvt.s64.s32 %rd82, %r225;\n" " add.s64 %rd83, %rd16, %rd82;\n" " shl.b64 %rd84, %rd83, 1;\n" " add.s64 %rd85, %rd3, %rd84;\n" " ld.global.u16 %rs15, [%rd85];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1289, %rs15;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_53:\n" " setp.lt.s32 %p47, %r84, 128;\n" " @%p47 bra $L__BB5_55;\n" "\n" " add.s32 %r226, %r24, 112;\n" " cvt.s64.s32 %rd86, %r226;\n" " add.s64 %rd87, %rd16, %rd86;\n" " shl.b64 %rd88, %rd87, 1;\n" " add.s64 %rd89, %rd3, %rd88;\n" " ld.global.u16 %rs16, [%rd89];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1288, %rs16;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_55:\n" " setp.lt.s32 %p48, %r84, 144;\n" " @%p48 bra $L__BB5_57;\n" "\n" " add.s32 %r227, %r24, 128;\n" " cvt.s64.s32 %rd90, %r227;\n" " add.s64 %rd91, %rd16, %rd90;\n" " shl.b64 %rd92, %rd91, 1;\n" " add.s64 %rd93, %rd3, %rd92;\n" " ld.global.u16 %rs17, [%rd93];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1287, %rs17;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_57:\n" " setp.lt.s32 %p49, %r84, 160;\n" " @%p49 bra $L__BB5_59;\n" "\n" " add.s32 %r228, %r24, 144;\n" " cvt.s64.s32 %rd94, %r228;\n" " add.s64 %rd95, %rd16, %rd94;\n" " shl.b64 %rd96, %rd95, 1;\n" " add.s64 %rd97, %rd3, %rd96;\n" " ld.global.u16 %rs18, [%rd97];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1286, %rs18;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_59:\n" " setp.lt.s32 %p50, %r84, 176;\n" " @%p50 bra $L__BB5_61;\n" "\n" " add.s32 %r229, %r24, 160;\n" " cvt.s64.s32 %rd98, %r229;\n" " add.s64 %rd99, %rd16, %rd98;\n" " shl.b64 %rd100, %rd99, 1;\n" " add.s64 %rd101, %rd3, %rd100;\n" " ld.global.u16 %rs19, [%rd101];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1285, %rs19;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_61:\n" " setp.lt.s32 %p51, %r84, 192;\n" " @%p51 bra $L__BB5_63;\n" "\n" " add.s32 %r230, %r24, 176;\n" " cvt.s64.s32 %rd102, %r230;\n" " add.s64 %rd103, %rd16, %rd102;\n" " shl.b64 %rd104, %rd103, 1;\n" " add.s64 %rd105, %rd3, %rd104;\n" " ld.global.u16 %rs20, [%rd105];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1284, %rs20;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_63:\n" " setp.lt.s32 %p52, %r84, 208;\n" " @%p52 bra $L__BB5_65;\n" "\n" " add.s32 %r231, %r24, 192;\n" " cvt.s64.s32 %rd106, %r231;\n" " add.s64 %rd107, %rd16, %rd106;\n" " shl.b64 %rd108, %rd107, 1;\n" " add.s64 %rd109, %rd3, %rd108;\n" " ld.global.u16 %rs21, [%rd109];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1283, %rs21;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_65:\n" " setp.lt.s32 %p53, %r84, 224;\n" " @%p53 bra $L__BB5_67;\n" "\n" " add.s32 %r232, %r24, 208;\n" " cvt.s64.s32 %rd110, %r232;\n" " add.s64 %rd111, %rd16, %rd110;\n" " shl.b64 %rd112, %rd111, 1;\n" " add.s64 %rd113, %rd3, %rd112;\n" " ld.global.u16 %rs22, [%rd113];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1282, %rs22;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_67:\n" " setp.lt.s32 %p54, %r84, 240;\n" " @%p54 bra $L__BB5_69;\n" "\n" " add.s32 %r233, %r24, 224;\n" " cvt.s64.s32 %rd114, %r233;\n" " add.s64 %rd115, %rd16, %rd114;\n" " shl.b64 %rd116, %rd115, 1;\n" " add.s64 %rd117, %rd3, %rd116;\n" " ld.global.u16 %rs23, [%rd117];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1281, %rs23;}\n" "\n" " // end inline asm\n" "\n" "$L__BB5_69:\n" " setp.lt.s32 %p55, %r84, 256;\n" " @%p55 bra $L__BB5_103;\n" "\n" " add.s32 %r234, %r24, 240;\n" " cvt.s64.s32 %rd118, %r234;\n" " add.s64 %rd119, %rd16, %rd118;\n" " shl.b64 %rd120, %rd119, 1;\n" " add.s64 %rd121, %rd3, %rd120;\n" " ld.global.u16 %rs24, [%rd121];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1280, %rs24;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB5_103;\n" "\n" "$L__BB5_15:\n" " cvt.s64.s32 %rd46, %r40;\n" " mul.lo.s64 %rd47, %rd46, %rd13;\n" " mul.wide.s32 %rd48, %r85, %r93;\n" " mul.lo.s64 %rd49, %rd48, %rd13;\n" " add.s64 %rd50, %rd47, %rd49;\n" " add.s64 %rd14, %rd51, %rd50;\n" " add.s32 %r157, %r24, 2;\n" " cvt.s64.s32 %rd52, %r157;\n" " add.s64 %rd15, %rd14, %rd52;\n" " @%p56 bra $L__BB5_18;\n" "\n" " ld.global.u8 %r158, [%rd14];\n" " ld.global.u8 %r159, [%rd14+1];\n" " prmt.b32 %r160, %r159, %r158, 30212;\n" " cvt.u16.u32 %rs1, %r160;\n" " // begin inline asm\n" " { cvt.f32.f16 %f696, %rs1;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r41, [%rd15];\n" " and.b32 %r161, %r41, 15;\n" " add.s32 %r162, %r161, -8;\n" " cvt.rn.f32.s32 %f697, %r162;\n" " mul.f32 %f1295, %f696, %f697;\n" " @%p57 bra $L__BB5_18;\n" "\n" " shr.u32 %r163, %r41, 4;\n" " add.s32 %r164, %r163, -8;\n" " cvt.rn.f32.s32 %f698, %r164;\n" " mul.f32 %f1294, %f696, %f698;\n" "\n" "$L__BB5_18:\n" " @%p58 bra $L__BB5_21;\n" "\n" " setp.lt.s32 %p27, %r84, 64;\n" " ld.global.u8 %r165, [%rd14+18];\n" " ld.global.u8 %r166, [%rd14+19];\n" " prmt.b32 %r167, %r166, %r165, 30212;\n" " cvt.u16.u32 %rs2, %r167;\n" " // begin inline asm\n" " { cvt.f32.f16 %f699, %rs2;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r42, [%rd15+18];\n" " and.b32 %r168, %r42, 15;\n" " add.s32 %r169, %r168, -8;\n" " cvt.rn.f32.s32 %f700, %r169;\n" " mul.f32 %f1293, %f699, %f700;\n" " @%p27 bra $L__BB5_21;\n" "\n" " shr.u32 %r170, %r42, 4;\n" " add.s32 %r171, %r170, -8;\n" " cvt.rn.f32.s32 %f701, %r171;\n" " mul.f32 %f1292, %f699, %f701;\n" "\n" "$L__BB5_21:\n" " setp.lt.s32 %p28, %r84, 80;\n" " @%p28 bra $L__BB5_24;\n" "\n" " setp.lt.s32 %p29, %r84, 96;\n" " ld.global.u8 %r172, [%rd14+36];\n" " ld.global.u8 %r173, [%rd14+37];\n" " prmt.b32 %r174, %r173, %r172, 30212;\n" " cvt.u16.u32 %rs3, %r174;\n" " // begin inline asm\n" " { cvt.f32.f16 %f702, %rs3;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r43, [%rd15+36];\n" " and.b32 %r175, %r43, 15;\n" " add.s32 %r176, %r175, -8;\n" " cvt.rn.f32.s32 %f703, %r176;\n" " mul.f32 %f1291, %f702, %f703;\n" " @%p29 bra $L__BB5_24;\n" "\n" " shr.u32 %r177, %r43, 4;\n" " add.s32 %r178, %r177, -8;\n" " cvt.rn.f32.s32 %f704, %r178;\n" " mul.f32 %f1290, %f702, %f704;\n" "\n" "$L__BB5_24:\n" " setp.lt.s32 %p30, %r84, 112;\n" " @%p30 bra $L__BB5_27;\n" "\n" " setp.lt.s32 %p31, %r84, 128;\n" " ld.global.u8 %r179, [%rd14+54];\n" " ld.global.u8 %r180, [%rd14+55];\n" " prmt.b32 %r181, %r180, %r179, 30212;\n" " cvt.u16.u32 %rs4, %r181;\n" " // begin inline asm\n" " { cvt.f32.f16 %f705, %rs4;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r44, [%rd15+54];\n" " and.b32 %r182, %r44, 15;\n" " add.s32 %r183, %r182, -8;\n" " cvt.rn.f32.s32 %f706, %r183;\n" " mul.f32 %f1289, %f705, %f706;\n" " @%p31 bra $L__BB5_27;\n" "\n" " shr.u32 %r184, %r44, 4;\n" " add.s32 %r185, %r184, -8;\n" " cvt.rn.f32.s32 %f707, %r185;\n" " mul.f32 %f1288, %f705, %f707;\n" "\n" "$L__BB5_27:\n" " setp.lt.s32 %p32, %r84, 144;\n" " @%p32 bra $L__BB5_30;\n" "\n" " setp.lt.s32 %p33, %r84, 160;\n" " ld.global.u8 %r186, [%rd14+72];\n" " ld.global.u8 %r187, [%rd14+73];\n" " prmt.b32 %r188, %r187, %r186, 30212;\n" " cvt.u16.u32 %rs5, %r188;\n" " // begin inline asm\n" " { cvt.f32.f16 %f708, %rs5;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r45, [%rd15+72];\n" " and.b32 %r189, %r45, 15;\n" " add.s32 %r190, %r189, -8;\n" " cvt.rn.f32.s32 %f709, %r190;\n" " mul.f32 %f1287, %f708, %f709;\n" " @%p33 bra $L__BB5_30;\n" "\n" " shr.u32 %r191, %r45, 4;\n" " add.s32 %r192, %r191, -8;\n" " cvt.rn.f32.s32 %f710, %r192;\n" " mul.f32 %f1286, %f708, %f710;\n" "\n" "$L__BB5_30:\n" " setp.lt.s32 %p34, %r84, 176;\n" " @%p34 bra $L__BB5_33;\n" "\n" " setp.lt.s32 %p35, %r84, 192;\n" " ld.global.u8 %r193, [%rd14+90];\n" " ld.global.u8 %r194, [%rd14+91];\n" " prmt.b32 %r195, %r194, %r193, 30212;\n" " cvt.u16.u32 %rs6, %r195;\n" " // begin inline asm\n" " { cvt.f32.f16 %f711, %rs6;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r46, [%rd15+90];\n" " and.b32 %r196, %r46, 15;\n" " add.s32 %r197, %r196, -8;\n" " cvt.rn.f32.s32 %f712, %r197;\n" " mul.f32 %f1285, %f711, %f712;\n" " @%p35 bra $L__BB5_33;\n" "\n" " shr.u32 %r198, %r46, 4;\n" " add.s32 %r199, %r198, -8;\n" " cvt.rn.f32.s32 %f713, %r199;\n" " mul.f32 %f1284, %f711, %f713;\n" "\n" "$L__BB5_33:\n" " setp.lt.s32 %p36, %r84, 208;\n" " @%p36 bra $L__BB5_36;\n" "\n" " setp.lt.s32 %p37, %r84, 224;\n" " ld.global.u8 %r200, [%rd14+108];\n" " ld.global.u8 %r201, [%rd14+109];\n" " prmt.b32 %r202, %r201, %r200, 30212;\n" " cvt.u16.u32 %rs7, %r202;\n" " // begin inline asm\n" " { cvt.f32.f16 %f714, %rs7;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r47, [%rd15+108];\n" " and.b32 %r203, %r47, 15;\n" " add.s32 %r204, %r203, -8;\n" " cvt.rn.f32.s32 %f715, %r204;\n" " mul.f32 %f1283, %f714, %f715;\n" " @%p37 bra $L__BB5_36;\n" "\n" " shr.u32 %r205, %r47, 4;\n" " add.s32 %r206, %r205, -8;\n" " cvt.rn.f32.s32 %f716, %r206;\n" " mul.f32 %f1282, %f714, %f716;\n" "\n" "$L__BB5_36:\n" " setp.lt.s32 %p38, %r84, 240;\n" " @%p38 bra $L__BB5_103;\n" "\n" " setp.lt.s32 %p39, %r84, 256;\n" " ld.global.u8 %r207, [%rd14+126];\n" " ld.global.u8 %r208, [%rd14+127];\n" " prmt.b32 %r209, %r208, %r207, 30212;\n" " cvt.u16.u32 %rs8, %r209;\n" " // begin inline asm\n" " { cvt.f32.f16 %f717, %rs8;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r48, [%rd15+126];\n" " and.b32 %r210, %r48, 15;\n" " add.s32 %r211, %r210, -8;\n" " cvt.rn.f32.s32 %f718, %r211;\n" " mul.f32 %f1281, %f717, %f718;\n" " @%p39 bra $L__BB5_103;\n" "\n" " shr.u32 %r212, %r48, 4;\n" " add.s32 %r213, %r212, -8;\n" " cvt.rn.f32.s32 %f719, %r213;\n" " mul.f32 %f1280, %f717, %f719;\n" "\n" "$L__BB5_103:\n" " mov.f32 %f1318, 0f00000000;\n" " mov.f32 %f1319, %f1318;\n" " mov.f32 %f1320, %f1318;\n" " mov.f32 %f1321, %f1318;\n" " mov.f32 %f1322, %f1318;\n" " mov.f32 %f1323, %f1318;\n" " mov.f32 %f1324, %f1318;\n" " mov.f32 %f1325, %f1318;\n" " @%p17 bra $L__BB5_376;\n" "\n" " mov.f32 %f1318, 0f00000000;\n" " mov.f32 %f1319, %f1318;\n" " mov.f32 %f1320, %f1318;\n" " mov.f32 %f1321, %f1318;\n" " mov.f32 %f1322, %f1318;\n" " mov.f32 %f1323, %f1318;\n" " mov.f32 %f1324, %f1318;\n" " mov.f32 %f1325, %f1318;\n" " @%p56 bra $L__BB5_121;\n" "\n" " mov.f32 %f1318, 0f00000000;\n" " mov.f32 %f1325, %f1318;\n" " @%p2 bra $L__BB5_107;\n" "\n" " ld.shared.f32 %f769, [%r26];\n" " fma.rn.f32 %f1325, %f1295, %f769, 0f00000000;\n" "\n" "$L__BB5_107:\n" " setp.lt.s32 %p75, %r83, 2;\n" " mov.f32 %f1324, %f1318;\n" " @%p75 bra $L__BB5_109;\n" "\n" " ld.shared.f32 %f771, [%r27];\n" " fma.rn.f32 %f1324, %f1295, %f771, 0f00000000;\n" "\n" "$L__BB5_109:\n" " setp.lt.s32 %p76, %r83, 3;\n" " mov.f32 %f1323, %f1318;\n" " @%p76 bra $L__BB5_111;\n" "\n" " ld.shared.f32 %f773, [%r28];\n" " fma.rn.f32 %f1323, %f1295, %f773, 0f00000000;\n" "\n" "$L__BB5_111:\n" " setp.lt.s32 %p77, %r83, 4;\n" " mov.f32 %f1322, %f1318;\n" " @%p77 bra $L__BB5_113;\n" "\n" " ld.shared.f32 %f775, [%r29];\n" " fma.rn.f32 %f1322, %f1295, %f775, 0f00000000;\n" "\n" "$L__BB5_113:\n" " setp.lt.s32 %p78, %r83, 5;\n" " mov.f32 %f1321, %f1318;\n" " @%p78 bra $L__BB5_115;\n" "\n" " ld.shared.f32 %f777, [%r30];\n" " fma.rn.f32 %f1321, %f1295, %f777, 0f00000000;\n" "\n" "$L__BB5_115:\n" " setp.lt.s32 %p79, %r83, 6;\n" " mov.f32 %f1320, %f1318;\n" " @%p79 bra $L__BB5_117;\n" "\n" " ld.shared.f32 %f779, [%r31];\n" " fma.rn.f32 %f1320, %f1295, %f779, 0f00000000;\n" "\n" "$L__BB5_117:\n" " setp.lt.s32 %p80, %r83, 7;\n" " mov.f32 %f1319, %f1318;\n" " @%p80 bra $L__BB5_119;\n" "\n" " ld.shared.f32 %f781, [%r32];\n" " fma.rn.f32 %f1319, %f1295, %f781, 0f00000000;\n" "\n" "$L__BB5_119:\n" " setp.lt.s32 %p81, %r83, 8;\n" " @%p81 bra $L__BB5_121;\n" "\n" " ld.shared.f32 %f783, [%r33];\n" " fma.rn.f32 %f1318, %f1295, %f783, 0f00000000;\n" "\n" "$L__BB5_121:\n" " @%p57 bra $L__BB5_138;\n" "\n" " @%p2 bra $L__BB5_124;\n" "\n" " ld.shared.f32 %f784, [%r26+64];\n" " fma.rn.f32 %f1325, %f1294, %f784, %f1325;\n" "\n" "$L__BB5_124:\n" " setp.lt.s32 %p84, %r83, 2;\n" " @%p84 bra $L__BB5_126;\n" "\n" " ld.shared.f32 %f785, [%r27+64];\n" " fma.rn.f32 %f1324, %f1294, %f785, %f1324;\n" "\n" "$L__BB5_126:\n" " setp.lt.s32 %p85, %r83, 3;\n" " @%p85 bra $L__BB5_128;\n" "\n" " ld.shared.f32 %f786, [%r28+64];\n" " fma.rn.f32 %f1323, %f1294, %f786, %f1323;\n" "\n" "$L__BB5_128:\n" " setp.lt.s32 %p86, %r83, 4;\n" " @%p86 bra $L__BB5_130;\n" "\n" " ld.shared.f32 %f787, [%r29+64];\n" " fma.rn.f32 %f1322, %f1294, %f787, %f1322;\n" "\n" "$L__BB5_130:\n" " setp.lt.s32 %p87, %r83, 5;\n" " @%p87 bra $L__BB5_132;\n" "\n" " ld.shared.f32 %f788, [%r30+64];\n" " fma.rn.f32 %f1321, %f1294, %f788, %f1321;\n" "\n" "$L__BB5_132:\n" " setp.lt.s32 %p88, %r83, 6;\n" " @%p88 bra $L__BB5_134;\n" "\n" " ld.shared.f32 %f789, [%r31+64];\n" " fma.rn.f32 %f1320, %f1294, %f789, %f1320;\n" "\n" "$L__BB5_134:\n" " setp.lt.s32 %p89, %r83, 7;\n" " @%p89 bra $L__BB5_136;\n" "\n" " ld.shared.f32 %f790, [%r32+64];\n" " fma.rn.f32 %f1319, %f1294, %f790, %f1319;\n" "\n" "$L__BB5_136:\n" " setp.lt.s32 %p90, %r83, 8;\n" " @%p90 bra $L__BB5_138;\n" "\n" " ld.shared.f32 %f791, [%r33+64];\n" " fma.rn.f32 %f1318, %f1294, %f791, %f1318;\n" "\n" "$L__BB5_138:\n" " @%p58 bra $L__BB5_155;\n" "\n" " @%p2 bra $L__BB5_141;\n" "\n" " ld.shared.f32 %f792, [%r26+128];\n" " fma.rn.f32 %f1325, %f1293, %f792, %f1325;\n" "\n" "$L__BB5_141:\n" " setp.lt.s32 %p93, %r83, 2;\n" " @%p93 bra $L__BB5_143;\n" "\n" " ld.shared.f32 %f793, [%r27+128];\n" " fma.rn.f32 %f1324, %f1293, %f793, %f1324;\n" "\n" "$L__BB5_143:\n" " setp.lt.s32 %p94, %r83, 3;\n" " @%p94 bra $L__BB5_145;\n" "\n" " ld.shared.f32 %f794, [%r28+128];\n" " fma.rn.f32 %f1323, %f1293, %f794, %f1323;\n" "\n" "$L__BB5_145:\n" " setp.lt.s32 %p95, %r83, 4;\n" " @%p95 bra $L__BB5_147;\n" "\n" " ld.shared.f32 %f795, [%r29+128];\n" " fma.rn.f32 %f1322, %f1293, %f795, %f1322;\n" "\n" "$L__BB5_147:\n" " setp.lt.s32 %p96, %r83, 5;\n" " @%p96 bra $L__BB5_149;\n" "\n" " ld.shared.f32 %f796, [%r30+128];\n" " fma.rn.f32 %f1321, %f1293, %f796, %f1321;\n" "\n" "$L__BB5_149:\n" " setp.lt.s32 %p97, %r83, 6;\n" " @%p97 bra $L__BB5_151;\n" "\n" " ld.shared.f32 %f797, [%r31+128];\n" " fma.rn.f32 %f1320, %f1293, %f797, %f1320;\n" "\n" "$L__BB5_151:\n" " setp.lt.s32 %p98, %r83, 7;\n" " @%p98 bra $L__BB5_153;\n" "\n" " ld.shared.f32 %f798, [%r32+128];\n" " fma.rn.f32 %f1319, %f1293, %f798, %f1319;\n" "\n" "$L__BB5_153:\n" " setp.lt.s32 %p99, %r83, 8;\n" " @%p99 bra $L__BB5_155;\n" "\n" " ld.shared.f32 %f799, [%r33+128];\n" " fma.rn.f32 %f1318, %f1293, %f799, %f1318;\n" "\n" "$L__BB5_155:\n" " setp.lt.s32 %p100, %r84, 64;\n" " @%p100 bra $L__BB5_172;\n" "\n" " @%p2 bra $L__BB5_158;\n" "\n" " ld.shared.f32 %f800, [%r26+192];\n" " fma.rn.f32 %f1325, %f1292, %f800, %f1325;\n" "\n" "$L__BB5_158:\n" " setp.lt.s32 %p102, %r83, 2;\n" " @%p102 bra $L__BB5_160;\n" "\n" " ld.shared.f32 %f801, [%r27+192];\n" " fma.rn.f32 %f1324, %f1292, %f801, %f1324;\n" "\n" "$L__BB5_160:\n" " setp.lt.s32 %p103, %r83, 3;\n" " @%p103 bra $L__BB5_162;\n" "\n" " ld.shared.f32 %f802, [%r28+192];\n" " fma.rn.f32 %f1323, %f1292, %f802, %f1323;\n" "\n" "$L__BB5_162:\n" " setp.lt.s32 %p104, %r83, 4;\n" " @%p104 bra $L__BB5_164;\n" "\n" " ld.shared.f32 %f803, [%r29+192];\n" " fma.rn.f32 %f1322, %f1292, %f803, %f1322;\n" "\n" "$L__BB5_164:\n" " setp.lt.s32 %p105, %r83, 5;\n" " @%p105 bra $L__BB5_166;\n" "\n" " ld.shared.f32 %f804, [%r30+192];\n" " fma.rn.f32 %f1321, %f1292, %f804, %f1321;\n" "\n" "$L__BB5_166:\n" " setp.lt.s32 %p106, %r83, 6;\n" " @%p106 bra $L__BB5_168;\n" "\n" " ld.shared.f32 %f805, [%r31+192];\n" " fma.rn.f32 %f1320, %f1292, %f805, %f1320;\n" "\n" "$L__BB5_168:\n" " setp.lt.s32 %p107, %r83, 7;\n" " @%p107 bra $L__BB5_170;\n" "\n" " ld.shared.f32 %f806, [%r32+192];\n" " fma.rn.f32 %f1319, %f1292, %f806, %f1319;\n" "\n" "$L__BB5_170:\n" " setp.lt.s32 %p108, %r83, 8;\n" " @%p108 bra $L__BB5_172;\n" "\n" " ld.shared.f32 %f807, [%r33+192];\n" " fma.rn.f32 %f1318, %f1292, %f807, %f1318;\n" "\n" "$L__BB5_172:\n" " setp.lt.s32 %p109, %r84, 80;\n" " @%p109 bra $L__BB5_189;\n" "\n" " @%p2 bra $L__BB5_175;\n" "\n" " ld.shared.f32 %f808, [%r26+256];\n" " fma.rn.f32 %f1325, %f1291, %f808, %f1325;\n" "\n" "$L__BB5_175:\n" " setp.lt.s32 %p111, %r83, 2;\n" " @%p111 bra $L__BB5_177;\n" "\n" " ld.shared.f32 %f809, [%r27+256];\n" " fma.rn.f32 %f1324, %f1291, %f809, %f1324;\n" "\n" "$L__BB5_177:\n" " setp.lt.s32 %p112, %r83, 3;\n" " @%p112 bra $L__BB5_179;\n" "\n" " ld.shared.f32 %f810, [%r28+256];\n" " fma.rn.f32 %f1323, %f1291, %f810, %f1323;\n" "\n" "$L__BB5_179:\n" " setp.lt.s32 %p113, %r83, 4;\n" " @%p113 bra $L__BB5_181;\n" "\n" " ld.shared.f32 %f811, [%r29+256];\n" " fma.rn.f32 %f1322, %f1291, %f811, %f1322;\n" "\n" "$L__BB5_181:\n" " setp.lt.s32 %p114, %r83, 5;\n" " @%p114 bra $L__BB5_183;\n" "\n" " ld.shared.f32 %f812, [%r30+256];\n" " fma.rn.f32 %f1321, %f1291, %f812, %f1321;\n" "\n" "$L__BB5_183:\n" " setp.lt.s32 %p115, %r83, 6;\n" " @%p115 bra $L__BB5_185;\n" "\n" " ld.shared.f32 %f813, [%r31+256];\n" " fma.rn.f32 %f1320, %f1291, %f813, %f1320;\n" "\n" "$L__BB5_185:\n" " setp.lt.s32 %p116, %r83, 7;\n" " @%p116 bra $L__BB5_187;\n" "\n" " ld.shared.f32 %f814, [%r32+256];\n" " fma.rn.f32 %f1319, %f1291, %f814, %f1319;\n" "\n" "$L__BB5_187:\n" " setp.lt.s32 %p117, %r83, 8;\n" " @%p117 bra $L__BB5_189;\n" "\n" " ld.shared.f32 %f815, [%r33+256];\n" " fma.rn.f32 %f1318, %f1291, %f815, %f1318;\n" "\n" "$L__BB5_189:\n" " setp.lt.s32 %p118, %r84, 96;\n" " @%p118 bra $L__BB5_206;\n" "\n" " @%p2 bra $L__BB5_192;\n" "\n" " ld.shared.f32 %f816, [%r26+320];\n" " fma.rn.f32 %f1325, %f1290, %f816, %f1325;\n" "\n" "$L__BB5_192:\n" " setp.lt.s32 %p120, %r83, 2;\n" " @%p120 bra $L__BB5_194;\n" "\n" " ld.shared.f32 %f817, [%r27+320];\n" " fma.rn.f32 %f1324, %f1290, %f817, %f1324;\n" "\n" "$L__BB5_194:\n" " setp.lt.s32 %p121, %r83, 3;\n" " @%p121 bra $L__BB5_196;\n" "\n" " ld.shared.f32 %f818, [%r28+320];\n" " fma.rn.f32 %f1323, %f1290, %f818, %f1323;\n" "\n" "$L__BB5_196:\n" " setp.lt.s32 %p122, %r83, 4;\n" " @%p122 bra $L__BB5_198;\n" "\n" " ld.shared.f32 %f819, [%r29+320];\n" " fma.rn.f32 %f1322, %f1290, %f819, %f1322;\n" "\n" "$L__BB5_198:\n" " setp.lt.s32 %p123, %r83, 5;\n" " @%p123 bra $L__BB5_200;\n" "\n" " ld.shared.f32 %f820, [%r30+320];\n" " fma.rn.f32 %f1321, %f1290, %f820, %f1321;\n" "\n" "$L__BB5_200:\n" " setp.lt.s32 %p124, %r83, 6;\n" " @%p124 bra $L__BB5_202;\n" "\n" " ld.shared.f32 %f821, [%r31+320];\n" " fma.rn.f32 %f1320, %f1290, %f821, %f1320;\n" "\n" "$L__BB5_202:\n" " setp.lt.s32 %p125, %r83, 7;\n" " @%p125 bra $L__BB5_204;\n" "\n" " ld.shared.f32 %f822, [%r32+320];\n" " fma.rn.f32 %f1319, %f1290, %f822, %f1319;\n" "\n" "$L__BB5_204:\n" " setp.lt.s32 %p126, %r83, 8;\n" " @%p126 bra $L__BB5_206;\n" "\n" " ld.shared.f32 %f823, [%r33+320];\n" " fma.rn.f32 %f1318, %f1290, %f823, %f1318;\n" "\n" "$L__BB5_206:\n" " setp.lt.s32 %p127, %r84, 112;\n" " @%p127 bra $L__BB5_223;\n" "\n" " @%p2 bra $L__BB5_209;\n" "\n" " ld.shared.f32 %f824, [%r26+384];\n" " fma.rn.f32 %f1325, %f1289, %f824, %f1325;\n" "\n" "$L__BB5_209:\n" " setp.lt.s32 %p129, %r83, 2;\n" " @%p129 bra $L__BB5_211;\n" "\n" " ld.shared.f32 %f825, [%r27+384];\n" " fma.rn.f32 %f1324, %f1289, %f825, %f1324;\n" "\n" "$L__BB5_211:\n" " setp.lt.s32 %p130, %r83, 3;\n" " @%p130 bra $L__BB5_213;\n" "\n" " ld.shared.f32 %f826, [%r28+384];\n" " fma.rn.f32 %f1323, %f1289, %f826, %f1323;\n" "\n" "$L__BB5_213:\n" " setp.lt.s32 %p131, %r83, 4;\n" " @%p131 bra $L__BB5_215;\n" "\n" " ld.shared.f32 %f827, [%r29+384];\n" " fma.rn.f32 %f1322, %f1289, %f827, %f1322;\n" "\n" "$L__BB5_215:\n" " setp.lt.s32 %p132, %r83, 5;\n" " @%p132 bra $L__BB5_217;\n" "\n" " ld.shared.f32 %f828, [%r30+384];\n" " fma.rn.f32 %f1321, %f1289, %f828, %f1321;\n" "\n" "$L__BB5_217:\n" " setp.lt.s32 %p133, %r83, 6;\n" " @%p133 bra $L__BB5_219;\n" "\n" " ld.shared.f32 %f829, [%r31+384];\n" " fma.rn.f32 %f1320, %f1289, %f829, %f1320;\n" "\n" "$L__BB5_219:\n" " setp.lt.s32 %p134, %r83, 7;\n" " @%p134 bra $L__BB5_221;\n" "\n" " ld.shared.f32 %f830, [%r32+384];\n" " fma.rn.f32 %f1319, %f1289, %f830, %f1319;\n" "\n" "$L__BB5_221:\n" " setp.lt.s32 %p135, %r83, 8;\n" " @%p135 bra $L__BB5_223;\n" "\n" " ld.shared.f32 %f831, [%r33+384];\n" " fma.rn.f32 %f1318, %f1289, %f831, %f1318;\n" "\n" "$L__BB5_223:\n" " setp.lt.s32 %p136, %r84, 128;\n" " @%p136 bra $L__BB5_240;\n" "\n" " @%p2 bra $L__BB5_226;\n" "\n" " ld.shared.f32 %f832, [%r26+448];\n" " fma.rn.f32 %f1325, %f1288, %f832, %f1325;\n" "\n" "$L__BB5_226:\n" " setp.lt.s32 %p138, %r83, 2;\n" " @%p138 bra $L__BB5_228;\n" "\n" " ld.shared.f32 %f833, [%r27+448];\n" " fma.rn.f32 %f1324, %f1288, %f833, %f1324;\n" "\n" "$L__BB5_228:\n" " setp.lt.s32 %p139, %r83, 3;\n" " @%p139 bra $L__BB5_230;\n" "\n" " ld.shared.f32 %f834, [%r28+448];\n" " fma.rn.f32 %f1323, %f1288, %f834, %f1323;\n" "\n" "$L__BB5_230:\n" " setp.lt.s32 %p140, %r83, 4;\n" " @%p140 bra $L__BB5_232;\n" "\n" " ld.shared.f32 %f835, [%r29+448];\n" " fma.rn.f32 %f1322, %f1288, %f835, %f1322;\n" "\n" "$L__BB5_232:\n" " setp.lt.s32 %p141, %r83, 5;\n" " @%p141 bra $L__BB5_234;\n" "\n" " ld.shared.f32 %f836, [%r30+448];\n" " fma.rn.f32 %f1321, %f1288, %f836, %f1321;\n" "\n" "$L__BB5_234:\n" " setp.lt.s32 %p142, %r83, 6;\n" " @%p142 bra $L__BB5_236;\n" "\n" " ld.shared.f32 %f837, [%r31+448];\n" " fma.rn.f32 %f1320, %f1288, %f837, %f1320;\n" "\n" "$L__BB5_236:\n" " setp.lt.s32 %p143, %r83, 7;\n" " @%p143 bra $L__BB5_238;\n" "\n" " ld.shared.f32 %f838, [%r32+448];\n" " fma.rn.f32 %f1319, %f1288, %f838, %f1319;\n" "\n" "$L__BB5_238:\n" " setp.lt.s32 %p144, %r83, 8;\n" " @%p144 bra $L__BB5_240;\n" "\n" " ld.shared.f32 %f839, [%r33+448];\n" " fma.rn.f32 %f1318, %f1288, %f839, %f1318;\n" "\n" "$L__BB5_240:\n" " setp.lt.s32 %p145, %r84, 144;\n" " @%p145 bra $L__BB5_257;\n" "\n" " @%p2 bra $L__BB5_243;\n" "\n" " ld.shared.f32 %f840, [%r26+512];\n" " fma.rn.f32 %f1325, %f1287, %f840, %f1325;\n" "\n" "$L__BB5_243:\n" " setp.lt.s32 %p147, %r83, 2;\n" " @%p147 bra $L__BB5_245;\n" "\n" " ld.shared.f32 %f841, [%r27+512];\n" " fma.rn.f32 %f1324, %f1287, %f841, %f1324;\n" "\n" "$L__BB5_245:\n" " setp.lt.s32 %p148, %r83, 3;\n" " @%p148 bra $L__BB5_247;\n" "\n" " ld.shared.f32 %f842, [%r28+512];\n" " fma.rn.f32 %f1323, %f1287, %f842, %f1323;\n" "\n" "$L__BB5_247:\n" " setp.lt.s32 %p149, %r83, 4;\n" " @%p149 bra $L__BB5_249;\n" "\n" " ld.shared.f32 %f843, [%r29+512];\n" " fma.rn.f32 %f1322, %f1287, %f843, %f1322;\n" "\n" "$L__BB5_249:\n" " setp.lt.s32 %p150, %r83, 5;\n" " @%p150 bra $L__BB5_251;\n" "\n" " ld.shared.f32 %f844, [%r30+512];\n" " fma.rn.f32 %f1321, %f1287, %f844, %f1321;\n" "\n" "$L__BB5_251:\n" " setp.lt.s32 %p151, %r83, 6;\n" " @%p151 bra $L__BB5_253;\n" "\n" " ld.shared.f32 %f845, [%r31+512];\n" " fma.rn.f32 %f1320, %f1287, %f845, %f1320;\n" "\n" "$L__BB5_253:\n" " setp.lt.s32 %p152, %r83, 7;\n" " @%p152 bra $L__BB5_255;\n" "\n" " ld.shared.f32 %f846, [%r32+512];\n" " fma.rn.f32 %f1319, %f1287, %f846, %f1319;\n" "\n" "$L__BB5_255:\n" " setp.lt.s32 %p153, %r83, 8;\n" " @%p153 bra $L__BB5_257;\n" "\n" " ld.shared.f32 %f847, [%r33+512];\n" " fma.rn.f32 %f1318, %f1287, %f847, %f1318;\n" "\n" "$L__BB5_257:\n" " setp.lt.s32 %p154, %r84, 160;\n" " @%p154 bra $L__BB5_274;\n" "\n" " @%p2 bra $L__BB5_260;\n" "\n" " ld.shared.f32 %f848, [%r26+576];\n" " fma.rn.f32 %f1325, %f1286, %f848, %f1325;\n" "\n" "$L__BB5_260:\n" " setp.lt.s32 %p156, %r83, 2;\n" " @%p156 bra $L__BB5_262;\n" "\n" " ld.shared.f32 %f849, [%r27+576];\n" " fma.rn.f32 %f1324, %f1286, %f849, %f1324;\n" "\n" "$L__BB5_262:\n" " setp.lt.s32 %p157, %r83, 3;\n" " @%p157 bra $L__BB5_264;\n" "\n" " ld.shared.f32 %f850, [%r28+576];\n" " fma.rn.f32 %f1323, %f1286, %f850, %f1323;\n" "\n" "$L__BB5_264:\n" " setp.lt.s32 %p158, %r83, 4;\n" " @%p158 bra $L__BB5_266;\n" "\n" " ld.shared.f32 %f851, [%r29+576];\n" " fma.rn.f32 %f1322, %f1286, %f851, %f1322;\n" "\n" "$L__BB5_266:\n" " setp.lt.s32 %p159, %r83, 5;\n" " @%p159 bra $L__BB5_268;\n" "\n" " ld.shared.f32 %f852, [%r30+576];\n" " fma.rn.f32 %f1321, %f1286, %f852, %f1321;\n" "\n" "$L__BB5_268:\n" " setp.lt.s32 %p160, %r83, 6;\n" " @%p160 bra $L__BB5_270;\n" "\n" " ld.shared.f32 %f853, [%r31+576];\n" " fma.rn.f32 %f1320, %f1286, %f853, %f1320;\n" "\n" "$L__BB5_270:\n" " setp.lt.s32 %p161, %r83, 7;\n" " @%p161 bra $L__BB5_272;\n" "\n" " ld.shared.f32 %f854, [%r32+576];\n" " fma.rn.f32 %f1319, %f1286, %f854, %f1319;\n" "\n" "$L__BB5_272:\n" " setp.lt.s32 %p162, %r83, 8;\n" " @%p162 bra $L__BB5_274;\n" "\n" " ld.shared.f32 %f855, [%r33+576];\n" " fma.rn.f32 %f1318, %f1286, %f855, %f1318;\n" "\n" "$L__BB5_274:\n" " setp.lt.s32 %p163, %r84, 176;\n" " @%p163 bra $L__BB5_291;\n" "\n" " @%p2 bra $L__BB5_277;\n" "\n" " ld.shared.f32 %f856, [%r26+640];\n" " fma.rn.f32 %f1325, %f1285, %f856, %f1325;\n" "\n" "$L__BB5_277:\n" " setp.lt.s32 %p165, %r83, 2;\n" " @%p165 bra $L__BB5_279;\n" "\n" " ld.shared.f32 %f857, [%r27+640];\n" " fma.rn.f32 %f1324, %f1285, %f857, %f1324;\n" "\n" "$L__BB5_279:\n" " setp.lt.s32 %p166, %r83, 3;\n" " @%p166 bra $L__BB5_281;\n" "\n" " ld.shared.f32 %f858, [%r28+640];\n" " fma.rn.f32 %f1323, %f1285, %f858, %f1323;\n" "\n" "$L__BB5_281:\n" " setp.lt.s32 %p167, %r83, 4;\n" " @%p167 bra $L__BB5_283;\n" "\n" " ld.shared.f32 %f859, [%r29+640];\n" " fma.rn.f32 %f1322, %f1285, %f859, %f1322;\n" "\n" "$L__BB5_283:\n" " setp.lt.s32 %p168, %r83, 5;\n" " @%p168 bra $L__BB5_285;\n" "\n" " ld.shared.f32 %f860, [%r30+640];\n" " fma.rn.f32 %f1321, %f1285, %f860, %f1321;\n" "\n" "$L__BB5_285:\n" " setp.lt.s32 %p169, %r83, 6;\n" " @%p169 bra $L__BB5_287;\n" "\n" " ld.shared.f32 %f861, [%r31+640];\n" " fma.rn.f32 %f1320, %f1285, %f861, %f1320;\n" "\n" "$L__BB5_287:\n" " setp.lt.s32 %p170, %r83, 7;\n" " @%p170 bra $L__BB5_289;\n" "\n" " ld.shared.f32 %f862, [%r32+640];\n" " fma.rn.f32 %f1319, %f1285, %f862, %f1319;\n" "\n" "$L__BB5_289:\n" " setp.lt.s32 %p171, %r83, 8;\n" " @%p171 bra $L__BB5_291;\n" "\n" " ld.shared.f32 %f863, [%r33+640];\n" " fma.rn.f32 %f1318, %f1285, %f863, %f1318;\n" "\n" "$L__BB5_291:\n" " setp.lt.s32 %p172, %r84, 192;\n" " @%p172 bra $L__BB5_308;\n" "\n" " @%p2 bra $L__BB5_294;\n" "\n" " ld.shared.f32 %f864, [%r26+704];\n" " fma.rn.f32 %f1325, %f1284, %f864, %f1325;\n" "\n" "$L__BB5_294:\n" " setp.lt.s32 %p174, %r83, 2;\n" " @%p174 bra $L__BB5_296;\n" "\n" " ld.shared.f32 %f865, [%r27+704];\n" " fma.rn.f32 %f1324, %f1284, %f865, %f1324;\n" "\n" "$L__BB5_296:\n" " setp.lt.s32 %p175, %r83, 3;\n" " @%p175 bra $L__BB5_298;\n" "\n" " ld.shared.f32 %f866, [%r28+704];\n" " fma.rn.f32 %f1323, %f1284, %f866, %f1323;\n" "\n" "$L__BB5_298:\n" " setp.lt.s32 %p176, %r83, 4;\n" " @%p176 bra $L__BB5_300;\n" "\n" " ld.shared.f32 %f867, [%r29+704];\n" " fma.rn.f32 %f1322, %f1284, %f867, %f1322;\n" "\n" "$L__BB5_300:\n" " setp.lt.s32 %p177, %r83, 5;\n" " @%p177 bra $L__BB5_302;\n" "\n" " ld.shared.f32 %f868, [%r30+704];\n" " fma.rn.f32 %f1321, %f1284, %f868, %f1321;\n" "\n" "$L__BB5_302:\n" " setp.lt.s32 %p178, %r83, 6;\n" " @%p178 bra $L__BB5_304;\n" "\n" " ld.shared.f32 %f869, [%r31+704];\n" " fma.rn.f32 %f1320, %f1284, %f869, %f1320;\n" "\n" "$L__BB5_304:\n" " setp.lt.s32 %p179, %r83, 7;\n" " @%p179 bra $L__BB5_306;\n" "\n" " ld.shared.f32 %f870, [%r32+704];\n" " fma.rn.f32 %f1319, %f1284, %f870, %f1319;\n" "\n" "$L__BB5_306:\n" " setp.lt.s32 %p180, %r83, 8;\n" " @%p180 bra $L__BB5_308;\n" "\n" " ld.shared.f32 %f871, [%r33+704];\n" " fma.rn.f32 %f1318, %f1284, %f871, %f1318;\n" "\n" "$L__BB5_308:\n" " setp.lt.s32 %p181, %r84, 208;\n" " @%p181 bra $L__BB5_325;\n" "\n" " @%p2 bra $L__BB5_311;\n" "\n" " ld.shared.f32 %f872, [%r26+768];\n" " fma.rn.f32 %f1325, %f1283, %f872, %f1325;\n" "\n" "$L__BB5_311:\n" " setp.lt.s32 %p183, %r83, 2;\n" " @%p183 bra $L__BB5_313;\n" "\n" " ld.shared.f32 %f873, [%r27+768];\n" " fma.rn.f32 %f1324, %f1283, %f873, %f1324;\n" "\n" "$L__BB5_313:\n" " setp.lt.s32 %p184, %r83, 3;\n" " @%p184 bra $L__BB5_315;\n" "\n" " ld.shared.f32 %f874, [%r28+768];\n" " fma.rn.f32 %f1323, %f1283, %f874, %f1323;\n" "\n" "$L__BB5_315:\n" " setp.lt.s32 %p185, %r83, 4;\n" " @%p185 bra $L__BB5_317;\n" "\n" " ld.shared.f32 %f875, [%r29+768];\n" " fma.rn.f32 %f1322, %f1283, %f875, %f1322;\n" "\n" "$L__BB5_317:\n" " setp.lt.s32 %p186, %r83, 5;\n" " @%p186 bra $L__BB5_319;\n" "\n" " ld.shared.f32 %f876, [%r30+768];\n" " fma.rn.f32 %f1321, %f1283, %f876, %f1321;\n" "\n" "$L__BB5_319:\n" " setp.lt.s32 %p187, %r83, 6;\n" " @%p187 bra $L__BB5_321;\n" "\n" " ld.shared.f32 %f877, [%r31+768];\n" " fma.rn.f32 %f1320, %f1283, %f877, %f1320;\n" "\n" "$L__BB5_321:\n" " setp.lt.s32 %p188, %r83, 7;\n" " @%p188 bra $L__BB5_323;\n" "\n" " ld.shared.f32 %f878, [%r32+768];\n" " fma.rn.f32 %f1319, %f1283, %f878, %f1319;\n" "\n" "$L__BB5_323:\n" " setp.lt.s32 %p189, %r83, 8;\n" " @%p189 bra $L__BB5_325;\n" "\n" " ld.shared.f32 %f879, [%r33+768];\n" " fma.rn.f32 %f1318, %f1283, %f879, %f1318;\n" "\n" "$L__BB5_325:\n" " setp.lt.s32 %p190, %r84, 224;\n" " @%p190 bra $L__BB5_342;\n" "\n" " @%p2 bra $L__BB5_328;\n" "\n" " ld.shared.f32 %f880, [%r26+832];\n" " fma.rn.f32 %f1325, %f1282, %f880, %f1325;\n" "\n" "$L__BB5_328:\n" " setp.lt.s32 %p192, %r83, 2;\n" " @%p192 bra $L__BB5_330;\n" "\n" " ld.shared.f32 %f881, [%r27+832];\n" " fma.rn.f32 %f1324, %f1282, %f881, %f1324;\n" "\n" "$L__BB5_330:\n" " setp.lt.s32 %p193, %r83, 3;\n" " @%p193 bra $L__BB5_332;\n" "\n" " ld.shared.f32 %f882, [%r28+832];\n" " fma.rn.f32 %f1323, %f1282, %f882, %f1323;\n" "\n" "$L__BB5_332:\n" " setp.lt.s32 %p194, %r83, 4;\n" " @%p194 bra $L__BB5_334;\n" "\n" " ld.shared.f32 %f883, [%r29+832];\n" " fma.rn.f32 %f1322, %f1282, %f883, %f1322;\n" "\n" "$L__BB5_334:\n" " setp.lt.s32 %p195, %r83, 5;\n" " @%p195 bra $L__BB5_336;\n" "\n" " ld.shared.f32 %f884, [%r30+832];\n" " fma.rn.f32 %f1321, %f1282, %f884, %f1321;\n" "\n" "$L__BB5_336:\n" " setp.lt.s32 %p196, %r83, 6;\n" " @%p196 bra $L__BB5_338;\n" "\n" " ld.shared.f32 %f885, [%r31+832];\n" " fma.rn.f32 %f1320, %f1282, %f885, %f1320;\n" "\n" "$L__BB5_338:\n" " setp.lt.s32 %p197, %r83, 7;\n" " @%p197 bra $L__BB5_340;\n" "\n" " ld.shared.f32 %f886, [%r32+832];\n" " fma.rn.f32 %f1319, %f1282, %f886, %f1319;\n" "\n" "$L__BB5_340:\n" " setp.lt.s32 %p198, %r83, 8;\n" " @%p198 bra $L__BB5_342;\n" "\n" " ld.shared.f32 %f887, [%r33+832];\n" " fma.rn.f32 %f1318, %f1282, %f887, %f1318;\n" "\n" "$L__BB5_342:\n" " setp.lt.s32 %p199, %r84, 240;\n" " @%p199 bra $L__BB5_359;\n" "\n" " @%p2 bra $L__BB5_345;\n" "\n" " ld.shared.f32 %f888, [%r26+896];\n" " fma.rn.f32 %f1325, %f1281, %f888, %f1325;\n" "\n" "$L__BB5_345:\n" " setp.lt.s32 %p201, %r83, 2;\n" " @%p201 bra $L__BB5_347;\n" "\n" " ld.shared.f32 %f889, [%r27+896];\n" " fma.rn.f32 %f1324, %f1281, %f889, %f1324;\n" "\n" "$L__BB5_347:\n" " setp.lt.s32 %p202, %r83, 3;\n" " @%p202 bra $L__BB5_349;\n" "\n" " ld.shared.f32 %f890, [%r28+896];\n" " fma.rn.f32 %f1323, %f1281, %f890, %f1323;\n" "\n" "$L__BB5_349:\n" " setp.lt.s32 %p203, %r83, 4;\n" " @%p203 bra $L__BB5_351;\n" "\n" " ld.shared.f32 %f891, [%r29+896];\n" " fma.rn.f32 %f1322, %f1281, %f891, %f1322;\n" "\n" "$L__BB5_351:\n" " setp.lt.s32 %p204, %r83, 5;\n" " @%p204 bra $L__BB5_353;\n" "\n" " ld.shared.f32 %f892, [%r30+896];\n" " fma.rn.f32 %f1321, %f1281, %f892, %f1321;\n" "\n" "$L__BB5_353:\n" " setp.lt.s32 %p205, %r83, 6;\n" " @%p205 bra $L__BB5_355;\n" "\n" " ld.shared.f32 %f893, [%r31+896];\n" " fma.rn.f32 %f1320, %f1281, %f893, %f1320;\n" "\n" "$L__BB5_355:\n" " setp.lt.s32 %p206, %r83, 7;\n" " @%p206 bra $L__BB5_357;\n" "\n" " ld.shared.f32 %f894, [%r32+896];\n" " fma.rn.f32 %f1319, %f1281, %f894, %f1319;\n" "\n" "$L__BB5_357:\n" " setp.lt.s32 %p207, %r83, 8;\n" " @%p207 bra $L__BB5_359;\n" "\n" " ld.shared.f32 %f895, [%r33+896];\n" " fma.rn.f32 %f1318, %f1281, %f895, %f1318;\n" "\n" "$L__BB5_359:\n" " setp.lt.s32 %p208, %r84, 256;\n" " @%p208 bra $L__BB5_376;\n" "\n" " @%p2 bra $L__BB5_362;\n" "\n" " ld.shared.f32 %f896, [%r26+960];\n" " fma.rn.f32 %f1325, %f1280, %f896, %f1325;\n" "\n" "$L__BB5_362:\n" " setp.lt.s32 %p210, %r83, 2;\n" " @%p210 bra $L__BB5_364;\n" "\n" " ld.shared.f32 %f897, [%r27+960];\n" " fma.rn.f32 %f1324, %f1280, %f897, %f1324;\n" "\n" "$L__BB5_364:\n" " setp.lt.s32 %p211, %r83, 3;\n" " @%p211 bra $L__BB5_366;\n" "\n" " ld.shared.f32 %f898, [%r28+960];\n" " fma.rn.f32 %f1323, %f1280, %f898, %f1323;\n" "\n" "$L__BB5_366:\n" " setp.lt.s32 %p212, %r83, 4;\n" " @%p212 bra $L__BB5_368;\n" "\n" " ld.shared.f32 %f899, [%r29+960];\n" " fma.rn.f32 %f1322, %f1280, %f899, %f1322;\n" "\n" "$L__BB5_368:\n" " setp.lt.s32 %p213, %r83, 5;\n" " @%p213 bra $L__BB5_370;\n" "\n" " ld.shared.f32 %f900, [%r30+960];\n" " fma.rn.f32 %f1321, %f1280, %f900, %f1321;\n" "\n" "$L__BB5_370:\n" " setp.lt.s32 %p214, %r83, 6;\n" " @%p214 bra $L__BB5_372;\n" "\n" " ld.shared.f32 %f901, [%r31+960];\n" " fma.rn.f32 %f1320, %f1280, %f901, %f1320;\n" "\n" "$L__BB5_372:\n" " setp.lt.s32 %p215, %r83, 7;\n" " @%p215 bra $L__BB5_374;\n" "\n" " ld.shared.f32 %f902, [%r32+960];\n" " fma.rn.f32 %f1319, %f1280, %f902, %f1319;\n" "\n" "$L__BB5_374:\n" " setp.lt.s32 %p216, %r83, 8;\n" " @%p216 bra $L__BB5_376;\n" "\n" " ld.shared.f32 %f903, [%r33+960];\n" " fma.rn.f32 %f1318, %f1280, %f903, %f1318;\n" "\n" "$L__BB5_376:\n" " setp.eq.s32 %p217, %r24, 0;\n" " setp.lt.s32 %p218, %r23, %r39;\n" " and.pred %p1, %p217, %p218;\n" " @%p2 bra $L__BB5_379;\n" "\n" " mov.b32 %r251, %f1325;\n" " mov.u32 %r252, 2;\n" " mov.u32 %r253, 4127;\n" " mov.u32 %r254, 8;\n" " mov.u32 %r255, -1;\n" " shfl.sync.down.b32 %r256|%p220, %r251, %r254, %r253, %r255;\n" " mov.b32 %f904, %r256;\n" " add.f32 %f905, %f1325, %f904;\n" " mov.b32 %r257, %f905;\n" " mov.u32 %r258, 4;\n" " shfl.sync.down.b32 %r259|%p221, %r257, %r258, %r253, %r255;\n" " mov.b32 %f906, %r259;\n" " add.f32 %f907, %f905, %f906;\n" " mov.b32 %r260, %f907;\n" " shfl.sync.down.b32 %r261|%p222, %r260, %r252, %r253, %r255;\n" " mov.b32 %f908, %r261;\n" " add.f32 %f909, %f907, %f908;\n" " mov.b32 %r262, %f909;\n" " mov.u32 %r263, 1;\n" " shfl.sync.down.b32 %r264|%p223, %r262, %r263, %r253, %r255;\n" " mov.b32 %f910, %r264;\n" " add.f32 %f509, %f909, %f910;\n" " not.pred %p224, %p1;\n" " @%p224 bra $L__BB5_379;\n" "\n" " mul.f32 %f911, %f509, %f673;\n" " mad.lo.s32 %r265, %r84, %r83, %r23;\n" " shl.b32 %r266, %r265, 2;\n" " add.s32 %r268, %r110, %r266;\n" " st.shared.f32 [%r268], %f911;\n" "\n" "$L__BB5_379:\n" " setp.lt.s32 %p225, %r83, 2;\n" " @%p225 bra $L__BB5_382;\n" "\n" " mov.b32 %r269, %f1324;\n" " mov.u32 %r270, 2;\n" " mov.u32 %r271, 4127;\n" " mov.u32 %r272, 8;\n" " mov.u32 %r273, -1;\n" " shfl.sync.down.b32 %r274|%p226, %r269, %r272, %r271, %r273;\n" " mov.b32 %f912, %r274;\n" " add.f32 %f913, %f1324, %f912;\n" " mov.b32 %r275, %f913;\n" " mov.u32 %r276, 4;\n" " shfl.sync.down.b32 %r277|%p227, %r275, %r276, %r271, %r273;\n" " mov.b32 %f914, %r277;\n" " add.f32 %f915, %f913, %f914;\n" " mov.b32 %r278, %f915;\n" " shfl.sync.down.b32 %r279|%p228, %r278, %r270, %r271, %r273;\n" " mov.b32 %f916, %r279;\n" " add.f32 %f917, %f915, %f916;\n" " mov.b32 %r280, %f917;\n" " mov.u32 %r281, 1;\n" " shfl.sync.down.b32 %r282|%p229, %r280, %r281, %r271, %r273;\n" " mov.b32 %f918, %r282;\n" " add.f32 %f510, %f917, %f918;\n" " not.pred %p230, %p1;\n" " @%p230 bra $L__BB5_382;\n" "\n" " mul.f32 %f919, %f510, %f673;\n" " mad.lo.s32 %r283, %r84, %r83, %r23;\n" " shl.b32 %r284, %r283, 2;\n" " add.s32 %r286, %r110, %r284;\n" " st.shared.f32 [%r286+64], %f919;\n" "\n" "$L__BB5_382:\n" " setp.lt.s32 %p231, %r83, 3;\n" " @%p231 bra $L__BB5_385;\n" "\n" " mov.b32 %r287, %f1323;\n" " mov.u32 %r288, 2;\n" " mov.u32 %r289, 4127;\n" " mov.u32 %r290, 8;\n" " mov.u32 %r291, -1;\n" " shfl.sync.down.b32 %r292|%p232, %r287, %r290, %r289, %r291;\n" " mov.b32 %f920, %r292;\n" " add.f32 %f921, %f1323, %f920;\n" " mov.b32 %r293, %f921;\n" " mov.u32 %r294, 4;\n" " shfl.sync.down.b32 %r295|%p233, %r293, %r294, %r289, %r291;\n" " mov.b32 %f922, %r295;\n" " add.f32 %f923, %f921, %f922;\n" " mov.b32 %r296, %f923;\n" " shfl.sync.down.b32 %r297|%p234, %r296, %r288, %r289, %r291;\n" " mov.b32 %f924, %r297;\n" " add.f32 %f925, %f923, %f924;\n" " mov.b32 %r298, %f925;\n" " mov.u32 %r299, 1;\n" " shfl.sync.down.b32 %r300|%p235, %r298, %r299, %r289, %r291;\n" " mov.b32 %f926, %r300;\n" " add.f32 %f511, %f925, %f926;\n" " not.pred %p236, %p1;\n" " @%p236 bra $L__BB5_385;\n" "\n" " mul.f32 %f927, %f511, %f673;\n" " mad.lo.s32 %r301, %r84, %r83, %r23;\n" " shl.b32 %r302, %r301, 2;\n" " add.s32 %r304, %r110, %r302;\n" " st.shared.f32 [%r304+128], %f927;\n" "\n" "$L__BB5_385:\n" " setp.lt.s32 %p237, %r83, 4;\n" " @%p237 bra $L__BB5_388;\n" "\n" " mov.b32 %r305, %f1322;\n" " mov.u32 %r306, 2;\n" " mov.u32 %r307, 4127;\n" " mov.u32 %r308, 8;\n" " mov.u32 %r309, -1;\n" " shfl.sync.down.b32 %r310|%p238, %r305, %r308, %r307, %r309;\n" " mov.b32 %f928, %r310;\n" " add.f32 %f929, %f1322, %f928;\n" " mov.b32 %r311, %f929;\n" " mov.u32 %r312, 4;\n" " shfl.sync.down.b32 %r313|%p239, %r311, %r312, %r307, %r309;\n" " mov.b32 %f930, %r313;\n" " add.f32 %f931, %f929, %f930;\n" " mov.b32 %r314, %f931;\n" " shfl.sync.down.b32 %r315|%p240, %r314, %r306, %r307, %r309;\n" " mov.b32 %f932, %r315;\n" " add.f32 %f933, %f931, %f932;\n" " mov.b32 %r316, %f933;\n" " mov.u32 %r317, 1;\n" " shfl.sync.down.b32 %r318|%p241, %r316, %r317, %r307, %r309;\n" " mov.b32 %f934, %r318;\n" " add.f32 %f512, %f933, %f934;\n" " not.pred %p242, %p1;\n" " @%p242 bra $L__BB5_388;\n" "\n" " mul.f32 %f935, %f512, %f673;\n" " mad.lo.s32 %r319, %r84, %r83, %r23;\n" " shl.b32 %r320, %r319, 2;\n" " add.s32 %r322, %r110, %r320;\n" " st.shared.f32 [%r322+192], %f935;\n" "\n" "$L__BB5_388:\n" " setp.lt.s32 %p243, %r83, 5;\n" " @%p243 bra $L__BB5_391;\n" "\n" " mov.b32 %r323, %f1321;\n" " mov.u32 %r324, 2;\n" " mov.u32 %r325, 4127;\n" " mov.u32 %r326, 8;\n" " mov.u32 %r327, -1;\n" " shfl.sync.down.b32 %r328|%p244, %r323, %r326, %r325, %r327;\n" " mov.b32 %f936, %r328;\n" " add.f32 %f937, %f1321, %f936;\n" " mov.b32 %r329, %f937;\n" " mov.u32 %r330, 4;\n" " shfl.sync.down.b32 %r331|%p245, %r329, %r330, %r325, %r327;\n" " mov.b32 %f938, %r331;\n" " add.f32 %f939, %f937, %f938;\n" " mov.b32 %r332, %f939;\n" " shfl.sync.down.b32 %r333|%p246, %r332, %r324, %r325, %r327;\n" " mov.b32 %f940, %r333;\n" " add.f32 %f941, %f939, %f940;\n" " mov.b32 %r334, %f941;\n" " mov.u32 %r335, 1;\n" " shfl.sync.down.b32 %r336|%p247, %r334, %r335, %r325, %r327;\n" " mov.b32 %f942, %r336;\n" " add.f32 %f513, %f941, %f942;\n" " not.pred %p248, %p1;\n" " @%p248 bra $L__BB5_391;\n" "\n" " mul.f32 %f943, %f513, %f673;\n" " mad.lo.s32 %r337, %r84, %r83, %r23;\n" " shl.b32 %r338, %r337, 2;\n" " add.s32 %r340, %r110, %r338;\n" " st.shared.f32 [%r340+256], %f943;\n" "\n" "$L__BB5_391:\n" " setp.lt.s32 %p249, %r83, 6;\n" " @%p249 bra $L__BB5_394;\n" "\n" " mov.b32 %r341, %f1320;\n" " mov.u32 %r342, 2;\n" " mov.u32 %r343, 4127;\n" " mov.u32 %r344, 8;\n" " mov.u32 %r345, -1;\n" " shfl.sync.down.b32 %r346|%p250, %r341, %r344, %r343, %r345;\n" " mov.b32 %f944, %r346;\n" " add.f32 %f945, %f1320, %f944;\n" " mov.b32 %r347, %f945;\n" " mov.u32 %r348, 4;\n" " shfl.sync.down.b32 %r349|%p251, %r347, %r348, %r343, %r345;\n" " mov.b32 %f946, %r349;\n" " add.f32 %f947, %f945, %f946;\n" " mov.b32 %r350, %f947;\n" " shfl.sync.down.b32 %r351|%p252, %r350, %r342, %r343, %r345;\n" " mov.b32 %f948, %r351;\n" " add.f32 %f949, %f947, %f948;\n" " mov.b32 %r352, %f949;\n" " mov.u32 %r353, 1;\n" " shfl.sync.down.b32 %r354|%p253, %r352, %r353, %r343, %r345;\n" " mov.b32 %f950, %r354;\n" " add.f32 %f514, %f949, %f950;\n" " not.pred %p254, %p1;\n" " @%p254 bra $L__BB5_394;\n" "\n" " mul.f32 %f951, %f514, %f673;\n" " mad.lo.s32 %r355, %r84, %r83, %r23;\n" " shl.b32 %r356, %r355, 2;\n" " add.s32 %r358, %r110, %r356;\n" " st.shared.f32 [%r358+320], %f951;\n" "\n" "$L__BB5_394:\n" " setp.lt.s32 %p255, %r83, 7;\n" " @%p255 bra $L__BB5_397;\n" "\n" " mov.b32 %r359, %f1319;\n" " mov.u32 %r360, 2;\n" " mov.u32 %r361, 4127;\n" " mov.u32 %r362, 8;\n" " mov.u32 %r363, -1;\n" " shfl.sync.down.b32 %r364|%p256, %r359, %r362, %r361, %r363;\n" " mov.b32 %f952, %r364;\n" " add.f32 %f953, %f1319, %f952;\n" " mov.b32 %r365, %f953;\n" " mov.u32 %r366, 4;\n" " shfl.sync.down.b32 %r367|%p257, %r365, %r366, %r361, %r363;\n" " mov.b32 %f954, %r367;\n" " add.f32 %f955, %f953, %f954;\n" " mov.b32 %r368, %f955;\n" " shfl.sync.down.b32 %r369|%p258, %r368, %r360, %r361, %r363;\n" " mov.b32 %f956, %r369;\n" " add.f32 %f957, %f955, %f956;\n" " mov.b32 %r370, %f957;\n" " mov.u32 %r371, 1;\n" " shfl.sync.down.b32 %r372|%p259, %r370, %r371, %r361, %r363;\n" " mov.b32 %f958, %r372;\n" " add.f32 %f515, %f957, %f958;\n" " not.pred %p260, %p1;\n" " @%p260 bra $L__BB5_397;\n" "\n" " mul.f32 %f959, %f515, %f673;\n" " mad.lo.s32 %r373, %r84, %r83, %r23;\n" " shl.b32 %r374, %r373, 2;\n" " add.s32 %r376, %r110, %r374;\n" " st.shared.f32 [%r376+384], %f959;\n" "\n" "$L__BB5_397:\n" " setp.lt.s32 %p261, %r83, 8;\n" " @%p261 bra $L__BB5_400;\n" "\n" " mov.b32 %r377, %f1318;\n" " mov.u32 %r378, 2;\n" " mov.u32 %r379, 4127;\n" " mov.u32 %r380, 8;\n" " mov.u32 %r381, -1;\n" " shfl.sync.down.b32 %r382|%p262, %r377, %r380, %r379, %r381;\n" " mov.b32 %f960, %r382;\n" " add.f32 %f961, %f1318, %f960;\n" " mov.b32 %r383, %f961;\n" " mov.u32 %r384, 4;\n" " shfl.sync.down.b32 %r385|%p263, %r383, %r384, %r379, %r381;\n" " mov.b32 %f962, %r385;\n" " add.f32 %f963, %f961, %f962;\n" " mov.b32 %r386, %f963;\n" " shfl.sync.down.b32 %r387|%p264, %r386, %r378, %r379, %r381;\n" " mov.b32 %f964, %r387;\n" " add.f32 %f965, %f963, %f964;\n" " mov.b32 %r388, %f965;\n" " mov.u32 %r389, 1;\n" " shfl.sync.down.b32 %r390|%p265, %r388, %r389, %r379, %r381;\n" " mov.b32 %f966, %r390;\n" " add.f32 %f516, %f965, %f966;\n" " not.pred %p266, %p1;\n" " @%p266 bra $L__BB5_400;\n" "\n" " mul.f32 %f967, %f516, %f673;\n" " mad.lo.s32 %r391, %r84, %r83, %r23;\n" " shl.b32 %r392, %r391, 2;\n" " add.s32 %r394, %r110, %r392;\n" " st.shared.f32 [%r394+448], %f967;\n" "\n" "$L__BB5_400:\n" " bar.sync 0;\n" " @%p7 bra $L__BB5_418;\n" "\n" " add.s32 %r552, %r2, 1;\n" " sub.s32 %r551, %r552, %r571;\n" " mad.lo.s32 %r397, %r84, %r83, %r1;\n" " shl.b32 %r398, %r83, 5;\n" " add.s32 %r399, %r397, %r398;\n" " shl.b32 %r400, %r399, 2;\n" " add.s32 %r402, %r110, %r400;\n" " ld.shared.f32 %f517, [%r402];\n" " setp.lt.s32 %p268, %r551, 1;\n" " mov.f32 %f1539, %f517;\n" " @%p268 bra $L__BB5_409;\n" "\n" " and.b32 %r49, %r37, 3;\n" " add.s32 %r404, %r37, -1;\n" " setp.lt.u32 %p269, %r404, 3;\n" " mov.u32 %r574, 0;\n" " mov.f32 %f1539, %f517;\n" " @%p269 bra $L__BB5_405;\n" "\n" " sub.s32 %r573, %r37, %r49;\n" " mad.lo.s32 %r51, %r1, 15, %r397;\n" " mov.u32 %r574, 0;\n" " mov.f32 %f1539, %f517;\n" "\n" "$L__BB5_404:\n" " add.s32 %r408, %r51, %r574;\n" " shl.b32 %r409, %r408, 2;\n" " add.s32 %r411, %r110, %r409;\n" " ld.shared.f32 %f969, [%r411];\n" " max.f32 %f970, %f1539, %f969;\n" " ld.shared.f32 %f971, [%r411+4];\n" " max.f32 %f972, %f970, %f971;\n" " ld.shared.f32 %f973, [%r411+8];\n" " max.f32 %f974, %f972, %f973;\n" " ld.shared.f32 %f975, [%r411+12];\n" " max.f32 %f1539, %f974, %f975;\n" " add.s32 %r574, %r574, 4;\n" " add.s32 %r573, %r573, -4;\n" " setp.ne.s32 %p270, %r573, 0;\n" " @%p270 bra $L__BB5_404;\n" "\n" "$L__BB5_405:\n" " setp.eq.s32 %p271, %r49, 0;\n" " @%p271 bra $L__BB5_409;\n" "\n" " mad.lo.s32 %r414, %r1, 15, %r397;\n" " add.s32 %r415, %r414, %r574;\n" " shl.b32 %r416, %r415, 2;\n" " add.s32 %r57, %r110, %r416;\n" " ld.shared.f32 %f976, [%r57];\n" " max.f32 %f1539, %f1539, %f976;\n" " setp.eq.s32 %p272, %r49, 1;\n" " @%p272 bra $L__BB5_409;\n" "\n" " ld.shared.f32 %f977, [%r57+4];\n" " max.f32 %f1539, %f1539, %f977;\n" " setp.eq.s32 %p273, %r49, 2;\n" " @%p273 bra $L__BB5_409;\n" "\n" " ld.shared.f32 %f978, [%r57+8];\n" " max.f32 %f1539, %f1539, %f978;\n" "\n" "$L__BB5_409:\n" " mov.f32 %f1543, 0f00000000;\n" " @%p268 bra $L__BB5_417;\n" "\n" " and.b32 %r58, %r37, 3;\n" " add.s32 %r419, %r37, -1;\n" " setp.lt.u32 %p275, %r419, 3;\n" " mov.f32 %f1543, 0f00000000;\n" " mov.u32 %r578, 0;\n" " @%p275 bra $L__BB5_413;\n" "\n" " sub.s32 %r576, %r37, %r58;\n" " add.s32 %r421, %r84, 16;\n" " mad.lo.s32 %r423, %r83, %r421, %r1;\n" " shl.b32 %r424, %r423, 2;\n" " add.s32 %r577, %r110, %r424;\n" " mov.f32 %f1543, 0f00000000;\n" " mov.u32 %r578, 0;\n" "\n" "$L__BB5_412:\n" " shl.b32 %r559, %r83, 2;\n" " mad.lo.s32 %r558, %r84, %r83, %r1;\n" " mad.lo.s32 %r557, %r1, 15, %r558;\n" " add.s32 %r427, %r557, %r578;\n" " shl.b32 %r428, %r427, 2;\n" " add.s32 %r430, %r110, %r428;\n" " ld.shared.f32 %f983, [%r430];\n" " sub.f32 %f984, %f983, %f1539;\n" " mov.f32 %f985, 0f3F000000;\n" " mov.f32 %f986, 0f3BBB989D;\n" " fma.rn.f32 %f987, %f984, %f986, %f985;\n" " mov.f32 %f988, 0f3FB8AA3B;\n" " mov.f32 %f989, 0f437C0000;\n" " cvt.sat.f32.f32 %f990, %f987;\n" " mov.f32 %f991, 0f4B400001;\n" " fma.rm.f32 %f992, %f990, %f989, %f991;\n" " add.f32 %f993, %f992, 0fCB40007F;\n" " neg.f32 %f994, %f993;\n" " fma.rn.f32 %f995, %f984, %f988, %f994;\n" " mov.f32 %f996, 0f32A57060;\n" " fma.rn.f32 %f997, %f984, %f996, %f995;\n" " mov.b32 %r431, %f992;\n" " shl.b32 %r432, %r431, 23;\n" " mov.b32 %f998, %r432;\n" " ex2.approx.ftz.f32 %f999, %f997;\n" " mul.f32 %f1000, %f999, %f998;\n" " st.shared.f32 [%r577], %f1000;\n" " add.f32 %f1001, %f1543, %f1000;\n" " ld.shared.f32 %f1002, [%r430+4];\n" " sub.f32 %f1003, %f1002, %f1539;\n" " fma.rn.f32 %f1004, %f1003, %f986, %f985;\n" " cvt.sat.f32.f32 %f1005, %f1004;\n" " fma.rm.f32 %f1006, %f1005, %f989, %f991;\n" " add.f32 %f1007, %f1006, 0fCB40007F;\n" " neg.f32 %f1008, %f1007;\n" " fma.rn.f32 %f1009, %f1003, %f988, %f1008;\n" " fma.rn.f32 %f1010, %f1003, %f996, %f1009;\n" " mov.b32 %r433, %f1006;\n" " shl.b32 %r434, %r433, 23;\n" " mov.b32 %f1011, %r434;\n" " ex2.approx.ftz.f32 %f1012, %f1010;\n" " mul.f32 %f1013, %f1012, %f1011;\n" " add.s32 %r435, %r577, %r559;\n" " st.shared.f32 [%r435], %f1013;\n" " add.f32 %f1014, %f1001, %f1013;\n" " ld.shared.f32 %f1015, [%r430+8];\n" " sub.f32 %f1016, %f1015, %f1539;\n" " fma.rn.f32 %f1017, %f1016, %f986, %f985;\n" " cvt.sat.f32.f32 %f1018, %f1017;\n" " fma.rm.f32 %f1019, %f1018, %f989, %f991;\n" " add.f32 %f1020, %f1019, 0fCB40007F;\n" " neg.f32 %f1021, %f1020;\n" " fma.rn.f32 %f1022, %f1016, %f988, %f1021;\n" " fma.rn.f32 %f1023, %f1016, %f996, %f1022;\n" " mov.b32 %r436, %f1019;\n" " shl.b32 %r437, %r436, 23;\n" " mov.b32 %f1024, %r437;\n" " ex2.approx.ftz.f32 %f1025, %f1023;\n" " mul.f32 %f1026, %f1025, %f1024;\n" " add.s32 %r438, %r435, %r559;\n" " st.shared.f32 [%r438], %f1026;\n" " add.f32 %f1027, %f1014, %f1026;\n" " ld.shared.f32 %f1028, [%r430+12];\n" " sub.f32 %f1029, %f1028, %f1539;\n" " fma.rn.f32 %f1030, %f1029, %f986, %f985;\n" " cvt.sat.f32.f32 %f1031, %f1030;\n" " fma.rm.f32 %f1032, %f1031, %f989, %f991;\n" " add.f32 %f1033, %f1032, 0fCB40007F;\n" " neg.f32 %f1034, %f1033;\n" " fma.rn.f32 %f1035, %f1029, %f988, %f1034;\n" " fma.rn.f32 %f1036, %f1029, %f996, %f1035;\n" " mov.b32 %r439, %f1032;\n" " shl.b32 %r440, %r439, 23;\n" " mov.b32 %f1037, %r440;\n" " ex2.approx.ftz.f32 %f1038, %f1036;\n" " mul.f32 %f1039, %f1038, %f1037;\n" " add.s32 %r441, %r438, %r559;\n" " add.s32 %r577, %r441, %r559;\n" " st.shared.f32 [%r441], %f1039;\n" " add.f32 %f1543, %f1027, %f1039;\n" " add.s32 %r578, %r578, 4;\n" " add.s32 %r576, %r576, -4;\n" " setp.ne.s32 %p276, %r576, 0;\n" " @%p276 bra $L__BB5_412;\n" "\n" "$L__BB5_413:\n" " setp.eq.s32 %p277, %r58, 0;\n" " @%p277 bra $L__BB5_417;\n" "\n" " mad.lo.s32 %r445, %r1, 15, %r106;\n" " add.s32 %r446, %r445, %r578;\n" " shl.b32 %r447, %r446, 2;\n" " add.s32 %r70, %r110, %r447;\n" " ld.shared.f32 %f1040, [%r70];\n" " sub.f32 %f1041, %f1040, %f1539;\n" " mov.f32 %f1042, 0f3F000000;\n" " mov.f32 %f1043, 0f3BBB989D;\n" " fma.rn.f32 %f1044, %f1041, %f1043, %f1042;\n" " mov.f32 %f1045, 0f3FB8AA3B;\n" " mov.f32 %f1046, 0f437C0000;\n" " cvt.sat.f32.f32 %f1047, %f1044;\n" " mov.f32 %f1048, 0f4B400001;\n" " fma.rm.f32 %f1049, %f1047, %f1046, %f1048;\n" " add.f32 %f1050, %f1049, 0fCB40007F;\n" " neg.f32 %f1051, %f1050;\n" " fma.rn.f32 %f1052, %f1041, %f1045, %f1051;\n" " mov.f32 %f1053, 0f32A57060;\n" " fma.rn.f32 %f1054, %f1041, %f1053, %f1052;\n" " mov.b32 %r449, %f1049;\n" " shl.b32 %r450, %r449, 23;\n" " mov.b32 %f1055, %r450;\n" " ex2.approx.ftz.f32 %f1056, %f1054;\n" " mul.f32 %f1057, %f1056, %f1055;\n" " add.s32 %r452, %r3, %r4;\n" " add.s32 %r453, %r452, %r1;\n" " mad.lo.s32 %r454, %r578, %r83, %r453;\n" " shl.b32 %r455, %r454, 2;\n" " add.s32 %r71, %r110, %r455;\n" " st.shared.f32 [%r71], %f1057;\n" " add.f32 %f1543, %f1543, %f1057;\n" " setp.eq.s32 %p278, %r58, 1;\n" " @%p278 bra $L__BB5_417;\n" "\n" " mov.f32 %f1205, 0f32A57060;\n" " mov.f32 %f1204, 0f3FB8AA3B;\n" " mov.f32 %f1203, 0f4B400001;\n" " mov.f32 %f1202, 0f437C0000;\n" " mov.f32 %f1201, 0f3F000000;\n" " mov.f32 %f1200, 0f3BBB989D;\n" " ld.shared.f32 %f1058, [%r70+4];\n" " sub.f32 %f1059, %f1058, %f1539;\n" " fma.rn.f32 %f1062, %f1059, %f1200, %f1201;\n" " cvt.sat.f32.f32 %f1065, %f1062;\n" " fma.rm.f32 %f1067, %f1065, %f1202, %f1203;\n" " add.f32 %f1068, %f1067, 0fCB40007F;\n" " neg.f32 %f1069, %f1068;\n" " fma.rn.f32 %f1070, %f1059, %f1204, %f1069;\n" " fma.rn.f32 %f1072, %f1059, %f1205, %f1070;\n" " mov.b32 %r456, %f1067;\n" " shl.b32 %r457, %r456, 23;\n" " mov.b32 %f1073, %r457;\n" " ex2.approx.ftz.f32 %f1074, %f1072;\n" " mul.f32 %f1075, %f1074, %f1073;\n" " shl.b32 %r458, %r83, 2;\n" " add.s32 %r72, %r71, %r458;\n" " st.shared.f32 [%r72], %f1075;\n" " add.f32 %f1543, %f1543, %f1075;\n" " setp.eq.s32 %p279, %r58, 2;\n" " @%p279 bra $L__BB5_417;\n" "\n" " mov.f32 %f1211, 0f32A57060;\n" " mov.f32 %f1210, 0f3FB8AA3B;\n" " mov.f32 %f1209, 0f4B400001;\n" " mov.f32 %f1208, 0f437C0000;\n" " mov.f32 %f1207, 0f3F000000;\n" " mov.f32 %f1206, 0f3BBB989D;\n" " ld.shared.f32 %f1076, [%r70+8];\n" " sub.f32 %f1077, %f1076, %f1539;\n" " fma.rn.f32 %f1080, %f1077, %f1206, %f1207;\n" " cvt.sat.f32.f32 %f1083, %f1080;\n" " fma.rm.f32 %f1085, %f1083, %f1208, %f1209;\n" " add.f32 %f1086, %f1085, 0fCB40007F;\n" " neg.f32 %f1087, %f1086;\n" " fma.rn.f32 %f1088, %f1077, %f1210, %f1087;\n" " fma.rn.f32 %f1090, %f1077, %f1211, %f1088;\n" " mov.b32 %r459, %f1085;\n" " shl.b32 %r460, %r459, 23;\n" " mov.b32 %f1091, %r460;\n" " ex2.approx.ftz.f32 %f1092, %f1090;\n" " mul.f32 %f1093, %f1092, %f1091;\n" " add.s32 %r462, %r72, %r458;\n" " st.shared.f32 [%r462], %f1093;\n" " add.f32 %f1543, %f1543, %f1093;\n" "\n" "$L__BB5_417:\n" " sub.f32 %f1094, %f517, %f1539;\n" " mov.f32 %f1095, 0f3F000000;\n" " mov.f32 %f1096, 0f3BBB989D;\n" " fma.rn.f32 %f1097, %f1094, %f1096, %f1095;\n" " mov.f32 %f1098, 0f3FB8AA3B;\n" " mov.f32 %f1099, 0f437C0000;\n" " cvt.sat.f32.f32 %f1100, %f1097;\n" " mov.f32 %f1101, 0f4B400001;\n" " fma.rm.f32 %f1102, %f1100, %f1099, %f1101;\n" " add.f32 %f1103, %f1102, 0fCB40007F;\n" " neg.f32 %f1104, %f1103;\n" " fma.rn.f32 %f1105, %f1094, %f1098, %f1104;\n" " mov.f32 %f1106, 0f32A57060;\n" " fma.rn.f32 %f1107, %f1094, %f1106, %f1105;\n" " ex2.approx.ftz.f32 %f1108, %f1107;\n" " mov.b32 %r463, %f1102;\n" " shl.b32 %r464, %r463, 23;\n" " mov.b32 %f1109, %r464;\n" " mul.f32 %f1110, %f1108, %f1109;\n" " cvt.u64.u32 %rd192, %r83;\n" " add.s64 %rd193, %rd44, %rd192;\n" " cvt.u32.u64 %r468, %rd193;\n" " add.s32 %r469, %r397, %r468;\n" " shl.b32 %r470, %r469, 2;\n" " add.s32 %r472, %r110, %r470;\n" " ld.shared.f32 %f1111, [%r472];\n" " fma.rn.f32 %f1112, %f1110, %f1111, %f1543;\n" " st.shared.f32 [%r472], %f1112;\n" " add.s32 %r474, %r397, %r107;\n" " shl.b32 %r475, %r474, 2;\n" " add.s32 %r476, %r110, %r475;\n" " st.shared.f32 [%r476], %f1539;\n" " add.s32 %r477, %r468, %r83;\n" " add.s32 %r478, %r397, %r477;\n" " shl.b32 %r479, %r478, 2;\n" " add.s32 %r480, %r110, %r479;\n" " st.shared.f32 [%r480], %f1110;\n" "\n" "$L__BB5_418:\n" " add.s32 %r547, %r2, 1;\n" " sub.s32 %r546, %r547, %r571;\n" " bar.sync 0;\n" " setp.lt.s32 %p280, %r546, 1;\n" " @%p280 bra $L__BB5_498;\n" "\n" " setp.gt.s32 %p281, %r88, 0;\n" " @%p281 bra $L__BB5_458;\n" " bra.uni $L__BB5_420;\n" "\n" "$L__BB5_458:\n" " mov.u32 %r580, 0;\n" "\n" "$L__BB5_459:\n" " add.s32 %r77, %r580, %r571;\n" " setp.lt.s32 %p302, %r77, %r87;\n" " @%p302 bra $L__BB5_463;\n" " bra.uni $L__BB5_460;\n" "\n" "$L__BB5_463:\n" " add.s32 %r563, %r580, %r571;\n" " cvt.s64.s32 %rd240, %r563;\n" " mul.lo.s64 %rd241, %rd240, %rd5;\n" " mul.lo.s64 %rd243, %rd31, %rd5;\n" " shl.b64 %rd244, %rd243, 3;\n" " cvt.s64.s32 %rd245, %r1;\n" " add.s64 %rd246, %rd244, %rd245;\n" " add.s64 %rd247, %rd246, %rd241;\n" " cvta.to.global.u64 %rd248, %rd22;\n" " shl.b64 %rd249, %rd247, 1;\n" " add.s64 %rd250, %rd248, %rd249;\n" " ld.global.u16 %rs45, [%rd250];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1583, %rs45;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB5_464;\n" "\n" "$L__BB5_460:\n" " add.s32 %r560, %r580, %r571;\n" " sub.s32 %r507, %r2, %r88;\n" " setp.gt.s32 %p303, %r560, %r507;\n" " mov.u32 %r508, %nctaid.x;\n" " add.s32 %r509, %r508, %r86;\n" " add.s32 %r510, %r509, -257;\n" " setp.gt.s32 %p304, %r560, %r510;\n" " and.pred %p305, %p304, %p303;\n" " @%p305 bra $L__BB5_462;\n" " bra.uni $L__BB5_461;\n" "\n" "$L__BB5_462:\n" " add.s32 %r562, %r580, %r571;\n" " and.b32 %r525, %r562, 255;\n" " cvt.u64.u32 %rd229, %r525;\n" " mul.lo.s64 %rd230, %rd229, %rd5;\n" " mul.lo.s64 %rd232, %rd31, %rd5;\n" " shl.b64 %rd233, %rd232, 8;\n" " cvt.s64.s32 %rd234, %r1;\n" " add.s64 %rd235, %rd233, %rd234;\n" " add.s64 %rd236, %rd235, %rd230;\n" " shl.b64 %rd238, %rd236, 1;\n" " add.s64 %rd239, %rd237, %rd238;\n" " ld.global.u16 %rs44, [%rd239];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1583, %rs44;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB5_464;\n" "\n" "$L__BB5_461:\n" " add.s32 %r561, %r580, %r571;\n" " cvt.s64.s32 %rd216, %r561;\n" " mul.lo.s64 %rd217, %rd216, %rd13;\n" " mul.wide.s32 %rd218, %r85, %r93;\n" " mul.lo.s64 %rd219, %rd218, %rd13;\n" " add.s64 %rd220, %rd217, %rd219;\n" " cvt.s64.s32 %rd221, %r25;\n" " add.s64 %rd222, %rd220, %rd221;\n" " cvta.to.global.u64 %rd223, %rd21;\n" " add.s64 %rd224, %rd223, %rd222;\n" " ld.global.u8 %r512, [%rd224];\n" " ld.global.u8 %r513, [%rd224+1];\n" " prmt.b32 %r514, %r513, %r512, 30212;\n" " cvt.u16.u32 %rs43, %r514;\n" " // begin inline asm\n" " { cvt.f32.f16 %f1148, %rs43;}\n" "\n" " // end inline asm\n" " and.b32 %r516, %r1, 31;\n" " add.s32 %r517, %r516, -16;\n" " setp.lt.u32 %p306, %r516, 16;\n" " selp.b32 %r518, %r516, %r517, %p306;\n" " add.s32 %r519, %r518, 2;\n" " cvt.s64.s32 %rd225, %r519;\n" " add.s64 %rd226, %rd225, %rd221;\n" " add.s64 %rd227, %rd226, %rd220;\n" " add.s64 %rd228, %rd223, %rd227;\n" " ld.global.u8 %r520, [%rd228];\n" " selp.b32 %r521, 0, 4, %p306;\n" " shr.u32 %r522, %r520, %r521;\n" " and.b32 %r523, %r522, 15;\n" " add.s32 %r524, %r523, -8;\n" " cvt.rn.f32.s32 %f1149, %r524;\n" " mul.f32 %f1583, %f1148, %f1149;\n" "\n" "$L__BB5_464:\n" " mad.lo.s32 %r531, %r84, %r83, %r4;\n" " mad.lo.s32 %r532, %r580, %r83, %r531;\n" " shl.b32 %r533, %r532, 2;\n" " add.s32 %r78, %r110, %r533;\n" " setp.eq.s32 %p307, %r580, 0;\n" " @%p307 bra $L__BB5_481;\n" "\n" " @%p2 bra $L__BB5_467;\n" "\n" " ld.shared.f32 %f1152, [%r78];\n" " fma.rn.f32 %f1574, %f1583, %f1152, %f1574;\n" "\n" "$L__BB5_467:\n" " @%p225 bra $L__BB5_469;\n" "\n" " ld.shared.f32 %f1153, [%r78+4];\n" " fma.rn.f32 %f1573, %f1583, %f1153, %f1573;\n" "\n" "$L__BB5_469:\n" " @%p231 bra $L__BB5_471;\n" "\n" " ld.shared.f32 %f1154, [%r78+8];\n" " fma.rn.f32 %f1572, %f1583, %f1154, %f1572;\n" "\n" "$L__BB5_471:\n" " @%p237 bra $L__BB5_473;\n" "\n" " ld.shared.f32 %f1155, [%r78+12];\n" " fma.rn.f32 %f1571, %f1583, %f1155, %f1571;\n" "\n" "$L__BB5_473:\n" " @%p243 bra $L__BB5_475;\n" "\n" " ld.shared.f32 %f1156, [%r78+16];\n" " fma.rn.f32 %f1570, %f1583, %f1156, %f1570;\n" "\n" "$L__BB5_475:\n" " @%p249 bra $L__BB5_477;\n" "\n" " ld.shared.f32 %f1157, [%r78+20];\n" " fma.rn.f32 %f1569, %f1583, %f1157, %f1569;\n" "\n" "$L__BB5_477:\n" " @%p255 bra $L__BB5_479;\n" "\n" " ld.shared.f32 %f1158, [%r78+24];\n" " fma.rn.f32 %f1568, %f1583, %f1158, %f1568;\n" "\n" "$L__BB5_479:\n" " @%p261 bra $L__BB5_497;\n" "\n" " ld.shared.f32 %f1159, [%r78+28];\n" " fma.rn.f32 %f1567, %f1583, %f1159, %f1567;\n" " bra.uni $L__BB5_497;\n" "\n" "$L__BB5_481:\n" " @%p2 bra $L__BB5_483;\n" "\n" " ld.shared.f32 %f1160, [%r34];\n" " ld.shared.f32 %f1161, [%r78];\n" " mul.f32 %f1162, %f1583, %f1161;\n" " fma.rn.f32 %f1574, %f1160, %f1574, %f1162;\n" "\n" "$L__BB5_483:\n" " @%p225 bra $L__BB5_485;\n" "\n" " ld.shared.f32 %f1163, [%r34+4];\n" " ld.shared.f32 %f1164, [%r78+4];\n" " mul.f32 %f1165, %f1583, %f1164;\n" " fma.rn.f32 %f1573, %f1163, %f1573, %f1165;\n" "\n" "$L__BB5_485:\n" " @%p231 bra $L__BB5_487;\n" "\n" " ld.shared.f32 %f1166, [%r34+8];\n" " ld.shared.f32 %f1167, [%r78+8];\n" " mul.f32 %f1168, %f1583, %f1167;\n" " fma.rn.f32 %f1572, %f1166, %f1572, %f1168;\n" "\n" "$L__BB5_487:\n" " @%p237 bra $L__BB5_489;\n" "\n" " ld.shared.f32 %f1169, [%r34+12];\n" " ld.shared.f32 %f1170, [%r78+12];\n" " mul.f32 %f1171, %f1583, %f1170;\n" " fma.rn.f32 %f1571, %f1169, %f1571, %f1171;\n" "\n" "$L__BB5_489:\n" " @%p243 bra $L__BB5_491;\n" "\n" " ld.shared.f32 %f1172, [%r34+16];\n" " ld.shared.f32 %f1173, [%r78+16];\n" " mul.f32 %f1174, %f1583, %f1173;\n" " fma.rn.f32 %f1570, %f1172, %f1570, %f1174;\n" "\n" "$L__BB5_491:\n" " @%p249 bra $L__BB5_493;\n" "\n" " ld.shared.f32 %f1175, [%r34+20];\n" " ld.shared.f32 %f1176, [%r78+20];\n" " mul.f32 %f1177, %f1583, %f1176;\n" " fma.rn.f32 %f1569, %f1175, %f1569, %f1177;\n" "\n" "$L__BB5_493:\n" " @%p255 bra $L__BB5_495;\n" "\n" " ld.shared.f32 %f1178, [%r34+24];\n" " ld.shared.f32 %f1179, [%r78+24];\n" " mul.f32 %f1180, %f1583, %f1179;\n" " fma.rn.f32 %f1568, %f1178, %f1568, %f1180;\n" "\n" "$L__BB5_495:\n" " @%p261 bra $L__BB5_497;\n" "\n" " ld.shared.f32 %f1181, [%r34+28];\n" " ld.shared.f32 %f1182, [%r78+28];\n" " mul.f32 %f1183, %f1583, %f1182;\n" " fma.rn.f32 %f1567, %f1181, %f1567, %f1183;\n" "\n" "$L__BB5_497:\n" " add.s32 %r580, %r580, 1;\n" " setp.lt.s32 %p324, %r580, %r39;\n" " @%p324 bra $L__BB5_459;\n" " bra.uni $L__BB5_498;\n" "\n" "$L__BB5_420:\n" " mov.u32 %r579, 0;\n" "\n" "$L__BB5_421:\n" " add.s32 %r482, %r579, %r571;\n" " setp.lt.s32 %p282, %r482, %r87;\n" " cvt.s64.s32 %rd18, %r482;\n" " @%p282 bra $L__BB5_423;\n" " bra.uni $L__BB5_422;\n" "\n" "$L__BB5_423:\n" " mul.lo.s64 %rd206, %rd18, %rd5;\n" " mul.lo.s64 %rd208, %rd31, %rd5;\n" " shl.b64 %rd209, %rd208, 3;\n" " cvt.s64.s32 %rd210, %r1;\n" " add.s64 %rd211, %rd209, %rd210;\n" " add.s64 %rd212, %rd211, %rd206;\n" " cvta.to.global.u64 %rd213, %rd22;\n" " shl.b64 %rd214, %rd212, 1;\n" " add.s64 %rd215, %rd213, %rd214;\n" " ld.global.u16 %rs42, [%rd215];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1552, %rs42;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB5_424;\n" "\n" "$L__BB5_422:\n" " mul.lo.s64 %rd194, %rd18, %rd13;\n" " mul.wide.s32 %rd195, %r85, %r93;\n" " mul.lo.s64 %rd196, %rd195, %rd13;\n" " add.s64 %rd197, %rd194, %rd196;\n" " cvt.s64.s32 %rd198, %r25;\n" " add.s64 %rd199, %rd197, %rd198;\n" " cvta.to.global.u64 %rd200, %rd21;\n" " add.s64 %rd201, %rd200, %rd199;\n" " ld.global.u8 %r484, [%rd201];\n" " ld.global.u8 %r485, [%rd201+1];\n" " prmt.b32 %r486, %r485, %r484, 30212;\n" " cvt.u16.u32 %rs41, %r486;\n" " // begin inline asm\n" " { cvt.f32.f16 %f1113, %rs41;}\n" "\n" " // end inline asm\n" " and.b32 %r488, %r1, 31;\n" " add.s32 %r489, %r488, -16;\n" " setp.lt.u32 %p283, %r488, 16;\n" " selp.b32 %r490, %r488, %r489, %p283;\n" " add.s32 %r491, %r490, 2;\n" " cvt.s64.s32 %rd202, %r491;\n" " add.s64 %rd203, %rd202, %rd198;\n" " add.s64 %rd204, %rd203, %rd197;\n" " add.s64 %rd205, %rd200, %rd204;\n" " ld.global.u8 %r492, [%rd205];\n" " selp.b32 %r493, 0, 4, %p283;\n" " shr.u32 %r494, %r492, %r493;\n" " and.b32 %r495, %r494, 15;\n" " add.s32 %r496, %r495, -8;\n" " cvt.rn.f32.s32 %f1114, %r496;\n" " mul.f32 %f1552, %f1113, %f1114;\n" "\n" "$L__BB5_424:\n" " mad.lo.s32 %r500, %r84, %r83, %r4;\n" " mad.lo.s32 %r501, %r579, %r83, %r500;\n" " shl.b32 %r502, %r501, 2;\n" " add.s32 %r74, %r110, %r502;\n" " setp.eq.s32 %p284, %r579, 0;\n" " @%p284 bra $L__BB5_441;\n" "\n" " @%p2 bra $L__BB5_427;\n" "\n" " ld.shared.f32 %f1116, [%r74];\n" " fma.rn.f32 %f1574, %f1552, %f1116, %f1574;\n" "\n" "$L__BB5_427:\n" " @%p225 bra $L__BB5_429;\n" "\n" " ld.shared.f32 %f1117, [%r74+4];\n" " fma.rn.f32 %f1573, %f1552, %f1117, %f1573;\n" "\n" "$L__BB5_429:\n" " @%p231 bra $L__BB5_431;\n" "\n" " ld.shared.f32 %f1118, [%r74+8];\n" " fma.rn.f32 %f1572, %f1552, %f1118, %f1572;\n" "\n" "$L__BB5_431:\n" " @%p237 bra $L__BB5_433;\n" "\n" " ld.shared.f32 %f1119, [%r74+12];\n" " fma.rn.f32 %f1571, %f1552, %f1119, %f1571;\n" "\n" "$L__BB5_433:\n" " @%p243 bra $L__BB5_435;\n" "\n" " ld.shared.f32 %f1120, [%r74+16];\n" " fma.rn.f32 %f1570, %f1552, %f1120, %f1570;\n" "\n" "$L__BB5_435:\n" " @%p249 bra $L__BB5_437;\n" "\n" " ld.shared.f32 %f1121, [%r74+20];\n" " fma.rn.f32 %f1569, %f1552, %f1121, %f1569;\n" "\n" "$L__BB5_437:\n" " @%p255 bra $L__BB5_439;\n" "\n" " ld.shared.f32 %f1122, [%r74+24];\n" " fma.rn.f32 %f1568, %f1552, %f1122, %f1568;\n" "\n" "$L__BB5_439:\n" " @%p261 bra $L__BB5_457;\n" "\n" " ld.shared.f32 %f1123, [%r74+28];\n" " fma.rn.f32 %f1567, %f1552, %f1123, %f1567;\n" " bra.uni $L__BB5_457;\n" "\n" "$L__BB5_441:\n" " @%p2 bra $L__BB5_443;\n" "\n" " ld.shared.f32 %f1124, [%r34];\n" " ld.shared.f32 %f1125, [%r74];\n" " mul.f32 %f1126, %f1552, %f1125;\n" " fma.rn.f32 %f1574, %f1124, %f1574, %f1126;\n" "\n" "$L__BB5_443:\n" " @%p225 bra $L__BB5_445;\n" "\n" " ld.shared.f32 %f1127, [%r34+4];\n" " ld.shared.f32 %f1128, [%r74+4];\n" " mul.f32 %f1129, %f1552, %f1128;\n" " fma.rn.f32 %f1573, %f1127, %f1573, %f1129;\n" "\n" "$L__BB5_445:\n" " @%p231 bra $L__BB5_447;\n" "\n" " ld.shared.f32 %f1130, [%r34+8];\n" " ld.shared.f32 %f1131, [%r74+8];\n" " mul.f32 %f1132, %f1552, %f1131;\n" " fma.rn.f32 %f1572, %f1130, %f1572, %f1132;\n" "\n" "$L__BB5_447:\n" " @%p237 bra $L__BB5_449;\n" "\n" " ld.shared.f32 %f1133, [%r34+12];\n" " ld.shared.f32 %f1134, [%r74+12];\n" " mul.f32 %f1135, %f1552, %f1134;\n" " fma.rn.f32 %f1571, %f1133, %f1571, %f1135;\n" "\n" "$L__BB5_449:\n" " @%p243 bra $L__BB5_451;\n" "\n" " ld.shared.f32 %f1136, [%r34+16];\n" " ld.shared.f32 %f1137, [%r74+16];\n" " mul.f32 %f1138, %f1552, %f1137;\n" " fma.rn.f32 %f1570, %f1136, %f1570, %f1138;\n" "\n" "$L__BB5_451:\n" " @%p249 bra $L__BB5_453;\n" "\n" " ld.shared.f32 %f1139, [%r34+20];\n" " ld.shared.f32 %f1140, [%r74+20];\n" " mul.f32 %f1141, %f1552, %f1140;\n" " fma.rn.f32 %f1569, %f1139, %f1569, %f1141;\n" "\n" "$L__BB5_453:\n" " @%p255 bra $L__BB5_455;\n" "\n" " ld.shared.f32 %f1142, [%r34+24];\n" " ld.shared.f32 %f1143, [%r74+24];\n" " mul.f32 %f1144, %f1552, %f1143;\n" " fma.rn.f32 %f1568, %f1142, %f1568, %f1144;\n" "\n" "$L__BB5_455:\n" " @%p261 bra $L__BB5_457;\n" "\n" " ld.shared.f32 %f1145, [%r34+28];\n" " ld.shared.f32 %f1146, [%r74+28];\n" " mul.f32 %f1147, %f1552, %f1146;\n" " fma.rn.f32 %f1567, %f1145, %f1567, %f1147;\n" "\n" "$L__BB5_457:\n" " add.s32 %r579, %r579, 1;\n" " setp.lt.s32 %p301, %r579, %r39;\n" " @%p301 bra $L__BB5_421;\n" "\n" "$L__BB5_498:\n" " bar.sync 0;\n" "\n" "$L__BB5_499:\n" " add.s32 %r571, %r571, 16;\n" " setp.ge.s32 %p325, %r2, %r571;\n" " add.s32 %r570, %r570, 1;\n" " @%p325 bra $L__BB5_11;\n" "\n" "$L__BB5_500:\n" " ld.param.u32 %r556, [fc_attn_flash_prefill_q4_kr_param_9];\n" " mov.u32 %r555, %ctaid.x;\n" " mov.u32 %r554, %nctaid.y;\n" " cvt.u64.u32 %rd305, %r554;\n" " cvt.s64.s32 %rd304, %r555;\n" " mov.u32 %r553, %ctaid.y;\n" " cvt.s64.s32 %rd303, %r553;\n" " mul.lo.s64 %rd302, %rd304, %rd305;\n" " add.s64 %rd301, %rd302, %rd303;\n" " mul.wide.s32 %rd300, %r556, %r83;\n" " mul.lo.s64 %rd299, %rd300, %rd301;\n" " shl.b32 %r537, %r83, 5;\n" " cvt.u64.u32 %rd251, %r537;\n" " cvt.u64.u32 %rd252, %r83;\n" " add.s64 %rd253, %rd251, %rd252;\n" " cvt.u32.u64 %r538, %rd253;\n" " mad.lo.s32 %r539, %r556, %r83, %r538;\n" " cvt.s64.s32 %rd261, %r1;\n" " add.s64 %rd19, %rd299, %rd261;\n" " shl.b32 %r544, %r539, 2;\n" " add.s32 %r82, %r110, %r544;\n" " @%p2 bra $L__BB5_502;\n" "\n" " ld.shared.f32 %f1184, [%r82];\n" " div.rn.f32 %f1185, %f1574, %f1184;\n" " cvta.to.global.u64 %rd262, %rd24;\n" " shl.b64 %rd263, %rd19, 2;\n" " add.s64 %rd264, %rd262, %rd263;\n" " st.global.f32 [%rd264], %f1185;\n" "\n" "$L__BB5_502:\n" " setp.lt.s32 %p327, %r83, 2;\n" " @%p327 bra $L__BB5_504;\n" "\n" " ld.shared.f32 %f1186, [%r82+4];\n" " div.rn.f32 %f1187, %f1573, %f1186;\n" " cvta.to.global.u64 %rd265, %rd24;\n" " add.s64 %rd266, %rd19, %rd5;\n" " shl.b64 %rd267, %rd266, 2;\n" " add.s64 %rd268, %rd265, %rd267;\n" " st.global.f32 [%rd268], %f1187;\n" "\n" "$L__BB5_504:\n" " setp.lt.s32 %p328, %r83, 3;\n" " @%p328 bra $L__BB5_506;\n" "\n" " ld.shared.f32 %f1188, [%r82+8];\n" " div.rn.f32 %f1189, %f1572, %f1188;\n" " shl.b64 %rd269, %rd5, 1;\n" " add.s64 %rd270, %rd19, %rd269;\n" " cvta.to.global.u64 %rd271, %rd24;\n" " shl.b64 %rd272, %rd270, 2;\n" " add.s64 %rd273, %rd271, %rd272;\n" " st.global.f32 [%rd273], %f1189;\n" "\n" "$L__BB5_506:\n" " setp.lt.s32 %p329, %r83, 4;\n" " @%p329 bra $L__BB5_508;\n" "\n" " ld.shared.f32 %f1190, [%r82+12];\n" " div.rn.f32 %f1191, %f1571, %f1190;\n" " mul.lo.s64 %rd274, %rd5, 3;\n" " add.s64 %rd275, %rd19, %rd274;\n" " cvta.to.global.u64 %rd276, %rd24;\n" " shl.b64 %rd277, %rd275, 2;\n" " add.s64 %rd278, %rd276, %rd277;\n" " st.global.f32 [%rd278], %f1191;\n" "\n" "$L__BB5_508:\n" " setp.lt.s32 %p330, %r83, 5;\n" " @%p330 bra $L__BB5_510;\n" "\n" " ld.shared.f32 %f1192, [%r82+16];\n" " div.rn.f32 %f1193, %f1570, %f1192;\n" " shl.b64 %rd279, %rd5, 2;\n" " add.s64 %rd280, %rd19, %rd279;\n" " cvta.to.global.u64 %rd281, %rd24;\n" " shl.b64 %rd282, %rd280, 2;\n" " add.s64 %rd283, %rd281, %rd282;\n" " st.global.f32 [%rd283], %f1193;\n" "\n" "$L__BB5_510:\n" " setp.lt.s32 %p331, %r83, 6;\n" " @%p331 bra $L__BB5_512;\n" "\n" " ld.shared.f32 %f1194, [%r82+20];\n" " div.rn.f32 %f1195, %f1569, %f1194;\n" " mul.lo.s64 %rd284, %rd5, 5;\n" " add.s64 %rd285, %rd19, %rd284;\n" " cvta.to.global.u64 %rd286, %rd24;\n" " shl.b64 %rd287, %rd285, 2;\n" " add.s64 %rd288, %rd286, %rd287;\n" " st.global.f32 [%rd288], %f1195;\n" "\n" "$L__BB5_512:\n" " setp.lt.s32 %p332, %r83, 7;\n" " @%p332 bra $L__BB5_514;\n" "\n" " ld.shared.f32 %f1196, [%r82+24];\n" " div.rn.f32 %f1197, %f1568, %f1196;\n" " mul.lo.s64 %rd289, %rd5, 6;\n" " add.s64 %rd290, %rd19, %rd289;\n" " cvta.to.global.u64 %rd291, %rd24;\n" " shl.b64 %rd292, %rd290, 2;\n" " add.s64 %rd293, %rd291, %rd292;\n" " st.global.f32 [%rd293], %f1197;\n" "\n" "$L__BB5_514:\n" " setp.lt.s32 %p333, %r83, 8;\n" " @%p333 bra $L__BB5_516;\n" "\n" " ld.shared.f32 %f1198, [%r82+28];\n" " div.rn.f32 %f1199, %f1567, %f1198;\n" " mul.lo.s64 %rd294, %rd5, 7;\n" " add.s64 %rd295, %rd19, %rd294;\n" " cvta.to.global.u64 %rd296, %rd24;\n" " shl.b64 %rd297, %rd295, 2;\n" " add.s64 %rd298, %rd296, %rd297;\n" " st.global.f32 [%rd298], %f1199;\n" "\n" "$L__BB5_516:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_prefill_q4_kvs\n" ".visible .entry fc_attn_flash_prefill_q4_kvs(\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_0,\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_1,\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_2,\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_3,\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_4,\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_5,\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_6,\n" " .param .u64 fc_attn_flash_prefill_q4_kvs_param_7,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_8,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_9,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_10,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_11,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_12,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_13,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_14,\n" " .param .u32 fc_attn_flash_prefill_q4_kvs_param_15,\n" " .param .f32 fc_attn_flash_prefill_q4_kvs_param_16\n" ")\n" "{\n" " .reg .pred %p<340>;\n" " .reg .b16 %rs<46>;\n" " .reg .f32 %f<1646>;\n" " .reg .b32 %r<619>;\n" " .reg .b64 %rd<299>;\n" "\n" "\n" " ld.param.u64 %rd24, [fc_attn_flash_prefill_q4_kvs_param_0];\n" " ld.param.u64 %rd19, [fc_attn_flash_prefill_q4_kvs_param_1];\n" " ld.param.u64 %rd20, [fc_attn_flash_prefill_q4_kvs_param_2];\n" " ld.param.u64 %rd25, [fc_attn_flash_prefill_q4_kvs_param_3];\n" " ld.param.u64 %rd21, [fc_attn_flash_prefill_q4_kvs_param_4];\n" " ld.param.u64 %rd26, [fc_attn_flash_prefill_q4_kvs_param_5];\n" " ld.param.u64 %rd22, [fc_attn_flash_prefill_q4_kvs_param_6];\n" " ld.param.u64 %rd23, [fc_attn_flash_prefill_q4_kvs_param_7];\n" " ld.param.u32 %r95, [fc_attn_flash_prefill_q4_kvs_param_8];\n" " ld.param.u32 %r96, [fc_attn_flash_prefill_q4_kvs_param_9];\n" " ld.param.u32 %r97, [fc_attn_flash_prefill_q4_kvs_param_10];\n" " ld.param.u32 %r98, [fc_attn_flash_prefill_q4_kvs_param_11];\n" " ld.param.u32 %r99, [fc_attn_flash_prefill_q4_kvs_param_12];\n" " ld.param.u32 %r100, [fc_attn_flash_prefill_q4_kvs_param_13];\n" " ld.param.u32 %r102, [fc_attn_flash_prefill_q4_kvs_param_15];\n" " ld.param.f32 %f673, [fc_attn_flash_prefill_q4_kvs_param_16];\n" " cvta.to.global.u64 %rd1, %rd24;\n" " cvta.to.global.u64 %rd2, %rd25;\n" " cvta.to.global.u64 %rd3, %rd26;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r103, %ctaid.x;\n" " add.s32 %r2, %r103, %r98;\n" " shr.s32 %r104, %r96, 31;\n" " shr.u32 %r105, %r104, 27;\n" " add.s32 %r106, %r96, %r105;\n" " shr.s32 %r107, %r106, 5;\n" " mul.lo.s32 %r3, %r107, 18;\n" " shl.b32 %r108, %r95, 4;\n" " mul.wide.s32 %rd4, %r108, 2;\n" " cvt.u64.u32 %rd27, %r95;\n" " add.s64 %rd28, %rd4, %rd27;\n" " cvt.u32.u64 %r4, %rd28;\n" " add.s32 %r5, %r4, %r95;\n" " mul.lo.s32 %r6, %r96, %r95;\n" " mov.u32 %r109, %nctaid.y;\n" " cvt.u64.u32 %rd29, %r109;\n" " cvt.s64.s32 %rd30, %r103;\n" " mul.lo.s64 %rd31, %rd30, %rd29;\n" " mov.u32 %r110, %ctaid.y;\n" " cvt.s64.s32 %rd32, %r110;\n" " add.s64 %rd33, %rd31, %rd32;\n" " cvt.s64.s32 %rd5, %r96;\n" " mul.wide.s32 %rd34, %r96, %r95;\n" " mul.lo.s64 %rd6, %rd34, %rd33;\n" " setp.lt.s32 %p2, %r95, 1;\n" " @%p2 bra $L__BB6_7;\n" "\n" " cvt.s64.s32 %rd7, %r1;\n" " and.b32 %r604, %r95, 3;\n" " add.s32 %r112, %r95, -1;\n" " setp.lt.u32 %p3, %r112, 3;\n" " mov.u32 %r602, 0;\n" " @%p3 bra $L__BB6_4;\n" "\n" " sub.s32 %r600, %r95, %r604;\n" " add.s64 %rd35, %rd6, %rd7;\n" " shl.b64 %rd36, %rd35, 2;\n" " add.s64 %rd298, %rd1, %rd36;\n" " shl.b64 %rd9, %rd5, 2;\n" " shl.b32 %r114, %r1, 2;\n" " mov.u32 %r115, shv;\n" " add.s32 %r601, %r115, %r114;\n" " shl.b32 %r10, %r96, 2;\n" " mov.u32 %r602, 0;\n" "\n" "$L__BB6_3:\n" " ld.global.f32 %f674, [%rd298];\n" " st.shared.f32 [%r601], %f674;\n" " add.s64 %rd37, %rd298, %rd9;\n" " ld.global.f32 %f675, [%rd37];\n" " add.s32 %r116, %r601, %r10;\n" " st.shared.f32 [%r116], %f675;\n" " add.s64 %rd38, %rd37, %rd9;\n" " ld.global.f32 %f676, [%rd38];\n" " add.s32 %r117, %r116, %r10;\n" " st.shared.f32 [%r117], %f676;\n" " add.s64 %rd39, %rd38, %rd9;\n" " add.s64 %rd298, %rd39, %rd9;\n" " ld.global.f32 %f677, [%rd39];\n" " add.s32 %r118, %r117, %r10;\n" " add.s32 %r601, %r118, %r10;\n" " st.shared.f32 [%r118], %f677;\n" " add.s32 %r602, %r602, 4;\n" " add.s32 %r600, %r600, -4;\n" " setp.ne.s32 %p4, %r600, 0;\n" " @%p4 bra $L__BB6_3;\n" "\n" "$L__BB6_4:\n" " setp.eq.s32 %p5, %r604, 0;\n" " add.s64 %rd12, %rd6, %rd7;\n" " @%p5 bra $L__BB6_7;\n" "\n" " mov.u32 %r121, shv;\n" "\n" "$L__BB6_6:\n" " .pragma \"nounroll\";\n" " cvt.s64.s32 %rd40, %r602;\n" " mul.lo.s64 %rd41, %rd40, %rd5;\n" " add.s64 %rd42, %rd12, %rd41;\n" " shl.b64 %rd43, %rd42, 2;\n" " add.s64 %rd44, %rd1, %rd43;\n" " ld.global.f32 %f678, [%rd44];\n" " mad.lo.s32 %r119, %r602, %r96, %r1;\n" " shl.b32 %r120, %r119, 2;\n" " add.s32 %r122, %r121, %r120;\n" " st.shared.f32 [%r122], %f678;\n" " add.s32 %r602, %r602, 1;\n" " add.s32 %r604, %r604, -1;\n" " setp.ne.s32 %p6, %r604, 0;\n" " @%p6 bra $L__BB6_6;\n" "\n" "$L__BB6_7:\n" " setp.ge.s32 %p7, %r1, %r95;\n" " add.s32 %r123, %r6, %r1;\n" " cvt.u32.u64 %r124, %rd4;\n" " add.s32 %r125, %r123, %r124;\n" " shl.b32 %r126, %r125, 2;\n" " mov.u32 %r127, shv;\n" " add.s32 %r22, %r127, %r126;\n" " add.s32 %r128, %r123, %r4;\n" " shl.b32 %r129, %r128, 2;\n" " add.s32 %r23, %r127, %r129;\n" " @%p7 bra $L__BB6_9;\n" "\n" " mov.u32 %r130, -8388608;\n" " st.shared.u32 [%r22], %r130;\n" " mov.u32 %r131, 0;\n" " st.shared.u32 [%r23], %r131;\n" "\n" "$L__BB6_9:\n" " bar.sync 0;\n" " shr.s32 %r132, %r1, 31;\n" " shr.u32 %r133, %r132, 28;\n" " add.s32 %r134, %r1, %r133;\n" " shr.s32 %r24, %r134, 4;\n" " and.b32 %r135, %r134, -16;\n" " sub.s32 %r25, %r1, %r135;\n" " shr.u32 %r136, %r132, 27;\n" " add.s32 %r137, %r1, %r136;\n" " shr.s32 %r138, %r137, 5;\n" " mul.lo.s32 %r26, %r138, 18;\n" " setp.lt.s32 %p8, %r2, 0;\n" " mov.f32 %f1567, 0f00000000;\n" " mov.f32 %f1568, %f1567;\n" " mov.f32 %f1569, %f1567;\n" " mov.f32 %f1570, %f1567;\n" " mov.f32 %f1571, %f1567;\n" " mov.f32 %f1572, %f1567;\n" " mov.f32 %f1573, %f1567;\n" " mov.f32 %f1574, %f1567;\n" " @%p8 bra $L__BB6_512;\n" "\n" " shl.b32 %r141, %r25, 2;\n" " add.s32 %r27, %r127, %r141;\n" " shl.b32 %r143, %r96, 2;\n" " add.s32 %r28, %r27, %r143;\n" " add.s32 %r29, %r28, %r143;\n" " add.s32 %r30, %r29, %r143;\n" " add.s32 %r31, %r30, %r143;\n" " add.s32 %r32, %r31, %r143;\n" " add.s32 %r33, %r32, %r143;\n" " add.s32 %r34, %r33, %r143;\n" " add.s32 %r144, %r6, %r5;\n" " shl.b32 %r145, %r144, 2;\n" " add.s32 %r35, %r127, %r145;\n" " mov.u32 %r605, 0;\n" " mov.f32 %f1567, 0f00000000;\n" " setp.lt.s32 %p10, %r102, 1;\n" " setp.lt.s32 %p56, %r96, 16;\n" " cvt.s64.s32 %rd126, %r25;\n" " setp.lt.s32 %p57, %r96, 32;\n" " setp.lt.s32 %p58, %r96, 48;\n" " cvt.s64.s32 %rd45, %r3;\n" " cvta.to.global.u64 %rd50, %rd19;\n" " cvta.to.global.u64 %rd198, %rd20;\n" " cvta.to.global.u64 %rd229, %rd22;\n" " mov.f32 %f1568, %f1567;\n" " mov.f32 %f1569, %f1567;\n" " mov.f32 %f1570, %f1567;\n" " mov.f32 %f1571, %f1567;\n" " mov.f32 %f1572, %f1567;\n" " mov.f32 %f1573, %f1567;\n" " mov.f32 %f1574, %f1567;\n" " mov.u32 %r606, %r605;\n" "\n" "$L__BB6_11:\n" " ld.param.u32 %r586, [fc_attn_flash_prefill_q4_kvs_param_14];\n" " mov.u32 %r585, %ctaid.x;\n" " ld.param.u32 %r584, [fc_attn_flash_prefill_q4_kvs_param_15];\n" " mov.u32 %r147, -2;\n" " sub.s32 %r148, %r147, %r585;\n" " sub.s32 %r149, %r148, %r98;\n" " shl.b32 %r150, %r605, 4;\n" " add.s32 %r151, %r149, %r150;\n" " max.s32 %r152, %r151, -17;\n" " not.b32 %r153, %r152;\n" " max.s32 %r38, %r153, 1;\n" " add.s32 %r155, %r2, 1;\n" " sub.s32 %r39, %r155, %r606;\n" " min.s32 %r40, %r39, 16;\n" " setp.lt.s32 %p9, %r606, %r586;\n" " or.pred %p11, %p10, %p9;\n" " add.s32 %r156, %r40, %r606;\n" " sub.s32 %r157, %r2, %r584;\n" " add.s32 %r158, %r157, 1;\n" " max.s32 %r159, %r158, 0;\n" " setp.gt.s32 %p12, %r156, %r159;\n" " setp.le.s32 %p13, %r606, %r2;\n" " and.pred %p14, %p13, %p12;\n" " or.pred %p15, %p11, %p14;\n" " not.pred %p16, %p15;\n" " @%p16 bra $L__BB6_511;\n" "\n" " setp.ge.s32 %p17, %r24, %r40;\n" " @%p17 bra $L__BB6_103;\n" "\n" " add.s32 %r41, %r606, %r24;\n" " setp.lt.s32 %p18, %r41, %r99;\n" " @%p18 bra $L__BB6_71;\n" " bra.uni $L__BB6_14;\n" "\n" "$L__BB6_71:\n" " cvt.s64.s32 %rd121, %r41;\n" " mul.lo.s64 %rd122, %rd121, %rd5;\n" " mul.lo.s64 %rd124, %rd32, %rd5;\n" " shl.b64 %rd125, %rd124, 3;\n" " add.s64 %rd16, %rd122, %rd125;\n" " @%p56 bra $L__BB6_73;\n" "\n" " add.s64 %rd127, %rd16, %rd126;\n" " shl.b64 %rd128, %rd127, 1;\n" " add.s64 %rd129, %rd2, %rd128;\n" " ld.global.u16 %rs25, [%rd129];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1295, %rs25;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_73:\n" " @%p57 bra $L__BB6_75;\n" "\n" " add.s32 %r246, %r25, 16;\n" " cvt.s64.s32 %rd130, %r246;\n" " add.s64 %rd131, %rd16, %rd130;\n" " shl.b64 %rd132, %rd131, 1;\n" " add.s64 %rd133, %rd2, %rd132;\n" " ld.global.u16 %rs26, [%rd133];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1294, %rs26;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_75:\n" " @%p58 bra $L__BB6_77;\n" "\n" " add.s32 %r247, %r25, 32;\n" " cvt.s64.s32 %rd134, %r247;\n" " add.s64 %rd135, %rd16, %rd134;\n" " shl.b64 %rd136, %rd135, 1;\n" " add.s64 %rd137, %rd2, %rd136;\n" " ld.global.u16 %rs27, [%rd137];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1293, %rs27;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_77:\n" " setp.lt.s32 %p59, %r96, 64;\n" " @%p59 bra $L__BB6_79;\n" "\n" " add.s32 %r248, %r25, 48;\n" " cvt.s64.s32 %rd138, %r248;\n" " add.s64 %rd139, %rd16, %rd138;\n" " shl.b64 %rd140, %rd139, 1;\n" " add.s64 %rd141, %rd2, %rd140;\n" " ld.global.u16 %rs28, [%rd141];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1292, %rs28;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_79:\n" " setp.lt.s32 %p60, %r96, 80;\n" " @%p60 bra $L__BB6_81;\n" "\n" " add.s32 %r249, %r25, 64;\n" " cvt.s64.s32 %rd142, %r249;\n" " add.s64 %rd143, %rd16, %rd142;\n" " shl.b64 %rd144, %rd143, 1;\n" " add.s64 %rd145, %rd2, %rd144;\n" " ld.global.u16 %rs29, [%rd145];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1291, %rs29;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_81:\n" " setp.lt.s32 %p61, %r96, 96;\n" " @%p61 bra $L__BB6_83;\n" "\n" " add.s32 %r250, %r25, 80;\n" " cvt.s64.s32 %rd146, %r250;\n" " add.s64 %rd147, %rd16, %rd146;\n" " shl.b64 %rd148, %rd147, 1;\n" " add.s64 %rd149, %rd2, %rd148;\n" " ld.global.u16 %rs30, [%rd149];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1290, %rs30;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_83:\n" " setp.lt.s32 %p62, %r96, 112;\n" " @%p62 bra $L__BB6_85;\n" "\n" " add.s32 %r251, %r25, 96;\n" " cvt.s64.s32 %rd150, %r251;\n" " add.s64 %rd151, %rd16, %rd150;\n" " shl.b64 %rd152, %rd151, 1;\n" " add.s64 %rd153, %rd2, %rd152;\n" " ld.global.u16 %rs31, [%rd153];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1289, %rs31;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_85:\n" " setp.lt.s32 %p63, %r96, 128;\n" " @%p63 bra $L__BB6_87;\n" "\n" " add.s32 %r252, %r25, 112;\n" " cvt.s64.s32 %rd154, %r252;\n" " add.s64 %rd155, %rd16, %rd154;\n" " shl.b64 %rd156, %rd155, 1;\n" " add.s64 %rd157, %rd2, %rd156;\n" " ld.global.u16 %rs32, [%rd157];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1288, %rs32;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_87:\n" " setp.lt.s32 %p64, %r96, 144;\n" " @%p64 bra $L__BB6_89;\n" "\n" " add.s32 %r253, %r25, 128;\n" " cvt.s64.s32 %rd158, %r253;\n" " add.s64 %rd159, %rd16, %rd158;\n" " shl.b64 %rd160, %rd159, 1;\n" " add.s64 %rd161, %rd2, %rd160;\n" " ld.global.u16 %rs33, [%rd161];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1287, %rs33;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_89:\n" " setp.lt.s32 %p65, %r96, 160;\n" " @%p65 bra $L__BB6_91;\n" "\n" " add.s32 %r254, %r25, 144;\n" " cvt.s64.s32 %rd162, %r254;\n" " add.s64 %rd163, %rd16, %rd162;\n" " shl.b64 %rd164, %rd163, 1;\n" " add.s64 %rd165, %rd2, %rd164;\n" " ld.global.u16 %rs34, [%rd165];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1286, %rs34;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_91:\n" " setp.lt.s32 %p66, %r96, 176;\n" " @%p66 bra $L__BB6_93;\n" "\n" " add.s32 %r255, %r25, 160;\n" " cvt.s64.s32 %rd166, %r255;\n" " add.s64 %rd167, %rd16, %rd166;\n" " shl.b64 %rd168, %rd167, 1;\n" " add.s64 %rd169, %rd2, %rd168;\n" " ld.global.u16 %rs35, [%rd169];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1285, %rs35;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_93:\n" " setp.lt.s32 %p67, %r96, 192;\n" " @%p67 bra $L__BB6_95;\n" "\n" " add.s32 %r256, %r25, 176;\n" " cvt.s64.s32 %rd170, %r256;\n" " add.s64 %rd171, %rd16, %rd170;\n" " shl.b64 %rd172, %rd171, 1;\n" " add.s64 %rd173, %rd2, %rd172;\n" " ld.global.u16 %rs36, [%rd173];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1284, %rs36;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_95:\n" " setp.lt.s32 %p68, %r96, 208;\n" " @%p68 bra $L__BB6_97;\n" "\n" " add.s32 %r257, %r25, 192;\n" " cvt.s64.s32 %rd174, %r257;\n" " add.s64 %rd175, %rd16, %rd174;\n" " shl.b64 %rd176, %rd175, 1;\n" " add.s64 %rd177, %rd2, %rd176;\n" " ld.global.u16 %rs37, [%rd177];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1283, %rs37;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_97:\n" " setp.lt.s32 %p69, %r96, 224;\n" " @%p69 bra $L__BB6_99;\n" "\n" " add.s32 %r258, %r25, 208;\n" " cvt.s64.s32 %rd178, %r258;\n" " add.s64 %rd179, %rd16, %rd178;\n" " shl.b64 %rd180, %rd179, 1;\n" " add.s64 %rd181, %rd2, %rd180;\n" " ld.global.u16 %rs38, [%rd181];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1282, %rs38;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_99:\n" " setp.lt.s32 %p70, %r96, 240;\n" " @%p70 bra $L__BB6_101;\n" "\n" " add.s32 %r259, %r25, 224;\n" " cvt.s64.s32 %rd182, %r259;\n" " add.s64 %rd183, %rd16, %rd182;\n" " shl.b64 %rd184, %rd183, 1;\n" " add.s64 %rd185, %rd2, %rd184;\n" " ld.global.u16 %rs39, [%rd185];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1281, %rs39;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_101:\n" " setp.lt.s32 %p71, %r96, 256;\n" " @%p71 bra $L__BB6_103;\n" "\n" " add.s32 %r260, %r25, 240;\n" " cvt.s64.s32 %rd186, %r260;\n" " add.s64 %rd187, %rd16, %rd186;\n" " shl.b64 %rd188, %rd187, 1;\n" " add.s64 %rd189, %rd2, %rd188;\n" " ld.global.u16 %rs40, [%rd189];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1280, %rs40;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB6_103;\n" "\n" "$L__BB6_14:\n" " setp.gt.s32 %p19, %r100, 0;\n" " sub.s32 %r162, %r2, %r100;\n" " setp.gt.s32 %p20, %r41, %r162;\n" " and.pred %p21, %p19, %p20;\n" " mov.u32 %r163, %nctaid.x;\n" " add.s32 %r164, %r163, %r98;\n" " add.s32 %r165, %r164, -257;\n" " setp.gt.s32 %p22, %r41, %r165;\n" " and.pred %p23, %p22, %p21;\n" " @%p23 bra $L__BB6_39;\n" " bra.uni $L__BB6_15;\n" "\n" "$L__BB6_39:\n" " shr.s32 %r224, %r41, 31;\n" " shr.u32 %r225, %r224, 24;\n" " add.s32 %r226, %r41, %r225;\n" " and.b32 %r227, %r226, -256;\n" " sub.s32 %r228, %r41, %r227;\n" " cvt.s64.s32 %rd52, %r228;\n" " mul.lo.s64 %rd53, %rd52, %rd5;\n" " mul.lo.s64 %rd55, %rd32, %rd5;\n" " shl.b64 %rd56, %rd55, 8;\n" " add.s64 %rd15, %rd53, %rd56;\n" " @%p56 bra $L__BB6_41;\n" "\n" " add.s64 %rd58, %rd15, %rd126;\n" " shl.b64 %rd59, %rd58, 1;\n" " add.s64 %rd60, %rd3, %rd59;\n" " ld.global.u16 %rs9, [%rd60];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1295, %rs9;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_41:\n" " @%p57 bra $L__BB6_43;\n" "\n" " add.s32 %r230, %r25, 16;\n" " cvt.s64.s32 %rd61, %r230;\n" " add.s64 %rd62, %rd15, %rd61;\n" " shl.b64 %rd63, %rd62, 1;\n" " add.s64 %rd64, %rd3, %rd63;\n" " ld.global.u16 %rs10, [%rd64];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1294, %rs10;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_43:\n" " @%p58 bra $L__BB6_45;\n" "\n" " add.s32 %r231, %r25, 32;\n" " cvt.s64.s32 %rd65, %r231;\n" " add.s64 %rd66, %rd15, %rd65;\n" " shl.b64 %rd67, %rd66, 1;\n" " add.s64 %rd68, %rd3, %rd67;\n" " ld.global.u16 %rs11, [%rd68];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1293, %rs11;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_45:\n" " setp.lt.s32 %p43, %r96, 64;\n" " @%p43 bra $L__BB6_47;\n" "\n" " add.s32 %r232, %r25, 48;\n" " cvt.s64.s32 %rd69, %r232;\n" " add.s64 %rd70, %rd15, %rd69;\n" " shl.b64 %rd71, %rd70, 1;\n" " add.s64 %rd72, %rd3, %rd71;\n" " ld.global.u16 %rs12, [%rd72];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1292, %rs12;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_47:\n" " setp.lt.s32 %p44, %r96, 80;\n" " @%p44 bra $L__BB6_49;\n" "\n" " add.s32 %r233, %r25, 64;\n" " cvt.s64.s32 %rd73, %r233;\n" " add.s64 %rd74, %rd15, %rd73;\n" " shl.b64 %rd75, %rd74, 1;\n" " add.s64 %rd76, %rd3, %rd75;\n" " ld.global.u16 %rs13, [%rd76];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1291, %rs13;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_49:\n" " setp.lt.s32 %p45, %r96, 96;\n" " @%p45 bra $L__BB6_51;\n" "\n" " add.s32 %r234, %r25, 80;\n" " cvt.s64.s32 %rd77, %r234;\n" " add.s64 %rd78, %rd15, %rd77;\n" " shl.b64 %rd79, %rd78, 1;\n" " add.s64 %rd80, %rd3, %rd79;\n" " ld.global.u16 %rs14, [%rd80];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1290, %rs14;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_51:\n" " setp.lt.s32 %p46, %r96, 112;\n" " @%p46 bra $L__BB6_53;\n" "\n" " add.s32 %r235, %r25, 96;\n" " cvt.s64.s32 %rd81, %r235;\n" " add.s64 %rd82, %rd15, %rd81;\n" " shl.b64 %rd83, %rd82, 1;\n" " add.s64 %rd84, %rd3, %rd83;\n" " ld.global.u16 %rs15, [%rd84];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1289, %rs15;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_53:\n" " setp.lt.s32 %p47, %r96, 128;\n" " @%p47 bra $L__BB6_55;\n" "\n" " add.s32 %r236, %r25, 112;\n" " cvt.s64.s32 %rd85, %r236;\n" " add.s64 %rd86, %rd15, %rd85;\n" " shl.b64 %rd87, %rd86, 1;\n" " add.s64 %rd88, %rd3, %rd87;\n" " ld.global.u16 %rs16, [%rd88];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1288, %rs16;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_55:\n" " setp.lt.s32 %p48, %r96, 144;\n" " @%p48 bra $L__BB6_57;\n" "\n" " add.s32 %r237, %r25, 128;\n" " cvt.s64.s32 %rd89, %r237;\n" " add.s64 %rd90, %rd15, %rd89;\n" " shl.b64 %rd91, %rd90, 1;\n" " add.s64 %rd92, %rd3, %rd91;\n" " ld.global.u16 %rs17, [%rd92];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1287, %rs17;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_57:\n" " setp.lt.s32 %p49, %r96, 160;\n" " @%p49 bra $L__BB6_59;\n" "\n" " add.s32 %r238, %r25, 144;\n" " cvt.s64.s32 %rd93, %r238;\n" " add.s64 %rd94, %rd15, %rd93;\n" " shl.b64 %rd95, %rd94, 1;\n" " add.s64 %rd96, %rd3, %rd95;\n" " ld.global.u16 %rs18, [%rd96];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1286, %rs18;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_59:\n" " setp.lt.s32 %p50, %r96, 176;\n" " @%p50 bra $L__BB6_61;\n" "\n" " add.s32 %r239, %r25, 160;\n" " cvt.s64.s32 %rd97, %r239;\n" " add.s64 %rd98, %rd15, %rd97;\n" " shl.b64 %rd99, %rd98, 1;\n" " add.s64 %rd100, %rd3, %rd99;\n" " ld.global.u16 %rs19, [%rd100];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1285, %rs19;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_61:\n" " setp.lt.s32 %p51, %r96, 192;\n" " @%p51 bra $L__BB6_63;\n" "\n" " add.s32 %r240, %r25, 176;\n" " cvt.s64.s32 %rd101, %r240;\n" " add.s64 %rd102, %rd15, %rd101;\n" " shl.b64 %rd103, %rd102, 1;\n" " add.s64 %rd104, %rd3, %rd103;\n" " ld.global.u16 %rs20, [%rd104];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1284, %rs20;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_63:\n" " setp.lt.s32 %p52, %r96, 208;\n" " @%p52 bra $L__BB6_65;\n" "\n" " add.s32 %r241, %r25, 192;\n" " cvt.s64.s32 %rd105, %r241;\n" " add.s64 %rd106, %rd15, %rd105;\n" " shl.b64 %rd107, %rd106, 1;\n" " add.s64 %rd108, %rd3, %rd107;\n" " ld.global.u16 %rs21, [%rd108];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1283, %rs21;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_65:\n" " setp.lt.s32 %p53, %r96, 224;\n" " @%p53 bra $L__BB6_67;\n" "\n" " add.s32 %r242, %r25, 208;\n" " cvt.s64.s32 %rd109, %r242;\n" " add.s64 %rd110, %rd15, %rd109;\n" " shl.b64 %rd111, %rd110, 1;\n" " add.s64 %rd112, %rd3, %rd111;\n" " ld.global.u16 %rs22, [%rd112];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1282, %rs22;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_67:\n" " setp.lt.s32 %p54, %r96, 240;\n" " @%p54 bra $L__BB6_69;\n" "\n" " add.s32 %r243, %r25, 224;\n" " cvt.s64.s32 %rd113, %r243;\n" " add.s64 %rd114, %rd15, %rd113;\n" " shl.b64 %rd115, %rd114, 1;\n" " add.s64 %rd116, %rd3, %rd115;\n" " ld.global.u16 %rs23, [%rd116];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1281, %rs23;}\n" "\n" " // end inline asm\n" "\n" "$L__BB6_69:\n" " setp.lt.s32 %p55, %r96, 256;\n" " @%p55 bra $L__BB6_103;\n" "\n" " add.s32 %r244, %r25, 240;\n" " cvt.s64.s32 %rd117, %r244;\n" " add.s64 %rd118, %rd15, %rd117;\n" " shl.b64 %rd119, %rd118, 1;\n" " add.s64 %rd120, %rd3, %rd119;\n" " ld.global.u16 %rs24, [%rd120];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1280, %rs24;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB6_103;\n" "\n" "$L__BB6_15:\n" " mul.wide.s32 %rd46, %r41, %r3;\n" " mul.wide.s32 %rd47, %r97, %r110;\n" " mul.lo.s64 %rd48, %rd47, %rd45;\n" " add.s64 %rd49, %rd46, %rd48;\n" " add.s64 %rd13, %rd50, %rd49;\n" " add.s32 %r167, %r25, 2;\n" " cvt.s64.s32 %rd51, %r167;\n" " add.s64 %rd14, %rd13, %rd51;\n" " @%p56 bra $L__BB6_18;\n" "\n" " ld.global.u8 %r168, [%rd13];\n" " ld.global.u8 %r169, [%rd13+1];\n" " prmt.b32 %r170, %r169, %r168, 30212;\n" " cvt.u16.u32 %rs1, %r170;\n" " // begin inline asm\n" " { cvt.f32.f16 %f696, %rs1;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r42, [%rd14];\n" " and.b32 %r171, %r42, 15;\n" " add.s32 %r172, %r171, -8;\n" " cvt.rn.f32.s32 %f697, %r172;\n" " mul.f32 %f1295, %f696, %f697;\n" " @%p57 bra $L__BB6_18;\n" "\n" " shr.u32 %r173, %r42, 4;\n" " add.s32 %r174, %r173, -8;\n" " cvt.rn.f32.s32 %f698, %r174;\n" " mul.f32 %f1294, %f696, %f698;\n" "\n" "$L__BB6_18:\n" " @%p58 bra $L__BB6_21;\n" "\n" " setp.lt.s32 %p27, %r96, 64;\n" " ld.global.u8 %r175, [%rd13+18];\n" " ld.global.u8 %r176, [%rd13+19];\n" " prmt.b32 %r177, %r176, %r175, 30212;\n" " cvt.u16.u32 %rs2, %r177;\n" " // begin inline asm\n" " { cvt.f32.f16 %f699, %rs2;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r43, [%rd14+18];\n" " and.b32 %r178, %r43, 15;\n" " add.s32 %r179, %r178, -8;\n" " cvt.rn.f32.s32 %f700, %r179;\n" " mul.f32 %f1293, %f699, %f700;\n" " @%p27 bra $L__BB6_21;\n" "\n" " shr.u32 %r180, %r43, 4;\n" " add.s32 %r181, %r180, -8;\n" " cvt.rn.f32.s32 %f701, %r181;\n" " mul.f32 %f1292, %f699, %f701;\n" "\n" "$L__BB6_21:\n" " setp.lt.s32 %p28, %r96, 80;\n" " @%p28 bra $L__BB6_24;\n" "\n" " setp.lt.s32 %p29, %r96, 96;\n" " ld.global.u8 %r182, [%rd13+36];\n" " ld.global.u8 %r183, [%rd13+37];\n" " prmt.b32 %r184, %r183, %r182, 30212;\n" " cvt.u16.u32 %rs3, %r184;\n" " // begin inline asm\n" " { cvt.f32.f16 %f702, %rs3;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r44, [%rd14+36];\n" " and.b32 %r185, %r44, 15;\n" " add.s32 %r186, %r185, -8;\n" " cvt.rn.f32.s32 %f703, %r186;\n" " mul.f32 %f1291, %f702, %f703;\n" " @%p29 bra $L__BB6_24;\n" "\n" " shr.u32 %r187, %r44, 4;\n" " add.s32 %r188, %r187, -8;\n" " cvt.rn.f32.s32 %f704, %r188;\n" " mul.f32 %f1290, %f702, %f704;\n" "\n" "$L__BB6_24:\n" " setp.lt.s32 %p30, %r96, 112;\n" " @%p30 bra $L__BB6_27;\n" "\n" " setp.lt.s32 %p31, %r96, 128;\n" " ld.global.u8 %r189, [%rd13+54];\n" " ld.global.u8 %r190, [%rd13+55];\n" " prmt.b32 %r191, %r190, %r189, 30212;\n" " cvt.u16.u32 %rs4, %r191;\n" " // begin inline asm\n" " { cvt.f32.f16 %f705, %rs4;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r45, [%rd14+54];\n" " and.b32 %r192, %r45, 15;\n" " add.s32 %r193, %r192, -8;\n" " cvt.rn.f32.s32 %f706, %r193;\n" " mul.f32 %f1289, %f705, %f706;\n" " @%p31 bra $L__BB6_27;\n" "\n" " shr.u32 %r194, %r45, 4;\n" " add.s32 %r195, %r194, -8;\n" " cvt.rn.f32.s32 %f707, %r195;\n" " mul.f32 %f1288, %f705, %f707;\n" "\n" "$L__BB6_27:\n" " setp.lt.s32 %p32, %r96, 144;\n" " @%p32 bra $L__BB6_30;\n" "\n" " setp.lt.s32 %p33, %r96, 160;\n" " ld.global.u8 %r196, [%rd13+72];\n" " ld.global.u8 %r197, [%rd13+73];\n" " prmt.b32 %r198, %r197, %r196, 30212;\n" " cvt.u16.u32 %rs5, %r198;\n" " // begin inline asm\n" " { cvt.f32.f16 %f708, %rs5;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r46, [%rd14+72];\n" " and.b32 %r199, %r46, 15;\n" " add.s32 %r200, %r199, -8;\n" " cvt.rn.f32.s32 %f709, %r200;\n" " mul.f32 %f1287, %f708, %f709;\n" " @%p33 bra $L__BB6_30;\n" "\n" " shr.u32 %r201, %r46, 4;\n" " add.s32 %r202, %r201, -8;\n" " cvt.rn.f32.s32 %f710, %r202;\n" " mul.f32 %f1286, %f708, %f710;\n" "\n" "$L__BB6_30:\n" " setp.lt.s32 %p34, %r96, 176;\n" " @%p34 bra $L__BB6_33;\n" "\n" " setp.lt.s32 %p35, %r96, 192;\n" " ld.global.u8 %r203, [%rd13+90];\n" " ld.global.u8 %r204, [%rd13+91];\n" " prmt.b32 %r205, %r204, %r203, 30212;\n" " cvt.u16.u32 %rs6, %r205;\n" " // begin inline asm\n" " { cvt.f32.f16 %f711, %rs6;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r47, [%rd14+90];\n" " and.b32 %r206, %r47, 15;\n" " add.s32 %r207, %r206, -8;\n" " cvt.rn.f32.s32 %f712, %r207;\n" " mul.f32 %f1285, %f711, %f712;\n" " @%p35 bra $L__BB6_33;\n" "\n" " shr.u32 %r208, %r47, 4;\n" " add.s32 %r209, %r208, -8;\n" " cvt.rn.f32.s32 %f713, %r209;\n" " mul.f32 %f1284, %f711, %f713;\n" "\n" "$L__BB6_33:\n" " setp.lt.s32 %p36, %r96, 208;\n" " @%p36 bra $L__BB6_36;\n" "\n" " setp.lt.s32 %p37, %r96, 224;\n" " ld.global.u8 %r210, [%rd13+108];\n" " ld.global.u8 %r211, [%rd13+109];\n" " prmt.b32 %r212, %r211, %r210, 30212;\n" " cvt.u16.u32 %rs7, %r212;\n" " // begin inline asm\n" " { cvt.f32.f16 %f714, %rs7;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r48, [%rd14+108];\n" " and.b32 %r213, %r48, 15;\n" " add.s32 %r214, %r213, -8;\n" " cvt.rn.f32.s32 %f715, %r214;\n" " mul.f32 %f1283, %f714, %f715;\n" " @%p37 bra $L__BB6_36;\n" "\n" " shr.u32 %r215, %r48, 4;\n" " add.s32 %r216, %r215, -8;\n" " cvt.rn.f32.s32 %f716, %r216;\n" " mul.f32 %f1282, %f714, %f716;\n" "\n" "$L__BB6_36:\n" " setp.lt.s32 %p38, %r96, 240;\n" " @%p38 bra $L__BB6_103;\n" "\n" " setp.lt.s32 %p39, %r96, 256;\n" " ld.global.u8 %r217, [%rd13+126];\n" " ld.global.u8 %r218, [%rd13+127];\n" " prmt.b32 %r219, %r218, %r217, 30212;\n" " cvt.u16.u32 %rs8, %r219;\n" " // begin inline asm\n" " { cvt.f32.f16 %f717, %rs8;}\n" "\n" " // end inline asm\n" " ld.global.u8 %r49, [%rd14+126];\n" " and.b32 %r220, %r49, 15;\n" " add.s32 %r221, %r220, -8;\n" " cvt.rn.f32.s32 %f718, %r221;\n" " mul.f32 %f1281, %f717, %f718;\n" " @%p39 bra $L__BB6_103;\n" "\n" " shr.u32 %r222, %r49, 4;\n" " add.s32 %r223, %r222, -8;\n" " cvt.rn.f32.s32 %f719, %r223;\n" " mul.f32 %f1280, %f717, %f719;\n" "\n" "$L__BB6_103:\n" " mov.f32 %f1318, 0f00000000;\n" " mov.f32 %f1319, %f1318;\n" " mov.f32 %f1320, %f1318;\n" " mov.f32 %f1321, %f1318;\n" " mov.f32 %f1322, %f1318;\n" " mov.f32 %f1323, %f1318;\n" " mov.f32 %f1324, %f1318;\n" " mov.f32 %f1325, %f1318;\n" " @%p17 bra $L__BB6_376;\n" "\n" " mov.f32 %f1318, 0f00000000;\n" " mov.f32 %f1319, %f1318;\n" " mov.f32 %f1320, %f1318;\n" " mov.f32 %f1321, %f1318;\n" " mov.f32 %f1322, %f1318;\n" " mov.f32 %f1323, %f1318;\n" " mov.f32 %f1324, %f1318;\n" " mov.f32 %f1325, %f1318;\n" " @%p56 bra $L__BB6_121;\n" "\n" " mov.f32 %f1318, 0f00000000;\n" " mov.f32 %f1325, %f1318;\n" " @%p2 bra $L__BB6_107;\n" "\n" " ld.shared.f32 %f769, [%r27];\n" " fma.rn.f32 %f1325, %f1295, %f769, 0f00000000;\n" "\n" "$L__BB6_107:\n" " setp.lt.s32 %p75, %r95, 2;\n" " mov.f32 %f1324, %f1318;\n" " @%p75 bra $L__BB6_109;\n" "\n" " ld.shared.f32 %f771, [%r28];\n" " fma.rn.f32 %f1324, %f1295, %f771, 0f00000000;\n" "\n" "$L__BB6_109:\n" " setp.lt.s32 %p76, %r95, 3;\n" " mov.f32 %f1323, %f1318;\n" " @%p76 bra $L__BB6_111;\n" "\n" " ld.shared.f32 %f773, [%r29];\n" " fma.rn.f32 %f1323, %f1295, %f773, 0f00000000;\n" "\n" "$L__BB6_111:\n" " setp.lt.s32 %p77, %r95, 4;\n" " mov.f32 %f1322, %f1318;\n" " @%p77 bra $L__BB6_113;\n" "\n" " ld.shared.f32 %f775, [%r30];\n" " fma.rn.f32 %f1322, %f1295, %f775, 0f00000000;\n" "\n" "$L__BB6_113:\n" " setp.lt.s32 %p78, %r95, 5;\n" " mov.f32 %f1321, %f1318;\n" " @%p78 bra $L__BB6_115;\n" "\n" " ld.shared.f32 %f777, [%r31];\n" " fma.rn.f32 %f1321, %f1295, %f777, 0f00000000;\n" "\n" "$L__BB6_115:\n" " setp.lt.s32 %p79, %r95, 6;\n" " mov.f32 %f1320, %f1318;\n" " @%p79 bra $L__BB6_117;\n" "\n" " ld.shared.f32 %f779, [%r32];\n" " fma.rn.f32 %f1320, %f1295, %f779, 0f00000000;\n" "\n" "$L__BB6_117:\n" " setp.lt.s32 %p80, %r95, 7;\n" " mov.f32 %f1319, %f1318;\n" " @%p80 bra $L__BB6_119;\n" "\n" " ld.shared.f32 %f781, [%r33];\n" " fma.rn.f32 %f1319, %f1295, %f781, 0f00000000;\n" "\n" "$L__BB6_119:\n" " setp.lt.s32 %p81, %r95, 8;\n" " @%p81 bra $L__BB6_121;\n" "\n" " ld.shared.f32 %f783, [%r34];\n" " fma.rn.f32 %f1318, %f1295, %f783, 0f00000000;\n" "\n" "$L__BB6_121:\n" " @%p57 bra $L__BB6_138;\n" "\n" " @%p2 bra $L__BB6_124;\n" "\n" " ld.shared.f32 %f784, [%r27+64];\n" " fma.rn.f32 %f1325, %f1294, %f784, %f1325;\n" "\n" "$L__BB6_124:\n" " setp.lt.s32 %p84, %r95, 2;\n" " @%p84 bra $L__BB6_126;\n" "\n" " ld.shared.f32 %f785, [%r28+64];\n" " fma.rn.f32 %f1324, %f1294, %f785, %f1324;\n" "\n" "$L__BB6_126:\n" " setp.lt.s32 %p85, %r95, 3;\n" " @%p85 bra $L__BB6_128;\n" "\n" " ld.shared.f32 %f786, [%r29+64];\n" " fma.rn.f32 %f1323, %f1294, %f786, %f1323;\n" "\n" "$L__BB6_128:\n" " setp.lt.s32 %p86, %r95, 4;\n" " @%p86 bra $L__BB6_130;\n" "\n" " ld.shared.f32 %f787, [%r30+64];\n" " fma.rn.f32 %f1322, %f1294, %f787, %f1322;\n" "\n" "$L__BB6_130:\n" " setp.lt.s32 %p87, %r95, 5;\n" " @%p87 bra $L__BB6_132;\n" "\n" " ld.shared.f32 %f788, [%r31+64];\n" " fma.rn.f32 %f1321, %f1294, %f788, %f1321;\n" "\n" "$L__BB6_132:\n" " setp.lt.s32 %p88, %r95, 6;\n" " @%p88 bra $L__BB6_134;\n" "\n" " ld.shared.f32 %f789, [%r32+64];\n" " fma.rn.f32 %f1320, %f1294, %f789, %f1320;\n" "\n" "$L__BB6_134:\n" " setp.lt.s32 %p89, %r95, 7;\n" " @%p89 bra $L__BB6_136;\n" "\n" " ld.shared.f32 %f790, [%r33+64];\n" " fma.rn.f32 %f1319, %f1294, %f790, %f1319;\n" "\n" "$L__BB6_136:\n" " setp.lt.s32 %p90, %r95, 8;\n" " @%p90 bra $L__BB6_138;\n" "\n" " ld.shared.f32 %f791, [%r34+64];\n" " fma.rn.f32 %f1318, %f1294, %f791, %f1318;\n" "\n" "$L__BB6_138:\n" " @%p58 bra $L__BB6_155;\n" "\n" " @%p2 bra $L__BB6_141;\n" "\n" " ld.shared.f32 %f792, [%r27+128];\n" " fma.rn.f32 %f1325, %f1293, %f792, %f1325;\n" "\n" "$L__BB6_141:\n" " setp.lt.s32 %p93, %r95, 2;\n" " @%p93 bra $L__BB6_143;\n" "\n" " ld.shared.f32 %f793, [%r28+128];\n" " fma.rn.f32 %f1324, %f1293, %f793, %f1324;\n" "\n" "$L__BB6_143:\n" " setp.lt.s32 %p94, %r95, 3;\n" " @%p94 bra $L__BB6_145;\n" "\n" " ld.shared.f32 %f794, [%r29+128];\n" " fma.rn.f32 %f1323, %f1293, %f794, %f1323;\n" "\n" "$L__BB6_145:\n" " setp.lt.s32 %p95, %r95, 4;\n" " @%p95 bra $L__BB6_147;\n" "\n" " ld.shared.f32 %f795, [%r30+128];\n" " fma.rn.f32 %f1322, %f1293, %f795, %f1322;\n" "\n" "$L__BB6_147:\n" " setp.lt.s32 %p96, %r95, 5;\n" " @%p96 bra $L__BB6_149;\n" "\n" " ld.shared.f32 %f796, [%r31+128];\n" " fma.rn.f32 %f1321, %f1293, %f796, %f1321;\n" "\n" "$L__BB6_149:\n" " setp.lt.s32 %p97, %r95, 6;\n" " @%p97 bra $L__BB6_151;\n" "\n" " ld.shared.f32 %f797, [%r32+128];\n" " fma.rn.f32 %f1320, %f1293, %f797, %f1320;\n" "\n" "$L__BB6_151:\n" " setp.lt.s32 %p98, %r95, 7;\n" " @%p98 bra $L__BB6_153;\n" "\n" " ld.shared.f32 %f798, [%r33+128];\n" " fma.rn.f32 %f1319, %f1293, %f798, %f1319;\n" "\n" "$L__BB6_153:\n" " setp.lt.s32 %p99, %r95, 8;\n" " @%p99 bra $L__BB6_155;\n" "\n" " ld.shared.f32 %f799, [%r34+128];\n" " fma.rn.f32 %f1318, %f1293, %f799, %f1318;\n" "\n" "$L__BB6_155:\n" " setp.lt.s32 %p100, %r96, 64;\n" " @%p100 bra $L__BB6_172;\n" "\n" " @%p2 bra $L__BB6_158;\n" "\n" " ld.shared.f32 %f800, [%r27+192];\n" " fma.rn.f32 %f1325, %f1292, %f800, %f1325;\n" "\n" "$L__BB6_158:\n" " setp.lt.s32 %p102, %r95, 2;\n" " @%p102 bra $L__BB6_160;\n" "\n" " ld.shared.f32 %f801, [%r28+192];\n" " fma.rn.f32 %f1324, %f1292, %f801, %f1324;\n" "\n" "$L__BB6_160:\n" " setp.lt.s32 %p103, %r95, 3;\n" " @%p103 bra $L__BB6_162;\n" "\n" " ld.shared.f32 %f802, [%r29+192];\n" " fma.rn.f32 %f1323, %f1292, %f802, %f1323;\n" "\n" "$L__BB6_162:\n" " setp.lt.s32 %p104, %r95, 4;\n" " @%p104 bra $L__BB6_164;\n" "\n" " ld.shared.f32 %f803, [%r30+192];\n" " fma.rn.f32 %f1322, %f1292, %f803, %f1322;\n" "\n" "$L__BB6_164:\n" " setp.lt.s32 %p105, %r95, 5;\n" " @%p105 bra $L__BB6_166;\n" "\n" " ld.shared.f32 %f804, [%r31+192];\n" " fma.rn.f32 %f1321, %f1292, %f804, %f1321;\n" "\n" "$L__BB6_166:\n" " setp.lt.s32 %p106, %r95, 6;\n" " @%p106 bra $L__BB6_168;\n" "\n" " ld.shared.f32 %f805, [%r32+192];\n" " fma.rn.f32 %f1320, %f1292, %f805, %f1320;\n" "\n" "$L__BB6_168:\n" " setp.lt.s32 %p107, %r95, 7;\n" " @%p107 bra $L__BB6_170;\n" "\n" " ld.shared.f32 %f806, [%r33+192];\n" " fma.rn.f32 %f1319, %f1292, %f806, %f1319;\n" "\n" "$L__BB6_170:\n" " setp.lt.s32 %p108, %r95, 8;\n" " @%p108 bra $L__BB6_172;\n" "\n" " ld.shared.f32 %f807, [%r34+192];\n" " fma.rn.f32 %f1318, %f1292, %f807, %f1318;\n" "\n" "$L__BB6_172:\n" " setp.lt.s32 %p109, %r96, 80;\n" " @%p109 bra $L__BB6_189;\n" "\n" " @%p2 bra $L__BB6_175;\n" "\n" " ld.shared.f32 %f808, [%r27+256];\n" " fma.rn.f32 %f1325, %f1291, %f808, %f1325;\n" "\n" "$L__BB6_175:\n" " setp.lt.s32 %p111, %r95, 2;\n" " @%p111 bra $L__BB6_177;\n" "\n" " ld.shared.f32 %f809, [%r28+256];\n" " fma.rn.f32 %f1324, %f1291, %f809, %f1324;\n" "\n" "$L__BB6_177:\n" " setp.lt.s32 %p112, %r95, 3;\n" " @%p112 bra $L__BB6_179;\n" "\n" " ld.shared.f32 %f810, [%r29+256];\n" " fma.rn.f32 %f1323, %f1291, %f810, %f1323;\n" "\n" "$L__BB6_179:\n" " setp.lt.s32 %p113, %r95, 4;\n" " @%p113 bra $L__BB6_181;\n" "\n" " ld.shared.f32 %f811, [%r30+256];\n" " fma.rn.f32 %f1322, %f1291, %f811, %f1322;\n" "\n" "$L__BB6_181:\n" " setp.lt.s32 %p114, %r95, 5;\n" " @%p114 bra $L__BB6_183;\n" "\n" " ld.shared.f32 %f812, [%r31+256];\n" " fma.rn.f32 %f1321, %f1291, %f812, %f1321;\n" "\n" "$L__BB6_183:\n" " setp.lt.s32 %p115, %r95, 6;\n" " @%p115 bra $L__BB6_185;\n" "\n" " ld.shared.f32 %f813, [%r32+256];\n" " fma.rn.f32 %f1320, %f1291, %f813, %f1320;\n" "\n" "$L__BB6_185:\n" " setp.lt.s32 %p116, %r95, 7;\n" " @%p116 bra $L__BB6_187;\n" "\n" " ld.shared.f32 %f814, [%r33+256];\n" " fma.rn.f32 %f1319, %f1291, %f814, %f1319;\n" "\n" "$L__BB6_187:\n" " setp.lt.s32 %p117, %r95, 8;\n" " @%p117 bra $L__BB6_189;\n" "\n" " ld.shared.f32 %f815, [%r34+256];\n" " fma.rn.f32 %f1318, %f1291, %f815, %f1318;\n" "\n" "$L__BB6_189:\n" " setp.lt.s32 %p118, %r96, 96;\n" " @%p118 bra $L__BB6_206;\n" "\n" " @%p2 bra $L__BB6_192;\n" "\n" " ld.shared.f32 %f816, [%r27+320];\n" " fma.rn.f32 %f1325, %f1290, %f816, %f1325;\n" "\n" "$L__BB6_192:\n" " setp.lt.s32 %p120, %r95, 2;\n" " @%p120 bra $L__BB6_194;\n" "\n" " ld.shared.f32 %f817, [%r28+320];\n" " fma.rn.f32 %f1324, %f1290, %f817, %f1324;\n" "\n" "$L__BB6_194:\n" " setp.lt.s32 %p121, %r95, 3;\n" " @%p121 bra $L__BB6_196;\n" "\n" " ld.shared.f32 %f818, [%r29+320];\n" " fma.rn.f32 %f1323, %f1290, %f818, %f1323;\n" "\n" "$L__BB6_196:\n" " setp.lt.s32 %p122, %r95, 4;\n" " @%p122 bra $L__BB6_198;\n" "\n" " ld.shared.f32 %f819, [%r30+320];\n" " fma.rn.f32 %f1322, %f1290, %f819, %f1322;\n" "\n" "$L__BB6_198:\n" " setp.lt.s32 %p123, %r95, 5;\n" " @%p123 bra $L__BB6_200;\n" "\n" " ld.shared.f32 %f820, [%r31+320];\n" " fma.rn.f32 %f1321, %f1290, %f820, %f1321;\n" "\n" "$L__BB6_200:\n" " setp.lt.s32 %p124, %r95, 6;\n" " @%p124 bra $L__BB6_202;\n" "\n" " ld.shared.f32 %f821, [%r32+320];\n" " fma.rn.f32 %f1320, %f1290, %f821, %f1320;\n" "\n" "$L__BB6_202:\n" " setp.lt.s32 %p125, %r95, 7;\n" " @%p125 bra $L__BB6_204;\n" "\n" " ld.shared.f32 %f822, [%r33+320];\n" " fma.rn.f32 %f1319, %f1290, %f822, %f1319;\n" "\n" "$L__BB6_204:\n" " setp.lt.s32 %p126, %r95, 8;\n" " @%p126 bra $L__BB6_206;\n" "\n" " ld.shared.f32 %f823, [%r34+320];\n" " fma.rn.f32 %f1318, %f1290, %f823, %f1318;\n" "\n" "$L__BB6_206:\n" " setp.lt.s32 %p127, %r96, 112;\n" " @%p127 bra $L__BB6_223;\n" "\n" " @%p2 bra $L__BB6_209;\n" "\n" " ld.shared.f32 %f824, [%r27+384];\n" " fma.rn.f32 %f1325, %f1289, %f824, %f1325;\n" "\n" "$L__BB6_209:\n" " setp.lt.s32 %p129, %r95, 2;\n" " @%p129 bra $L__BB6_211;\n" "\n" " ld.shared.f32 %f825, [%r28+384];\n" " fma.rn.f32 %f1324, %f1289, %f825, %f1324;\n" "\n" "$L__BB6_211:\n" " setp.lt.s32 %p130, %r95, 3;\n" " @%p130 bra $L__BB6_213;\n" "\n" " ld.shared.f32 %f826, [%r29+384];\n" " fma.rn.f32 %f1323, %f1289, %f826, %f1323;\n" "\n" "$L__BB6_213:\n" " setp.lt.s32 %p131, %r95, 4;\n" " @%p131 bra $L__BB6_215;\n" "\n" " ld.shared.f32 %f827, [%r30+384];\n" " fma.rn.f32 %f1322, %f1289, %f827, %f1322;\n" "\n" "$L__BB6_215:\n" " setp.lt.s32 %p132, %r95, 5;\n" " @%p132 bra $L__BB6_217;\n" "\n" " ld.shared.f32 %f828, [%r31+384];\n" " fma.rn.f32 %f1321, %f1289, %f828, %f1321;\n" "\n" "$L__BB6_217:\n" " setp.lt.s32 %p133, %r95, 6;\n" " @%p133 bra $L__BB6_219;\n" "\n" " ld.shared.f32 %f829, [%r32+384];\n" " fma.rn.f32 %f1320, %f1289, %f829, %f1320;\n" "\n" "$L__BB6_219:\n" " setp.lt.s32 %p134, %r95, 7;\n" " @%p134 bra $L__BB6_221;\n" "\n" " ld.shared.f32 %f830, [%r33+384];\n" " fma.rn.f32 %f1319, %f1289, %f830, %f1319;\n" "\n" "$L__BB6_221:\n" " setp.lt.s32 %p135, %r95, 8;\n" " @%p135 bra $L__BB6_223;\n" "\n" " ld.shared.f32 %f831, [%r34+384];\n" " fma.rn.f32 %f1318, %f1289, %f831, %f1318;\n" "\n" "$L__BB6_223:\n" " setp.lt.s32 %p136, %r96, 128;\n" " @%p136 bra $L__BB6_240;\n" "\n" " @%p2 bra $L__BB6_226;\n" "\n" " ld.shared.f32 %f832, [%r27+448];\n" " fma.rn.f32 %f1325, %f1288, %f832, %f1325;\n" "\n" "$L__BB6_226:\n" " setp.lt.s32 %p138, %r95, 2;\n" " @%p138 bra $L__BB6_228;\n" "\n" " ld.shared.f32 %f833, [%r28+448];\n" " fma.rn.f32 %f1324, %f1288, %f833, %f1324;\n" "\n" "$L__BB6_228:\n" " setp.lt.s32 %p139, %r95, 3;\n" " @%p139 bra $L__BB6_230;\n" "\n" " ld.shared.f32 %f834, [%r29+448];\n" " fma.rn.f32 %f1323, %f1288, %f834, %f1323;\n" "\n" "$L__BB6_230:\n" " setp.lt.s32 %p140, %r95, 4;\n" " @%p140 bra $L__BB6_232;\n" "\n" " ld.shared.f32 %f835, [%r30+448];\n" " fma.rn.f32 %f1322, %f1288, %f835, %f1322;\n" "\n" "$L__BB6_232:\n" " setp.lt.s32 %p141, %r95, 5;\n" " @%p141 bra $L__BB6_234;\n" "\n" " ld.shared.f32 %f836, [%r31+448];\n" " fma.rn.f32 %f1321, %f1288, %f836, %f1321;\n" "\n" "$L__BB6_234:\n" " setp.lt.s32 %p142, %r95, 6;\n" " @%p142 bra $L__BB6_236;\n" "\n" " ld.shared.f32 %f837, [%r32+448];\n" " fma.rn.f32 %f1320, %f1288, %f837, %f1320;\n" "\n" "$L__BB6_236:\n" " setp.lt.s32 %p143, %r95, 7;\n" " @%p143 bra $L__BB6_238;\n" "\n" " ld.shared.f32 %f838, [%r33+448];\n" " fma.rn.f32 %f1319, %f1288, %f838, %f1319;\n" "\n" "$L__BB6_238:\n" " setp.lt.s32 %p144, %r95, 8;\n" " @%p144 bra $L__BB6_240;\n" "\n" " ld.shared.f32 %f839, [%r34+448];\n" " fma.rn.f32 %f1318, %f1288, %f839, %f1318;\n" "\n" "$L__BB6_240:\n" " setp.lt.s32 %p145, %r96, 144;\n" " @%p145 bra $L__BB6_257;\n" "\n" " @%p2 bra $L__BB6_243;\n" "\n" " ld.shared.f32 %f840, [%r27+512];\n" " fma.rn.f32 %f1325, %f1287, %f840, %f1325;\n" "\n" "$L__BB6_243:\n" " setp.lt.s32 %p147, %r95, 2;\n" " @%p147 bra $L__BB6_245;\n" "\n" " ld.shared.f32 %f841, [%r28+512];\n" " fma.rn.f32 %f1324, %f1287, %f841, %f1324;\n" "\n" "$L__BB6_245:\n" " setp.lt.s32 %p148, %r95, 3;\n" " @%p148 bra $L__BB6_247;\n" "\n" " ld.shared.f32 %f842, [%r29+512];\n" " fma.rn.f32 %f1323, %f1287, %f842, %f1323;\n" "\n" "$L__BB6_247:\n" " setp.lt.s32 %p149, %r95, 4;\n" " @%p149 bra $L__BB6_249;\n" "\n" " ld.shared.f32 %f843, [%r30+512];\n" " fma.rn.f32 %f1322, %f1287, %f843, %f1322;\n" "\n" "$L__BB6_249:\n" " setp.lt.s32 %p150, %r95, 5;\n" " @%p150 bra $L__BB6_251;\n" "\n" " ld.shared.f32 %f844, [%r31+512];\n" " fma.rn.f32 %f1321, %f1287, %f844, %f1321;\n" "\n" "$L__BB6_251:\n" " setp.lt.s32 %p151, %r95, 6;\n" " @%p151 bra $L__BB6_253;\n" "\n" " ld.shared.f32 %f845, [%r32+512];\n" " fma.rn.f32 %f1320, %f1287, %f845, %f1320;\n" "\n" "$L__BB6_253:\n" " setp.lt.s32 %p152, %r95, 7;\n" " @%p152 bra $L__BB6_255;\n" "\n" " ld.shared.f32 %f846, [%r33+512];\n" " fma.rn.f32 %f1319, %f1287, %f846, %f1319;\n" "\n" "$L__BB6_255:\n" " setp.lt.s32 %p153, %r95, 8;\n" " @%p153 bra $L__BB6_257;\n" "\n" " ld.shared.f32 %f847, [%r34+512];\n" " fma.rn.f32 %f1318, %f1287, %f847, %f1318;\n" "\n" "$L__BB6_257:\n" " setp.lt.s32 %p154, %r96, 160;\n" " @%p154 bra $L__BB6_274;\n" "\n" " @%p2 bra $L__BB6_260;\n" "\n" " ld.shared.f32 %f848, [%r27+576];\n" " fma.rn.f32 %f1325, %f1286, %f848, %f1325;\n" "\n" "$L__BB6_260:\n" " setp.lt.s32 %p156, %r95, 2;\n" " @%p156 bra $L__BB6_262;\n" "\n" " ld.shared.f32 %f849, [%r28+576];\n" " fma.rn.f32 %f1324, %f1286, %f849, %f1324;\n" "\n" "$L__BB6_262:\n" " setp.lt.s32 %p157, %r95, 3;\n" " @%p157 bra $L__BB6_264;\n" "\n" " ld.shared.f32 %f850, [%r29+576];\n" " fma.rn.f32 %f1323, %f1286, %f850, %f1323;\n" "\n" "$L__BB6_264:\n" " setp.lt.s32 %p158, %r95, 4;\n" " @%p158 bra $L__BB6_266;\n" "\n" " ld.shared.f32 %f851, [%r30+576];\n" " fma.rn.f32 %f1322, %f1286, %f851, %f1322;\n" "\n" "$L__BB6_266:\n" " setp.lt.s32 %p159, %r95, 5;\n" " @%p159 bra $L__BB6_268;\n" "\n" " ld.shared.f32 %f852, [%r31+576];\n" " fma.rn.f32 %f1321, %f1286, %f852, %f1321;\n" "\n" "$L__BB6_268:\n" " setp.lt.s32 %p160, %r95, 6;\n" " @%p160 bra $L__BB6_270;\n" "\n" " ld.shared.f32 %f853, [%r32+576];\n" " fma.rn.f32 %f1320, %f1286, %f853, %f1320;\n" "\n" "$L__BB6_270:\n" " setp.lt.s32 %p161, %r95, 7;\n" " @%p161 bra $L__BB6_272;\n" "\n" " ld.shared.f32 %f854, [%r33+576];\n" " fma.rn.f32 %f1319, %f1286, %f854, %f1319;\n" "\n" "$L__BB6_272:\n" " setp.lt.s32 %p162, %r95, 8;\n" " @%p162 bra $L__BB6_274;\n" "\n" " ld.shared.f32 %f855, [%r34+576];\n" " fma.rn.f32 %f1318, %f1286, %f855, %f1318;\n" "\n" "$L__BB6_274:\n" " setp.lt.s32 %p163, %r96, 176;\n" " @%p163 bra $L__BB6_291;\n" "\n" " @%p2 bra $L__BB6_277;\n" "\n" " ld.shared.f32 %f856, [%r27+640];\n" " fma.rn.f32 %f1325, %f1285, %f856, %f1325;\n" "\n" "$L__BB6_277:\n" " setp.lt.s32 %p165, %r95, 2;\n" " @%p165 bra $L__BB6_279;\n" "\n" " ld.shared.f32 %f857, [%r28+640];\n" " fma.rn.f32 %f1324, %f1285, %f857, %f1324;\n" "\n" "$L__BB6_279:\n" " setp.lt.s32 %p166, %r95, 3;\n" " @%p166 bra $L__BB6_281;\n" "\n" " ld.shared.f32 %f858, [%r29+640];\n" " fma.rn.f32 %f1323, %f1285, %f858, %f1323;\n" "\n" "$L__BB6_281:\n" " setp.lt.s32 %p167, %r95, 4;\n" " @%p167 bra $L__BB6_283;\n" "\n" " ld.shared.f32 %f859, [%r30+640];\n" " fma.rn.f32 %f1322, %f1285, %f859, %f1322;\n" "\n" "$L__BB6_283:\n" " setp.lt.s32 %p168, %r95, 5;\n" " @%p168 bra $L__BB6_285;\n" "\n" " ld.shared.f32 %f860, [%r31+640];\n" " fma.rn.f32 %f1321, %f1285, %f860, %f1321;\n" "\n" "$L__BB6_285:\n" " setp.lt.s32 %p169, %r95, 6;\n" " @%p169 bra $L__BB6_287;\n" "\n" " ld.shared.f32 %f861, [%r32+640];\n" " fma.rn.f32 %f1320, %f1285, %f861, %f1320;\n" "\n" "$L__BB6_287:\n" " setp.lt.s32 %p170, %r95, 7;\n" " @%p170 bra $L__BB6_289;\n" "\n" " ld.shared.f32 %f862, [%r33+640];\n" " fma.rn.f32 %f1319, %f1285, %f862, %f1319;\n" "\n" "$L__BB6_289:\n" " setp.lt.s32 %p171, %r95, 8;\n" " @%p171 bra $L__BB6_291;\n" "\n" " ld.shared.f32 %f863, [%r34+640];\n" " fma.rn.f32 %f1318, %f1285, %f863, %f1318;\n" "\n" "$L__BB6_291:\n" " setp.lt.s32 %p172, %r96, 192;\n" " @%p172 bra $L__BB6_308;\n" "\n" " @%p2 bra $L__BB6_294;\n" "\n" " ld.shared.f32 %f864, [%r27+704];\n" " fma.rn.f32 %f1325, %f1284, %f864, %f1325;\n" "\n" "$L__BB6_294:\n" " setp.lt.s32 %p174, %r95, 2;\n" " @%p174 bra $L__BB6_296;\n" "\n" " ld.shared.f32 %f865, [%r28+704];\n" " fma.rn.f32 %f1324, %f1284, %f865, %f1324;\n" "\n" "$L__BB6_296:\n" " setp.lt.s32 %p175, %r95, 3;\n" " @%p175 bra $L__BB6_298;\n" "\n" " ld.shared.f32 %f866, [%r29+704];\n" " fma.rn.f32 %f1323, %f1284, %f866, %f1323;\n" "\n" "$L__BB6_298:\n" " setp.lt.s32 %p176, %r95, 4;\n" " @%p176 bra $L__BB6_300;\n" "\n" " ld.shared.f32 %f867, [%r30+704];\n" " fma.rn.f32 %f1322, %f1284, %f867, %f1322;\n" "\n" "$L__BB6_300:\n" " setp.lt.s32 %p177, %r95, 5;\n" " @%p177 bra $L__BB6_302;\n" "\n" " ld.shared.f32 %f868, [%r31+704];\n" " fma.rn.f32 %f1321, %f1284, %f868, %f1321;\n" "\n" "$L__BB6_302:\n" " setp.lt.s32 %p178, %r95, 6;\n" " @%p178 bra $L__BB6_304;\n" "\n" " ld.shared.f32 %f869, [%r32+704];\n" " fma.rn.f32 %f1320, %f1284, %f869, %f1320;\n" "\n" "$L__BB6_304:\n" " setp.lt.s32 %p179, %r95, 7;\n" " @%p179 bra $L__BB6_306;\n" "\n" " ld.shared.f32 %f870, [%r33+704];\n" " fma.rn.f32 %f1319, %f1284, %f870, %f1319;\n" "\n" "$L__BB6_306:\n" " setp.lt.s32 %p180, %r95, 8;\n" " @%p180 bra $L__BB6_308;\n" "\n" " ld.shared.f32 %f871, [%r34+704];\n" " fma.rn.f32 %f1318, %f1284, %f871, %f1318;\n" "\n" "$L__BB6_308:\n" " setp.lt.s32 %p181, %r96, 208;\n" " @%p181 bra $L__BB6_325;\n" "\n" " @%p2 bra $L__BB6_311;\n" "\n" " ld.shared.f32 %f872, [%r27+768];\n" " fma.rn.f32 %f1325, %f1283, %f872, %f1325;\n" "\n" "$L__BB6_311:\n" " setp.lt.s32 %p183, %r95, 2;\n" " @%p183 bra $L__BB6_313;\n" "\n" " ld.shared.f32 %f873, [%r28+768];\n" " fma.rn.f32 %f1324, %f1283, %f873, %f1324;\n" "\n" "$L__BB6_313:\n" " setp.lt.s32 %p184, %r95, 3;\n" " @%p184 bra $L__BB6_315;\n" "\n" " ld.shared.f32 %f874, [%r29+768];\n" " fma.rn.f32 %f1323, %f1283, %f874, %f1323;\n" "\n" "$L__BB6_315:\n" " setp.lt.s32 %p185, %r95, 4;\n" " @%p185 bra $L__BB6_317;\n" "\n" " ld.shared.f32 %f875, [%r30+768];\n" " fma.rn.f32 %f1322, %f1283, %f875, %f1322;\n" "\n" "$L__BB6_317:\n" " setp.lt.s32 %p186, %r95, 5;\n" " @%p186 bra $L__BB6_319;\n" "\n" " ld.shared.f32 %f876, [%r31+768];\n" " fma.rn.f32 %f1321, %f1283, %f876, %f1321;\n" "\n" "$L__BB6_319:\n" " setp.lt.s32 %p187, %r95, 6;\n" " @%p187 bra $L__BB6_321;\n" "\n" " ld.shared.f32 %f877, [%r32+768];\n" " fma.rn.f32 %f1320, %f1283, %f877, %f1320;\n" "\n" "$L__BB6_321:\n" " setp.lt.s32 %p188, %r95, 7;\n" " @%p188 bra $L__BB6_323;\n" "\n" " ld.shared.f32 %f878, [%r33+768];\n" " fma.rn.f32 %f1319, %f1283, %f878, %f1319;\n" "\n" "$L__BB6_323:\n" " setp.lt.s32 %p189, %r95, 8;\n" " @%p189 bra $L__BB6_325;\n" "\n" " ld.shared.f32 %f879, [%r34+768];\n" " fma.rn.f32 %f1318, %f1283, %f879, %f1318;\n" "\n" "$L__BB6_325:\n" " setp.lt.s32 %p190, %r96, 224;\n" " @%p190 bra $L__BB6_342;\n" "\n" " @%p2 bra $L__BB6_328;\n" "\n" " ld.shared.f32 %f880, [%r27+832];\n" " fma.rn.f32 %f1325, %f1282, %f880, %f1325;\n" "\n" "$L__BB6_328:\n" " setp.lt.s32 %p192, %r95, 2;\n" " @%p192 bra $L__BB6_330;\n" "\n" " ld.shared.f32 %f881, [%r28+832];\n" " fma.rn.f32 %f1324, %f1282, %f881, %f1324;\n" "\n" "$L__BB6_330:\n" " setp.lt.s32 %p193, %r95, 3;\n" " @%p193 bra $L__BB6_332;\n" "\n" " ld.shared.f32 %f882, [%r29+832];\n" " fma.rn.f32 %f1323, %f1282, %f882, %f1323;\n" "\n" "$L__BB6_332:\n" " setp.lt.s32 %p194, %r95, 4;\n" " @%p194 bra $L__BB6_334;\n" "\n" " ld.shared.f32 %f883, [%r30+832];\n" " fma.rn.f32 %f1322, %f1282, %f883, %f1322;\n" "\n" "$L__BB6_334:\n" " setp.lt.s32 %p195, %r95, 5;\n" " @%p195 bra $L__BB6_336;\n" "\n" " ld.shared.f32 %f884, [%r31+832];\n" " fma.rn.f32 %f1321, %f1282, %f884, %f1321;\n" "\n" "$L__BB6_336:\n" " setp.lt.s32 %p196, %r95, 6;\n" " @%p196 bra $L__BB6_338;\n" "\n" " ld.shared.f32 %f885, [%r32+832];\n" " fma.rn.f32 %f1320, %f1282, %f885, %f1320;\n" "\n" "$L__BB6_338:\n" " setp.lt.s32 %p197, %r95, 7;\n" " @%p197 bra $L__BB6_340;\n" "\n" " ld.shared.f32 %f886, [%r33+832];\n" " fma.rn.f32 %f1319, %f1282, %f886, %f1319;\n" "\n" "$L__BB6_340:\n" " setp.lt.s32 %p198, %r95, 8;\n" " @%p198 bra $L__BB6_342;\n" "\n" " ld.shared.f32 %f887, [%r34+832];\n" " fma.rn.f32 %f1318, %f1282, %f887, %f1318;\n" "\n" "$L__BB6_342:\n" " setp.lt.s32 %p199, %r96, 240;\n" " @%p199 bra $L__BB6_359;\n" "\n" " @%p2 bra $L__BB6_345;\n" "\n" " ld.shared.f32 %f888, [%r27+896];\n" " fma.rn.f32 %f1325, %f1281, %f888, %f1325;\n" "\n" "$L__BB6_345:\n" " setp.lt.s32 %p201, %r95, 2;\n" " @%p201 bra $L__BB6_347;\n" "\n" " ld.shared.f32 %f889, [%r28+896];\n" " fma.rn.f32 %f1324, %f1281, %f889, %f1324;\n" "\n" "$L__BB6_347:\n" " setp.lt.s32 %p202, %r95, 3;\n" " @%p202 bra $L__BB6_349;\n" "\n" " ld.shared.f32 %f890, [%r29+896];\n" " fma.rn.f32 %f1323, %f1281, %f890, %f1323;\n" "\n" "$L__BB6_349:\n" " setp.lt.s32 %p203, %r95, 4;\n" " @%p203 bra $L__BB6_351;\n" "\n" " ld.shared.f32 %f891, [%r30+896];\n" " fma.rn.f32 %f1322, %f1281, %f891, %f1322;\n" "\n" "$L__BB6_351:\n" " setp.lt.s32 %p204, %r95, 5;\n" " @%p204 bra $L__BB6_353;\n" "\n" " ld.shared.f32 %f892, [%r31+896];\n" " fma.rn.f32 %f1321, %f1281, %f892, %f1321;\n" "\n" "$L__BB6_353:\n" " setp.lt.s32 %p205, %r95, 6;\n" " @%p205 bra $L__BB6_355;\n" "\n" " ld.shared.f32 %f893, [%r32+896];\n" " fma.rn.f32 %f1320, %f1281, %f893, %f1320;\n" "\n" "$L__BB6_355:\n" " setp.lt.s32 %p206, %r95, 7;\n" " @%p206 bra $L__BB6_357;\n" "\n" " ld.shared.f32 %f894, [%r33+896];\n" " fma.rn.f32 %f1319, %f1281, %f894, %f1319;\n" "\n" "$L__BB6_357:\n" " setp.lt.s32 %p207, %r95, 8;\n" " @%p207 bra $L__BB6_359;\n" "\n" " ld.shared.f32 %f895, [%r34+896];\n" " fma.rn.f32 %f1318, %f1281, %f895, %f1318;\n" "\n" "$L__BB6_359:\n" " setp.lt.s32 %p208, %r96, 256;\n" " @%p208 bra $L__BB6_376;\n" "\n" " @%p2 bra $L__BB6_362;\n" "\n" " ld.shared.f32 %f896, [%r27+960];\n" " fma.rn.f32 %f1325, %f1280, %f896, %f1325;\n" "\n" "$L__BB6_362:\n" " setp.lt.s32 %p210, %r95, 2;\n" " @%p210 bra $L__BB6_364;\n" "\n" " ld.shared.f32 %f897, [%r28+960];\n" " fma.rn.f32 %f1324, %f1280, %f897, %f1324;\n" "\n" "$L__BB6_364:\n" " setp.lt.s32 %p211, %r95, 3;\n" " @%p211 bra $L__BB6_366;\n" "\n" " ld.shared.f32 %f898, [%r29+960];\n" " fma.rn.f32 %f1323, %f1280, %f898, %f1323;\n" "\n" "$L__BB6_366:\n" " setp.lt.s32 %p212, %r95, 4;\n" " @%p212 bra $L__BB6_368;\n" "\n" " ld.shared.f32 %f899, [%r30+960];\n" " fma.rn.f32 %f1322, %f1280, %f899, %f1322;\n" "\n" "$L__BB6_368:\n" " setp.lt.s32 %p213, %r95, 5;\n" " @%p213 bra $L__BB6_370;\n" "\n" " ld.shared.f32 %f900, [%r31+960];\n" " fma.rn.f32 %f1321, %f1280, %f900, %f1321;\n" "\n" "$L__BB6_370:\n" " setp.lt.s32 %p214, %r95, 6;\n" " @%p214 bra $L__BB6_372;\n" "\n" " ld.shared.f32 %f901, [%r32+960];\n" " fma.rn.f32 %f1320, %f1280, %f901, %f1320;\n" "\n" "$L__BB6_372:\n" " setp.lt.s32 %p215, %r95, 7;\n" " @%p215 bra $L__BB6_374;\n" "\n" " ld.shared.f32 %f902, [%r33+960];\n" " fma.rn.f32 %f1319, %f1280, %f902, %f1319;\n" "\n" "$L__BB6_374:\n" " setp.lt.s32 %p216, %r95, 8;\n" " @%p216 bra $L__BB6_376;\n" "\n" " ld.shared.f32 %f903, [%r34+960];\n" " fma.rn.f32 %f1318, %f1280, %f903, %f1318;\n" "\n" "$L__BB6_376:\n" " setp.eq.s32 %p217, %r25, 0;\n" " setp.lt.s32 %p218, %r24, %r40;\n" " and.pred %p1, %p217, %p218;\n" " @%p2 bra $L__BB6_379;\n" "\n" " mov.b32 %r261, %f1325;\n" " mov.u32 %r262, 2;\n" " mov.u32 %r263, 4127;\n" " mov.u32 %r264, 8;\n" " mov.u32 %r265, -1;\n" " shfl.sync.down.b32 %r266|%p220, %r261, %r264, %r263, %r265;\n" " mov.b32 %f904, %r266;\n" " add.f32 %f905, %f1325, %f904;\n" " mov.b32 %r267, %f905;\n" " mov.u32 %r268, 4;\n" " shfl.sync.down.b32 %r269|%p221, %r267, %r268, %r263, %r265;\n" " mov.b32 %f906, %r269;\n" " add.f32 %f907, %f905, %f906;\n" " mov.b32 %r270, %f907;\n" " shfl.sync.down.b32 %r271|%p222, %r270, %r262, %r263, %r265;\n" " mov.b32 %f908, %r271;\n" " add.f32 %f909, %f907, %f908;\n" " mov.b32 %r272, %f909;\n" " mov.u32 %r273, 1;\n" " shfl.sync.down.b32 %r274|%p223, %r272, %r273, %r263, %r265;\n" " mov.b32 %f910, %r274;\n" " add.f32 %f509, %f909, %f910;\n" " not.pred %p224, %p1;\n" " @%p224 bra $L__BB6_379;\n" "\n" " mul.f32 %f911, %f509, %f673;\n" " mad.lo.s32 %r275, %r96, %r95, %r24;\n" " shl.b32 %r276, %r275, 2;\n" " add.s32 %r278, %r127, %r276;\n" " st.shared.f32 [%r278], %f911;\n" "\n" "$L__BB6_379:\n" " setp.lt.s32 %p225, %r95, 2;\n" " @%p225 bra $L__BB6_382;\n" "\n" " mov.b32 %r279, %f1324;\n" " mov.u32 %r280, 2;\n" " mov.u32 %r281, 4127;\n" " mov.u32 %r282, 8;\n" " mov.u32 %r283, -1;\n" " shfl.sync.down.b32 %r284|%p226, %r279, %r282, %r281, %r283;\n" " mov.b32 %f912, %r284;\n" " add.f32 %f913, %f1324, %f912;\n" " mov.b32 %r285, %f913;\n" " mov.u32 %r286, 4;\n" " shfl.sync.down.b32 %r287|%p227, %r285, %r286, %r281, %r283;\n" " mov.b32 %f914, %r287;\n" " add.f32 %f915, %f913, %f914;\n" " mov.b32 %r288, %f915;\n" " shfl.sync.down.b32 %r289|%p228, %r288, %r280, %r281, %r283;\n" " mov.b32 %f916, %r289;\n" " add.f32 %f917, %f915, %f916;\n" " mov.b32 %r290, %f917;\n" " mov.u32 %r291, 1;\n" " shfl.sync.down.b32 %r292|%p229, %r290, %r291, %r281, %r283;\n" " mov.b32 %f918, %r292;\n" " add.f32 %f510, %f917, %f918;\n" " not.pred %p230, %p1;\n" " @%p230 bra $L__BB6_382;\n" "\n" " mul.f32 %f919, %f510, %f673;\n" " mad.lo.s32 %r293, %r96, %r95, %r24;\n" " shl.b32 %r294, %r293, 2;\n" " add.s32 %r296, %r127, %r294;\n" " st.shared.f32 [%r296+64], %f919;\n" "\n" "$L__BB6_382:\n" " setp.lt.s32 %p231, %r95, 3;\n" " @%p231 bra $L__BB6_385;\n" "\n" " mov.b32 %r297, %f1323;\n" " mov.u32 %r298, 2;\n" " mov.u32 %r299, 4127;\n" " mov.u32 %r300, 8;\n" " mov.u32 %r301, -1;\n" " shfl.sync.down.b32 %r302|%p232, %r297, %r300, %r299, %r301;\n" " mov.b32 %f920, %r302;\n" " add.f32 %f921, %f1323, %f920;\n" " mov.b32 %r303, %f921;\n" " mov.u32 %r304, 4;\n" " shfl.sync.down.b32 %r305|%p233, %r303, %r304, %r299, %r301;\n" " mov.b32 %f922, %r305;\n" " add.f32 %f923, %f921, %f922;\n" " mov.b32 %r306, %f923;\n" " shfl.sync.down.b32 %r307|%p234, %r306, %r298, %r299, %r301;\n" " mov.b32 %f924, %r307;\n" " add.f32 %f925, %f923, %f924;\n" " mov.b32 %r308, %f925;\n" " mov.u32 %r309, 1;\n" " shfl.sync.down.b32 %r310|%p235, %r308, %r309, %r299, %r301;\n" " mov.b32 %f926, %r310;\n" " add.f32 %f511, %f925, %f926;\n" " not.pred %p236, %p1;\n" " @%p236 bra $L__BB6_385;\n" "\n" " mul.f32 %f927, %f511, %f673;\n" " mad.lo.s32 %r311, %r96, %r95, %r24;\n" " shl.b32 %r312, %r311, 2;\n" " add.s32 %r314, %r127, %r312;\n" " st.shared.f32 [%r314+128], %f927;\n" "\n" "$L__BB6_385:\n" " setp.lt.s32 %p237, %r95, 4;\n" " @%p237 bra $L__BB6_388;\n" "\n" " mov.b32 %r315, %f1322;\n" " mov.u32 %r316, 2;\n" " mov.u32 %r317, 4127;\n" " mov.u32 %r318, 8;\n" " mov.u32 %r319, -1;\n" " shfl.sync.down.b32 %r320|%p238, %r315, %r318, %r317, %r319;\n" " mov.b32 %f928, %r320;\n" " add.f32 %f929, %f1322, %f928;\n" " mov.b32 %r321, %f929;\n" " mov.u32 %r322, 4;\n" " shfl.sync.down.b32 %r323|%p239, %r321, %r322, %r317, %r319;\n" " mov.b32 %f930, %r323;\n" " add.f32 %f931, %f929, %f930;\n" " mov.b32 %r324, %f931;\n" " shfl.sync.down.b32 %r325|%p240, %r324, %r316, %r317, %r319;\n" " mov.b32 %f932, %r325;\n" " add.f32 %f933, %f931, %f932;\n" " mov.b32 %r326, %f933;\n" " mov.u32 %r327, 1;\n" " shfl.sync.down.b32 %r328|%p241, %r326, %r327, %r317, %r319;\n" " mov.b32 %f934, %r328;\n" " add.f32 %f512, %f933, %f934;\n" " not.pred %p242, %p1;\n" " @%p242 bra $L__BB6_388;\n" "\n" " mul.f32 %f935, %f512, %f673;\n" " mad.lo.s32 %r329, %r96, %r95, %r24;\n" " shl.b32 %r330, %r329, 2;\n" " add.s32 %r332, %r127, %r330;\n" " st.shared.f32 [%r332+192], %f935;\n" "\n" "$L__BB6_388:\n" " setp.lt.s32 %p243, %r95, 5;\n" " @%p243 bra $L__BB6_391;\n" "\n" " mov.b32 %r333, %f1321;\n" " mov.u32 %r334, 2;\n" " mov.u32 %r335, 4127;\n" " mov.u32 %r336, 8;\n" " mov.u32 %r337, -1;\n" " shfl.sync.down.b32 %r338|%p244, %r333, %r336, %r335, %r337;\n" " mov.b32 %f936, %r338;\n" " add.f32 %f937, %f1321, %f936;\n" " mov.b32 %r339, %f937;\n" " mov.u32 %r340, 4;\n" " shfl.sync.down.b32 %r341|%p245, %r339, %r340, %r335, %r337;\n" " mov.b32 %f938, %r341;\n" " add.f32 %f939, %f937, %f938;\n" " mov.b32 %r342, %f939;\n" " shfl.sync.down.b32 %r343|%p246, %r342, %r334, %r335, %r337;\n" " mov.b32 %f940, %r343;\n" " add.f32 %f941, %f939, %f940;\n" " mov.b32 %r344, %f941;\n" " mov.u32 %r345, 1;\n" " shfl.sync.down.b32 %r346|%p247, %r344, %r345, %r335, %r337;\n" " mov.b32 %f942, %r346;\n" " add.f32 %f513, %f941, %f942;\n" " not.pred %p248, %p1;\n" " @%p248 bra $L__BB6_391;\n" "\n" " mul.f32 %f943, %f513, %f673;\n" " mad.lo.s32 %r347, %r96, %r95, %r24;\n" " shl.b32 %r348, %r347, 2;\n" " add.s32 %r350, %r127, %r348;\n" " st.shared.f32 [%r350+256], %f943;\n" "\n" "$L__BB6_391:\n" " setp.lt.s32 %p249, %r95, 6;\n" " @%p249 bra $L__BB6_394;\n" "\n" " mov.b32 %r351, %f1320;\n" " mov.u32 %r352, 2;\n" " mov.u32 %r353, 4127;\n" " mov.u32 %r354, 8;\n" " mov.u32 %r355, -1;\n" " shfl.sync.down.b32 %r356|%p250, %r351, %r354, %r353, %r355;\n" " mov.b32 %f944, %r356;\n" " add.f32 %f945, %f1320, %f944;\n" " mov.b32 %r357, %f945;\n" " mov.u32 %r358, 4;\n" " shfl.sync.down.b32 %r359|%p251, %r357, %r358, %r353, %r355;\n" " mov.b32 %f946, %r359;\n" " add.f32 %f947, %f945, %f946;\n" " mov.b32 %r360, %f947;\n" " shfl.sync.down.b32 %r361|%p252, %r360, %r352, %r353, %r355;\n" " mov.b32 %f948, %r361;\n" " add.f32 %f949, %f947, %f948;\n" " mov.b32 %r362, %f949;\n" " mov.u32 %r363, 1;\n" " shfl.sync.down.b32 %r364|%p253, %r362, %r363, %r353, %r355;\n" " mov.b32 %f950, %r364;\n" " add.f32 %f514, %f949, %f950;\n" " not.pred %p254, %p1;\n" " @%p254 bra $L__BB6_394;\n" "\n" " mul.f32 %f951, %f514, %f673;\n" " mad.lo.s32 %r365, %r96, %r95, %r24;\n" " shl.b32 %r366, %r365, 2;\n" " add.s32 %r368, %r127, %r366;\n" " st.shared.f32 [%r368+320], %f951;\n" "\n" "$L__BB6_394:\n" " setp.lt.s32 %p255, %r95, 7;\n" " @%p255 bra $L__BB6_397;\n" "\n" " mov.b32 %r369, %f1319;\n" " mov.u32 %r370, 2;\n" " mov.u32 %r371, 4127;\n" " mov.u32 %r372, 8;\n" " mov.u32 %r373, -1;\n" " shfl.sync.down.b32 %r374|%p256, %r369, %r372, %r371, %r373;\n" " mov.b32 %f952, %r374;\n" " add.f32 %f953, %f1319, %f952;\n" " mov.b32 %r375, %f953;\n" " mov.u32 %r376, 4;\n" " shfl.sync.down.b32 %r377|%p257, %r375, %r376, %r371, %r373;\n" " mov.b32 %f954, %r377;\n" " add.f32 %f955, %f953, %f954;\n" " mov.b32 %r378, %f955;\n" " shfl.sync.down.b32 %r379|%p258, %r378, %r370, %r371, %r373;\n" " mov.b32 %f956, %r379;\n" " add.f32 %f957, %f955, %f956;\n" " mov.b32 %r380, %f957;\n" " mov.u32 %r381, 1;\n" " shfl.sync.down.b32 %r382|%p259, %r380, %r381, %r371, %r373;\n" " mov.b32 %f958, %r382;\n" " add.f32 %f515, %f957, %f958;\n" " not.pred %p260, %p1;\n" " @%p260 bra $L__BB6_397;\n" "\n" " mul.f32 %f959, %f515, %f673;\n" " mad.lo.s32 %r383, %r96, %r95, %r24;\n" " shl.b32 %r384, %r383, 2;\n" " add.s32 %r386, %r127, %r384;\n" " st.shared.f32 [%r386+384], %f959;\n" "\n" "$L__BB6_397:\n" " setp.lt.s32 %p261, %r95, 8;\n" " @%p261 bra $L__BB6_400;\n" "\n" " mov.b32 %r387, %f1318;\n" " mov.u32 %r388, 2;\n" " mov.u32 %r389, 4127;\n" " mov.u32 %r390, 8;\n" " mov.u32 %r391, -1;\n" " shfl.sync.down.b32 %r392|%p262, %r387, %r390, %r389, %r391;\n" " mov.b32 %f960, %r392;\n" " add.f32 %f961, %f1318, %f960;\n" " mov.b32 %r393, %f961;\n" " mov.u32 %r394, 4;\n" " shfl.sync.down.b32 %r395|%p263, %r393, %r394, %r389, %r391;\n" " mov.b32 %f962, %r395;\n" " add.f32 %f963, %f961, %f962;\n" " mov.b32 %r396, %f963;\n" " shfl.sync.down.b32 %r397|%p264, %r396, %r388, %r389, %r391;\n" " mov.b32 %f964, %r397;\n" " add.f32 %f965, %f963, %f964;\n" " mov.b32 %r398, %f965;\n" " mov.u32 %r399, 1;\n" " shfl.sync.down.b32 %r400|%p265, %r398, %r399, %r389, %r391;\n" " mov.b32 %f966, %r400;\n" " add.f32 %f516, %f965, %f966;\n" " not.pred %p266, %p1;\n" " @%p266 bra $L__BB6_400;\n" "\n" " mul.f32 %f967, %f516, %f673;\n" " mad.lo.s32 %r401, %r96, %r95, %r24;\n" " shl.b32 %r402, %r401, 2;\n" " add.s32 %r404, %r127, %r402;\n" " st.shared.f32 [%r404+448], %f967;\n" "\n" "$L__BB6_400:\n" " bar.sync 0;\n" " @%p7 bra $L__BB6_418;\n" "\n" " add.s32 %r588, %r2, 1;\n" " sub.s32 %r587, %r588, %r606;\n" " mad.lo.s32 %r407, %r96, %r95, %r1;\n" " shl.b32 %r408, %r95, 5;\n" " add.s32 %r409, %r407, %r408;\n" " shl.b32 %r410, %r409, 2;\n" " add.s32 %r412, %r127, %r410;\n" " ld.shared.f32 %f517, [%r412];\n" " setp.lt.s32 %p268, %r587, 1;\n" " mov.f32 %f1539, %f517;\n" " @%p268 bra $L__BB6_409;\n" "\n" " and.b32 %r50, %r38, 3;\n" " add.s32 %r414, %r38, -1;\n" " setp.lt.u32 %p269, %r414, 3;\n" " mov.u32 %r609, 0;\n" " mov.f32 %f1539, %f517;\n" " @%p269 bra $L__BB6_405;\n" "\n" " sub.s32 %r608, %r38, %r50;\n" " mad.lo.s32 %r52, %r1, 15, %r407;\n" " mov.u32 %r609, 0;\n" " mov.f32 %f1539, %f517;\n" "\n" "$L__BB6_404:\n" " add.s32 %r418, %r52, %r609;\n" " shl.b32 %r419, %r418, 2;\n" " add.s32 %r421, %r127, %r419;\n" " ld.shared.f32 %f969, [%r421];\n" " max.f32 %f970, %f1539, %f969;\n" " ld.shared.f32 %f971, [%r421+4];\n" " max.f32 %f972, %f970, %f971;\n" " ld.shared.f32 %f973, [%r421+8];\n" " max.f32 %f974, %f972, %f973;\n" " ld.shared.f32 %f975, [%r421+12];\n" " max.f32 %f1539, %f974, %f975;\n" " add.s32 %r609, %r609, 4;\n" " add.s32 %r608, %r608, -4;\n" " setp.ne.s32 %p270, %r608, 0;\n" " @%p270 bra $L__BB6_404;\n" "\n" "$L__BB6_405:\n" " setp.eq.s32 %p271, %r50, 0;\n" " @%p271 bra $L__BB6_409;\n" "\n" " mad.lo.s32 %r424, %r1, 15, %r407;\n" " add.s32 %r425, %r424, %r609;\n" " shl.b32 %r426, %r425, 2;\n" " add.s32 %r58, %r127, %r426;\n" " ld.shared.f32 %f976, [%r58];\n" " max.f32 %f1539, %f1539, %f976;\n" " setp.eq.s32 %p272, %r50, 1;\n" " @%p272 bra $L__BB6_409;\n" "\n" " ld.shared.f32 %f977, [%r58+4];\n" " max.f32 %f1539, %f1539, %f977;\n" " setp.eq.s32 %p273, %r50, 2;\n" " @%p273 bra $L__BB6_409;\n" "\n" " ld.shared.f32 %f978, [%r58+8];\n" " max.f32 %f1539, %f1539, %f978;\n" "\n" "$L__BB6_409:\n" " mov.f32 %f1543, 0f00000000;\n" " @%p268 bra $L__BB6_417;\n" "\n" " and.b32 %r59, %r38, 3;\n" " add.s32 %r429, %r38, -1;\n" " setp.lt.u32 %p275, %r429, 3;\n" " mov.f32 %f1543, 0f00000000;\n" " mov.u32 %r613, 0;\n" " @%p275 bra $L__BB6_413;\n" "\n" " sub.s32 %r611, %r38, %r59;\n" " add.s32 %r431, %r96, 16;\n" " mad.lo.s32 %r433, %r95, %r431, %r1;\n" " shl.b32 %r434, %r433, 2;\n" " add.s32 %r612, %r127, %r434;\n" " mov.f32 %f1543, 0f00000000;\n" " mov.u32 %r613, 0;\n" "\n" "$L__BB6_412:\n" " shl.b32 %r595, %r95, 2;\n" " mad.lo.s32 %r594, %r96, %r95, %r1;\n" " mad.lo.s32 %r593, %r1, 15, %r594;\n" " add.s32 %r437, %r593, %r613;\n" " shl.b32 %r438, %r437, 2;\n" " add.s32 %r440, %r127, %r438;\n" " ld.shared.f32 %f983, [%r440];\n" " sub.f32 %f984, %f983, %f1539;\n" " mov.f32 %f985, 0f3F000000;\n" " mov.f32 %f986, 0f3BBB989D;\n" " fma.rn.f32 %f987, %f984, %f986, %f985;\n" " mov.f32 %f988, 0f3FB8AA3B;\n" " mov.f32 %f989, 0f437C0000;\n" " cvt.sat.f32.f32 %f990, %f987;\n" " mov.f32 %f991, 0f4B400001;\n" " fma.rm.f32 %f992, %f990, %f989, %f991;\n" " add.f32 %f993, %f992, 0fCB40007F;\n" " neg.f32 %f994, %f993;\n" " fma.rn.f32 %f995, %f984, %f988, %f994;\n" " mov.f32 %f996, 0f32A57060;\n" " fma.rn.f32 %f997, %f984, %f996, %f995;\n" " mov.b32 %r441, %f992;\n" " shl.b32 %r442, %r441, 23;\n" " mov.b32 %f998, %r442;\n" " ex2.approx.ftz.f32 %f999, %f997;\n" " mul.f32 %f1000, %f999, %f998;\n" " st.shared.f32 [%r612], %f1000;\n" " add.f32 %f1001, %f1543, %f1000;\n" " ld.shared.f32 %f1002, [%r440+4];\n" " sub.f32 %f1003, %f1002, %f1539;\n" " fma.rn.f32 %f1004, %f1003, %f986, %f985;\n" " cvt.sat.f32.f32 %f1005, %f1004;\n" " fma.rm.f32 %f1006, %f1005, %f989, %f991;\n" " add.f32 %f1007, %f1006, 0fCB40007F;\n" " neg.f32 %f1008, %f1007;\n" " fma.rn.f32 %f1009, %f1003, %f988, %f1008;\n" " fma.rn.f32 %f1010, %f1003, %f996, %f1009;\n" " mov.b32 %r443, %f1006;\n" " shl.b32 %r444, %r443, 23;\n" " mov.b32 %f1011, %r444;\n" " ex2.approx.ftz.f32 %f1012, %f1010;\n" " mul.f32 %f1013, %f1012, %f1011;\n" " add.s32 %r445, %r612, %r595;\n" " st.shared.f32 [%r445], %f1013;\n" " add.f32 %f1014, %f1001, %f1013;\n" " ld.shared.f32 %f1015, [%r440+8];\n" " sub.f32 %f1016, %f1015, %f1539;\n" " fma.rn.f32 %f1017, %f1016, %f986, %f985;\n" " cvt.sat.f32.f32 %f1018, %f1017;\n" " fma.rm.f32 %f1019, %f1018, %f989, %f991;\n" " add.f32 %f1020, %f1019, 0fCB40007F;\n" " neg.f32 %f1021, %f1020;\n" " fma.rn.f32 %f1022, %f1016, %f988, %f1021;\n" " fma.rn.f32 %f1023, %f1016, %f996, %f1022;\n" " mov.b32 %r446, %f1019;\n" " shl.b32 %r447, %r446, 23;\n" " mov.b32 %f1024, %r447;\n" " ex2.approx.ftz.f32 %f1025, %f1023;\n" " mul.f32 %f1026, %f1025, %f1024;\n" " add.s32 %r448, %r445, %r595;\n" " st.shared.f32 [%r448], %f1026;\n" " add.f32 %f1027, %f1014, %f1026;\n" " ld.shared.f32 %f1028, [%r440+12];\n" " sub.f32 %f1029, %f1028, %f1539;\n" " fma.rn.f32 %f1030, %f1029, %f986, %f985;\n" " cvt.sat.f32.f32 %f1031, %f1030;\n" " fma.rm.f32 %f1032, %f1031, %f989, %f991;\n" " add.f32 %f1033, %f1032, 0fCB40007F;\n" " neg.f32 %f1034, %f1033;\n" " fma.rn.f32 %f1035, %f1029, %f988, %f1034;\n" " fma.rn.f32 %f1036, %f1029, %f996, %f1035;\n" " mov.b32 %r449, %f1032;\n" " shl.b32 %r450, %r449, 23;\n" " mov.b32 %f1037, %r450;\n" " ex2.approx.ftz.f32 %f1038, %f1036;\n" " mul.f32 %f1039, %f1038, %f1037;\n" " add.s32 %r451, %r448, %r595;\n" " add.s32 %r612, %r451, %r595;\n" " st.shared.f32 [%r451], %f1039;\n" " add.f32 %f1543, %f1027, %f1039;\n" " add.s32 %r613, %r613, 4;\n" " add.s32 %r611, %r611, -4;\n" " setp.ne.s32 %p276, %r611, 0;\n" " @%p276 bra $L__BB6_412;\n" "\n" "$L__BB6_413:\n" " setp.eq.s32 %p277, %r59, 0;\n" " @%p277 bra $L__BB6_417;\n" "\n" " mad.lo.s32 %r454, %r1, 15, %r407;\n" " add.s32 %r455, %r454, %r613;\n" " shl.b32 %r456, %r455, 2;\n" " add.s32 %r71, %r127, %r456;\n" " ld.shared.f32 %f1040, [%r71];\n" " sub.f32 %f1041, %f1040, %f1539;\n" " mov.f32 %f1042, 0f3F000000;\n" " mov.f32 %f1043, 0f3BBB989D;\n" " fma.rn.f32 %f1044, %f1041, %f1043, %f1042;\n" " mov.f32 %f1045, 0f3FB8AA3B;\n" " mov.f32 %f1046, 0f437C0000;\n" " cvt.sat.f32.f32 %f1047, %f1044;\n" " mov.f32 %f1048, 0f4B400001;\n" " fma.rm.f32 %f1049, %f1047, %f1046, %f1048;\n" " add.f32 %f1050, %f1049, 0fCB40007F;\n" " neg.f32 %f1051, %f1050;\n" " fma.rn.f32 %f1052, %f1041, %f1045, %f1051;\n" " mov.f32 %f1053, 0f32A57060;\n" " fma.rn.f32 %f1054, %f1041, %f1053, %f1052;\n" " mov.b32 %r458, %f1049;\n" " shl.b32 %r459, %r458, 23;\n" " mov.b32 %f1055, %r459;\n" " ex2.approx.ftz.f32 %f1056, %f1054;\n" " mul.f32 %f1057, %f1056, %f1055;\n" " add.s32 %r461, %r407, %r108;\n" " mad.lo.s32 %r462, %r613, %r95, %r461;\n" " shl.b32 %r463, %r462, 2;\n" " add.s32 %r72, %r127, %r463;\n" " st.shared.f32 [%r72], %f1057;\n" " add.f32 %f1543, %f1543, %f1057;\n" " setp.eq.s32 %p278, %r59, 1;\n" " @%p278 bra $L__BB6_417;\n" "\n" " mov.f32 %f1205, 0f32A57060;\n" " mov.f32 %f1204, 0f3FB8AA3B;\n" " mov.f32 %f1203, 0f4B400001;\n" " mov.f32 %f1202, 0f437C0000;\n" " mov.f32 %f1201, 0f3F000000;\n" " mov.f32 %f1200, 0f3BBB989D;\n" " ld.shared.f32 %f1058, [%r71+4];\n" " sub.f32 %f1059, %f1058, %f1539;\n" " fma.rn.f32 %f1062, %f1059, %f1200, %f1201;\n" " cvt.sat.f32.f32 %f1065, %f1062;\n" " fma.rm.f32 %f1067, %f1065, %f1202, %f1203;\n" " add.f32 %f1068, %f1067, 0fCB40007F;\n" " neg.f32 %f1069, %f1068;\n" " fma.rn.f32 %f1070, %f1059, %f1204, %f1069;\n" " fma.rn.f32 %f1072, %f1059, %f1205, %f1070;\n" " mov.b32 %r464, %f1067;\n" " shl.b32 %r465, %r464, 23;\n" " mov.b32 %f1073, %r465;\n" " ex2.approx.ftz.f32 %f1074, %f1072;\n" " mul.f32 %f1075, %f1074, %f1073;\n" " shl.b32 %r466, %r95, 2;\n" " add.s32 %r73, %r72, %r466;\n" " st.shared.f32 [%r73], %f1075;\n" " add.f32 %f1543, %f1543, %f1075;\n" " setp.eq.s32 %p279, %r59, 2;\n" " @%p279 bra $L__BB6_417;\n" "\n" " mov.f32 %f1211, 0f32A57060;\n" " mov.f32 %f1210, 0f3FB8AA3B;\n" " mov.f32 %f1209, 0f4B400001;\n" " mov.f32 %f1208, 0f437C0000;\n" " mov.f32 %f1207, 0f3F000000;\n" " mov.f32 %f1206, 0f3BBB989D;\n" " ld.shared.f32 %f1076, [%r71+8];\n" " sub.f32 %f1077, %f1076, %f1539;\n" " fma.rn.f32 %f1080, %f1077, %f1206, %f1207;\n" " cvt.sat.f32.f32 %f1083, %f1080;\n" " fma.rm.f32 %f1085, %f1083, %f1208, %f1209;\n" " add.f32 %f1086, %f1085, 0fCB40007F;\n" " neg.f32 %f1087, %f1086;\n" " fma.rn.f32 %f1088, %f1077, %f1210, %f1087;\n" " fma.rn.f32 %f1090, %f1077, %f1211, %f1088;\n" " mov.b32 %r467, %f1085;\n" " shl.b32 %r468, %r467, 23;\n" " mov.b32 %f1091, %r468;\n" " ex2.approx.ftz.f32 %f1092, %f1090;\n" " mul.f32 %f1093, %f1092, %f1091;\n" " add.s32 %r470, %r73, %r466;\n" " st.shared.f32 [%r470], %f1093;\n" " add.f32 %f1543, %f1543, %f1093;\n" "\n" "$L__BB6_417:\n" " sub.f32 %f1094, %f517, %f1539;\n" " mov.f32 %f1095, 0f3F000000;\n" " mov.f32 %f1096, 0f3BBB989D;\n" " fma.rn.f32 %f1097, %f1094, %f1096, %f1095;\n" " mov.f32 %f1098, 0f3FB8AA3B;\n" " mov.f32 %f1099, 0f437C0000;\n" " cvt.sat.f32.f32 %f1100, %f1097;\n" " mov.f32 %f1101, 0f4B400001;\n" " fma.rm.f32 %f1102, %f1100, %f1099, %f1101;\n" " add.f32 %f1103, %f1102, 0fCB40007F;\n" " neg.f32 %f1104, %f1103;\n" " fma.rn.f32 %f1105, %f1094, %f1098, %f1104;\n" " mov.f32 %f1106, 0f32A57060;\n" " fma.rn.f32 %f1107, %f1094, %f1106, %f1105;\n" " ex2.approx.ftz.f32 %f1108, %f1107;\n" " mov.b32 %r471, %f1102;\n" " shl.b32 %r472, %r471, 23;\n" " mov.b32 %f1109, %r472;\n" " mul.f32 %f1110, %f1108, %f1109;\n" " add.s32 %r477, %r407, %r4;\n" " shl.b32 %r478, %r477, 2;\n" " add.s32 %r480, %r127, %r478;\n" " ld.shared.f32 %f1111, [%r480];\n" " fma.rn.f32 %f1112, %f1110, %f1111, %f1543;\n" " st.shared.f32 [%r480], %f1112;\n" " add.s32 %r482, %r407, %r124;\n" " shl.b32 %r483, %r482, 2;\n" " add.s32 %r484, %r127, %r483;\n" " st.shared.f32 [%r484], %f1539;\n" " add.s32 %r486, %r407, %r5;\n" " shl.b32 %r487, %r486, 2;\n" " add.s32 %r488, %r127, %r487;\n" " st.shared.f32 [%r488], %f1110;\n" "\n" "$L__BB6_418:\n" " mul.wide.s32 %rd194, %r606, %r3;\n" " mul.wide.s32 %rd195, %r97, %r110;\n" " mul.lo.s64 %rd196, %rd195, %rd45;\n" " add.s64 %rd197, %rd194, %rd196;\n" " add.s64 %rd199, %rd198, %rd197;\n" " mul.lo.s32 %r491, %r40, %r3;\n" " shr.s32 %r74, %r491, 2;\n" " setp.ge.s32 %p280, %r1, %r74;\n" " mul.wide.s32 %rd200, %r1, 4;\n" " add.s64 %rd17, %rd199, %rd200;\n" " mov.u32 %r616, 0;\n" " mov.u32 %r614, %r616;\n" " @%p280 bra $L__BB6_420;\n" "\n" " ld.global.u32 %r614, [%rd17];\n" "\n" "$L__BB6_420:\n" " add.s32 %r77, %r1, 256;\n" " setp.ge.s32 %p281, %r77, %r74;\n" " mov.u32 %r615, %r616;\n" " @%p281 bra $L__BB6_422;\n" "\n" " ld.global.u32 %r615, [%rd17+1024];\n" "\n" "$L__BB6_422:\n" " add.s32 %r80, %r1, 512;\n" " setp.ge.s32 %p282, %r80, %r74;\n" " @%p282 bra $L__BB6_424;\n" "\n" " ld.global.u32 %r616, [%rd17+2048];\n" "\n" "$L__BB6_424:\n" " mad.lo.s32 %r498, %r96, %r95, %r1;\n" " shl.b32 %r499, %r95, 5;\n" " cvt.u64.u32 %rd201, %r499;\n" " add.s64 %rd203, %rd201, %rd27;\n" " cvt.u32.u64 %r500, %rd203;\n" " add.s32 %r501, %r500, %r95;\n" " add.s32 %r502, %r501, %r95;\n" " add.s32 %r503, %r498, %r502;\n" " shl.b32 %r504, %r503, 2;\n" " add.s32 %r83, %r127, %r504;\n" " @%p280 bra $L__BB6_426;\n" "\n" " st.shared.u32 [%r83], %r614;\n" "\n" "$L__BB6_426:\n" " @%p281 bra $L__BB6_428;\n" "\n" " st.shared.u32 [%r83+1024], %r615;\n" "\n" "$L__BB6_428:\n" " @%p282 bra $L__BB6_430;\n" "\n" " st.shared.u32 [%r83+2048], %r616;\n" "\n" "$L__BB6_430:\n" " add.s32 %r583, %r2, 1;\n" " sub.s32 %r582, %r583, %r606;\n" " bar.sync 0;\n" " setp.lt.s32 %p286, %r582, 1;\n" " @%p286 bra $L__BB6_510;\n" "\n" " setp.gt.s32 %p287, %r100, 0;\n" " @%p287 bra $L__BB6_470;\n" " bra.uni $L__BB6_432;\n" "\n" "$L__BB6_470:\n" " mov.u32 %r618, 0;\n" "\n" "$L__BB6_471:\n" " add.s32 %r89, %r618, %r606;\n" " setp.lt.s32 %p308, %r89, %r99;\n" " @%p308 bra $L__BB6_475;\n" " bra.uni $L__BB6_472;\n" "\n" "$L__BB6_475:\n" " add.s32 %r598, %r618, %r606;\n" " cvt.s64.s32 %rd232, %r598;\n" " mul.lo.s64 %rd233, %rd232, %rd5;\n" " mul.lo.s64 %rd235, %rd32, %rd5;\n" " shl.b64 %rd236, %rd235, 3;\n" " cvt.s64.s32 %rd237, %r1;\n" " add.s64 %rd238, %rd236, %rd237;\n" " add.s64 %rd239, %rd238, %rd233;\n" " cvta.to.global.u64 %rd240, %rd21;\n" " shl.b64 %rd241, %rd239, 1;\n" " add.s64 %rd242, %rd240, %rd241;\n" " ld.global.u16 %rs45, [%rd242];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1583, %rs45;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB6_476;\n" "\n" "$L__BB6_472:\n" " add.s32 %r596, %r618, %r606;\n" " sub.s32 %r537, %r2, %r100;\n" " setp.gt.s32 %p309, %r596, %r537;\n" " mov.u32 %r538, %nctaid.x;\n" " add.s32 %r539, %r538, %r98;\n" " add.s32 %r540, %r539, -257;\n" " setp.gt.s32 %p310, %r596, %r540;\n" " and.pred %p311, %p310, %p309;\n" " @%p311 bra $L__BB6_474;\n" " bra.uni $L__BB6_473;\n" "\n" "$L__BB6_474:\n" " add.s32 %r597, %r618, %r606;\n" " and.b32 %r561, %r597, 255;\n" " cvt.u64.u32 %rd221, %r561;\n" " mul.lo.s64 %rd222, %rd221, %rd5;\n" " mul.lo.s64 %rd224, %rd32, %rd5;\n" " shl.b64 %rd225, %rd224, 8;\n" " cvt.s64.s32 %rd226, %r1;\n" " add.s64 %rd227, %rd225, %rd226;\n" " add.s64 %rd228, %rd227, %rd222;\n" " shl.b64 %rd230, %rd228, 1;\n" " add.s64 %rd231, %rd229, %rd230;\n" " ld.global.u16 %rs44, [%rd231];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1583, %rs44;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB6_476;\n" "\n" "$L__BB6_473:\n" " mad.lo.s32 %r545, %r96, %r95, %r502;\n" " shl.b32 %r546, %r545, 2;\n" " add.s32 %r548, %r127, %r546;\n" " mad.lo.s32 %r549, %r618, %r3, %r26;\n" " add.s32 %r550, %r548, %r549;\n" " ld.shared.u16 %rs43, [%r550];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1148, %rs43;}\n" "\n" " // end inline asm\n" " and.b32 %r552, %r1, 31;\n" " add.s32 %r553, %r552, -16;\n" " setp.lt.u32 %p312, %r552, 16;\n" " selp.b32 %r554, %r552, %r553, %p312;\n" " add.s32 %r555, %r550, %r554;\n" " ld.shared.u8 %r556, [%r555+2];\n" " selp.b32 %r557, 0, 4, %p312;\n" " shr.u32 %r558, %r556, %r557;\n" " and.b32 %r559, %r558, 15;\n" " add.s32 %r560, %r559, -8;\n" " cvt.rn.f32.s32 %f1149, %r560;\n" " mul.f32 %f1583, %f1148, %f1149;\n" "\n" "$L__BB6_476:\n" " mad.lo.s32 %r567, %r96, %r95, %r108;\n" " mad.lo.s32 %r568, %r618, %r95, %r567;\n" " shl.b32 %r569, %r568, 2;\n" " add.s32 %r90, %r127, %r569;\n" " setp.eq.s32 %p313, %r618, 0;\n" " @%p313 bra $L__BB6_493;\n" "\n" " @%p2 bra $L__BB6_479;\n" "\n" " ld.shared.f32 %f1152, [%r90];\n" " fma.rn.f32 %f1574, %f1583, %f1152, %f1574;\n" "\n" "$L__BB6_479:\n" " @%p225 bra $L__BB6_481;\n" "\n" " ld.shared.f32 %f1153, [%r90+4];\n" " fma.rn.f32 %f1573, %f1583, %f1153, %f1573;\n" "\n" "$L__BB6_481:\n" " @%p231 bra $L__BB6_483;\n" "\n" " ld.shared.f32 %f1154, [%r90+8];\n" " fma.rn.f32 %f1572, %f1583, %f1154, %f1572;\n" "\n" "$L__BB6_483:\n" " @%p237 bra $L__BB6_485;\n" "\n" " ld.shared.f32 %f1155, [%r90+12];\n" " fma.rn.f32 %f1571, %f1583, %f1155, %f1571;\n" "\n" "$L__BB6_485:\n" " @%p243 bra $L__BB6_487;\n" "\n" " ld.shared.f32 %f1156, [%r90+16];\n" " fma.rn.f32 %f1570, %f1583, %f1156, %f1570;\n" "\n" "$L__BB6_487:\n" " @%p249 bra $L__BB6_489;\n" "\n" " ld.shared.f32 %f1157, [%r90+20];\n" " fma.rn.f32 %f1569, %f1583, %f1157, %f1569;\n" "\n" "$L__BB6_489:\n" " @%p255 bra $L__BB6_491;\n" "\n" " ld.shared.f32 %f1158, [%r90+24];\n" " fma.rn.f32 %f1568, %f1583, %f1158, %f1568;\n" "\n" "$L__BB6_491:\n" " @%p261 bra $L__BB6_509;\n" "\n" " ld.shared.f32 %f1159, [%r90+28];\n" " fma.rn.f32 %f1567, %f1583, %f1159, %f1567;\n" " bra.uni $L__BB6_509;\n" "\n" "$L__BB6_493:\n" " @%p2 bra $L__BB6_495;\n" "\n" " ld.shared.f32 %f1160, [%r35];\n" " ld.shared.f32 %f1161, [%r90];\n" " mul.f32 %f1162, %f1583, %f1161;\n" " fma.rn.f32 %f1574, %f1160, %f1574, %f1162;\n" "\n" "$L__BB6_495:\n" " @%p225 bra $L__BB6_497;\n" "\n" " ld.shared.f32 %f1163, [%r35+4];\n" " ld.shared.f32 %f1164, [%r90+4];\n" " mul.f32 %f1165, %f1583, %f1164;\n" " fma.rn.f32 %f1573, %f1163, %f1573, %f1165;\n" "\n" "$L__BB6_497:\n" " @%p231 bra $L__BB6_499;\n" "\n" " ld.shared.f32 %f1166, [%r35+8];\n" " ld.shared.f32 %f1167, [%r90+8];\n" " mul.f32 %f1168, %f1583, %f1167;\n" " fma.rn.f32 %f1572, %f1166, %f1572, %f1168;\n" "\n" "$L__BB6_499:\n" " @%p237 bra $L__BB6_501;\n" "\n" " ld.shared.f32 %f1169, [%r35+12];\n" " ld.shared.f32 %f1170, [%r90+12];\n" " mul.f32 %f1171, %f1583, %f1170;\n" " fma.rn.f32 %f1571, %f1169, %f1571, %f1171;\n" "\n" "$L__BB6_501:\n" " @%p243 bra $L__BB6_503;\n" "\n" " ld.shared.f32 %f1172, [%r35+16];\n" " ld.shared.f32 %f1173, [%r90+16];\n" " mul.f32 %f1174, %f1583, %f1173;\n" " fma.rn.f32 %f1570, %f1172, %f1570, %f1174;\n" "\n" "$L__BB6_503:\n" " @%p249 bra $L__BB6_505;\n" "\n" " ld.shared.f32 %f1175, [%r35+20];\n" " ld.shared.f32 %f1176, [%r90+20];\n" " mul.f32 %f1177, %f1583, %f1176;\n" " fma.rn.f32 %f1569, %f1175, %f1569, %f1177;\n" "\n" "$L__BB6_505:\n" " @%p255 bra $L__BB6_507;\n" "\n" " ld.shared.f32 %f1178, [%r35+24];\n" " ld.shared.f32 %f1179, [%r90+24];\n" " mul.f32 %f1180, %f1583, %f1179;\n" " fma.rn.f32 %f1568, %f1178, %f1568, %f1180;\n" "\n" "$L__BB6_507:\n" " @%p261 bra $L__BB6_509;\n" "\n" " ld.shared.f32 %f1181, [%r35+28];\n" " ld.shared.f32 %f1182, [%r90+28];\n" " mul.f32 %f1183, %f1583, %f1182;\n" " fma.rn.f32 %f1567, %f1181, %f1567, %f1183;\n" "\n" "$L__BB6_509:\n" " add.s32 %r618, %r618, 1;\n" " setp.lt.s32 %p330, %r618, %r40;\n" " @%p330 bra $L__BB6_471;\n" " bra.uni $L__BB6_510;\n" "\n" "$L__BB6_432:\n" " mov.u32 %r617, 0;\n" "\n" "$L__BB6_433:\n" " add.s32 %r85, %r617, %r606;\n" " setp.lt.s32 %p288, %r85, %r99;\n" " @%p288 bra $L__BB6_435;\n" " bra.uni $L__BB6_434;\n" "\n" "$L__BB6_435:\n" " cvt.s64.s32 %rd207, %r85;\n" " mul.lo.s64 %rd208, %rd207, %rd5;\n" " mul.lo.s64 %rd210, %rd32, %rd5;\n" " shl.b64 %rd211, %rd210, 3;\n" " cvt.s64.s32 %rd212, %r1;\n" " add.s64 %rd213, %rd211, %rd212;\n" " add.s64 %rd214, %rd213, %rd208;\n" " cvta.to.global.u64 %rd215, %rd21;\n" " shl.b64 %rd216, %rd214, 1;\n" " add.s64 %rd217, %rd215, %rd216;\n" " ld.global.u16 %rs42, [%rd217];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1552, %rs42;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB6_436;\n" "\n" "$L__BB6_434:\n" " mad.lo.s32 %r511, %r96, %r95, %r502;\n" " shl.b32 %r512, %r511, 2;\n" " add.s32 %r514, %r127, %r512;\n" " mad.lo.s32 %r515, %r617, %r3, %r26;\n" " add.s32 %r516, %r514, %r515;\n" " ld.shared.u16 %rs41, [%r516];\n" " // begin inline asm\n" " { cvt.f32.f16 %f1113, %rs41;}\n" "\n" " // end inline asm\n" " and.b32 %r518, %r1, 31;\n" " add.s32 %r519, %r518, -16;\n" " setp.lt.u32 %p289, %r518, 16;\n" " selp.b32 %r520, %r518, %r519, %p289;\n" " add.s32 %r521, %r516, %r520;\n" " ld.shared.u8 %r522, [%r521+2];\n" " selp.b32 %r523, 0, 4, %p289;\n" " shr.u32 %r524, %r522, %r523;\n" " and.b32 %r525, %r524, 15;\n" " add.s32 %r526, %r525, -8;\n" " cvt.rn.f32.s32 %f1114, %r526;\n" " mul.f32 %f1552, %f1113, %f1114;\n" "\n" "$L__BB6_436:\n" " mad.lo.s32 %r530, %r96, %r95, %r108;\n" " mad.lo.s32 %r531, %r617, %r95, %r530;\n" " shl.b32 %r532, %r531, 2;\n" " add.s32 %r86, %r127, %r532;\n" " setp.eq.s32 %p290, %r617, 0;\n" " @%p290 bra $L__BB6_453;\n" "\n" " @%p2 bra $L__BB6_439;\n" "\n" " ld.shared.f32 %f1116, [%r86];\n" " fma.rn.f32 %f1574, %f1552, %f1116, %f1574;\n" "\n" "$L__BB6_439:\n" " @%p225 bra $L__BB6_441;\n" "\n" " ld.shared.f32 %f1117, [%r86+4];\n" " fma.rn.f32 %f1573, %f1552, %f1117, %f1573;\n" "\n" "$L__BB6_441:\n" " @%p231 bra $L__BB6_443;\n" "\n" " ld.shared.f32 %f1118, [%r86+8];\n" " fma.rn.f32 %f1572, %f1552, %f1118, %f1572;\n" "\n" "$L__BB6_443:\n" " @%p237 bra $L__BB6_445;\n" "\n" " ld.shared.f32 %f1119, [%r86+12];\n" " fma.rn.f32 %f1571, %f1552, %f1119, %f1571;\n" "\n" "$L__BB6_445:\n" " @%p243 bra $L__BB6_447;\n" "\n" " ld.shared.f32 %f1120, [%r86+16];\n" " fma.rn.f32 %f1570, %f1552, %f1120, %f1570;\n" "\n" "$L__BB6_447:\n" " @%p249 bra $L__BB6_449;\n" "\n" " ld.shared.f32 %f1121, [%r86+20];\n" " fma.rn.f32 %f1569, %f1552, %f1121, %f1569;\n" "\n" "$L__BB6_449:\n" " @%p255 bra $L__BB6_451;\n" "\n" " ld.shared.f32 %f1122, [%r86+24];\n" " fma.rn.f32 %f1568, %f1552, %f1122, %f1568;\n" "\n" "$L__BB6_451:\n" " @%p261 bra $L__BB6_469;\n" "\n" " ld.shared.f32 %f1123, [%r86+28];\n" " fma.rn.f32 %f1567, %f1552, %f1123, %f1567;\n" " bra.uni $L__BB6_469;\n" "\n" "$L__BB6_453:\n" " @%p2 bra $L__BB6_455;\n" "\n" " ld.shared.f32 %f1124, [%r35];\n" " ld.shared.f32 %f1125, [%r86];\n" " mul.f32 %f1126, %f1552, %f1125;\n" " fma.rn.f32 %f1574, %f1124, %f1574, %f1126;\n" "\n" "$L__BB6_455:\n" " @%p225 bra $L__BB6_457;\n" "\n" " ld.shared.f32 %f1127, [%r35+4];\n" " ld.shared.f32 %f1128, [%r86+4];\n" " mul.f32 %f1129, %f1552, %f1128;\n" " fma.rn.f32 %f1573, %f1127, %f1573, %f1129;\n" "\n" "$L__BB6_457:\n" " @%p231 bra $L__BB6_459;\n" "\n" " ld.shared.f32 %f1130, [%r35+8];\n" " ld.shared.f32 %f1131, [%r86+8];\n" " mul.f32 %f1132, %f1552, %f1131;\n" " fma.rn.f32 %f1572, %f1130, %f1572, %f1132;\n" "\n" "$L__BB6_459:\n" " @%p237 bra $L__BB6_461;\n" "\n" " ld.shared.f32 %f1133, [%r35+12];\n" " ld.shared.f32 %f1134, [%r86+12];\n" " mul.f32 %f1135, %f1552, %f1134;\n" " fma.rn.f32 %f1571, %f1133, %f1571, %f1135;\n" "\n" "$L__BB6_461:\n" " @%p243 bra $L__BB6_463;\n" "\n" " ld.shared.f32 %f1136, [%r35+16];\n" " ld.shared.f32 %f1137, [%r86+16];\n" " mul.f32 %f1138, %f1552, %f1137;\n" " fma.rn.f32 %f1570, %f1136, %f1570, %f1138;\n" "\n" "$L__BB6_463:\n" " @%p249 bra $L__BB6_465;\n" "\n" " ld.shared.f32 %f1139, [%r35+20];\n" " ld.shared.f32 %f1140, [%r86+20];\n" " mul.f32 %f1141, %f1552, %f1140;\n" " fma.rn.f32 %f1569, %f1139, %f1569, %f1141;\n" "\n" "$L__BB6_465:\n" " @%p255 bra $L__BB6_467;\n" "\n" " ld.shared.f32 %f1142, [%r35+24];\n" " ld.shared.f32 %f1143, [%r86+24];\n" " mul.f32 %f1144, %f1552, %f1143;\n" " fma.rn.f32 %f1568, %f1142, %f1568, %f1144;\n" "\n" "$L__BB6_467:\n" " @%p261 bra $L__BB6_469;\n" "\n" " ld.shared.f32 %f1145, [%r35+28];\n" " ld.shared.f32 %f1146, [%r86+28];\n" " mul.f32 %f1147, %f1552, %f1146;\n" " fma.rn.f32 %f1567, %f1145, %f1567, %f1147;\n" "\n" "$L__BB6_469:\n" " add.s32 %r617, %r617, 1;\n" " setp.lt.s32 %p307, %r617, %r40;\n" " @%p307 bra $L__BB6_433;\n" "\n" "$L__BB6_510:\n" " bar.sync 0;\n" "\n" "$L__BB6_511:\n" " add.s32 %r606, %r606, 16;\n" " setp.ge.s32 %p331, %r2, %r606;\n" " add.s32 %r605, %r605, 1;\n" " @%p331 bra $L__BB6_11;\n" "\n" "$L__BB6_512:\n" " ld.param.u32 %r592, [fc_attn_flash_prefill_q4_kvs_param_9];\n" " mov.u32 %r591, %ctaid.x;\n" " mov.u32 %r590, %nctaid.y;\n" " cvt.u64.u32 %rd297, %r590;\n" " cvt.s64.s32 %rd296, %r591;\n" " mov.u32 %r589, %ctaid.y;\n" " cvt.s64.s32 %rd295, %r589;\n" " mul.lo.s64 %rd294, %rd296, %rd297;\n" " add.s64 %rd293, %rd294, %rd295;\n" " mul.wide.s32 %rd292, %r592, %r95;\n" " mul.lo.s64 %rd291, %rd292, %rd293;\n" " shl.b32 %r573, %r95, 5;\n" " cvt.u64.u32 %rd243, %r573;\n" " add.s64 %rd245, %rd243, %rd27;\n" " cvt.u32.u64 %r574, %rd245;\n" " mad.lo.s32 %r575, %r592, %r95, %r574;\n" " cvt.s64.s32 %rd253, %r1;\n" " add.s64 %rd18, %rd291, %rd253;\n" " shl.b32 %r580, %r575, 2;\n" " add.s32 %r94, %r127, %r580;\n" " @%p2 bra $L__BB6_514;\n" "\n" " ld.shared.f32 %f1184, [%r94];\n" " div.rn.f32 %f1185, %f1574, %f1184;\n" " cvta.to.global.u64 %rd254, %rd23;\n" " shl.b64 %rd255, %rd18, 2;\n" " add.s64 %rd256, %rd254, %rd255;\n" " st.global.f32 [%rd256], %f1185;\n" "\n" "$L__BB6_514:\n" " setp.lt.s32 %p333, %r95, 2;\n" " @%p333 bra $L__BB6_516;\n" "\n" " ld.shared.f32 %f1186, [%r94+4];\n" " div.rn.f32 %f1187, %f1573, %f1186;\n" " cvta.to.global.u64 %rd257, %rd23;\n" " add.s64 %rd258, %rd18, %rd5;\n" " shl.b64 %rd259, %rd258, 2;\n" " add.s64 %rd260, %rd257, %rd259;\n" " st.global.f32 [%rd260], %f1187;\n" "\n" "$L__BB6_516:\n" " setp.lt.s32 %p334, %r95, 3;\n" " @%p334 bra $L__BB6_518;\n" "\n" " ld.shared.f32 %f1188, [%r94+8];\n" " div.rn.f32 %f1189, %f1572, %f1188;\n" " shl.b64 %rd261, %rd5, 1;\n" " add.s64 %rd262, %rd18, %rd261;\n" " cvta.to.global.u64 %rd263, %rd23;\n" " shl.b64 %rd264, %rd262, 2;\n" " add.s64 %rd265, %rd263, %rd264;\n" " st.global.f32 [%rd265], %f1189;\n" "\n" "$L__BB6_518:\n" " setp.lt.s32 %p335, %r95, 4;\n" " @%p335 bra $L__BB6_520;\n" "\n" " ld.shared.f32 %f1190, [%r94+12];\n" " div.rn.f32 %f1191, %f1571, %f1190;\n" " mul.lo.s64 %rd266, %rd5, 3;\n" " add.s64 %rd267, %rd18, %rd266;\n" " cvta.to.global.u64 %rd268, %rd23;\n" " shl.b64 %rd269, %rd267, 2;\n" " add.s64 %rd270, %rd268, %rd269;\n" " st.global.f32 [%rd270], %f1191;\n" "\n" "$L__BB6_520:\n" " setp.lt.s32 %p336, %r95, 5;\n" " @%p336 bra $L__BB6_522;\n" "\n" " ld.shared.f32 %f1192, [%r94+16];\n" " div.rn.f32 %f1193, %f1570, %f1192;\n" " shl.b64 %rd271, %rd5, 2;\n" " add.s64 %rd272, %rd18, %rd271;\n" " cvta.to.global.u64 %rd273, %rd23;\n" " shl.b64 %rd274, %rd272, 2;\n" " add.s64 %rd275, %rd273, %rd274;\n" " st.global.f32 [%rd275], %f1193;\n" "\n" "$L__BB6_522:\n" " setp.lt.s32 %p337, %r95, 6;\n" " @%p337 bra $L__BB6_524;\n" "\n" " ld.shared.f32 %f1194, [%r94+20];\n" " div.rn.f32 %f1195, %f1569, %f1194;\n" " mul.lo.s64 %rd276, %rd5, 5;\n" " add.s64 %rd277, %rd18, %rd276;\n" " cvta.to.global.u64 %rd278, %rd23;\n" " shl.b64 %rd279, %rd277, 2;\n" " add.s64 %rd280, %rd278, %rd279;\n" " st.global.f32 [%rd280], %f1195;\n" "\n" "$L__BB6_524:\n" " setp.lt.s32 %p338, %r95, 7;\n" " @%p338 bra $L__BB6_526;\n" "\n" " ld.shared.f32 %f1196, [%r94+24];\n" " div.rn.f32 %f1197, %f1568, %f1196;\n" " mul.lo.s64 %rd281, %rd5, 6;\n" " add.s64 %rd282, %rd18, %rd281;\n" " cvta.to.global.u64 %rd283, %rd23;\n" " shl.b64 %rd284, %rd282, 2;\n" " add.s64 %rd285, %rd283, %rd284;\n" " st.global.f32 [%rd285], %f1197;\n" "\n" "$L__BB6_526:\n" " setp.lt.s32 %p339, %r95, 8;\n" " @%p339 bra $L__BB6_528;\n" "\n" " ld.shared.f32 %f1198, [%r94+28];\n" " div.rn.f32 %f1199, %f1567, %f1198;\n" " mul.lo.s64 %rd286, %rd5, 7;\n" " add.s64 %rd287, %rd18, %rd286;\n" " cvta.to.global.u64 %rd288, %rd23;\n" " shl.b64 %rd289, %rd287, 2;\n" " add.s64 %rd290, %rd288, %rd289;\n" " st.global.f32 [%rd290], %f1199;\n" "\n" "$L__BB6_528:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_prefill_q4_kh\n" ".visible .entry fc_attn_flash_prefill_q4_kh(\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_0,\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_1,\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_2,\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_3,\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_4,\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_5,\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_6,\n" " .param .u64 fc_attn_flash_prefill_q4_kh_param_7,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_8,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_9,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_10,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_11,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_12,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_13,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_14,\n" " .param .u32 fc_attn_flash_prefill_q4_kh_param_15,\n" " .param .f32 fc_attn_flash_prefill_q4_kh_param_16\n" ")\n" "{\n" " .reg .pred %p<191>;\n" " .reg .b16 %rs<16>;\n" " .reg .f32 %f<767>;\n" " .reg .b32 %r<630>;\n" " .reg .b64 %rd<171>;\n" "\n" "\n" " ld.param.u64 %rd45, [fc_attn_flash_prefill_q4_kh_param_0];\n" " ld.param.u64 %rd46, [fc_attn_flash_prefill_q4_kh_param_1];\n" " ld.param.u64 %rd47, [fc_attn_flash_prefill_q4_kh_param_2];\n" " ld.param.u64 %rd48, [fc_attn_flash_prefill_q4_kh_param_3];\n" " ld.param.u64 %rd49, [fc_attn_flash_prefill_q4_kh_param_4];\n" " ld.param.u64 %rd43, [fc_attn_flash_prefill_q4_kh_param_5];\n" " ld.param.u64 %rd44, [fc_attn_flash_prefill_q4_kh_param_6];\n" " ld.param.u64 %rd50, [fc_attn_flash_prefill_q4_kh_param_7];\n" " ld.param.u32 %r126, [fc_attn_flash_prefill_q4_kh_param_8];\n" " ld.param.u32 %r127, [fc_attn_flash_prefill_q4_kh_param_9];\n" " ld.param.u32 %r128, [fc_attn_flash_prefill_q4_kh_param_10];\n" " ld.param.u32 %r129, [fc_attn_flash_prefill_q4_kh_param_11];\n" " ld.param.u32 %r130, [fc_attn_flash_prefill_q4_kh_param_12];\n" " ld.param.u32 %r131, [fc_attn_flash_prefill_q4_kh_param_13];\n" " ld.param.u32 %r133, [fc_attn_flash_prefill_q4_kh_param_15];\n" " ld.param.f32 %f231, [fc_attn_flash_prefill_q4_kh_param_16];\n" " cvta.to.global.u64 %rd1, %rd45;\n" " cvta.to.global.u64 %rd2, %rd50;\n" " cvta.to.global.u64 %rd3, %rd48;\n" " cvta.to.global.u64 %rd4, %rd46;\n" " cvta.to.global.u64 %rd5, %rd49;\n" " cvta.to.global.u64 %rd6, %rd47;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %ctaid.x;\n" " add.s32 %r3, %r2, %r129;\n" " mov.u32 %r134, %nctaid.x;\n" " add.s32 %r135, %r134, %r129;\n" " add.s32 %r4, %r135, -257;\n" " shl.b32 %r5, %r126, 2;\n" " add.s32 %r6, %r127, 16;\n" " shl.b32 %r7, %r126, 4;\n" " mul.wide.s32 %rd7, %r7, 2;\n" " cvt.s64.s32 %rd51, %r126;\n" " add.s64 %rd8, %rd7, %rd51;\n" " mov.u32 %r136, %nctaid.y;\n" " cvt.u64.u32 %rd52, %r136;\n" " cvt.s64.s32 %rd53, %r2;\n" " mul.lo.s64 %rd54, %rd53, %rd52;\n" " mov.u32 %r137, %ctaid.y;\n" " cvt.s64.s32 %rd9, %r137;\n" " add.s64 %rd55, %rd54, %rd9;\n" " cvt.s64.s32 %rd10, %r127;\n" " mul.wide.s32 %rd56, %r127, %r126;\n" " mul.lo.s64 %rd11, %rd56, %rd55;\n" " setp.lt.s32 %p2, %r126, 1;\n" " @%p2 bra $L__BB7_7;\n" "\n" " cvt.s64.s32 %rd12, %r1;\n" " and.b32 %r608, %r126, 3;\n" " add.s32 %r139, %r126, -1;\n" " setp.lt.u32 %p3, %r139, 3;\n" " mov.u32 %r606, 0;\n" " @%p3 bra $L__BB7_4;\n" "\n" " sub.s32 %r604, %r126, %r608;\n" " add.s64 %rd57, %rd11, %rd12;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd169, %rd1, %rd58;\n" " shl.b64 %rd14, %rd10, 2;\n" " shl.b32 %r141, %r1, 2;\n" " mov.u32 %r142, shb;\n" " add.s32 %r605, %r142, %r141;\n" " shl.b32 %r11, %r127, 2;\n" " mov.u32 %r606, 0;\n" "\n" "$L__BB7_3:\n" " ld.global.f32 %f232, [%rd169];\n" " st.shared.f32 [%r605], %f232;\n" " add.s64 %rd59, %rd169, %rd14;\n" " ld.global.f32 %f233, [%rd59];\n" " add.s32 %r143, %r605, %r11;\n" " st.shared.f32 [%r143], %f233;\n" " add.s64 %rd60, %rd59, %rd14;\n" " ld.global.f32 %f234, [%rd60];\n" " add.s32 %r144, %r143, %r11;\n" " st.shared.f32 [%r144], %f234;\n" " add.s64 %rd61, %rd60, %rd14;\n" " add.s64 %rd169, %rd61, %rd14;\n" " ld.global.f32 %f235, [%rd61];\n" " add.s32 %r145, %r144, %r11;\n" " add.s32 %r605, %r145, %r11;\n" " st.shared.f32 [%r145], %f235;\n" " add.s32 %r606, %r606, 4;\n" " add.s32 %r604, %r604, -4;\n" " setp.ne.s32 %p4, %r604, 0;\n" " @%p4 bra $L__BB7_3;\n" "\n" "$L__BB7_4:\n" " setp.eq.s32 %p5, %r608, 0;\n" " @%p5 bra $L__BB7_7;\n" "\n" " mul.lo.s32 %r146, %r606, %r127;\n" " shl.b32 %r147, %r146, 2;\n" " shl.b32 %r148, %r1, 2;\n" " add.s32 %r149, %r147, %r148;\n" " mov.u32 %r150, shb;\n" " add.s32 %r607, %r150, %r149;\n" " shl.b32 %r20, %r127, 2;\n" " cvt.s64.s32 %rd170, %r606;\n" " add.s64 %rd18, %rd11, %rd12;\n" "\n" "$L__BB7_6:\n" " .pragma \"nounroll\";\n" " mul.lo.s64 %rd62, %rd170, %rd10;\n" " add.s64 %rd63, %rd18, %rd62;\n" " shl.b64 %rd64, %rd63, 2;\n" " add.s64 %rd65, %rd1, %rd64;\n" " ld.global.f32 %f236, [%rd65];\n" " st.shared.f32 [%r607], %f236;\n" " add.s32 %r607, %r607, %r20;\n" " add.s64 %rd170, %rd170, 1;\n" " add.s32 %r608, %r608, -1;\n" " setp.ne.s32 %p6, %r608, 0;\n" " @%p6 bra $L__BB7_6;\n" "\n" "$L__BB7_7:\n" " mov.u32 %r151, shb;\n" " mad.lo.s32 %r25, %r5, %r127, %r151;\n" " shl.b32 %r152, %r6, 5;\n" " add.s32 %r26, %r25, %r152;\n" " setp.ge.s32 %p7, %r1, %r126;\n" " cvt.u32.u64 %r153, %rd7;\n" " add.s32 %r154, %r1, %r153;\n" " shl.b32 %r155, %r154, 2;\n" " add.s32 %r27, %r26, %r155;\n" " cvt.u32.u64 %r156, %rd8;\n" " add.s32 %r157, %r1, %r156;\n" " shl.b32 %r158, %r157, 2;\n" " add.s32 %r28, %r26, %r158;\n" " @%p7 bra $L__BB7_9;\n" "\n" " mov.u32 %r159, -8388608;\n" " st.shared.u32 [%r27], %r159;\n" " mov.u32 %r160, 0;\n" " st.shared.u32 [%r28], %r160;\n" "\n" "$L__BB7_9:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd66, %r128;\n" " mul.lo.s64 %rd67, %rd66, %rd9;\n" " shr.s32 %r161, %r127, 31;\n" " shr.u32 %r162, %r161, 27;\n" " add.s32 %r163, %r127, %r162;\n" " shr.s32 %r164, %r163, 5;\n" " mul.lo.s32 %r165, %r164, 18;\n" " cvt.s64.s32 %rd21, %r165;\n" " mul.lo.s64 %rd22, %rd67, %rd21;\n" " mul.lo.s64 %rd68, %rd9, %rd10;\n" " shl.b64 %rd23, %rd68, 8;\n" " shl.b64 %rd24, %rd68, 3;\n" " shr.s32 %r166, %r1, 31;\n" " shr.u32 %r167, %r166, 28;\n" " add.s32 %r168, %r1, %r167;\n" " shr.s32 %r30, %r168, 4;\n" " and.b32 %r169, %r168, -16;\n" " sub.s32 %r31, %r1, %r169;\n" " cvt.s64.s32 %rd25, %r1;\n" " setp.lt.s32 %p8, %r3, 0;\n" " mov.f32 %f704, 0f00000000;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " @%p8 bra $L__BB7_213;\n" "\n" " shr.u32 %r173, %r161, 28;\n" " add.s32 %r174, %r127, %r173;\n" " shr.s32 %r175, %r174, 4;\n" " sub.s32 %r176, %r3, %r133;\n" " add.s32 %r177, %r176, 1;\n" " mov.u32 %r609, 0;\n" " max.s32 %r32, %r177, 0;\n" " cvt.s64.s32 %rd26, %r6;\n" " sub.s32 %r33, %r3, %r131;\n" " mov.u32 %r34, %ntid.x;\n" " mul.lo.s32 %r35, %r6, %r30;\n" " shl.b32 %r36, %r1, 4;\n" " add.s64 %rd27, %rd24, %rd25;\n" " add.s64 %rd28, %rd23, %rd25;\n" " shr.u32 %r179, %r166, 27;\n" " add.s32 %r180, %r1, %r179;\n" " shr.s32 %r181, %r180, 5;\n" " mul.lo.s32 %r182, %r181, 18;\n" " cvt.s64.s32 %rd29, %r182;\n" " and.b32 %r183, %r1, 31;\n" " setp.lt.u32 %p9, %r183, 16;\n" " add.s32 %r184, %r183, -16;\n" " selp.b32 %r185, %r183, %r184, %p9;\n" " selp.b32 %r37, 0, 4, %p9;\n" " add.s32 %r186, %r185, 2;\n" " cvt.s64.s32 %rd69, %r186;\n" " add.s64 %rd30, %rd69, %rd29;\n" " add.s32 %r187, %r156, %r126;\n" " add.s32 %r188, %r1, %r187;\n" " shl.b32 %r189, %r188, 2;\n" " add.s32 %r38, %r26, %r189;\n" " mov.u32 %r190, -2;\n" " sub.s32 %r191, %r190, %r2;\n" " sub.s32 %r40, %r191, %r129;\n" " bfe.u32 %r42, %r174, 4, 1;\n" " sub.s32 %r43, %r175, %r42;\n" " shl.b32 %r44, %r127, 1;\n" " mul.lo.s32 %r45, %r127, 3;\n" " shl.b32 %r46, %r127, 2;\n" " mul.lo.s32 %r47, %r127, 5;\n" " mul.lo.s32 %r48, %r127, 6;\n" " mul.lo.s32 %r49, %r127, 7;\n" " shl.b32 %r192, %r30, 2;\n" " add.s32 %r50, %r26, %r192;\n" " shl.b32 %r193, %r187, 2;\n" " add.s32 %r51, %r26, %r193;\n" " add.s32 %r194, %r5, 32;\n" " shl.b32 %r195, %r126, 6;\n" " mad.lo.s32 %r196, %r127, %r194, %r195;\n" " shl.b32 %r197, %r1, 2;\n" " add.s32 %r198, %r196, %r197;\n" " add.s32 %r52, %r151, %r198;\n" " cvta.to.global.u64 %rd31, %rd44;\n" " cvta.to.global.u64 %rd32, %rd43;\n" " mov.f32 %f704, 0f00000000;\n" " setp.lt.s32 %p11, %r133, 1;\n" " setp.gt.s32 %p19, %r131, 0;\n" " setp.ge.s32 %p36, %r1, %r127;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " mov.u32 %r610, %r609;\n" "\n" "$L__BB7_11:\n" " ld.param.u32 %r506, [fc_attn_flash_prefill_q4_kh_param_14];\n" " add.s32 %r505, %r3, 1;\n" " shl.b32 %r200, %r609, 4;\n" " add.s32 %r201, %r40, %r200;\n" " max.s32 %r202, %r201, -17;\n" " not.b32 %r203, %r202;\n" " max.s32 %r57, %r203, 1;\n" " sub.s32 %r58, %r505, %r610;\n" " min.s32 %r59, %r58, 16;\n" " setp.lt.s32 %p10, %r610, %r506;\n" " or.pred %p12, %p11, %p10;\n" " add.s32 %r204, %r59, %r610;\n" " setp.gt.s32 %p13, %r204, %r32;\n" " setp.le.s32 %p14, %r610, %r3;\n" " and.pred %p15, %p14, %p13;\n" " or.pred %p16, %p12, %p15;\n" " not.pred %p17, %p16;\n" " @%p17 bra $L__BB7_212;\n" "\n" " setp.lt.s32 %p18, %r58, 1;\n" " @%p18 bra $L__BB7_36;\n" "\n" " @%p19 bra $L__BB7_23;\n" " bra.uni $L__BB7_14;\n" "\n" "$L__BB7_23:\n" " mov.u32 %r614, 0;\n" "\n" "$L__BB7_24:\n" " cvt.s64.s32 %rd82, %r614;\n" " mul.lo.s64 %rd37, %rd82, %rd26;\n" " add.s32 %r69, %r614, %r610;\n" " setp.lt.s32 %p27, %r69, %r130;\n" " @%p27 bra $L__BB7_32;\n" " bra.uni $L__BB7_25;\n" "\n" "$L__BB7_32:\n" " @%p36 bra $L__BB7_35;\n" "\n" " cvt.u32.u64 %r76, %rd37;\n" " cvt.s64.s32 %rd96, %r69;\n" " mul.lo.s64 %rd97, %rd96, %rd10;\n" " add.s64 %rd40, %rd97, %rd24;\n" " mov.u32 %r617, %r1;\n" "\n" "$L__BB7_34:\n" " cvt.s64.s32 %rd98, %r617;\n" " add.s64 %rd99, %rd40, %rd98;\n" " shl.b64 %rd100, %rd99, 1;\n" " add.s64 %rd101, %rd3, %rd100;\n" " ld.global.u16 %rs7, [%rd101];\n" " add.s32 %r255, %r617, %r76;\n" " shl.b32 %r256, %r255, 1;\n" " add.s32 %r257, %r25, %r256;\n" " st.shared.u16 [%r257], %rs7;\n" " add.s32 %r617, %r617, %r34;\n" " setp.lt.s32 %p37, %r617, %r127;\n" " @%p37 bra $L__BB7_34;\n" " bra.uni $L__BB7_35;\n" "\n" "$L__BB7_25:\n" " setp.gt.s32 %p28, %r69, %r33;\n" " setp.gt.s32 %p29, %r69, %r4;\n" " and.pred %p30, %p29, %p28;\n" " @%p30 bra $L__BB7_29;\n" " bra.uni $L__BB7_26;\n" "\n" "$L__BB7_29:\n" " @%p36 bra $L__BB7_35;\n" "\n" " cvt.u32.u64 %r73, %rd37;\n" " and.b32 %r251, %r69, 255;\n" " cvt.u64.u32 %rd90, %r251;\n" " mul.lo.s64 %rd91, %rd90, %rd10;\n" " add.s64 %rd39, %rd91, %rd23;\n" " mov.u32 %r616, %r1;\n" "\n" "$L__BB7_31:\n" " cvt.s64.s32 %rd92, %r616;\n" " add.s64 %rd93, %rd39, %rd92;\n" " shl.b64 %rd94, %rd93, 1;\n" " add.s64 %rd95, %rd32, %rd94;\n" " ld.global.u16 %rs6, [%rd95];\n" " add.s32 %r252, %r616, %r73;\n" " shl.b32 %r253, %r252, 1;\n" " add.s32 %r254, %r25, %r253;\n" " st.shared.u16 [%r254], %rs6;\n" " add.s32 %r616, %r616, %r34;\n" " setp.lt.s32 %p35, %r616, %r127;\n" " @%p35 bra $L__BB7_31;\n" " bra.uni $L__BB7_35;\n" "\n" "$L__BB7_26:\n" " @%p36 bra $L__BB7_35;\n" "\n" " cvt.u32.u64 %r70, %rd37;\n" " cvt.s64.s32 %rd83, %r69;\n" " mul.lo.s64 %rd84, %rd83, %rd21;\n" " add.s64 %rd38, %rd84, %rd22;\n" " mov.u32 %r615, %r1;\n" "\n" "$L__BB7_28:\n" " shr.s32 %r231, %r615, 31;\n" " shr.u32 %r232, %r231, 27;\n" " add.s32 %r233, %r615, %r232;\n" " shr.s32 %r234, %r233, 5;\n" " mul.lo.s32 %r235, %r234, 18;\n" " cvt.s64.s32 %rd85, %r235;\n" " add.s64 %rd86, %rd38, %rd85;\n" " add.s64 %rd87, %rd4, %rd86;\n" " ld.global.u8 %r236, [%rd87];\n" " ld.global.u8 %r237, [%rd87+1];\n" " prmt.b32 %r238, %r237, %r236, 30212;\n" " cvt.u16.u32 %rs4, %r238;\n" " // begin inline asm\n" " { cvt.f32.f16 %f256, %rs4;}\n" "\n" " // end inline asm\n" " and.b32 %r239, %r615, 31;\n" " setp.lt.u32 %p32, %r239, 16;\n" " add.s32 %r240, %r239, -16;\n" " selp.b32 %r241, %r239, %r240, %p32;\n" " selp.b32 %r242, 0, 4, %p32;\n" " add.s32 %r243, %r241, 2;\n" " cvt.s64.s32 %rd88, %r243;\n" " add.s64 %rd89, %rd87, %rd88;\n" " ld.global.u8 %r244, [%rd89];\n" " shr.u32 %r245, %r244, %r242;\n" " and.b32 %r246, %r245, 15;\n" " add.s32 %r247, %r246, -8;\n" " cvt.rn.f32.s32 %f258, %r247;\n" " mul.f32 %f257, %f256, %f258;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs5, %f257;}\n" "\n" " // end inline asm\n" " add.s32 %r248, %r615, %r70;\n" " shl.b32 %r249, %r248, 1;\n" " add.s32 %r250, %r25, %r249;\n" " st.shared.u16 [%r250], %rs5;\n" " add.s32 %r615, %r615, %r34;\n" " setp.lt.s32 %p33, %r615, %r127;\n" " @%p33 bra $L__BB7_28;\n" "\n" "$L__BB7_35:\n" " add.s32 %r614, %r614, 1;\n" " setp.lt.s32 %p38, %r614, %r59;\n" " @%p38 bra $L__BB7_24;\n" " bra.uni $L__BB7_36;\n" "\n" "$L__BB7_14:\n" " mov.u32 %r611, 0;\n" "\n" "$L__BB7_15:\n" " cvt.s64.s32 %rd70, %r611;\n" " mul.lo.s64 %rd33, %rd70, %rd26;\n" " add.s32 %r206, %r611, %r610;\n" " setp.lt.s32 %p20, %r206, %r130;\n" " cvt.s64.s32 %rd34, %r206;\n" " @%p20 bra $L__BB7_19;\n" " bra.uni $L__BB7_16;\n" "\n" "$L__BB7_19:\n" " @%p36 bra $L__BB7_22;\n" "\n" " cvt.u32.u64 %r64, %rd33;\n" " mul.lo.s64 %rd77, %rd34, %rd10;\n" " add.s64 %rd36, %rd77, %rd24;\n" " mov.u32 %r613, %r1;\n" "\n" "$L__BB7_21:\n" " cvt.s64.s32 %rd78, %r613;\n" " add.s64 %rd79, %rd36, %rd78;\n" " shl.b64 %rd80, %rd79, 1;\n" " add.s64 %rd81, %rd3, %rd80;\n" " ld.global.u16 %rs3, [%rd81];\n" " add.s32 %r227, %r613, %r64;\n" " shl.b32 %r228, %r227, 1;\n" " add.s32 %r229, %r25, %r228;\n" " st.shared.u16 [%r229], %rs3;\n" " add.s32 %r613, %r613, %r34;\n" " setp.lt.s32 %p25, %r613, %r127;\n" " @%p25 bra $L__BB7_21;\n" " bra.uni $L__BB7_22;\n" "\n" "$L__BB7_16:\n" " @%p36 bra $L__BB7_22;\n" "\n" " cvt.u32.u64 %r61, %rd33;\n" " mul.lo.s64 %rd71, %rd34, %rd21;\n" " add.s64 %rd35, %rd71, %rd22;\n" " mov.u32 %r612, %r1;\n" "\n" "$L__BB7_18:\n" " shr.s32 %r207, %r612, 31;\n" " shr.u32 %r208, %r207, 27;\n" " add.s32 %r209, %r612, %r208;\n" " shr.s32 %r210, %r209, 5;\n" " mul.lo.s32 %r211, %r210, 18;\n" " cvt.s64.s32 %rd72, %r211;\n" " add.s64 %rd73, %rd35, %rd72;\n" " add.s64 %rd74, %rd4, %rd73;\n" " ld.global.u8 %r212, [%rd74];\n" " ld.global.u8 %r213, [%rd74+1];\n" " prmt.b32 %r214, %r213, %r212, 30212;\n" " cvt.u16.u32 %rs1, %r214;\n" " // begin inline asm\n" " { cvt.f32.f16 %f253, %rs1;}\n" "\n" " // end inline asm\n" " and.b32 %r215, %r612, 31;\n" " setp.lt.u32 %p22, %r215, 16;\n" " add.s32 %r216, %r215, -16;\n" " selp.b32 %r217, %r215, %r216, %p22;\n" " selp.b32 %r218, 0, 4, %p22;\n" " add.s32 %r219, %r217, 2;\n" " cvt.s64.s32 %rd75, %r219;\n" " add.s64 %rd76, %rd74, %rd75;\n" " ld.global.u8 %r220, [%rd76];\n" " shr.u32 %r221, %r220, %r218;\n" " and.b32 %r222, %r221, 15;\n" " add.s32 %r223, %r222, -8;\n" " cvt.rn.f32.s32 %f255, %r223;\n" " mul.f32 %f254, %f253, %f255;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs2, %f254;}\n" "\n" " // end inline asm\n" " add.s32 %r224, %r612, %r61;\n" " shl.b32 %r225, %r224, 1;\n" " add.s32 %r226, %r25, %r225;\n" " st.shared.u16 [%r226], %rs2;\n" " add.s32 %r612, %r612, %r34;\n" " setp.lt.s32 %p23, %r612, %r127;\n" " @%p23 bra $L__BB7_18;\n" "\n" "$L__BB7_22:\n" " add.s32 %r611, %r611, 1;\n" " setp.lt.s32 %p26, %r611, %r59;\n" " @%p26 bra $L__BB7_15;\n" "\n" "$L__BB7_36:\n" " setp.lt.s32 %p39, %r127, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p40, %r30, %r59;\n" " or.pred %p41, %p39, %p40;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p41 bra $L__BB7_89;\n" "\n" " and.b32 %r507, %r127, -16;\n" " setp.eq.s32 %p42, %r507, 16;\n" " mov.u32 %r620, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p42 bra $L__BB7_72;\n" "\n" " mov.u32 %r620, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " mov.u32 %r619, %r43;\n" "\n" "$L__BB7_39:\n" " shl.b32 %r260, %r620, 4;\n" " add.s32 %r82, %r260, %r31;\n" " add.s32 %r261, %r82, %r35;\n" " shl.b32 %r262, %r261, 1;\n" " add.s32 %r83, %r25, %r262;\n" " ld.shared.u16 %rs8, [%r83];\n" " // begin inline asm\n" " { cvt.f32.f16 %f283, %rs8;}\n" "\n" " // end inline asm\n" " @%p2 bra $L__BB7_41;\n" "\n" " shl.b32 %r593, %r620, 4;\n" " add.s32 %r592, %r593, %r31;\n" " shl.b32 %r591, %r592, 2;\n" " add.s32 %r590, %r151, %r591;\n" " ld.shared.f32 %f284, [%r590];\n" " fma.rn.f32 %f634, %f283, %f284, %f634;\n" "\n" "$L__BB7_41:\n" " setp.lt.s32 %p44, %r126, 2;\n" " @%p44 bra $L__BB7_43;\n" "\n" " shl.b32 %r598, %r620, 4;\n" " add.s32 %r597, %r598, %r31;\n" " add.s32 %r596, %r597, %r127;\n" " shl.b32 %r595, %r596, 2;\n" " add.s32 %r594, %r151, %r595;\n" " ld.shared.f32 %f285, [%r594];\n" " fma.rn.f32 %f635, %f283, %f285, %f635;\n" "\n" "$L__BB7_43:\n" " setp.lt.s32 %p45, %r126, 3;\n" " @%p45 bra $L__BB7_45;\n" "\n" " shl.b32 %r539, %r620, 4;\n" " add.s32 %r538, %r539, %r31;\n" " add.s32 %r537, %r44, %r538;\n" " shl.b32 %r536, %r537, 2;\n" " add.s32 %r535, %r151, %r536;\n" " ld.shared.f32 %f286, [%r535];\n" " fma.rn.f32 %f636, %f283, %f286, %f636;\n" "\n" "$L__BB7_45:\n" " setp.lt.s32 %p46, %r126, 4;\n" " @%p46 bra $L__BB7_47;\n" "\n" " shl.b32 %r549, %r620, 4;\n" " add.s32 %r548, %r549, %r31;\n" " add.s32 %r547, %r45, %r548;\n" " shl.b32 %r546, %r547, 2;\n" " add.s32 %r545, %r151, %r546;\n" " ld.shared.f32 %f287, [%r545];\n" " fma.rn.f32 %f637, %f283, %f287, %f637;\n" "\n" "$L__BB7_47:\n" " setp.lt.s32 %p47, %r126, 5;\n" " @%p47 bra $L__BB7_49;\n" "\n" " shl.b32 %r559, %r620, 4;\n" " add.s32 %r558, %r559, %r31;\n" " add.s32 %r557, %r46, %r558;\n" " shl.b32 %r556, %r557, 2;\n" " add.s32 %r555, %r151, %r556;\n" " ld.shared.f32 %f288, [%r555];\n" " fma.rn.f32 %f638, %f283, %f288, %f638;\n" "\n" "$L__BB7_49:\n" " setp.lt.s32 %p48, %r126, 6;\n" " @%p48 bra $L__BB7_51;\n" "\n" " shl.b32 %r569, %r620, 4;\n" " add.s32 %r568, %r569, %r31;\n" " add.s32 %r567, %r47, %r568;\n" " shl.b32 %r566, %r567, 2;\n" " add.s32 %r565, %r151, %r566;\n" " ld.shared.f32 %f289, [%r565];\n" " fma.rn.f32 %f639, %f283, %f289, %f639;\n" "\n" "$L__BB7_51:\n" " setp.lt.s32 %p49, %r126, 7;\n" " @%p49 bra $L__BB7_53;\n" "\n" " shl.b32 %r579, %r620, 4;\n" " add.s32 %r578, %r579, %r31;\n" " add.s32 %r577, %r48, %r578;\n" " shl.b32 %r576, %r577, 2;\n" " add.s32 %r575, %r151, %r576;\n" " ld.shared.f32 %f290, [%r575];\n" " fma.rn.f32 %f640, %f283, %f290, %f640;\n" "\n" "$L__BB7_53:\n" " setp.lt.s32 %p50, %r126, 8;\n" " @%p50 bra $L__BB7_55;\n" "\n" " shl.b32 %r589, %r620, 4;\n" " add.s32 %r588, %r589, %r31;\n" " add.s32 %r587, %r49, %r588;\n" " shl.b32 %r586, %r587, 2;\n" " add.s32 %r585, %r151, %r586;\n" " ld.shared.f32 %f291, [%r585];\n" " fma.rn.f32 %f641, %f283, %f291, %f641;\n" "\n" "$L__BB7_55:\n" " shl.b32 %r520, %r620, 4;\n" " add.s32 %r519, %r520, %r31;\n" " add.s32 %r518, %r519, %r35;\n" " shl.b32 %r517, %r518, 1;\n" " add.s32 %r516, %r25, %r517;\n" " ld.shared.u16 %rs9, [%r516+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f292, %rs9;}\n" "\n" " // end inline asm\n" " @%p2 bra $L__BB7_57;\n" "\n" " shl.b32 %r534, %r620, 4;\n" " add.s32 %r533, %r534, %r31;\n" " shl.b32 %r532, %r533, 2;\n" " add.s32 %r531, %r151, %r532;\n" " ld.shared.f32 %f293, [%r531+64];\n" " fma.rn.f32 %f634, %f292, %f293, %f634;\n" "\n" "$L__BB7_57:\n" " setp.lt.s32 %p184, %r126, 2;\n" " @%p184 bra $L__BB7_59;\n" "\n" " shl.b32 %r530, %r620, 4;\n" " add.s32 %r529, %r530, %r31;\n" " add.s32 %r528, %r529, %r127;\n" " shl.b32 %r527, %r528, 2;\n" " add.s32 %r526, %r151, %r527;\n" " ld.shared.f32 %f294, [%r526+64];\n" " fma.rn.f32 %f635, %f292, %f294, %f635;\n" "\n" "$L__BB7_59:\n" " setp.lt.s32 %p185, %r126, 3;\n" " @%p185 bra $L__BB7_61;\n" "\n" " shl.b32 %r525, %r620, 4;\n" " add.s32 %r524, %r525, %r31;\n" " add.s32 %r523, %r44, %r524;\n" " shl.b32 %r522, %r523, 2;\n" " add.s32 %r521, %r151, %r522;\n" " ld.shared.f32 %f295, [%r521+64];\n" " fma.rn.f32 %f636, %f292, %f295, %f636;\n" "\n" "$L__BB7_61:\n" " setp.lt.s32 %p186, %r126, 4;\n" " @%p186 bra $L__BB7_63;\n" "\n" " shl.b32 %r544, %r620, 4;\n" " add.s32 %r543, %r544, %r31;\n" " add.s32 %r542, %r45, %r543;\n" " shl.b32 %r541, %r542, 2;\n" " add.s32 %r540, %r151, %r541;\n" " ld.shared.f32 %f296, [%r540+64];\n" " fma.rn.f32 %f637, %f292, %f296, %f637;\n" "\n" "$L__BB7_63:\n" " setp.lt.s32 %p187, %r126, 5;\n" " @%p187 bra $L__BB7_65;\n" "\n" " shl.b32 %r554, %r620, 4;\n" " add.s32 %r553, %r554, %r31;\n" " add.s32 %r552, %r46, %r553;\n" " shl.b32 %r551, %r552, 2;\n" " add.s32 %r550, %r151, %r551;\n" " ld.shared.f32 %f297, [%r550+64];\n" " fma.rn.f32 %f638, %f292, %f297, %f638;\n" "\n" "$L__BB7_65:\n" " setp.lt.s32 %p188, %r126, 6;\n" " @%p188 bra $L__BB7_67;\n" "\n" " shl.b32 %r564, %r620, 4;\n" " add.s32 %r563, %r564, %r31;\n" " add.s32 %r562, %r47, %r563;\n" " shl.b32 %r561, %r562, 2;\n" " add.s32 %r560, %r151, %r561;\n" " ld.shared.f32 %f298, [%r560+64];\n" " fma.rn.f32 %f639, %f292, %f298, %f639;\n" "\n" "$L__BB7_67:\n" " setp.lt.s32 %p189, %r126, 7;\n" " @%p189 bra $L__BB7_69;\n" "\n" " shl.b32 %r574, %r620, 4;\n" " add.s32 %r573, %r574, %r31;\n" " add.s32 %r572, %r48, %r573;\n" " shl.b32 %r571, %r572, 2;\n" " add.s32 %r570, %r151, %r571;\n" " ld.shared.f32 %f299, [%r570+64];\n" " fma.rn.f32 %f640, %f292, %f299, %f640;\n" "\n" "$L__BB7_69:\n" " setp.lt.s32 %p190, %r126, 8;\n" " @%p190 bra $L__BB7_71;\n" "\n" " shl.b32 %r584, %r620, 4;\n" " add.s32 %r583, %r584, %r31;\n" " add.s32 %r582, %r49, %r583;\n" " shl.b32 %r581, %r582, 2;\n" " add.s32 %r580, %r151, %r581;\n" " ld.shared.f32 %f300, [%r580+64];\n" " fma.rn.f32 %f641, %f292, %f300, %f641;\n" "\n" "$L__BB7_71:\n" " add.s32 %r620, %r620, 2;\n" " add.s32 %r619, %r619, -2;\n" " setp.ne.s32 %p59, %r619, 0;\n" " @%p59 bra $L__BB7_39;\n" "\n" "$L__BB7_72:\n" " setp.eq.s32 %p60, %r42, 0;\n" " @%p60 bra $L__BB7_89;\n" "\n" " shl.b32 %r286, %r620, 4;\n" " add.s32 %r95, %r286, %r31;\n" " add.s32 %r287, %r95, %r35;\n" " shl.b32 %r288, %r287, 1;\n" " add.s32 %r289, %r25, %r288;\n" " ld.shared.u16 %rs10, [%r289];\n" " // begin inline asm\n" " { cvt.f32.f16 %f301, %rs10;}\n" "\n" " // end inline asm\n" " @%p2 bra $L__BB7_75;\n" "\n" " shl.b32 %r290, %r95, 2;\n" " add.s32 %r292, %r151, %r290;\n" " ld.shared.f32 %f302, [%r292];\n" " fma.rn.f32 %f634, %f301, %f302, %f634;\n" "\n" "$L__BB7_75:\n" " setp.lt.s32 %p62, %r126, 2;\n" " @%p62 bra $L__BB7_77;\n" "\n" " add.s32 %r293, %r95, %r127;\n" " shl.b32 %r294, %r293, 2;\n" " add.s32 %r296, %r151, %r294;\n" " ld.shared.f32 %f303, [%r296];\n" " fma.rn.f32 %f635, %f301, %f303, %f635;\n" "\n" "$L__BB7_77:\n" " setp.lt.s32 %p63, %r126, 3;\n" " @%p63 bra $L__BB7_79;\n" "\n" " add.s32 %r297, %r44, %r95;\n" " shl.b32 %r298, %r297, 2;\n" " add.s32 %r300, %r151, %r298;\n" " ld.shared.f32 %f304, [%r300];\n" " fma.rn.f32 %f636, %f301, %f304, %f636;\n" "\n" "$L__BB7_79:\n" " setp.lt.s32 %p64, %r126, 4;\n" " @%p64 bra $L__BB7_81;\n" "\n" " add.s32 %r301, %r45, %r95;\n" " shl.b32 %r302, %r301, 2;\n" " add.s32 %r304, %r151, %r302;\n" " ld.shared.f32 %f305, [%r304];\n" " fma.rn.f32 %f637, %f301, %f305, %f637;\n" "\n" "$L__BB7_81:\n" " setp.lt.s32 %p65, %r126, 5;\n" " @%p65 bra $L__BB7_83;\n" "\n" " add.s32 %r305, %r46, %r95;\n" " shl.b32 %r306, %r305, 2;\n" " add.s32 %r308, %r151, %r306;\n" " ld.shared.f32 %f306, [%r308];\n" " fma.rn.f32 %f638, %f301, %f306, %f638;\n" "\n" "$L__BB7_83:\n" " setp.lt.s32 %p66, %r126, 6;\n" " @%p66 bra $L__BB7_85;\n" "\n" " add.s32 %r309, %r47, %r95;\n" " shl.b32 %r310, %r309, 2;\n" " add.s32 %r312, %r151, %r310;\n" " ld.shared.f32 %f307, [%r312];\n" " fma.rn.f32 %f639, %f301, %f307, %f639;\n" "\n" "$L__BB7_85:\n" " setp.lt.s32 %p67, %r126, 7;\n" " @%p67 bra $L__BB7_87;\n" "\n" " add.s32 %r313, %r48, %r95;\n" " shl.b32 %r314, %r313, 2;\n" " add.s32 %r316, %r151, %r314;\n" " ld.shared.f32 %f308, [%r316];\n" " fma.rn.f32 %f640, %f301, %f308, %f640;\n" "\n" "$L__BB7_87:\n" " setp.lt.s32 %p68, %r126, 8;\n" " @%p68 bra $L__BB7_89;\n" "\n" " add.s32 %r317, %r49, %r95;\n" " shl.b32 %r318, %r317, 2;\n" " add.s32 %r320, %r151, %r318;\n" " ld.shared.f32 %f309, [%r320];\n" " fma.rn.f32 %f641, %f301, %f309, %f641;\n" "\n" "$L__BB7_89:\n" " setp.eq.s32 %p69, %r31, 0;\n" " setp.lt.s32 %p70, %r30, %r59;\n" " and.pred %p1, %p69, %p70;\n" " @%p2 bra $L__BB7_92;\n" "\n" " mov.b32 %r321, %f634;\n" " mov.u32 %r322, 2;\n" " mov.u32 %r323, 4127;\n" " mov.u32 %r324, 8;\n" " mov.u32 %r325, -1;\n" " shfl.sync.down.b32 %r326|%p72, %r321, %r324, %r323, %r325;\n" " mov.b32 %f310, %r326;\n" " add.f32 %f311, %f634, %f310;\n" " mov.b32 %r327, %f311;\n" " mov.u32 %r328, 4;\n" " shfl.sync.down.b32 %r329|%p73, %r327, %r328, %r323, %r325;\n" " mov.b32 %f312, %r329;\n" " add.f32 %f313, %f311, %f312;\n" " mov.b32 %r330, %f313;\n" " shfl.sync.down.b32 %r331|%p74, %r330, %r322, %r323, %r325;\n" " mov.b32 %f314, %r331;\n" " add.f32 %f315, %f313, %f314;\n" " mov.b32 %r332, %f315;\n" " mov.u32 %r333, 1;\n" " shfl.sync.down.b32 %r334|%p75, %r332, %r333, %r323, %r325;\n" " mov.b32 %f316, %r334;\n" " add.f32 %f83, %f315, %f316;\n" " not.pred %p76, %p1;\n" " @%p76 bra $L__BB7_92;\n" "\n" " mul.f32 %f317, %f83, %f231;\n" " st.shared.f32 [%r50], %f317;\n" "\n" "$L__BB7_92:\n" " setp.lt.s32 %p77, %r126, 2;\n" " @%p77 bra $L__BB7_95;\n" "\n" " mov.b32 %r335, %f635;\n" " mov.u32 %r336, 2;\n" " mov.u32 %r337, 4127;\n" " mov.u32 %r338, 8;\n" " mov.u32 %r339, -1;\n" " shfl.sync.down.b32 %r340|%p78, %r335, %r338, %r337, %r339;\n" " mov.b32 %f318, %r340;\n" " add.f32 %f319, %f635, %f318;\n" " mov.b32 %r341, %f319;\n" " mov.u32 %r342, 4;\n" " shfl.sync.down.b32 %r343|%p79, %r341, %r342, %r337, %r339;\n" " mov.b32 %f320, %r343;\n" " add.f32 %f321, %f319, %f320;\n" " mov.b32 %r344, %f321;\n" " shfl.sync.down.b32 %r345|%p80, %r344, %r336, %r337, %r339;\n" " mov.b32 %f322, %r345;\n" " add.f32 %f323, %f321, %f322;\n" " mov.b32 %r346, %f323;\n" " mov.u32 %r347, 1;\n" " shfl.sync.down.b32 %r348|%p81, %r346, %r347, %r337, %r339;\n" " mov.b32 %f324, %r348;\n" " add.f32 %f84, %f323, %f324;\n" " not.pred %p82, %p1;\n" " @%p82 bra $L__BB7_95;\n" "\n" " mul.f32 %f325, %f84, %f231;\n" " st.shared.f32 [%r50+64], %f325;\n" "\n" "$L__BB7_95:\n" " setp.lt.s32 %p83, %r126, 3;\n" " @%p83 bra $L__BB7_98;\n" "\n" " mov.b32 %r349, %f636;\n" " mov.u32 %r350, 2;\n" " mov.u32 %r351, 4127;\n" " mov.u32 %r352, 8;\n" " mov.u32 %r353, -1;\n" " shfl.sync.down.b32 %r354|%p84, %r349, %r352, %r351, %r353;\n" " mov.b32 %f326, %r354;\n" " add.f32 %f327, %f636, %f326;\n" " mov.b32 %r355, %f327;\n" " mov.u32 %r356, 4;\n" " shfl.sync.down.b32 %r357|%p85, %r355, %r356, %r351, %r353;\n" " mov.b32 %f328, %r357;\n" " add.f32 %f329, %f327, %f328;\n" " mov.b32 %r358, %f329;\n" " shfl.sync.down.b32 %r359|%p86, %r358, %r350, %r351, %r353;\n" " mov.b32 %f330, %r359;\n" " add.f32 %f331, %f329, %f330;\n" " mov.b32 %r360, %f331;\n" " mov.u32 %r361, 1;\n" " shfl.sync.down.b32 %r362|%p87, %r360, %r361, %r351, %r353;\n" " mov.b32 %f332, %r362;\n" " add.f32 %f85, %f331, %f332;\n" " not.pred %p88, %p1;\n" " @%p88 bra $L__BB7_98;\n" "\n" " mul.f32 %f333, %f85, %f231;\n" " st.shared.f32 [%r50+128], %f333;\n" "\n" "$L__BB7_98:\n" " setp.lt.s32 %p89, %r126, 4;\n" " @%p89 bra $L__BB7_101;\n" "\n" " mov.b32 %r363, %f637;\n" " mov.u32 %r364, 2;\n" " mov.u32 %r365, 4127;\n" " mov.u32 %r366, 8;\n" " mov.u32 %r367, -1;\n" " shfl.sync.down.b32 %r368|%p90, %r363, %r366, %r365, %r367;\n" " mov.b32 %f334, %r368;\n" " add.f32 %f335, %f637, %f334;\n" " mov.b32 %r369, %f335;\n" " mov.u32 %r370, 4;\n" " shfl.sync.down.b32 %r371|%p91, %r369, %r370, %r365, %r367;\n" " mov.b32 %f336, %r371;\n" " add.f32 %f337, %f335, %f336;\n" " mov.b32 %r372, %f337;\n" " shfl.sync.down.b32 %r373|%p92, %r372, %r364, %r365, %r367;\n" " mov.b32 %f338, %r373;\n" " add.f32 %f339, %f337, %f338;\n" " mov.b32 %r374, %f339;\n" " mov.u32 %r375, 1;\n" " shfl.sync.down.b32 %r376|%p93, %r374, %r375, %r365, %r367;\n" " mov.b32 %f340, %r376;\n" " add.f32 %f86, %f339, %f340;\n" " not.pred %p94, %p1;\n" " @%p94 bra $L__BB7_101;\n" "\n" " mul.f32 %f341, %f86, %f231;\n" " st.shared.f32 [%r50+192], %f341;\n" "\n" "$L__BB7_101:\n" " setp.lt.s32 %p95, %r126, 5;\n" " @%p95 bra $L__BB7_104;\n" "\n" " mov.b32 %r377, %f638;\n" " mov.u32 %r378, 2;\n" " mov.u32 %r379, 4127;\n" " mov.u32 %r380, 8;\n" " mov.u32 %r381, -1;\n" " shfl.sync.down.b32 %r382|%p96, %r377, %r380, %r379, %r381;\n" " mov.b32 %f342, %r382;\n" " add.f32 %f343, %f638, %f342;\n" " mov.b32 %r383, %f343;\n" " mov.u32 %r384, 4;\n" " shfl.sync.down.b32 %r385|%p97, %r383, %r384, %r379, %r381;\n" " mov.b32 %f344, %r385;\n" " add.f32 %f345, %f343, %f344;\n" " mov.b32 %r386, %f345;\n" " shfl.sync.down.b32 %r387|%p98, %r386, %r378, %r379, %r381;\n" " mov.b32 %f346, %r387;\n" " add.f32 %f347, %f345, %f346;\n" " mov.b32 %r388, %f347;\n" " mov.u32 %r389, 1;\n" " shfl.sync.down.b32 %r390|%p99, %r388, %r389, %r379, %r381;\n" " mov.b32 %f348, %r390;\n" " add.f32 %f87, %f347, %f348;\n" " not.pred %p100, %p1;\n" " @%p100 bra $L__BB7_104;\n" "\n" " mul.f32 %f349, %f87, %f231;\n" " st.shared.f32 [%r50+256], %f349;\n" "\n" "$L__BB7_104:\n" " setp.lt.s32 %p101, %r126, 6;\n" " @%p101 bra $L__BB7_107;\n" "\n" " mov.b32 %r391, %f639;\n" " mov.u32 %r392, 2;\n" " mov.u32 %r393, 4127;\n" " mov.u32 %r394, 8;\n" " mov.u32 %r395, -1;\n" " shfl.sync.down.b32 %r396|%p102, %r391, %r394, %r393, %r395;\n" " mov.b32 %f350, %r396;\n" " add.f32 %f351, %f639, %f350;\n" " mov.b32 %r397, %f351;\n" " mov.u32 %r398, 4;\n" " shfl.sync.down.b32 %r399|%p103, %r397, %r398, %r393, %r395;\n" " mov.b32 %f352, %r399;\n" " add.f32 %f353, %f351, %f352;\n" " mov.b32 %r400, %f353;\n" " shfl.sync.down.b32 %r401|%p104, %r400, %r392, %r393, %r395;\n" " mov.b32 %f354, %r401;\n" " add.f32 %f355, %f353, %f354;\n" " mov.b32 %r402, %f355;\n" " mov.u32 %r403, 1;\n" " shfl.sync.down.b32 %r404|%p105, %r402, %r403, %r393, %r395;\n" " mov.b32 %f356, %r404;\n" " add.f32 %f88, %f355, %f356;\n" " not.pred %p106, %p1;\n" " @%p106 bra $L__BB7_107;\n" "\n" " mul.f32 %f357, %f88, %f231;\n" " st.shared.f32 [%r50+320], %f357;\n" "\n" "$L__BB7_107:\n" " setp.lt.s32 %p107, %r126, 7;\n" " @%p107 bra $L__BB7_110;\n" "\n" " mov.b32 %r405, %f640;\n" " mov.u32 %r406, 2;\n" " mov.u32 %r407, 4127;\n" " mov.u32 %r408, 8;\n" " mov.u32 %r409, -1;\n" " shfl.sync.down.b32 %r410|%p108, %r405, %r408, %r407, %r409;\n" " mov.b32 %f358, %r410;\n" " add.f32 %f359, %f640, %f358;\n" " mov.b32 %r411, %f359;\n" " mov.u32 %r412, 4;\n" " shfl.sync.down.b32 %r413|%p109, %r411, %r412, %r407, %r409;\n" " mov.b32 %f360, %r413;\n" " add.f32 %f361, %f359, %f360;\n" " mov.b32 %r414, %f361;\n" " shfl.sync.down.b32 %r415|%p110, %r414, %r406, %r407, %r409;\n" " mov.b32 %f362, %r415;\n" " add.f32 %f363, %f361, %f362;\n" " mov.b32 %r416, %f363;\n" " mov.u32 %r417, 1;\n" " shfl.sync.down.b32 %r418|%p111, %r416, %r417, %r407, %r409;\n" " mov.b32 %f364, %r418;\n" " add.f32 %f89, %f363, %f364;\n" " not.pred %p112, %p1;\n" " @%p112 bra $L__BB7_110;\n" "\n" " mul.f32 %f365, %f89, %f231;\n" " st.shared.f32 [%r50+384], %f365;\n" "\n" "$L__BB7_110:\n" " setp.lt.s32 %p113, %r126, 8;\n" " @%p113 bra $L__BB7_113;\n" "\n" " mov.b32 %r419, %f641;\n" " mov.u32 %r420, 2;\n" " mov.u32 %r421, 4127;\n" " mov.u32 %r422, 8;\n" " mov.u32 %r423, -1;\n" " shfl.sync.down.b32 %r424|%p114, %r419, %r422, %r421, %r423;\n" " mov.b32 %f366, %r424;\n" " add.f32 %f367, %f641, %f366;\n" " mov.b32 %r425, %f367;\n" " mov.u32 %r426, 4;\n" " shfl.sync.down.b32 %r427|%p115, %r425, %r426, %r421, %r423;\n" " mov.b32 %f368, %r427;\n" " add.f32 %f369, %f367, %f368;\n" " mov.b32 %r428, %f369;\n" " shfl.sync.down.b32 %r429|%p116, %r428, %r420, %r421, %r423;\n" " mov.b32 %f370, %r429;\n" " add.f32 %f371, %f369, %f370;\n" " mov.b32 %r430, %f371;\n" " mov.u32 %r431, 1;\n" " shfl.sync.down.b32 %r432|%p117, %r430, %r431, %r421, %r423;\n" " mov.b32 %f372, %r432;\n" " add.f32 %f90, %f371, %f372;\n" " not.pred %p118, %p1;\n" " @%p118 bra $L__BB7_113;\n" "\n" " mul.f32 %f373, %f90, %f231;\n" " st.shared.f32 [%r50+448], %f373;\n" "\n" "$L__BB7_113:\n" " bar.sync 0;\n" " @%p7 bra $L__BB7_131;\n" "\n" " ld.shared.f32 %f91, [%r27];\n" " mov.f32 %f676, %f91;\n" " @%p18 bra $L__BB7_122;\n" "\n" " and.b32 %r96, %r57, 3;\n" " add.s32 %r434, %r57, -1;\n" " setp.lt.u32 %p121, %r434, 3;\n" " mov.u32 %r623, 0;\n" " mov.f32 %f676, %f91;\n" " @%p121 bra $L__BB7_118;\n" "\n" " sub.s32 %r622, %r57, %r96;\n" " mov.u32 %r623, 0;\n" " mov.f32 %f676, %f91;\n" "\n" "$L__BB7_117:\n" " add.s32 %r436, %r623, %r36;\n" " shl.b32 %r437, %r436, 2;\n" " add.s32 %r438, %r26, %r437;\n" " ld.shared.f32 %f375, [%r438];\n" " max.f32 %f376, %f676, %f375;\n" " ld.shared.f32 %f377, [%r438+4];\n" " max.f32 %f378, %f376, %f377;\n" " ld.shared.f32 %f379, [%r438+8];\n" " max.f32 %f380, %f378, %f379;\n" " ld.shared.f32 %f381, [%r438+12];\n" " max.f32 %f676, %f380, %f381;\n" " add.s32 %r623, %r623, 4;\n" " add.s32 %r622, %r622, -4;\n" " setp.ne.s32 %p122, %r622, 0;\n" " @%p122 bra $L__BB7_117;\n" "\n" "$L__BB7_118:\n" " setp.eq.s32 %p123, %r96, 0;\n" " @%p123 bra $L__BB7_122;\n" "\n" " add.s32 %r439, %r623, %r36;\n" " shl.b32 %r440, %r439, 2;\n" " add.s32 %r103, %r26, %r440;\n" " ld.shared.f32 %f382, [%r103];\n" " max.f32 %f676, %f676, %f382;\n" " setp.eq.s32 %p124, %r96, 1;\n" " @%p124 bra $L__BB7_122;\n" "\n" " ld.shared.f32 %f383, [%r103+4];\n" " max.f32 %f676, %f676, %f383;\n" " setp.eq.s32 %p125, %r96, 2;\n" " @%p125 bra $L__BB7_122;\n" "\n" " ld.shared.f32 %f384, [%r103+8];\n" " max.f32 %f676, %f676, %f384;\n" "\n" "$L__BB7_122:\n" " mov.f32 %f680, 0f00000000;\n" " @%p18 bra $L__BB7_130;\n" "\n" " and.b32 %r104, %r57, 3;\n" " add.s32 %r442, %r57, -1;\n" " setp.lt.u32 %p127, %r442, 3;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r627, 0;\n" " @%p127 bra $L__BB7_126;\n" "\n" " sub.s32 %r625, %r57, %r104;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r627, 0;\n" " mov.u32 %r626, %r52;\n" "\n" "$L__BB7_125:\n" " add.s32 %r508, %r5, 512;\n" " add.s32 %r444, %r627, %r36;\n" " shl.b32 %r445, %r444, 2;\n" " add.s32 %r446, %r26, %r445;\n" " ld.shared.f32 %f389, [%r446];\n" " sub.f32 %f390, %f389, %f676;\n" " mov.f32 %f391, 0f3F000000;\n" " mov.f32 %f392, 0f3BBB989D;\n" " fma.rn.f32 %f393, %f390, %f392, %f391;\n" " mov.f32 %f394, 0f3FB8AA3B;\n" " mov.f32 %f395, 0f437C0000;\n" " cvt.sat.f32.f32 %f396, %f393;\n" " mov.f32 %f397, 0f4B400001;\n" " fma.rm.f32 %f398, %f396, %f395, %f397;\n" " add.f32 %f399, %f398, 0fCB40007F;\n" " neg.f32 %f400, %f399;\n" " fma.rn.f32 %f401, %f390, %f394, %f400;\n" " mov.f32 %f402, 0f32A57060;\n" " fma.rn.f32 %f403, %f390, %f402, %f401;\n" " mov.b32 %r447, %f398;\n" " shl.b32 %r448, %r447, 23;\n" " mov.b32 %f404, %r448;\n" " ex2.approx.ftz.f32 %f405, %f403;\n" " mul.f32 %f406, %f405, %f404;\n" " st.shared.f32 [%r626+512], %f406;\n" " add.f32 %f407, %f680, %f406;\n" " ld.shared.f32 %f408, [%r446+4];\n" " sub.f32 %f409, %f408, %f676;\n" " fma.rn.f32 %f410, %f409, %f392, %f391;\n" " cvt.sat.f32.f32 %f411, %f410;\n" " fma.rm.f32 %f412, %f411, %f395, %f397;\n" " add.f32 %f413, %f412, 0fCB40007F;\n" " neg.f32 %f414, %f413;\n" " fma.rn.f32 %f415, %f409, %f394, %f414;\n" " fma.rn.f32 %f416, %f409, %f402, %f415;\n" " mov.b32 %r449, %f412;\n" " shl.b32 %r450, %r449, 23;\n" " mov.b32 %f417, %r450;\n" " ex2.approx.ftz.f32 %f418, %f416;\n" " mul.f32 %f419, %f418, %f417;\n" " add.s32 %r451, %r626, %r508;\n" " st.shared.f32 [%r451], %f419;\n" " add.f32 %f420, %f407, %f419;\n" " ld.shared.f32 %f421, [%r446+8];\n" " sub.f32 %f422, %f421, %f676;\n" " fma.rn.f32 %f423, %f422, %f392, %f391;\n" " cvt.sat.f32.f32 %f424, %f423;\n" " fma.rm.f32 %f425, %f424, %f395, %f397;\n" " add.f32 %f426, %f425, 0fCB40007F;\n" " neg.f32 %f427, %f426;\n" " fma.rn.f32 %f428, %f422, %f394, %f427;\n" " fma.rn.f32 %f429, %f422, %f402, %f428;\n" " mov.b32 %r452, %f425;\n" " shl.b32 %r453, %r452, 23;\n" " mov.b32 %f430, %r453;\n" " ex2.approx.ftz.f32 %f431, %f429;\n" " mul.f32 %f432, %f431, %f430;\n" " add.s32 %r454, %r626, %r5;\n" " add.s32 %r455, %r454, %r5;\n" " add.s32 %r456, %r454, %r508;\n" " st.shared.f32 [%r456], %f432;\n" " add.f32 %f433, %f420, %f432;\n" " ld.shared.f32 %f434, [%r446+12];\n" " sub.f32 %f435, %f434, %f676;\n" " fma.rn.f32 %f436, %f435, %f392, %f391;\n" " cvt.sat.f32.f32 %f437, %f436;\n" " fma.rm.f32 %f438, %f437, %f395, %f397;\n" " add.f32 %f439, %f438, 0fCB40007F;\n" " neg.f32 %f440, %f439;\n" " fma.rn.f32 %f441, %f435, %f394, %f440;\n" " fma.rn.f32 %f442, %f435, %f402, %f441;\n" " mov.b32 %r457, %f438;\n" " shl.b32 %r458, %r457, 23;\n" " mov.b32 %f443, %r458;\n" " ex2.approx.ftz.f32 %f444, %f442;\n" " mul.f32 %f445, %f444, %f443;\n" " add.s32 %r459, %r455, %r5;\n" " add.s32 %r460, %r455, %r508;\n" " add.s32 %r626, %r459, %r5;\n" " st.shared.f32 [%r460], %f445;\n" " add.f32 %f680, %f433, %f445;\n" " add.s32 %r627, %r627, 4;\n" " add.s32 %r625, %r625, -4;\n" " setp.ne.s32 %p128, %r625, 0;\n" " @%p128 bra $L__BB7_125;\n" "\n" "$L__BB7_126:\n" " setp.eq.s32 %p129, %r104, 0;\n" " @%p129 bra $L__BB7_130;\n" "\n" " add.s32 %r509, %r7, %r1;\n" " add.s32 %r461, %r627, %r36;\n" " shl.b32 %r462, %r461, 2;\n" " add.s32 %r113, %r26, %r462;\n" " ld.shared.f32 %f446, [%r113];\n" " sub.f32 %f447, %f446, %f676;\n" " mov.f32 %f448, 0f3F000000;\n" " mov.f32 %f449, 0f3BBB989D;\n" " fma.rn.f32 %f450, %f447, %f449, %f448;\n" " mov.f32 %f451, 0f3FB8AA3B;\n" " mov.f32 %f452, 0f437C0000;\n" " cvt.sat.f32.f32 %f453, %f450;\n" " mov.f32 %f454, 0f4B400001;\n" " fma.rm.f32 %f455, %f453, %f452, %f454;\n" " add.f32 %f456, %f455, 0fCB40007F;\n" " neg.f32 %f457, %f456;\n" " fma.rn.f32 %f458, %f447, %f451, %f457;\n" " mov.f32 %f459, 0f32A57060;\n" " fma.rn.f32 %f460, %f447, %f459, %f458;\n" " mov.b32 %r463, %f455;\n" " shl.b32 %r464, %r463, 23;\n" " mov.b32 %f461, %r464;\n" " ex2.approx.ftz.f32 %f462, %f460;\n" " mul.f32 %f463, %f462, %f461;\n" " mad.lo.s32 %r465, %r627, %r126, %r509;\n" " shl.b32 %r466, %r465, 2;\n" " add.s32 %r114, %r26, %r466;\n" " st.shared.f32 [%r114], %f463;\n" " add.f32 %f680, %f680, %f463;\n" " setp.eq.s32 %p130, %r104, 1;\n" " @%p130 bra $L__BB7_130;\n" "\n" " add.s32 %r501, %r627, %r36;\n" " shl.b32 %r500, %r501, 2;\n" " add.s32 %r499, %r26, %r500;\n" " mov.f32 %f611, 0f32A57060;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f609, 0f4B400001;\n" " mov.f32 %f608, 0f437C0000;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f606, 0f3BBB989D;\n" " ld.shared.f32 %f464, [%r499+4];\n" " sub.f32 %f465, %f464, %f676;\n" " fma.rn.f32 %f468, %f465, %f606, %f607;\n" " cvt.sat.f32.f32 %f471, %f468;\n" " fma.rm.f32 %f473, %f471, %f608, %f609;\n" " add.f32 %f474, %f473, 0fCB40007F;\n" " neg.f32 %f475, %f474;\n" " fma.rn.f32 %f476, %f465, %f610, %f475;\n" " fma.rn.f32 %f478, %f465, %f611, %f476;\n" " mov.b32 %r467, %f473;\n" " shl.b32 %r468, %r467, 23;\n" " mov.b32 %f479, %r468;\n" " ex2.approx.ftz.f32 %f480, %f478;\n" " mul.f32 %f481, %f480, %f479;\n" " add.s32 %r115, %r114, %r5;\n" " st.shared.f32 [%r115], %f481;\n" " add.f32 %f680, %f680, %f481;\n" " setp.eq.s32 %p131, %r104, 2;\n" " @%p131 bra $L__BB7_130;\n" "\n" " add.s32 %r504, %r627, %r36;\n" " shl.b32 %r503, %r504, 2;\n" " add.s32 %r502, %r26, %r503;\n" " mov.f32 %f617, 0f32A57060;\n" " mov.f32 %f616, 0f3FB8AA3B;\n" " mov.f32 %f615, 0f4B400001;\n" " mov.f32 %f614, 0f437C0000;\n" " mov.f32 %f613, 0f3F000000;\n" " mov.f32 %f612, 0f3BBB989D;\n" " ld.shared.f32 %f482, [%r502+8];\n" " sub.f32 %f483, %f482, %f676;\n" " fma.rn.f32 %f486, %f483, %f612, %f613;\n" " cvt.sat.f32.f32 %f489, %f486;\n" " fma.rm.f32 %f491, %f489, %f614, %f615;\n" " add.f32 %f492, %f491, 0fCB40007F;\n" " neg.f32 %f493, %f492;\n" " fma.rn.f32 %f494, %f483, %f616, %f493;\n" " fma.rn.f32 %f496, %f483, %f617, %f494;\n" " mov.b32 %r470, %f491;\n" " shl.b32 %r471, %r470, 23;\n" " mov.b32 %f497, %r471;\n" " ex2.approx.ftz.f32 %f498, %f496;\n" " mul.f32 %f499, %f498, %f497;\n" " add.s32 %r473, %r115, %r5;\n" " st.shared.f32 [%r473], %f499;\n" " add.f32 %f680, %f680, %f499;\n" "\n" "$L__BB7_130:\n" " sub.f32 %f500, %f91, %f676;\n" " mov.f32 %f501, 0f3F000000;\n" " mov.f32 %f502, 0f3BBB989D;\n" " fma.rn.f32 %f503, %f500, %f502, %f501;\n" " mov.f32 %f504, 0f3FB8AA3B;\n" " mov.f32 %f505, 0f437C0000;\n" " cvt.sat.f32.f32 %f506, %f503;\n" " mov.f32 %f507, 0f4B400001;\n" " fma.rm.f32 %f508, %f506, %f505, %f507;\n" " add.f32 %f509, %f508, 0fCB40007F;\n" " neg.f32 %f510, %f509;\n" " fma.rn.f32 %f511, %f500, %f504, %f510;\n" " mov.f32 %f512, 0f32A57060;\n" " fma.rn.f32 %f513, %f500, %f512, %f511;\n" " ex2.approx.ftz.f32 %f514, %f513;\n" " mov.b32 %r474, %f508;\n" " shl.b32 %r475, %r474, 23;\n" " mov.b32 %f515, %r475;\n" " mul.f32 %f516, %f514, %f515;\n" " ld.shared.f32 %f517, [%r28];\n" " fma.rn.f32 %f518, %f516, %f517, %f680;\n" " st.shared.f32 [%r28], %f518;\n" " st.shared.f32 [%r27], %f676;\n" " st.shared.f32 [%r38], %f516;\n" "\n" "$L__BB7_131:\n" " bar.sync 0;\n" " @%p18 bra $L__BB7_211;\n" "\n" " @%p19 bra $L__BB7_171;\n" " bra.uni $L__BB7_133;\n" "\n" "$L__BB7_171:\n" " mov.u32 %r629, 0;\n" "\n" "$L__BB7_172:\n" " add.s32 %r120, %r629, %r610;\n" " setp.lt.s32 %p153, %r120, %r130;\n" " @%p153 bra $L__BB7_176;\n" " bra.uni $L__BB7_173;\n" "\n" "$L__BB7_176:\n" " add.s32 %r602, %r629, %r610;\n" " cvt.s64.s32 %rd124, %r602;\n" " mul.lo.s64 %rd125, %rd124, %rd10;\n" " add.s64 %rd126, %rd27, %rd125;\n" " shl.b64 %rd127, %rd126, 1;\n" " add.s64 %rd128, %rd5, %rd127;\n" " ld.global.u16 %rs15, [%rd128];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs15;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB7_177;\n" "\n" "$L__BB7_173:\n" " add.s32 %r599, %r629, %r610;\n" " setp.gt.s32 %p154, %r599, %r33;\n" " setp.gt.s32 %p155, %r599, %r4;\n" " and.pred %p156, %p155, %p154;\n" " @%p156 bra $L__BB7_175;\n" " bra.uni $L__BB7_174;\n" "\n" "$L__BB7_175:\n" " add.s32 %r601, %r629, %r610;\n" " and.b32 %r495, %r601, 255;\n" " cvt.u64.u32 %rd119, %r495;\n" " mul.lo.s64 %rd120, %rd119, %rd10;\n" " add.s64 %rd121, %rd28, %rd120;\n" " shl.b64 %rd122, %rd121, 1;\n" " add.s64 %rd123, %rd31, %rd122;\n" " ld.global.u16 %rs14, [%rd123];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs14;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB7_177;\n" "\n" "$L__BB7_174:\n" " add.s32 %r600, %r629, %r610;\n" " cvt.s64.s32 %rd112, %r600;\n" " mul.lo.s64 %rd113, %rd112, %rd21;\n" " add.s64 %rd114, %rd113, %rd22;\n" " add.s64 %rd115, %rd114, %rd29;\n" " add.s64 %rd116, %rd6, %rd115;\n" " ld.global.u8 %r488, [%rd116];\n" " ld.global.u8 %r489, [%rd116+1];\n" " prmt.b32 %r490, %r489, %r488, 30212;\n" " cvt.u16.u32 %rs13, %r490;\n" " // begin inline asm\n" " { cvt.f32.f16 %f554, %rs13;}\n" "\n" " // end inline asm\n" " add.s64 %rd117, %rd30, %rd114;\n" " add.s64 %rd118, %rd6, %rd117;\n" " ld.global.u8 %r491, [%rd118];\n" " shr.u32 %r492, %r491, %r37;\n" " and.b32 %r493, %r492, 15;\n" " add.s32 %r494, %r493, -8;\n" " cvt.rn.f32.s32 %f555, %r494;\n" " mul.f32 %f720, %f554, %f555;\n" "\n" "$L__BB7_177:\n" " mad.lo.s32 %r496, %r629, %r126, %r7;\n" " shl.b32 %r497, %r496, 2;\n" " add.s32 %r121, %r26, %r497;\n" " setp.eq.s32 %p157, %r629, 0;\n" " @%p157 bra $L__BB7_194;\n" "\n" " @%p2 bra $L__BB7_180;\n" "\n" " ld.shared.f32 %f558, [%r121];\n" " fma.rn.f32 %f711, %f720, %f558, %f711;\n" "\n" "$L__BB7_180:\n" " @%p77 bra $L__BB7_182;\n" "\n" " ld.shared.f32 %f559, [%r121+4];\n" " fma.rn.f32 %f710, %f720, %f559, %f710;\n" "\n" "$L__BB7_182:\n" " @%p83 bra $L__BB7_184;\n" "\n" " ld.shared.f32 %f560, [%r121+8];\n" " fma.rn.f32 %f709, %f720, %f560, %f709;\n" "\n" "$L__BB7_184:\n" " @%p89 bra $L__BB7_186;\n" "\n" " ld.shared.f32 %f561, [%r121+12];\n" " fma.rn.f32 %f708, %f720, %f561, %f708;\n" "\n" "$L__BB7_186:\n" " @%p95 bra $L__BB7_188;\n" "\n" " ld.shared.f32 %f562, [%r121+16];\n" " fma.rn.f32 %f707, %f720, %f562, %f707;\n" "\n" "$L__BB7_188:\n" " @%p101 bra $L__BB7_190;\n" "\n" " ld.shared.f32 %f563, [%r121+20];\n" " fma.rn.f32 %f706, %f720, %f563, %f706;\n" "\n" "$L__BB7_190:\n" " @%p107 bra $L__BB7_192;\n" "\n" " ld.shared.f32 %f564, [%r121+24];\n" " fma.rn.f32 %f705, %f720, %f564, %f705;\n" "\n" "$L__BB7_192:\n" " @%p113 bra $L__BB7_210;\n" "\n" " ld.shared.f32 %f565, [%r121+28];\n" " fma.rn.f32 %f704, %f720, %f565, %f704;\n" " bra.uni $L__BB7_210;\n" "\n" "$L__BB7_194:\n" " @%p2 bra $L__BB7_196;\n" "\n" " ld.shared.f32 %f566, [%r51];\n" " ld.shared.f32 %f567, [%r121];\n" " mul.f32 %f568, %f720, %f567;\n" " fma.rn.f32 %f711, %f566, %f711, %f568;\n" "\n" "$L__BB7_196:\n" " @%p77 bra $L__BB7_198;\n" "\n" " ld.shared.f32 %f569, [%r51+4];\n" " ld.shared.f32 %f570, [%r121+4];\n" " mul.f32 %f571, %f720, %f570;\n" " fma.rn.f32 %f710, %f569, %f710, %f571;\n" "\n" "$L__BB7_198:\n" " @%p83 bra $L__BB7_200;\n" "\n" " ld.shared.f32 %f572, [%r51+8];\n" " ld.shared.f32 %f573, [%r121+8];\n" " mul.f32 %f574, %f720, %f573;\n" " fma.rn.f32 %f709, %f572, %f709, %f574;\n" "\n" "$L__BB7_200:\n" " @%p89 bra $L__BB7_202;\n" "\n" " ld.shared.f32 %f575, [%r51+12];\n" " ld.shared.f32 %f576, [%r121+12];\n" " mul.f32 %f577, %f720, %f576;\n" " fma.rn.f32 %f708, %f575, %f708, %f577;\n" "\n" "$L__BB7_202:\n" " @%p95 bra $L__BB7_204;\n" "\n" " ld.shared.f32 %f578, [%r51+16];\n" " ld.shared.f32 %f579, [%r121+16];\n" " mul.f32 %f580, %f720, %f579;\n" " fma.rn.f32 %f707, %f578, %f707, %f580;\n" "\n" "$L__BB7_204:\n" " @%p101 bra $L__BB7_206;\n" "\n" " ld.shared.f32 %f581, [%r51+20];\n" " ld.shared.f32 %f582, [%r121+20];\n" " mul.f32 %f583, %f720, %f582;\n" " fma.rn.f32 %f706, %f581, %f706, %f583;\n" "\n" "$L__BB7_206:\n" " @%p107 bra $L__BB7_208;\n" "\n" " ld.shared.f32 %f584, [%r51+24];\n" " ld.shared.f32 %f585, [%r121+24];\n" " mul.f32 %f586, %f720, %f585;\n" " fma.rn.f32 %f705, %f584, %f705, %f586;\n" "\n" "$L__BB7_208:\n" " @%p113 bra $L__BB7_210;\n" "\n" " ld.shared.f32 %f587, [%r51+28];\n" " ld.shared.f32 %f588, [%r121+28];\n" " mul.f32 %f589, %f720, %f588;\n" " fma.rn.f32 %f704, %f587, %f704, %f589;\n" "\n" "$L__BB7_210:\n" " add.s32 %r629, %r629, 1;\n" " setp.lt.s32 %p174, %r629, %r59;\n" " @%p174 bra $L__BB7_172;\n" " bra.uni $L__BB7_211;\n" "\n" "$L__BB7_133:\n" " mov.u32 %r628, 0;\n" "\n" "$L__BB7_134:\n" " add.s32 %r477, %r628, %r610;\n" " setp.lt.s32 %p134, %r477, %r130;\n" " cvt.s64.s32 %rd41, %r477;\n" " @%p134 bra $L__BB7_136;\n" " bra.uni $L__BB7_135;\n" "\n" "$L__BB7_136:\n" " mul.lo.s64 %rd108, %rd41, %rd10;\n" " add.s64 %rd109, %rd27, %rd108;\n" " shl.b64 %rd110, %rd109, 1;\n" " add.s64 %rd111, %rd5, %rd110;\n" " ld.global.u16 %rs12, [%rd111];\n" " // begin inline asm\n" " { cvt.f32.f16 %f689, %rs12;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB7_137;\n" "\n" "$L__BB7_135:\n" " mul.lo.s64 %rd102, %rd41, %rd21;\n" " add.s64 %rd103, %rd102, %rd22;\n" " add.s64 %rd104, %rd103, %rd29;\n" " add.s64 %rd105, %rd6, %rd104;\n" " ld.global.u8 %r478, [%rd105];\n" " ld.global.u8 %r479, [%rd105+1];\n" " prmt.b32 %r480, %r479, %r478, 30212;\n" " cvt.u16.u32 %rs11, %r480;\n" " // begin inline asm\n" " { cvt.f32.f16 %f519, %rs11;}\n" "\n" " // end inline asm\n" " add.s64 %rd106, %rd30, %rd103;\n" " add.s64 %rd107, %rd6, %rd106;\n" " ld.global.u8 %r481, [%rd107];\n" " shr.u32 %r482, %r481, %r37;\n" " and.b32 %r483, %r482, 15;\n" " add.s32 %r484, %r483, -8;\n" " cvt.rn.f32.s32 %f520, %r484;\n" " mul.f32 %f689, %f519, %f520;\n" "\n" "$L__BB7_137:\n" " mad.lo.s32 %r485, %r628, %r126, %r7;\n" " shl.b32 %r486, %r485, 2;\n" " add.s32 %r117, %r26, %r486;\n" " setp.eq.s32 %p135, %r628, 0;\n" " @%p135 bra $L__BB7_154;\n" "\n" " @%p2 bra $L__BB7_140;\n" "\n" " ld.shared.f32 %f522, [%r117];\n" " fma.rn.f32 %f711, %f689, %f522, %f711;\n" "\n" "$L__BB7_140:\n" " @%p77 bra $L__BB7_142;\n" "\n" " ld.shared.f32 %f523, [%r117+4];\n" " fma.rn.f32 %f710, %f689, %f523, %f710;\n" "\n" "$L__BB7_142:\n" " @%p83 bra $L__BB7_144;\n" "\n" " ld.shared.f32 %f524, [%r117+8];\n" " fma.rn.f32 %f709, %f689, %f524, %f709;\n" "\n" "$L__BB7_144:\n" " @%p89 bra $L__BB7_146;\n" "\n" " ld.shared.f32 %f525, [%r117+12];\n" " fma.rn.f32 %f708, %f689, %f525, %f708;\n" "\n" "$L__BB7_146:\n" " @%p95 bra $L__BB7_148;\n" "\n" " ld.shared.f32 %f526, [%r117+16];\n" " fma.rn.f32 %f707, %f689, %f526, %f707;\n" "\n" "$L__BB7_148:\n" " @%p101 bra $L__BB7_150;\n" "\n" " ld.shared.f32 %f527, [%r117+20];\n" " fma.rn.f32 %f706, %f689, %f527, %f706;\n" "\n" "$L__BB7_150:\n" " @%p107 bra $L__BB7_152;\n" "\n" " ld.shared.f32 %f528, [%r117+24];\n" " fma.rn.f32 %f705, %f689, %f528, %f705;\n" "\n" "$L__BB7_152:\n" " @%p113 bra $L__BB7_170;\n" "\n" " ld.shared.f32 %f529, [%r117+28];\n" " fma.rn.f32 %f704, %f689, %f529, %f704;\n" " bra.uni $L__BB7_170;\n" "\n" "$L__BB7_154:\n" " @%p2 bra $L__BB7_156;\n" "\n" " ld.shared.f32 %f530, [%r51];\n" " ld.shared.f32 %f531, [%r117];\n" " mul.f32 %f532, %f689, %f531;\n" " fma.rn.f32 %f711, %f530, %f711, %f532;\n" "\n" "$L__BB7_156:\n" " @%p77 bra $L__BB7_158;\n" "\n" " ld.shared.f32 %f533, [%r51+4];\n" " ld.shared.f32 %f534, [%r117+4];\n" " mul.f32 %f535, %f689, %f534;\n" " fma.rn.f32 %f710, %f533, %f710, %f535;\n" "\n" "$L__BB7_158:\n" " @%p83 bra $L__BB7_160;\n" "\n" " ld.shared.f32 %f536, [%r51+8];\n" " ld.shared.f32 %f537, [%r117+8];\n" " mul.f32 %f538, %f689, %f537;\n" " fma.rn.f32 %f709, %f536, %f709, %f538;\n" "\n" "$L__BB7_160:\n" " @%p89 bra $L__BB7_162;\n" "\n" " ld.shared.f32 %f539, [%r51+12];\n" " ld.shared.f32 %f540, [%r117+12];\n" " mul.f32 %f541, %f689, %f540;\n" " fma.rn.f32 %f708, %f539, %f708, %f541;\n" "\n" "$L__BB7_162:\n" " @%p95 bra $L__BB7_164;\n" "\n" " ld.shared.f32 %f542, [%r51+16];\n" " ld.shared.f32 %f543, [%r117+16];\n" " mul.f32 %f544, %f689, %f543;\n" " fma.rn.f32 %f707, %f542, %f707, %f544;\n" "\n" "$L__BB7_164:\n" " @%p101 bra $L__BB7_166;\n" "\n" " ld.shared.f32 %f545, [%r51+20];\n" " ld.shared.f32 %f546, [%r117+20];\n" " mul.f32 %f547, %f689, %f546;\n" " fma.rn.f32 %f706, %f545, %f706, %f547;\n" "\n" "$L__BB7_166:\n" " @%p107 bra $L__BB7_168;\n" "\n" " ld.shared.f32 %f548, [%r51+24];\n" " ld.shared.f32 %f549, [%r117+24];\n" " mul.f32 %f550, %f689, %f549;\n" " fma.rn.f32 %f705, %f548, %f705, %f550;\n" "\n" "$L__BB7_168:\n" " @%p113 bra $L__BB7_170;\n" "\n" " ld.shared.f32 %f551, [%r51+28];\n" " ld.shared.f32 %f552, [%r117+28];\n" " mul.f32 %f553, %f689, %f552;\n" " fma.rn.f32 %f704, %f551, %f704, %f553;\n" "\n" "$L__BB7_170:\n" " add.s32 %r628, %r628, 1;\n" " setp.lt.s32 %p152, %r628, %r59;\n" " @%p152 bra $L__BB7_134;\n" "\n" "$L__BB7_211:\n" " bar.sync 0;\n" "\n" "$L__BB7_212:\n" " add.s32 %r610, %r610, 16;\n" " setp.ge.s32 %p175, %r3, %r610;\n" " add.s32 %r609, %r609, 1;\n" " @%p175 bra $L__BB7_11;\n" "\n" "$L__BB7_213:\n" " shl.b32 %r515, %r126, 4;\n" " cvt.s64.s32 %rd168, %r126;\n" " mul.wide.s32 %rd167, %r515, 2;\n" " add.s64 %rd166, %rd167, %rd168;\n" " cvt.u32.u64 %r514, %rd166;\n" " cvt.s64.s32 %rd165, %r1;\n" " ld.param.u32 %r513, [fc_attn_flash_prefill_q4_kh_param_9];\n" " mov.u32 %r512, %ctaid.x;\n" " mov.u32 %r511, %nctaid.y;\n" " cvt.u64.u32 %rd164, %r511;\n" " cvt.s64.s32 %rd163, %r512;\n" " mov.u32 %r510, %ctaid.y;\n" " cvt.s64.s32 %rd162, %r510;\n" " mul.lo.s64 %rd161, %rd163, %rd164;\n" " add.s64 %rd160, %rd161, %rd162;\n" " mul.wide.s32 %rd159, %r513, %r126;\n" " mul.lo.s64 %rd158, %rd159, %rd160;\n" " add.s64 %rd42, %rd158, %rd165;\n" " shl.b32 %r498, %r514, 2;\n" " add.s32 %r125, %r26, %r498;\n" " @%p2 bra $L__BB7_215;\n" "\n" " ld.shared.f32 %f590, [%r125];\n" " div.rn.f32 %f591, %f711, %f590;\n" " shl.b64 %rd129, %rd42, 2;\n" " add.s64 %rd130, %rd2, %rd129;\n" " st.global.f32 [%rd130], %f591;\n" "\n" "$L__BB7_215:\n" " setp.lt.s32 %p177, %r126, 2;\n" " @%p177 bra $L__BB7_217;\n" "\n" " ld.shared.f32 %f592, [%r125+4];\n" " div.rn.f32 %f593, %f710, %f592;\n" " add.s64 %rd131, %rd42, %rd10;\n" " shl.b64 %rd132, %rd131, 2;\n" " add.s64 %rd133, %rd2, %rd132;\n" " st.global.f32 [%rd133], %f593;\n" "\n" "$L__BB7_217:\n" " setp.lt.s32 %p178, %r126, 3;\n" " @%p178 bra $L__BB7_219;\n" "\n" " ld.shared.f32 %f594, [%r125+8];\n" " div.rn.f32 %f595, %f709, %f594;\n" " shl.b64 %rd134, %rd10, 1;\n" " add.s64 %rd135, %rd42, %rd134;\n" " shl.b64 %rd136, %rd135, 2;\n" " add.s64 %rd137, %rd2, %rd136;\n" " st.global.f32 [%rd137], %f595;\n" "\n" "$L__BB7_219:\n" " setp.lt.s32 %p179, %r126, 4;\n" " @%p179 bra $L__BB7_221;\n" "\n" " ld.shared.f32 %f596, [%r125+12];\n" " div.rn.f32 %f597, %f708, %f596;\n" " mul.lo.s64 %rd138, %rd10, 3;\n" " add.s64 %rd139, %rd42, %rd138;\n" " shl.b64 %rd140, %rd139, 2;\n" " add.s64 %rd141, %rd2, %rd140;\n" " st.global.f32 [%rd141], %f597;\n" "\n" "$L__BB7_221:\n" " setp.lt.s32 %p180, %r126, 5;\n" " @%p180 bra $L__BB7_223;\n" "\n" " ld.shared.f32 %f598, [%r125+16];\n" " div.rn.f32 %f599, %f707, %f598;\n" " shl.b64 %rd142, %rd10, 2;\n" " add.s64 %rd143, %rd42, %rd142;\n" " shl.b64 %rd144, %rd143, 2;\n" " add.s64 %rd145, %rd2, %rd144;\n" " st.global.f32 [%rd145], %f599;\n" "\n" "$L__BB7_223:\n" " setp.lt.s32 %p181, %r126, 6;\n" " @%p181 bra $L__BB7_225;\n" "\n" " ld.shared.f32 %f600, [%r125+20];\n" " div.rn.f32 %f601, %f706, %f600;\n" " mul.lo.s64 %rd146, %rd10, 5;\n" " add.s64 %rd147, %rd42, %rd146;\n" " shl.b64 %rd148, %rd147, 2;\n" " add.s64 %rd149, %rd2, %rd148;\n" " st.global.f32 [%rd149], %f601;\n" "\n" "$L__BB7_225:\n" " setp.lt.s32 %p182, %r126, 7;\n" " @%p182 bra $L__BB7_227;\n" "\n" " ld.shared.f32 %f602, [%r125+24];\n" " div.rn.f32 %f603, %f705, %f602;\n" " mul.lo.s64 %rd150, %rd10, 6;\n" " add.s64 %rd151, %rd42, %rd150;\n" " shl.b64 %rd152, %rd151, 2;\n" " add.s64 %rd153, %rd2, %rd152;\n" " st.global.f32 [%rd153], %f603;\n" "\n" "$L__BB7_227:\n" " setp.lt.s32 %p183, %r126, 8;\n" " @%p183 bra $L__BB7_229;\n" "\n" " ld.shared.f32 %f604, [%r125+28];\n" " div.rn.f32 %f605, %f704, %f604;\n" " mul.lo.s64 %rd154, %rd10, 7;\n" " add.s64 %rd155, %rd42, %rd154;\n" " shl.b64 %rd156, %rd155, 2;\n" " add.s64 %rd157, %rd2, %rd156;\n" " st.global.f32 [%rd157], %f605;\n" "\n" "$L__BB7_229:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_prefill_q4_kqh\n" ".visible .entry fc_attn_flash_prefill_q4_kqh(\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_0,\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_1,\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_2,\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_3,\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_4,\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_5,\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_6,\n" " .param .u64 fc_attn_flash_prefill_q4_kqh_param_7,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_8,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_9,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_10,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_11,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_12,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_13,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_14,\n" " .param .u32 fc_attn_flash_prefill_q4_kqh_param_15,\n" " .param .f32 fc_attn_flash_prefill_q4_kqh_param_16\n" ")\n" "{\n" " .reg .pred %p<191>;\n" " .reg .b16 %rs<45>;\n" " .reg .f32 %f<767>;\n" " .reg .b32 %r<632>;\n" " .reg .b64 %rd<171>;\n" "\n" "\n" " ld.param.u64 %rd45, [fc_attn_flash_prefill_q4_kqh_param_0];\n" " ld.param.u64 %rd46, [fc_attn_flash_prefill_q4_kqh_param_1];\n" " ld.param.u64 %rd47, [fc_attn_flash_prefill_q4_kqh_param_2];\n" " ld.param.u64 %rd48, [fc_attn_flash_prefill_q4_kqh_param_3];\n" " ld.param.u64 %rd49, [fc_attn_flash_prefill_q4_kqh_param_4];\n" " ld.param.u64 %rd43, [fc_attn_flash_prefill_q4_kqh_param_5];\n" " ld.param.u64 %rd44, [fc_attn_flash_prefill_q4_kqh_param_6];\n" " ld.param.u64 %rd50, [fc_attn_flash_prefill_q4_kqh_param_7];\n" " ld.param.u32 %r127, [fc_attn_flash_prefill_q4_kqh_param_8];\n" " ld.param.u32 %r128, [fc_attn_flash_prefill_q4_kqh_param_9];\n" " ld.param.u32 %r129, [fc_attn_flash_prefill_q4_kqh_param_10];\n" " ld.param.u32 %r130, [fc_attn_flash_prefill_q4_kqh_param_11];\n" " ld.param.u32 %r131, [fc_attn_flash_prefill_q4_kqh_param_12];\n" " ld.param.u32 %r132, [fc_attn_flash_prefill_q4_kqh_param_13];\n" " ld.param.u32 %r134, [fc_attn_flash_prefill_q4_kqh_param_15];\n" " ld.param.f32 %f231, [fc_attn_flash_prefill_q4_kqh_param_16];\n" " cvta.to.global.u64 %rd1, %rd45;\n" " cvta.to.global.u64 %rd2, %rd50;\n" " cvta.to.global.u64 %rd3, %rd48;\n" " cvta.to.global.u64 %rd4, %rd46;\n" " cvta.to.global.u64 %rd5, %rd49;\n" " cvta.to.global.u64 %rd6, %rd47;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %ctaid.x;\n" " add.s32 %r3, %r2, %r130;\n" " mov.u32 %r135, %nctaid.x;\n" " add.s32 %r136, %r135, %r130;\n" " add.s32 %r4, %r136, -257;\n" " shl.b32 %r5, %r127, 1;\n" " add.s32 %r6, %r128, 16;\n" " shl.b32 %r7, %r127, 4;\n" " mul.wide.s32 %rd7, %r7, 2;\n" " cvt.s64.s32 %rd51, %r127;\n" " add.s64 %rd8, %rd7, %rd51;\n" " mov.u32 %r137, %nctaid.y;\n" " cvt.u64.u32 %rd52, %r137;\n" " cvt.s64.s32 %rd53, %r2;\n" " mul.lo.s64 %rd54, %rd53, %rd52;\n" " mov.u32 %r138, %ctaid.y;\n" " cvt.s64.s32 %rd9, %r138;\n" " add.s64 %rd55, %rd54, %rd9;\n" " cvt.s64.s32 %rd10, %r128;\n" " mul.wide.s32 %rd56, %r128, %r127;\n" " mul.lo.s64 %rd11, %rd56, %rd55;\n" " setp.lt.s32 %p2, %r127, 1;\n" " @%p2 bra $L__BB8_7;\n" "\n" " cvt.s64.s32 %rd12, %r1;\n" " and.b32 %r610, %r127, 3;\n" " add.s32 %r140, %r127, -1;\n" " setp.lt.u32 %p3, %r140, 3;\n" " mov.u32 %r608, 0;\n" " @%p3 bra $L__BB8_4;\n" "\n" " sub.s32 %r606, %r127, %r610;\n" " add.s64 %rd57, %rd11, %rd12;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd169, %rd1, %rd58;\n" " shl.b64 %rd14, %rd10, 2;\n" " shl.b32 %r142, %r1, 1;\n" " mov.u32 %r143, shb;\n" " add.s32 %r607, %r143, %r142;\n" " shl.b32 %r11, %r128, 1;\n" " mov.u32 %r608, 0;\n" "\n" "$L__BB8_3:\n" " ld.global.f32 %f232, [%rd169];\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs1, %f232;}\n" "\n" " // end inline asm\n" " st.shared.u16 [%r607], %rs1;\n" " add.s64 %rd59, %rd169, %rd14;\n" " ld.global.f32 %f233, [%rd59];\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs2, %f233;}\n" "\n" " // end inline asm\n" " add.s32 %r144, %r607, %r11;\n" " st.shared.u16 [%r144], %rs2;\n" " add.s64 %rd60, %rd59, %rd14;\n" " ld.global.f32 %f234, [%rd60];\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs3, %f234;}\n" "\n" " // end inline asm\n" " add.s32 %r145, %r144, %r11;\n" " st.shared.u16 [%r145], %rs3;\n" " add.s64 %rd61, %rd60, %rd14;\n" " add.s64 %rd169, %rd61, %rd14;\n" " ld.global.f32 %f235, [%rd61];\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs4, %f235;}\n" "\n" " // end inline asm\n" " add.s32 %r146, %r145, %r11;\n" " add.s32 %r607, %r146, %r11;\n" " st.shared.u16 [%r146], %rs4;\n" " add.s32 %r608, %r608, 4;\n" " add.s32 %r606, %r606, -4;\n" " setp.ne.s32 %p4, %r606, 0;\n" " @%p4 bra $L__BB8_3;\n" "\n" "$L__BB8_4:\n" " setp.eq.s32 %p5, %r610, 0;\n" " @%p5 bra $L__BB8_7;\n" "\n" " mul.lo.s32 %r147, %r608, %r128;\n" " shl.b32 %r148, %r147, 1;\n" " shl.b32 %r149, %r1, 1;\n" " add.s32 %r150, %r148, %r149;\n" " mov.u32 %r151, shb;\n" " add.s32 %r609, %r151, %r150;\n" " shl.b32 %r20, %r128, 1;\n" " cvt.s64.s32 %rd170, %r608;\n" " add.s64 %rd18, %rd11, %rd12;\n" "\n" "$L__BB8_6:\n" " .pragma \"nounroll\";\n" " mul.lo.s64 %rd62, %rd170, %rd10;\n" " add.s64 %rd63, %rd18, %rd62;\n" " shl.b64 %rd64, %rd63, 2;\n" " add.s64 %rd65, %rd1, %rd64;\n" " ld.global.f32 %f236, [%rd65];\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs5, %f236;}\n" "\n" " // end inline asm\n" " st.shared.u16 [%r609], %rs5;\n" " add.s32 %r609, %r609, %r20;\n" " add.s64 %rd170, %rd170, 1;\n" " add.s32 %r610, %r610, -1;\n" " setp.ne.s32 %p6, %r610, 0;\n" " @%p6 bra $L__BB8_6;\n" "\n" "$L__BB8_7:\n" " mov.u32 %r152, shb;\n" " mad.lo.s32 %r25, %r5, %r128, %r152;\n" " shl.b32 %r153, %r6, 5;\n" " add.s32 %r26, %r25, %r153;\n" " setp.ge.s32 %p7, %r1, %r127;\n" " cvt.u32.u64 %r154, %rd7;\n" " add.s32 %r155, %r1, %r154;\n" " shl.b32 %r156, %r155, 2;\n" " add.s32 %r27, %r26, %r156;\n" " cvt.u32.u64 %r157, %rd8;\n" " add.s32 %r158, %r1, %r157;\n" " shl.b32 %r159, %r158, 2;\n" " add.s32 %r28, %r26, %r159;\n" " @%p7 bra $L__BB8_9;\n" "\n" " mov.u32 %r160, -8388608;\n" " st.shared.u32 [%r27], %r160;\n" " mov.u32 %r161, 0;\n" " st.shared.u32 [%r28], %r161;\n" "\n" "$L__BB8_9:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd66, %r129;\n" " mul.lo.s64 %rd67, %rd66, %rd9;\n" " shr.s32 %r162, %r128, 31;\n" " shr.u32 %r163, %r162, 27;\n" " add.s32 %r164, %r128, %r163;\n" " shr.s32 %r165, %r164, 5;\n" " mul.lo.s32 %r166, %r165, 18;\n" " cvt.s64.s32 %rd21, %r166;\n" " mul.lo.s64 %rd22, %rd67, %rd21;\n" " mul.lo.s64 %rd68, %rd9, %rd10;\n" " shl.b64 %rd23, %rd68, 8;\n" " shl.b64 %rd24, %rd68, 3;\n" " shr.s32 %r167, %r1, 31;\n" " shr.u32 %r168, %r167, 28;\n" " add.s32 %r169, %r1, %r168;\n" " shr.s32 %r30, %r169, 4;\n" " and.b32 %r170, %r169, -16;\n" " sub.s32 %r31, %r1, %r170;\n" " cvt.s64.s32 %rd25, %r1;\n" " setp.lt.s32 %p8, %r3, 0;\n" " mov.f32 %f704, 0f00000000;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " @%p8 bra $L__BB8_213;\n" "\n" " shr.u32 %r174, %r162, 28;\n" " add.s32 %r175, %r128, %r174;\n" " shr.s32 %r176, %r175, 4;\n" " sub.s32 %r177, %r3, %r134;\n" " add.s32 %r178, %r177, 1;\n" " mov.u32 %r611, 0;\n" " max.s32 %r32, %r178, 0;\n" " cvt.s64.s32 %rd26, %r6;\n" " sub.s32 %r33, %r3, %r132;\n" " mov.u32 %r34, %ntid.x;\n" " mul.lo.s32 %r35, %r6, %r30;\n" " shl.b32 %r36, %r1, 4;\n" " add.s64 %rd27, %rd24, %rd25;\n" " add.s64 %rd28, %rd23, %rd25;\n" " shr.u32 %r180, %r167, 27;\n" " add.s32 %r181, %r1, %r180;\n" " shr.s32 %r182, %r181, 5;\n" " mul.lo.s32 %r183, %r182, 18;\n" " cvt.s64.s32 %rd29, %r183;\n" " and.b32 %r184, %r1, 31;\n" " setp.lt.u32 %p9, %r184, 16;\n" " add.s32 %r185, %r184, -16;\n" " selp.b32 %r186, %r184, %r185, %p9;\n" " selp.b32 %r37, 0, 4, %p9;\n" " add.s32 %r187, %r186, 2;\n" " cvt.s64.s32 %rd69, %r187;\n" " add.s64 %rd30, %rd69, %rd29;\n" " add.s32 %r188, %r157, %r127;\n" " add.s32 %r189, %r1, %r188;\n" " shl.b32 %r190, %r189, 2;\n" " add.s32 %r38, %r26, %r190;\n" " mov.u32 %r191, -2;\n" " sub.s32 %r192, %r191, %r2;\n" " sub.s32 %r40, %r192, %r130;\n" " bfe.u32 %r42, %r175, 4, 1;\n" " sub.s32 %r43, %r176, %r42;\n" " shl.b32 %r44, %r128, 1;\n" " mul.lo.s32 %r45, %r128, 3;\n" " shl.b32 %r46, %r128, 2;\n" " mul.lo.s32 %r47, %r128, 5;\n" " mul.lo.s32 %r48, %r128, 6;\n" " mul.lo.s32 %r49, %r128, 7;\n" " shl.b32 %r193, %r30, 2;\n" " add.s32 %r50, %r26, %r193;\n" " shl.b32 %r194, %r188, 2;\n" " add.s32 %r51, %r26, %r194;\n" " add.s32 %r195, %r5, 32;\n" " shl.b32 %r196, %r127, 6;\n" " mad.lo.s32 %r197, %r128, %r195, %r196;\n" " shl.b32 %r198, %r1, 2;\n" " add.s32 %r199, %r197, %r198;\n" " add.s32 %r52, %r152, %r199;\n" " shl.b32 %r53, %r127, 2;\n" " cvta.to.global.u64 %rd31, %rd44;\n" " cvta.to.global.u64 %rd32, %rd43;\n" " mov.f32 %f704, 0f00000000;\n" " setp.lt.s32 %p11, %r134, 1;\n" " setp.gt.s32 %p19, %r132, 0;\n" " setp.ge.s32 %p36, %r1, %r128;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " mov.u32 %r612, %r611;\n" "\n" "$L__BB8_11:\n" " ld.param.u32 %r507, [fc_attn_flash_prefill_q4_kqh_param_14];\n" " add.s32 %r506, %r3, 1;\n" " shl.b32 %r201, %r611, 4;\n" " add.s32 %r202, %r40, %r201;\n" " max.s32 %r203, %r202, -17;\n" " not.b32 %r204, %r203;\n" " max.s32 %r58, %r204, 1;\n" " sub.s32 %r59, %r506, %r612;\n" " min.s32 %r60, %r59, 16;\n" " setp.lt.s32 %p10, %r612, %r507;\n" " or.pred %p12, %p11, %p10;\n" " add.s32 %r205, %r60, %r612;\n" " setp.gt.s32 %p13, %r205, %r32;\n" " setp.le.s32 %p14, %r612, %r3;\n" " and.pred %p15, %p14, %p13;\n" " or.pred %p16, %p12, %p15;\n" " not.pred %p17, %p16;\n" " @%p17 bra $L__BB8_212;\n" "\n" " setp.lt.s32 %p18, %r59, 1;\n" " @%p18 bra $L__BB8_36;\n" "\n" " @%p19 bra $L__BB8_23;\n" " bra.uni $L__BB8_14;\n" "\n" "$L__BB8_23:\n" " mov.u32 %r616, 0;\n" "\n" "$L__BB8_24:\n" " cvt.s64.s32 %rd82, %r616;\n" " mul.lo.s64 %rd37, %rd82, %rd26;\n" " add.s32 %r70, %r616, %r612;\n" " setp.lt.s32 %p27, %r70, %r131;\n" " @%p27 bra $L__BB8_32;\n" " bra.uni $L__BB8_25;\n" "\n" "$L__BB8_32:\n" " @%p36 bra $L__BB8_35;\n" "\n" " cvt.u32.u64 %r77, %rd37;\n" " cvt.s64.s32 %rd96, %r70;\n" " mul.lo.s64 %rd97, %rd96, %rd10;\n" " add.s64 %rd40, %rd97, %rd24;\n" " mov.u32 %r619, %r1;\n" "\n" "$L__BB8_34:\n" " cvt.s64.s32 %rd98, %r619;\n" " add.s64 %rd99, %rd40, %rd98;\n" " shl.b64 %rd100, %rd99, 1;\n" " add.s64 %rd101, %rd3, %rd100;\n" " ld.global.u16 %rs12, [%rd101];\n" " add.s32 %r256, %r619, %r77;\n" " shl.b32 %r257, %r256, 1;\n" " add.s32 %r258, %r25, %r257;\n" " st.shared.u16 [%r258], %rs12;\n" " add.s32 %r619, %r619, %r34;\n" " setp.lt.s32 %p37, %r619, %r128;\n" " @%p37 bra $L__BB8_34;\n" " bra.uni $L__BB8_35;\n" "\n" "$L__BB8_25:\n" " setp.gt.s32 %p28, %r70, %r33;\n" " setp.gt.s32 %p29, %r70, %r4;\n" " and.pred %p30, %p29, %p28;\n" " @%p30 bra $L__BB8_29;\n" " bra.uni $L__BB8_26;\n" "\n" "$L__BB8_29:\n" " @%p36 bra $L__BB8_35;\n" "\n" " cvt.u32.u64 %r74, %rd37;\n" " and.b32 %r252, %r70, 255;\n" " cvt.u64.u32 %rd90, %r252;\n" " mul.lo.s64 %rd91, %rd90, %rd10;\n" " add.s64 %rd39, %rd91, %rd23;\n" " mov.u32 %r618, %r1;\n" "\n" "$L__BB8_31:\n" " cvt.s64.s32 %rd92, %r618;\n" " add.s64 %rd93, %rd39, %rd92;\n" " shl.b64 %rd94, %rd93, 1;\n" " add.s64 %rd95, %rd32, %rd94;\n" " ld.global.u16 %rs11, [%rd95];\n" " add.s32 %r253, %r618, %r74;\n" " shl.b32 %r254, %r253, 1;\n" " add.s32 %r255, %r25, %r254;\n" " st.shared.u16 [%r255], %rs11;\n" " add.s32 %r618, %r618, %r34;\n" " setp.lt.s32 %p35, %r618, %r128;\n" " @%p35 bra $L__BB8_31;\n" " bra.uni $L__BB8_35;\n" "\n" "$L__BB8_26:\n" " @%p36 bra $L__BB8_35;\n" "\n" " cvt.u32.u64 %r71, %rd37;\n" " cvt.s64.s32 %rd83, %r70;\n" " mul.lo.s64 %rd84, %rd83, %rd21;\n" " add.s64 %rd38, %rd84, %rd22;\n" " mov.u32 %r617, %r1;\n" "\n" "$L__BB8_28:\n" " shr.s32 %r232, %r617, 31;\n" " shr.u32 %r233, %r232, 27;\n" " add.s32 %r234, %r617, %r233;\n" " shr.s32 %r235, %r234, 5;\n" " mul.lo.s32 %r236, %r235, 18;\n" " cvt.s64.s32 %rd85, %r236;\n" " add.s64 %rd86, %rd38, %rd85;\n" " add.s64 %rd87, %rd4, %rd86;\n" " ld.global.u8 %r237, [%rd87];\n" " ld.global.u8 %r238, [%rd87+1];\n" " prmt.b32 %r239, %r238, %r237, 30212;\n" " cvt.u16.u32 %rs9, %r239;\n" " // begin inline asm\n" " { cvt.f32.f16 %f256, %rs9;}\n" "\n" " // end inline asm\n" " and.b32 %r240, %r617, 31;\n" " setp.lt.u32 %p32, %r240, 16;\n" " add.s32 %r241, %r240, -16;\n" " selp.b32 %r242, %r240, %r241, %p32;\n" " selp.b32 %r243, 0, 4, %p32;\n" " add.s32 %r244, %r242, 2;\n" " cvt.s64.s32 %rd88, %r244;\n" " add.s64 %rd89, %rd87, %rd88;\n" " ld.global.u8 %r245, [%rd89];\n" " shr.u32 %r246, %r245, %r243;\n" " and.b32 %r247, %r246, 15;\n" " add.s32 %r248, %r247, -8;\n" " cvt.rn.f32.s32 %f258, %r248;\n" " mul.f32 %f257, %f256, %f258;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs10, %f257;}\n" "\n" " // end inline asm\n" " add.s32 %r249, %r617, %r71;\n" " shl.b32 %r250, %r249, 1;\n" " add.s32 %r251, %r25, %r250;\n" " st.shared.u16 [%r251], %rs10;\n" " add.s32 %r617, %r617, %r34;\n" " setp.lt.s32 %p33, %r617, %r128;\n" " @%p33 bra $L__BB8_28;\n" "\n" "$L__BB8_35:\n" " add.s32 %r616, %r616, 1;\n" " setp.lt.s32 %p38, %r616, %r60;\n" " @%p38 bra $L__BB8_24;\n" " bra.uni $L__BB8_36;\n" "\n" "$L__BB8_14:\n" " mov.u32 %r613, 0;\n" "\n" "$L__BB8_15:\n" " cvt.s64.s32 %rd70, %r613;\n" " mul.lo.s64 %rd33, %rd70, %rd26;\n" " add.s32 %r207, %r613, %r612;\n" " setp.lt.s32 %p20, %r207, %r131;\n" " cvt.s64.s32 %rd34, %r207;\n" " @%p20 bra $L__BB8_19;\n" " bra.uni $L__BB8_16;\n" "\n" "$L__BB8_19:\n" " @%p36 bra $L__BB8_22;\n" "\n" " cvt.u32.u64 %r65, %rd33;\n" " mul.lo.s64 %rd77, %rd34, %rd10;\n" " add.s64 %rd36, %rd77, %rd24;\n" " mov.u32 %r615, %r1;\n" "\n" "$L__BB8_21:\n" " cvt.s64.s32 %rd78, %r615;\n" " add.s64 %rd79, %rd36, %rd78;\n" " shl.b64 %rd80, %rd79, 1;\n" " add.s64 %rd81, %rd3, %rd80;\n" " ld.global.u16 %rs8, [%rd81];\n" " add.s32 %r228, %r615, %r65;\n" " shl.b32 %r229, %r228, 1;\n" " add.s32 %r230, %r25, %r229;\n" " st.shared.u16 [%r230], %rs8;\n" " add.s32 %r615, %r615, %r34;\n" " setp.lt.s32 %p25, %r615, %r128;\n" " @%p25 bra $L__BB8_21;\n" " bra.uni $L__BB8_22;\n" "\n" "$L__BB8_16:\n" " @%p36 bra $L__BB8_22;\n" "\n" " cvt.u32.u64 %r62, %rd33;\n" " mul.lo.s64 %rd71, %rd34, %rd21;\n" " add.s64 %rd35, %rd71, %rd22;\n" " mov.u32 %r614, %r1;\n" "\n" "$L__BB8_18:\n" " shr.s32 %r208, %r614, 31;\n" " shr.u32 %r209, %r208, 27;\n" " add.s32 %r210, %r614, %r209;\n" " shr.s32 %r211, %r210, 5;\n" " mul.lo.s32 %r212, %r211, 18;\n" " cvt.s64.s32 %rd72, %r212;\n" " add.s64 %rd73, %rd35, %rd72;\n" " add.s64 %rd74, %rd4, %rd73;\n" " ld.global.u8 %r213, [%rd74];\n" " ld.global.u8 %r214, [%rd74+1];\n" " prmt.b32 %r215, %r214, %r213, 30212;\n" " cvt.u16.u32 %rs6, %r215;\n" " // begin inline asm\n" " { cvt.f32.f16 %f253, %rs6;}\n" "\n" " // end inline asm\n" " and.b32 %r216, %r614, 31;\n" " setp.lt.u32 %p22, %r216, 16;\n" " add.s32 %r217, %r216, -16;\n" " selp.b32 %r218, %r216, %r217, %p22;\n" " selp.b32 %r219, 0, 4, %p22;\n" " add.s32 %r220, %r218, 2;\n" " cvt.s64.s32 %rd75, %r220;\n" " add.s64 %rd76, %rd74, %rd75;\n" " ld.global.u8 %r221, [%rd76];\n" " shr.u32 %r222, %r221, %r219;\n" " and.b32 %r223, %r222, 15;\n" " add.s32 %r224, %r223, -8;\n" " cvt.rn.f32.s32 %f255, %r224;\n" " mul.f32 %f254, %f253, %f255;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs7, %f254;}\n" "\n" " // end inline asm\n" " add.s32 %r225, %r614, %r62;\n" " shl.b32 %r226, %r225, 1;\n" " add.s32 %r227, %r25, %r226;\n" " st.shared.u16 [%r227], %rs7;\n" " add.s32 %r614, %r614, %r34;\n" " setp.lt.s32 %p23, %r614, %r128;\n" " @%p23 bra $L__BB8_18;\n" "\n" "$L__BB8_22:\n" " add.s32 %r613, %r613, 1;\n" " setp.lt.s32 %p26, %r613, %r60;\n" " @%p26 bra $L__BB8_15;\n" "\n" "$L__BB8_36:\n" " setp.lt.s32 %p39, %r128, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p40, %r30, %r60;\n" " or.pred %p41, %p39, %p40;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p41 bra $L__BB8_89;\n" "\n" " and.b32 %r508, %r128, -16;\n" " setp.eq.s32 %p42, %r508, 16;\n" " mov.u32 %r622, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p42 bra $L__BB8_72;\n" "\n" " mov.u32 %r622, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " mov.u32 %r621, %r43;\n" "\n" "$L__BB8_39:\n" " shl.b32 %r261, %r622, 4;\n" " add.s32 %r83, %r261, %r31;\n" " add.s32 %r262, %r83, %r35;\n" " shl.b32 %r263, %r262, 1;\n" " add.s32 %r84, %r25, %r263;\n" " ld.shared.u16 %rs13, [%r84];\n" " // begin inline asm\n" " { cvt.f32.f16 %f283, %rs13;}\n" "\n" " // end inline asm\n" " @%p2 bra $L__BB8_41;\n" "\n" " shl.b32 %r595, %r622, 4;\n" " add.s32 %r594, %r595, %r31;\n" " shl.b32 %r593, %r594, 1;\n" " add.s32 %r592, %r152, %r593;\n" " ld.shared.u16 %rs14, [%r592];\n" " // begin inline asm\n" " { cvt.f32.f16 %f284, %rs14;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f634, %f283, %f284, %f634;\n" "\n" "$L__BB8_41:\n" " setp.lt.s32 %p44, %r127, 2;\n" " @%p44 bra $L__BB8_43;\n" "\n" " shl.b32 %r600, %r622, 4;\n" " add.s32 %r599, %r600, %r31;\n" " add.s32 %r598, %r599, %r128;\n" " shl.b32 %r597, %r598, 1;\n" " add.s32 %r596, %r152, %r597;\n" " ld.shared.u16 %rs15, [%r596];\n" " // begin inline asm\n" " { cvt.f32.f16 %f285, %rs15;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f635, %f283, %f285, %f635;\n" "\n" "$L__BB8_43:\n" " setp.lt.s32 %p45, %r127, 3;\n" " @%p45 bra $L__BB8_45;\n" "\n" " shl.b32 %r541, %r622, 4;\n" " add.s32 %r540, %r541, %r31;\n" " add.s32 %r539, %r44, %r540;\n" " shl.b32 %r538, %r539, 1;\n" " add.s32 %r537, %r152, %r538;\n" " ld.shared.u16 %rs16, [%r537];\n" " // begin inline asm\n" " { cvt.f32.f16 %f286, %rs16;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f636, %f283, %f286, %f636;\n" "\n" "$L__BB8_45:\n" " setp.lt.s32 %p46, %r127, 4;\n" " @%p46 bra $L__BB8_47;\n" "\n" " shl.b32 %r551, %r622, 4;\n" " add.s32 %r550, %r551, %r31;\n" " add.s32 %r549, %r45, %r550;\n" " shl.b32 %r548, %r549, 1;\n" " add.s32 %r547, %r152, %r548;\n" " ld.shared.u16 %rs17, [%r547];\n" " // begin inline asm\n" " { cvt.f32.f16 %f287, %rs17;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f637, %f283, %f287, %f637;\n" "\n" "$L__BB8_47:\n" " setp.lt.s32 %p47, %r127, 5;\n" " @%p47 bra $L__BB8_49;\n" "\n" " shl.b32 %r561, %r622, 4;\n" " add.s32 %r560, %r561, %r31;\n" " add.s32 %r559, %r46, %r560;\n" " shl.b32 %r558, %r559, 1;\n" " add.s32 %r557, %r152, %r558;\n" " ld.shared.u16 %rs18, [%r557];\n" " // begin inline asm\n" " { cvt.f32.f16 %f288, %rs18;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f638, %f283, %f288, %f638;\n" "\n" "$L__BB8_49:\n" " setp.lt.s32 %p48, %r127, 6;\n" " @%p48 bra $L__BB8_51;\n" "\n" " shl.b32 %r571, %r622, 4;\n" " add.s32 %r570, %r571, %r31;\n" " add.s32 %r569, %r47, %r570;\n" " shl.b32 %r568, %r569, 1;\n" " add.s32 %r567, %r152, %r568;\n" " ld.shared.u16 %rs19, [%r567];\n" " // begin inline asm\n" " { cvt.f32.f16 %f289, %rs19;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f639, %f283, %f289, %f639;\n" "\n" "$L__BB8_51:\n" " setp.lt.s32 %p49, %r127, 7;\n" " @%p49 bra $L__BB8_53;\n" "\n" " shl.b32 %r581, %r622, 4;\n" " add.s32 %r580, %r581, %r31;\n" " add.s32 %r579, %r48, %r580;\n" " shl.b32 %r578, %r579, 1;\n" " add.s32 %r577, %r152, %r578;\n" " ld.shared.u16 %rs20, [%r577];\n" " // begin inline asm\n" " { cvt.f32.f16 %f290, %rs20;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f640, %f283, %f290, %f640;\n" "\n" "$L__BB8_53:\n" " setp.lt.s32 %p50, %r127, 8;\n" " @%p50 bra $L__BB8_55;\n" "\n" " shl.b32 %r591, %r622, 4;\n" " add.s32 %r590, %r591, %r31;\n" " add.s32 %r589, %r49, %r590;\n" " shl.b32 %r588, %r589, 1;\n" " add.s32 %r587, %r152, %r588;\n" " ld.shared.u16 %rs21, [%r587];\n" " // begin inline asm\n" " { cvt.f32.f16 %f291, %rs21;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f641, %f283, %f291, %f641;\n" "\n" "$L__BB8_55:\n" " shl.b32 %r522, %r622, 4;\n" " add.s32 %r521, %r522, %r31;\n" " add.s32 %r520, %r521, %r35;\n" " shl.b32 %r519, %r520, 1;\n" " add.s32 %r518, %r25, %r519;\n" " ld.shared.u16 %rs22, [%r518+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f292, %rs22;}\n" "\n" " // end inline asm\n" " @%p2 bra $L__BB8_57;\n" "\n" " shl.b32 %r536, %r622, 4;\n" " add.s32 %r535, %r536, %r31;\n" " shl.b32 %r534, %r535, 1;\n" " add.s32 %r533, %r152, %r534;\n" " ld.shared.u16 %rs23, [%r533+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f293, %rs23;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f634, %f292, %f293, %f634;\n" "\n" "$L__BB8_57:\n" " setp.lt.s32 %p184, %r127, 2;\n" " @%p184 bra $L__BB8_59;\n" "\n" " shl.b32 %r532, %r622, 4;\n" " add.s32 %r531, %r532, %r31;\n" " add.s32 %r530, %r531, %r128;\n" " shl.b32 %r529, %r530, 1;\n" " add.s32 %r528, %r152, %r529;\n" " ld.shared.u16 %rs24, [%r528+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f294, %rs24;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f635, %f292, %f294, %f635;\n" "\n" "$L__BB8_59:\n" " setp.lt.s32 %p185, %r127, 3;\n" " @%p185 bra $L__BB8_61;\n" "\n" " shl.b32 %r527, %r622, 4;\n" " add.s32 %r526, %r527, %r31;\n" " add.s32 %r525, %r44, %r526;\n" " shl.b32 %r524, %r525, 1;\n" " add.s32 %r523, %r152, %r524;\n" " ld.shared.u16 %rs25, [%r523+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f295, %rs25;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f636, %f292, %f295, %f636;\n" "\n" "$L__BB8_61:\n" " setp.lt.s32 %p186, %r127, 4;\n" " @%p186 bra $L__BB8_63;\n" "\n" " shl.b32 %r546, %r622, 4;\n" " add.s32 %r545, %r546, %r31;\n" " add.s32 %r544, %r45, %r545;\n" " shl.b32 %r543, %r544, 1;\n" " add.s32 %r542, %r152, %r543;\n" " ld.shared.u16 %rs26, [%r542+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f296, %rs26;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f637, %f292, %f296, %f637;\n" "\n" "$L__BB8_63:\n" " setp.lt.s32 %p187, %r127, 5;\n" " @%p187 bra $L__BB8_65;\n" "\n" " shl.b32 %r556, %r622, 4;\n" " add.s32 %r555, %r556, %r31;\n" " add.s32 %r554, %r46, %r555;\n" " shl.b32 %r553, %r554, 1;\n" " add.s32 %r552, %r152, %r553;\n" " ld.shared.u16 %rs27, [%r552+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f297, %rs27;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f638, %f292, %f297, %f638;\n" "\n" "$L__BB8_65:\n" " setp.lt.s32 %p188, %r127, 6;\n" " @%p188 bra $L__BB8_67;\n" "\n" " shl.b32 %r566, %r622, 4;\n" " add.s32 %r565, %r566, %r31;\n" " add.s32 %r564, %r47, %r565;\n" " shl.b32 %r563, %r564, 1;\n" " add.s32 %r562, %r152, %r563;\n" " ld.shared.u16 %rs28, [%r562+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f298, %rs28;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f639, %f292, %f298, %f639;\n" "\n" "$L__BB8_67:\n" " setp.lt.s32 %p189, %r127, 7;\n" " @%p189 bra $L__BB8_69;\n" "\n" " shl.b32 %r576, %r622, 4;\n" " add.s32 %r575, %r576, %r31;\n" " add.s32 %r574, %r48, %r575;\n" " shl.b32 %r573, %r574, 1;\n" " add.s32 %r572, %r152, %r573;\n" " ld.shared.u16 %rs29, [%r572+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f299, %rs29;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f640, %f292, %f299, %f640;\n" "\n" "$L__BB8_69:\n" " setp.lt.s32 %p190, %r127, 8;\n" " @%p190 bra $L__BB8_71;\n" "\n" " shl.b32 %r586, %r622, 4;\n" " add.s32 %r585, %r586, %r31;\n" " add.s32 %r584, %r49, %r585;\n" " shl.b32 %r583, %r584, 1;\n" " add.s32 %r582, %r152, %r583;\n" " ld.shared.u16 %rs30, [%r582+32];\n" " // begin inline asm\n" " { cvt.f32.f16 %f300, %rs30;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f641, %f292, %f300, %f641;\n" "\n" "$L__BB8_71:\n" " add.s32 %r622, %r622, 2;\n" " add.s32 %r621, %r621, -2;\n" " setp.ne.s32 %p59, %r621, 0;\n" " @%p59 bra $L__BB8_39;\n" "\n" "$L__BB8_72:\n" " setp.eq.s32 %p60, %r42, 0;\n" " @%p60 bra $L__BB8_89;\n" "\n" " shl.b32 %r287, %r622, 4;\n" " add.s32 %r96, %r287, %r31;\n" " add.s32 %r288, %r96, %r35;\n" " shl.b32 %r289, %r288, 1;\n" " add.s32 %r290, %r25, %r289;\n" " ld.shared.u16 %rs31, [%r290];\n" " // begin inline asm\n" " { cvt.f32.f16 %f301, %rs31;}\n" "\n" " // end inline asm\n" " @%p2 bra $L__BB8_75;\n" "\n" " shl.b32 %r291, %r96, 1;\n" " add.s32 %r293, %r152, %r291;\n" " ld.shared.u16 %rs32, [%r293];\n" " // begin inline asm\n" " { cvt.f32.f16 %f302, %rs32;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f634, %f301, %f302, %f634;\n" "\n" "$L__BB8_75:\n" " setp.lt.s32 %p62, %r127, 2;\n" " @%p62 bra $L__BB8_77;\n" "\n" " add.s32 %r294, %r96, %r128;\n" " shl.b32 %r295, %r294, 1;\n" " add.s32 %r297, %r152, %r295;\n" " ld.shared.u16 %rs33, [%r297];\n" " // begin inline asm\n" " { cvt.f32.f16 %f303, %rs33;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f635, %f301, %f303, %f635;\n" "\n" "$L__BB8_77:\n" " setp.lt.s32 %p63, %r127, 3;\n" " @%p63 bra $L__BB8_79;\n" "\n" " add.s32 %r298, %r44, %r96;\n" " shl.b32 %r299, %r298, 1;\n" " add.s32 %r301, %r152, %r299;\n" " ld.shared.u16 %rs34, [%r301];\n" " // begin inline asm\n" " { cvt.f32.f16 %f304, %rs34;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f636, %f301, %f304, %f636;\n" "\n" "$L__BB8_79:\n" " setp.lt.s32 %p64, %r127, 4;\n" " @%p64 bra $L__BB8_81;\n" "\n" " add.s32 %r302, %r45, %r96;\n" " shl.b32 %r303, %r302, 1;\n" " add.s32 %r305, %r152, %r303;\n" " ld.shared.u16 %rs35, [%r305];\n" " // begin inline asm\n" " { cvt.f32.f16 %f305, %rs35;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f637, %f301, %f305, %f637;\n" "\n" "$L__BB8_81:\n" " setp.lt.s32 %p65, %r127, 5;\n" " @%p65 bra $L__BB8_83;\n" "\n" " add.s32 %r306, %r46, %r96;\n" " shl.b32 %r307, %r306, 1;\n" " add.s32 %r309, %r152, %r307;\n" " ld.shared.u16 %rs36, [%r309];\n" " // begin inline asm\n" " { cvt.f32.f16 %f306, %rs36;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f638, %f301, %f306, %f638;\n" "\n" "$L__BB8_83:\n" " setp.lt.s32 %p66, %r127, 6;\n" " @%p66 bra $L__BB8_85;\n" "\n" " add.s32 %r310, %r47, %r96;\n" " shl.b32 %r311, %r310, 1;\n" " add.s32 %r313, %r152, %r311;\n" " ld.shared.u16 %rs37, [%r313];\n" " // begin inline asm\n" " { cvt.f32.f16 %f307, %rs37;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f639, %f301, %f307, %f639;\n" "\n" "$L__BB8_85:\n" " setp.lt.s32 %p67, %r127, 7;\n" " @%p67 bra $L__BB8_87;\n" "\n" " add.s32 %r314, %r48, %r96;\n" " shl.b32 %r315, %r314, 1;\n" " add.s32 %r317, %r152, %r315;\n" " ld.shared.u16 %rs38, [%r317];\n" " // begin inline asm\n" " { cvt.f32.f16 %f308, %rs38;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f640, %f301, %f308, %f640;\n" "\n" "$L__BB8_87:\n" " setp.lt.s32 %p68, %r127, 8;\n" " @%p68 bra $L__BB8_89;\n" "\n" " add.s32 %r318, %r49, %r96;\n" " shl.b32 %r319, %r318, 1;\n" " add.s32 %r321, %r152, %r319;\n" " ld.shared.u16 %rs39, [%r321];\n" " // begin inline asm\n" " { cvt.f32.f16 %f309, %rs39;}\n" "\n" " // end inline asm\n" " fma.rn.f32 %f641, %f301, %f309, %f641;\n" "\n" "$L__BB8_89:\n" " setp.eq.s32 %p69, %r31, 0;\n" " setp.lt.s32 %p70, %r30, %r60;\n" " and.pred %p1, %p69, %p70;\n" " @%p2 bra $L__BB8_92;\n" "\n" " mov.b32 %r322, %f634;\n" " mov.u32 %r323, 2;\n" " mov.u32 %r324, 4127;\n" " mov.u32 %r325, 8;\n" " mov.u32 %r326, -1;\n" " shfl.sync.down.b32 %r327|%p72, %r322, %r325, %r324, %r326;\n" " mov.b32 %f310, %r327;\n" " add.f32 %f311, %f634, %f310;\n" " mov.b32 %r328, %f311;\n" " mov.u32 %r329, 4;\n" " shfl.sync.down.b32 %r330|%p73, %r328, %r329, %r324, %r326;\n" " mov.b32 %f312, %r330;\n" " add.f32 %f313, %f311, %f312;\n" " mov.b32 %r331, %f313;\n" " shfl.sync.down.b32 %r332|%p74, %r331, %r323, %r324, %r326;\n" " mov.b32 %f314, %r332;\n" " add.f32 %f315, %f313, %f314;\n" " mov.b32 %r333, %f315;\n" " mov.u32 %r334, 1;\n" " shfl.sync.down.b32 %r335|%p75, %r333, %r334, %r324, %r326;\n" " mov.b32 %f316, %r335;\n" " add.f32 %f83, %f315, %f316;\n" " not.pred %p76, %p1;\n" " @%p76 bra $L__BB8_92;\n" "\n" " mul.f32 %f317, %f83, %f231;\n" " st.shared.f32 [%r50], %f317;\n" "\n" "$L__BB8_92:\n" " setp.lt.s32 %p77, %r127, 2;\n" " @%p77 bra $L__BB8_95;\n" "\n" " mov.b32 %r336, %f635;\n" " mov.u32 %r337, 2;\n" " mov.u32 %r338, 4127;\n" " mov.u32 %r339, 8;\n" " mov.u32 %r340, -1;\n" " shfl.sync.down.b32 %r341|%p78, %r336, %r339, %r338, %r340;\n" " mov.b32 %f318, %r341;\n" " add.f32 %f319, %f635, %f318;\n" " mov.b32 %r342, %f319;\n" " mov.u32 %r343, 4;\n" " shfl.sync.down.b32 %r344|%p79, %r342, %r343, %r338, %r340;\n" " mov.b32 %f320, %r344;\n" " add.f32 %f321, %f319, %f320;\n" " mov.b32 %r345, %f321;\n" " shfl.sync.down.b32 %r346|%p80, %r345, %r337, %r338, %r340;\n" " mov.b32 %f322, %r346;\n" " add.f32 %f323, %f321, %f322;\n" " mov.b32 %r347, %f323;\n" " mov.u32 %r348, 1;\n" " shfl.sync.down.b32 %r349|%p81, %r347, %r348, %r338, %r340;\n" " mov.b32 %f324, %r349;\n" " add.f32 %f84, %f323, %f324;\n" " not.pred %p82, %p1;\n" " @%p82 bra $L__BB8_95;\n" "\n" " mul.f32 %f325, %f84, %f231;\n" " st.shared.f32 [%r50+64], %f325;\n" "\n" "$L__BB8_95:\n" " setp.lt.s32 %p83, %r127, 3;\n" " @%p83 bra $L__BB8_98;\n" "\n" " mov.b32 %r350, %f636;\n" " mov.u32 %r351, 2;\n" " mov.u32 %r352, 4127;\n" " mov.u32 %r353, 8;\n" " mov.u32 %r354, -1;\n" " shfl.sync.down.b32 %r355|%p84, %r350, %r353, %r352, %r354;\n" " mov.b32 %f326, %r355;\n" " add.f32 %f327, %f636, %f326;\n" " mov.b32 %r356, %f327;\n" " mov.u32 %r357, 4;\n" " shfl.sync.down.b32 %r358|%p85, %r356, %r357, %r352, %r354;\n" " mov.b32 %f328, %r358;\n" " add.f32 %f329, %f327, %f328;\n" " mov.b32 %r359, %f329;\n" " shfl.sync.down.b32 %r360|%p86, %r359, %r351, %r352, %r354;\n" " mov.b32 %f330, %r360;\n" " add.f32 %f331, %f329, %f330;\n" " mov.b32 %r361, %f331;\n" " mov.u32 %r362, 1;\n" " shfl.sync.down.b32 %r363|%p87, %r361, %r362, %r352, %r354;\n" " mov.b32 %f332, %r363;\n" " add.f32 %f85, %f331, %f332;\n" " not.pred %p88, %p1;\n" " @%p88 bra $L__BB8_98;\n" "\n" " mul.f32 %f333, %f85, %f231;\n" " st.shared.f32 [%r50+128], %f333;\n" "\n" "$L__BB8_98:\n" " setp.lt.s32 %p89, %r127, 4;\n" " @%p89 bra $L__BB8_101;\n" "\n" " mov.b32 %r364, %f637;\n" " mov.u32 %r365, 2;\n" " mov.u32 %r366, 4127;\n" " mov.u32 %r367, 8;\n" " mov.u32 %r368, -1;\n" " shfl.sync.down.b32 %r369|%p90, %r364, %r367, %r366, %r368;\n" " mov.b32 %f334, %r369;\n" " add.f32 %f335, %f637, %f334;\n" " mov.b32 %r370, %f335;\n" " mov.u32 %r371, 4;\n" " shfl.sync.down.b32 %r372|%p91, %r370, %r371, %r366, %r368;\n" " mov.b32 %f336, %r372;\n" " add.f32 %f337, %f335, %f336;\n" " mov.b32 %r373, %f337;\n" " shfl.sync.down.b32 %r374|%p92, %r373, %r365, %r366, %r368;\n" " mov.b32 %f338, %r374;\n" " add.f32 %f339, %f337, %f338;\n" " mov.b32 %r375, %f339;\n" " mov.u32 %r376, 1;\n" " shfl.sync.down.b32 %r377|%p93, %r375, %r376, %r366, %r368;\n" " mov.b32 %f340, %r377;\n" " add.f32 %f86, %f339, %f340;\n" " not.pred %p94, %p1;\n" " @%p94 bra $L__BB8_101;\n" "\n" " mul.f32 %f341, %f86, %f231;\n" " st.shared.f32 [%r50+192], %f341;\n" "\n" "$L__BB8_101:\n" " setp.lt.s32 %p95, %r127, 5;\n" " @%p95 bra $L__BB8_104;\n" "\n" " mov.b32 %r378, %f638;\n" " mov.u32 %r379, 2;\n" " mov.u32 %r380, 4127;\n" " mov.u32 %r381, 8;\n" " mov.u32 %r382, -1;\n" " shfl.sync.down.b32 %r383|%p96, %r378, %r381, %r380, %r382;\n" " mov.b32 %f342, %r383;\n" " add.f32 %f343, %f638, %f342;\n" " mov.b32 %r384, %f343;\n" " mov.u32 %r385, 4;\n" " shfl.sync.down.b32 %r386|%p97, %r384, %r385, %r380, %r382;\n" " mov.b32 %f344, %r386;\n" " add.f32 %f345, %f343, %f344;\n" " mov.b32 %r387, %f345;\n" " shfl.sync.down.b32 %r388|%p98, %r387, %r379, %r380, %r382;\n" " mov.b32 %f346, %r388;\n" " add.f32 %f347, %f345, %f346;\n" " mov.b32 %r389, %f347;\n" " mov.u32 %r390, 1;\n" " shfl.sync.down.b32 %r391|%p99, %r389, %r390, %r380, %r382;\n" " mov.b32 %f348, %r391;\n" " add.f32 %f87, %f347, %f348;\n" " not.pred %p100, %p1;\n" " @%p100 bra $L__BB8_104;\n" "\n" " mul.f32 %f349, %f87, %f231;\n" " st.shared.f32 [%r50+256], %f349;\n" "\n" "$L__BB8_104:\n" " setp.lt.s32 %p101, %r127, 6;\n" " @%p101 bra $L__BB8_107;\n" "\n" " mov.b32 %r392, %f639;\n" " mov.u32 %r393, 2;\n" " mov.u32 %r394, 4127;\n" " mov.u32 %r395, 8;\n" " mov.u32 %r396, -1;\n" " shfl.sync.down.b32 %r397|%p102, %r392, %r395, %r394, %r396;\n" " mov.b32 %f350, %r397;\n" " add.f32 %f351, %f639, %f350;\n" " mov.b32 %r398, %f351;\n" " mov.u32 %r399, 4;\n" " shfl.sync.down.b32 %r400|%p103, %r398, %r399, %r394, %r396;\n" " mov.b32 %f352, %r400;\n" " add.f32 %f353, %f351, %f352;\n" " mov.b32 %r401, %f353;\n" " shfl.sync.down.b32 %r402|%p104, %r401, %r393, %r394, %r396;\n" " mov.b32 %f354, %r402;\n" " add.f32 %f355, %f353, %f354;\n" " mov.b32 %r403, %f355;\n" " mov.u32 %r404, 1;\n" " shfl.sync.down.b32 %r405|%p105, %r403, %r404, %r394, %r396;\n" " mov.b32 %f356, %r405;\n" " add.f32 %f88, %f355, %f356;\n" " not.pred %p106, %p1;\n" " @%p106 bra $L__BB8_107;\n" "\n" " mul.f32 %f357, %f88, %f231;\n" " st.shared.f32 [%r50+320], %f357;\n" "\n" "$L__BB8_107:\n" " setp.lt.s32 %p107, %r127, 7;\n" " @%p107 bra $L__BB8_110;\n" "\n" " mov.b32 %r406, %f640;\n" " mov.u32 %r407, 2;\n" " mov.u32 %r408, 4127;\n" " mov.u32 %r409, 8;\n" " mov.u32 %r410, -1;\n" " shfl.sync.down.b32 %r411|%p108, %r406, %r409, %r408, %r410;\n" " mov.b32 %f358, %r411;\n" " add.f32 %f359, %f640, %f358;\n" " mov.b32 %r412, %f359;\n" " mov.u32 %r413, 4;\n" " shfl.sync.down.b32 %r414|%p109, %r412, %r413, %r408, %r410;\n" " mov.b32 %f360, %r414;\n" " add.f32 %f361, %f359, %f360;\n" " mov.b32 %r415, %f361;\n" " shfl.sync.down.b32 %r416|%p110, %r415, %r407, %r408, %r410;\n" " mov.b32 %f362, %r416;\n" " add.f32 %f363, %f361, %f362;\n" " mov.b32 %r417, %f363;\n" " mov.u32 %r418, 1;\n" " shfl.sync.down.b32 %r419|%p111, %r417, %r418, %r408, %r410;\n" " mov.b32 %f364, %r419;\n" " add.f32 %f89, %f363, %f364;\n" " not.pred %p112, %p1;\n" " @%p112 bra $L__BB8_110;\n" "\n" " mul.f32 %f365, %f89, %f231;\n" " st.shared.f32 [%r50+384], %f365;\n" "\n" "$L__BB8_110:\n" " setp.lt.s32 %p113, %r127, 8;\n" " @%p113 bra $L__BB8_113;\n" "\n" " mov.b32 %r420, %f641;\n" " mov.u32 %r421, 2;\n" " mov.u32 %r422, 4127;\n" " mov.u32 %r423, 8;\n" " mov.u32 %r424, -1;\n" " shfl.sync.down.b32 %r425|%p114, %r420, %r423, %r422, %r424;\n" " mov.b32 %f366, %r425;\n" " add.f32 %f367, %f641, %f366;\n" " mov.b32 %r426, %f367;\n" " mov.u32 %r427, 4;\n" " shfl.sync.down.b32 %r428|%p115, %r426, %r427, %r422, %r424;\n" " mov.b32 %f368, %r428;\n" " add.f32 %f369, %f367, %f368;\n" " mov.b32 %r429, %f369;\n" " shfl.sync.down.b32 %r430|%p116, %r429, %r421, %r422, %r424;\n" " mov.b32 %f370, %r430;\n" " add.f32 %f371, %f369, %f370;\n" " mov.b32 %r431, %f371;\n" " mov.u32 %r432, 1;\n" " shfl.sync.down.b32 %r433|%p117, %r431, %r432, %r422, %r424;\n" " mov.b32 %f372, %r433;\n" " add.f32 %f90, %f371, %f372;\n" " not.pred %p118, %p1;\n" " @%p118 bra $L__BB8_113;\n" "\n" " mul.f32 %f373, %f90, %f231;\n" " st.shared.f32 [%r50+448], %f373;\n" "\n" "$L__BB8_113:\n" " bar.sync 0;\n" " @%p7 bra $L__BB8_131;\n" "\n" " ld.shared.f32 %f91, [%r27];\n" " mov.f32 %f676, %f91;\n" " @%p18 bra $L__BB8_122;\n" "\n" " and.b32 %r97, %r58, 3;\n" " add.s32 %r435, %r58, -1;\n" " setp.lt.u32 %p121, %r435, 3;\n" " mov.u32 %r625, 0;\n" " mov.f32 %f676, %f91;\n" " @%p121 bra $L__BB8_118;\n" "\n" " sub.s32 %r624, %r58, %r97;\n" " mov.u32 %r625, 0;\n" " mov.f32 %f676, %f91;\n" "\n" "$L__BB8_117:\n" " add.s32 %r437, %r625, %r36;\n" " shl.b32 %r438, %r437, 2;\n" " add.s32 %r439, %r26, %r438;\n" " ld.shared.f32 %f375, [%r439];\n" " max.f32 %f376, %f676, %f375;\n" " ld.shared.f32 %f377, [%r439+4];\n" " max.f32 %f378, %f376, %f377;\n" " ld.shared.f32 %f379, [%r439+8];\n" " max.f32 %f380, %f378, %f379;\n" " ld.shared.f32 %f381, [%r439+12];\n" " max.f32 %f676, %f380, %f381;\n" " add.s32 %r625, %r625, 4;\n" " add.s32 %r624, %r624, -4;\n" " setp.ne.s32 %p122, %r624, 0;\n" " @%p122 bra $L__BB8_117;\n" "\n" "$L__BB8_118:\n" " setp.eq.s32 %p123, %r97, 0;\n" " @%p123 bra $L__BB8_122;\n" "\n" " add.s32 %r440, %r625, %r36;\n" " shl.b32 %r441, %r440, 2;\n" " add.s32 %r104, %r26, %r441;\n" " ld.shared.f32 %f382, [%r104];\n" " max.f32 %f676, %f676, %f382;\n" " setp.eq.s32 %p124, %r97, 1;\n" " @%p124 bra $L__BB8_122;\n" "\n" " ld.shared.f32 %f383, [%r104+4];\n" " max.f32 %f676, %f676, %f383;\n" " setp.eq.s32 %p125, %r97, 2;\n" " @%p125 bra $L__BB8_122;\n" "\n" " ld.shared.f32 %f384, [%r104+8];\n" " max.f32 %f676, %f676, %f384;\n" "\n" "$L__BB8_122:\n" " mov.f32 %f680, 0f00000000;\n" " @%p18 bra $L__BB8_130;\n" "\n" " and.b32 %r105, %r58, 3;\n" " add.s32 %r443, %r58, -1;\n" " setp.lt.u32 %p127, %r443, 3;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r629, 0;\n" " @%p127 bra $L__BB8_126;\n" "\n" " sub.s32 %r627, %r58, %r105;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r629, 0;\n" " mov.u32 %r628, %r52;\n" "\n" "$L__BB8_125:\n" " shl.b32 %r510, %r127, 2;\n" " add.s32 %r509, %r510, 512;\n" " add.s32 %r445, %r629, %r36;\n" " shl.b32 %r446, %r445, 2;\n" " add.s32 %r447, %r26, %r446;\n" " ld.shared.f32 %f389, [%r447];\n" " sub.f32 %f390, %f389, %f676;\n" " mov.f32 %f391, 0f3F000000;\n" " mov.f32 %f392, 0f3BBB989D;\n" " fma.rn.f32 %f393, %f390, %f392, %f391;\n" " mov.f32 %f394, 0f3FB8AA3B;\n" " mov.f32 %f395, 0f437C0000;\n" " cvt.sat.f32.f32 %f396, %f393;\n" " mov.f32 %f397, 0f4B400001;\n" " fma.rm.f32 %f398, %f396, %f395, %f397;\n" " add.f32 %f399, %f398, 0fCB40007F;\n" " neg.f32 %f400, %f399;\n" " fma.rn.f32 %f401, %f390, %f394, %f400;\n" " mov.f32 %f402, 0f32A57060;\n" " fma.rn.f32 %f403, %f390, %f402, %f401;\n" " mov.b32 %r448, %f398;\n" " shl.b32 %r449, %r448, 23;\n" " mov.b32 %f404, %r449;\n" " ex2.approx.ftz.f32 %f405, %f403;\n" " mul.f32 %f406, %f405, %f404;\n" " st.shared.f32 [%r628+512], %f406;\n" " add.f32 %f407, %f680, %f406;\n" " ld.shared.f32 %f408, [%r447+4];\n" " sub.f32 %f409, %f408, %f676;\n" " fma.rn.f32 %f410, %f409, %f392, %f391;\n" " cvt.sat.f32.f32 %f411, %f410;\n" " fma.rm.f32 %f412, %f411, %f395, %f397;\n" " add.f32 %f413, %f412, 0fCB40007F;\n" " neg.f32 %f414, %f413;\n" " fma.rn.f32 %f415, %f409, %f394, %f414;\n" " fma.rn.f32 %f416, %f409, %f402, %f415;\n" " mov.b32 %r450, %f412;\n" " shl.b32 %r451, %r450, 23;\n" " mov.b32 %f417, %r451;\n" " ex2.approx.ftz.f32 %f418, %f416;\n" " mul.f32 %f419, %f418, %f417;\n" " add.s32 %r452, %r628, %r509;\n" " st.shared.f32 [%r452], %f419;\n" " add.f32 %f420, %f407, %f419;\n" " ld.shared.f32 %f421, [%r447+8];\n" " sub.f32 %f422, %f421, %f676;\n" " fma.rn.f32 %f423, %f422, %f392, %f391;\n" " cvt.sat.f32.f32 %f424, %f423;\n" " fma.rm.f32 %f425, %f424, %f395, %f397;\n" " add.f32 %f426, %f425, 0fCB40007F;\n" " neg.f32 %f427, %f426;\n" " fma.rn.f32 %f428, %f422, %f394, %f427;\n" " fma.rn.f32 %f429, %f422, %f402, %f428;\n" " mov.b32 %r453, %f425;\n" " shl.b32 %r454, %r453, 23;\n" " mov.b32 %f430, %r454;\n" " ex2.approx.ftz.f32 %f431, %f429;\n" " mul.f32 %f432, %f431, %f430;\n" " add.s32 %r455, %r628, %r510;\n" " add.s32 %r456, %r455, %r510;\n" " add.s32 %r457, %r455, %r509;\n" " st.shared.f32 [%r457], %f432;\n" " add.f32 %f433, %f420, %f432;\n" " ld.shared.f32 %f434, [%r447+12];\n" " sub.f32 %f435, %f434, %f676;\n" " fma.rn.f32 %f436, %f435, %f392, %f391;\n" " cvt.sat.f32.f32 %f437, %f436;\n" " fma.rm.f32 %f438, %f437, %f395, %f397;\n" " add.f32 %f439, %f438, 0fCB40007F;\n" " neg.f32 %f440, %f439;\n" " fma.rn.f32 %f441, %f435, %f394, %f440;\n" " fma.rn.f32 %f442, %f435, %f402, %f441;\n" " mov.b32 %r458, %f438;\n" " shl.b32 %r459, %r458, 23;\n" " mov.b32 %f443, %r459;\n" " ex2.approx.ftz.f32 %f444, %f442;\n" " mul.f32 %f445, %f444, %f443;\n" " add.s32 %r460, %r456, %r510;\n" " add.s32 %r461, %r456, %r509;\n" " add.s32 %r628, %r460, %r510;\n" " st.shared.f32 [%r461], %f445;\n" " add.f32 %f680, %f433, %f445;\n" " add.s32 %r629, %r629, 4;\n" " add.s32 %r627, %r627, -4;\n" " setp.ne.s32 %p128, %r627, 0;\n" " @%p128 bra $L__BB8_125;\n" "\n" "$L__BB8_126:\n" " setp.eq.s32 %p129, %r105, 0;\n" " @%p129 bra $L__BB8_130;\n" "\n" " add.s32 %r511, %r7, %r1;\n" " add.s32 %r462, %r629, %r36;\n" " shl.b32 %r463, %r462, 2;\n" " add.s32 %r114, %r26, %r463;\n" " ld.shared.f32 %f446, [%r114];\n" " sub.f32 %f447, %f446, %f676;\n" " mov.f32 %f448, 0f3F000000;\n" " mov.f32 %f449, 0f3BBB989D;\n" " fma.rn.f32 %f450, %f447, %f449, %f448;\n" " mov.f32 %f451, 0f3FB8AA3B;\n" " mov.f32 %f452, 0f437C0000;\n" " cvt.sat.f32.f32 %f453, %f450;\n" " mov.f32 %f454, 0f4B400001;\n" " fma.rm.f32 %f455, %f453, %f452, %f454;\n" " add.f32 %f456, %f455, 0fCB40007F;\n" " neg.f32 %f457, %f456;\n" " fma.rn.f32 %f458, %f447, %f451, %f457;\n" " mov.f32 %f459, 0f32A57060;\n" " fma.rn.f32 %f460, %f447, %f459, %f458;\n" " mov.b32 %r464, %f455;\n" " shl.b32 %r465, %r464, 23;\n" " mov.b32 %f461, %r465;\n" " ex2.approx.ftz.f32 %f462, %f460;\n" " mul.f32 %f463, %f462, %f461;\n" " mad.lo.s32 %r466, %r629, %r127, %r511;\n" " shl.b32 %r467, %r466, 2;\n" " add.s32 %r115, %r26, %r467;\n" " st.shared.f32 [%r115], %f463;\n" " add.f32 %f680, %f680, %f463;\n" " setp.eq.s32 %p130, %r105, 1;\n" " @%p130 bra $L__BB8_130;\n" "\n" " add.s32 %r502, %r629, %r36;\n" " shl.b32 %r501, %r502, 2;\n" " add.s32 %r500, %r26, %r501;\n" " mov.f32 %f611, 0f32A57060;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f609, 0f4B400001;\n" " mov.f32 %f608, 0f437C0000;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f606, 0f3BBB989D;\n" " ld.shared.f32 %f464, [%r500+4];\n" " sub.f32 %f465, %f464, %f676;\n" " fma.rn.f32 %f468, %f465, %f606, %f607;\n" " cvt.sat.f32.f32 %f471, %f468;\n" " fma.rm.f32 %f473, %f471, %f608, %f609;\n" " add.f32 %f474, %f473, 0fCB40007F;\n" " neg.f32 %f475, %f474;\n" " fma.rn.f32 %f476, %f465, %f610, %f475;\n" " fma.rn.f32 %f478, %f465, %f611, %f476;\n" " mov.b32 %r468, %f473;\n" " shl.b32 %r469, %r468, 23;\n" " mov.b32 %f479, %r469;\n" " ex2.approx.ftz.f32 %f480, %f478;\n" " mul.f32 %f481, %f480, %f479;\n" " add.s32 %r116, %r115, %r53;\n" " st.shared.f32 [%r116], %f481;\n" " add.f32 %f680, %f680, %f481;\n" " setp.eq.s32 %p131, %r105, 2;\n" " @%p131 bra $L__BB8_130;\n" "\n" " add.s32 %r505, %r629, %r36;\n" " shl.b32 %r504, %r505, 2;\n" " add.s32 %r503, %r26, %r504;\n" " mov.f32 %f617, 0f32A57060;\n" " mov.f32 %f616, 0f3FB8AA3B;\n" " mov.f32 %f615, 0f4B400001;\n" " mov.f32 %f614, 0f437C0000;\n" " mov.f32 %f613, 0f3F000000;\n" " mov.f32 %f612, 0f3BBB989D;\n" " ld.shared.f32 %f482, [%r503+8];\n" " sub.f32 %f483, %f482, %f676;\n" " fma.rn.f32 %f486, %f483, %f612, %f613;\n" " cvt.sat.f32.f32 %f489, %f486;\n" " fma.rm.f32 %f491, %f489, %f614, %f615;\n" " add.f32 %f492, %f491, 0fCB40007F;\n" " neg.f32 %f493, %f492;\n" " fma.rn.f32 %f494, %f483, %f616, %f493;\n" " fma.rn.f32 %f496, %f483, %f617, %f494;\n" " mov.b32 %r471, %f491;\n" " shl.b32 %r472, %r471, 23;\n" " mov.b32 %f497, %r472;\n" " ex2.approx.ftz.f32 %f498, %f496;\n" " mul.f32 %f499, %f498, %f497;\n" " add.s32 %r474, %r116, %r53;\n" " st.shared.f32 [%r474], %f499;\n" " add.f32 %f680, %f680, %f499;\n" "\n" "$L__BB8_130:\n" " sub.f32 %f500, %f91, %f676;\n" " mov.f32 %f501, 0f3F000000;\n" " mov.f32 %f502, 0f3BBB989D;\n" " fma.rn.f32 %f503, %f500, %f502, %f501;\n" " mov.f32 %f504, 0f3FB8AA3B;\n" " mov.f32 %f505, 0f437C0000;\n" " cvt.sat.f32.f32 %f506, %f503;\n" " mov.f32 %f507, 0f4B400001;\n" " fma.rm.f32 %f508, %f506, %f505, %f507;\n" " add.f32 %f509, %f508, 0fCB40007F;\n" " neg.f32 %f510, %f509;\n" " fma.rn.f32 %f511, %f500, %f504, %f510;\n" " mov.f32 %f512, 0f32A57060;\n" " fma.rn.f32 %f513, %f500, %f512, %f511;\n" " ex2.approx.ftz.f32 %f514, %f513;\n" " mov.b32 %r475, %f508;\n" " shl.b32 %r476, %r475, 23;\n" " mov.b32 %f515, %r476;\n" " mul.f32 %f516, %f514, %f515;\n" " ld.shared.f32 %f517, [%r28];\n" " fma.rn.f32 %f518, %f516, %f517, %f680;\n" " st.shared.f32 [%r28], %f518;\n" " st.shared.f32 [%r27], %f676;\n" " st.shared.f32 [%r38], %f516;\n" "\n" "$L__BB8_131:\n" " bar.sync 0;\n" " @%p18 bra $L__BB8_211;\n" "\n" " @%p19 bra $L__BB8_171;\n" " bra.uni $L__BB8_133;\n" "\n" "$L__BB8_171:\n" " mov.u32 %r631, 0;\n" "\n" "$L__BB8_172:\n" " add.s32 %r121, %r631, %r612;\n" " setp.lt.s32 %p153, %r121, %r131;\n" " @%p153 bra $L__BB8_176;\n" " bra.uni $L__BB8_173;\n" "\n" "$L__BB8_176:\n" " add.s32 %r604, %r631, %r612;\n" " cvt.s64.s32 %rd124, %r604;\n" " mul.lo.s64 %rd125, %rd124, %rd10;\n" " add.s64 %rd126, %rd27, %rd125;\n" " shl.b64 %rd127, %rd126, 1;\n" " add.s64 %rd128, %rd5, %rd127;\n" " ld.global.u16 %rs44, [%rd128];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs44;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB8_177;\n" "\n" "$L__BB8_173:\n" " add.s32 %r601, %r631, %r612;\n" " setp.gt.s32 %p154, %r601, %r33;\n" " setp.gt.s32 %p155, %r601, %r4;\n" " and.pred %p156, %p155, %p154;\n" " @%p156 bra $L__BB8_175;\n" " bra.uni $L__BB8_174;\n" "\n" "$L__BB8_175:\n" " add.s32 %r603, %r631, %r612;\n" " and.b32 %r496, %r603, 255;\n" " cvt.u64.u32 %rd119, %r496;\n" " mul.lo.s64 %rd120, %rd119, %rd10;\n" " add.s64 %rd121, %rd28, %rd120;\n" " shl.b64 %rd122, %rd121, 1;\n" " add.s64 %rd123, %rd31, %rd122;\n" " ld.global.u16 %rs43, [%rd123];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs43;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB8_177;\n" "\n" "$L__BB8_174:\n" " add.s32 %r602, %r631, %r612;\n" " cvt.s64.s32 %rd112, %r602;\n" " mul.lo.s64 %rd113, %rd112, %rd21;\n" " add.s64 %rd114, %rd113, %rd22;\n" " add.s64 %rd115, %rd114, %rd29;\n" " add.s64 %rd116, %rd6, %rd115;\n" " ld.global.u8 %r489, [%rd116];\n" " ld.global.u8 %r490, [%rd116+1];\n" " prmt.b32 %r491, %r490, %r489, 30212;\n" " cvt.u16.u32 %rs42, %r491;\n" " // begin inline asm\n" " { cvt.f32.f16 %f554, %rs42;}\n" "\n" " // end inline asm\n" " add.s64 %rd117, %rd30, %rd114;\n" " add.s64 %rd118, %rd6, %rd117;\n" " ld.global.u8 %r492, [%rd118];\n" " shr.u32 %r493, %r492, %r37;\n" " and.b32 %r494, %r493, 15;\n" " add.s32 %r495, %r494, -8;\n" " cvt.rn.f32.s32 %f555, %r495;\n" " mul.f32 %f720, %f554, %f555;\n" "\n" "$L__BB8_177:\n" " mad.lo.s32 %r497, %r631, %r127, %r7;\n" " shl.b32 %r498, %r497, 2;\n" " add.s32 %r122, %r26, %r498;\n" " setp.eq.s32 %p157, %r631, 0;\n" " @%p157 bra $L__BB8_194;\n" "\n" " @%p2 bra $L__BB8_180;\n" "\n" " ld.shared.f32 %f558, [%r122];\n" " fma.rn.f32 %f711, %f720, %f558, %f711;\n" "\n" "$L__BB8_180:\n" " @%p77 bra $L__BB8_182;\n" "\n" " ld.shared.f32 %f559, [%r122+4];\n" " fma.rn.f32 %f710, %f720, %f559, %f710;\n" "\n" "$L__BB8_182:\n" " @%p83 bra $L__BB8_184;\n" "\n" " ld.shared.f32 %f560, [%r122+8];\n" " fma.rn.f32 %f709, %f720, %f560, %f709;\n" "\n" "$L__BB8_184:\n" " @%p89 bra $L__BB8_186;\n" "\n" " ld.shared.f32 %f561, [%r122+12];\n" " fma.rn.f32 %f708, %f720, %f561, %f708;\n" "\n" "$L__BB8_186:\n" " @%p95 bra $L__BB8_188;\n" "\n" " ld.shared.f32 %f562, [%r122+16];\n" " fma.rn.f32 %f707, %f720, %f562, %f707;\n" "\n" "$L__BB8_188:\n" " @%p101 bra $L__BB8_190;\n" "\n" " ld.shared.f32 %f563, [%r122+20];\n" " fma.rn.f32 %f706, %f720, %f563, %f706;\n" "\n" "$L__BB8_190:\n" " @%p107 bra $L__BB8_192;\n" "\n" " ld.shared.f32 %f564, [%r122+24];\n" " fma.rn.f32 %f705, %f720, %f564, %f705;\n" "\n" "$L__BB8_192:\n" " @%p113 bra $L__BB8_210;\n" "\n" " ld.shared.f32 %f565, [%r122+28];\n" " fma.rn.f32 %f704, %f720, %f565, %f704;\n" " bra.uni $L__BB8_210;\n" "\n" "$L__BB8_194:\n" " @%p2 bra $L__BB8_196;\n" "\n" " ld.shared.f32 %f566, [%r51];\n" " ld.shared.f32 %f567, [%r122];\n" " mul.f32 %f568, %f720, %f567;\n" " fma.rn.f32 %f711, %f566, %f711, %f568;\n" "\n" "$L__BB8_196:\n" " @%p77 bra $L__BB8_198;\n" "\n" " ld.shared.f32 %f569, [%r51+4];\n" " ld.shared.f32 %f570, [%r122+4];\n" " mul.f32 %f571, %f720, %f570;\n" " fma.rn.f32 %f710, %f569, %f710, %f571;\n" "\n" "$L__BB8_198:\n" " @%p83 bra $L__BB8_200;\n" "\n" " ld.shared.f32 %f572, [%r51+8];\n" " ld.shared.f32 %f573, [%r122+8];\n" " mul.f32 %f574, %f720, %f573;\n" " fma.rn.f32 %f709, %f572, %f709, %f574;\n" "\n" "$L__BB8_200:\n" " @%p89 bra $L__BB8_202;\n" "\n" " ld.shared.f32 %f575, [%r51+12];\n" " ld.shared.f32 %f576, [%r122+12];\n" " mul.f32 %f577, %f720, %f576;\n" " fma.rn.f32 %f708, %f575, %f708, %f577;\n" "\n" "$L__BB8_202:\n" " @%p95 bra $L__BB8_204;\n" "\n" " ld.shared.f32 %f578, [%r51+16];\n" " ld.shared.f32 %f579, [%r122+16];\n" " mul.f32 %f580, %f720, %f579;\n" " fma.rn.f32 %f707, %f578, %f707, %f580;\n" "\n" "$L__BB8_204:\n" " @%p101 bra $L__BB8_206;\n" "\n" " ld.shared.f32 %f581, [%r51+20];\n" " ld.shared.f32 %f582, [%r122+20];\n" " mul.f32 %f583, %f720, %f582;\n" " fma.rn.f32 %f706, %f581, %f706, %f583;\n" "\n" "$L__BB8_206:\n" " @%p107 bra $L__BB8_208;\n" "\n" " ld.shared.f32 %f584, [%r51+24];\n" " ld.shared.f32 %f585, [%r122+24];\n" " mul.f32 %f586, %f720, %f585;\n" " fma.rn.f32 %f705, %f584, %f705, %f586;\n" "\n" "$L__BB8_208:\n" " @%p113 bra $L__BB8_210;\n" "\n" " ld.shared.f32 %f587, [%r51+28];\n" " ld.shared.f32 %f588, [%r122+28];\n" " mul.f32 %f589, %f720, %f588;\n" " fma.rn.f32 %f704, %f587, %f704, %f589;\n" "\n" "$L__BB8_210:\n" " add.s32 %r631, %r631, 1;\n" " setp.lt.s32 %p174, %r631, %r60;\n" " @%p174 bra $L__BB8_172;\n" " bra.uni $L__BB8_211;\n" "\n" "$L__BB8_133:\n" " mov.u32 %r630, 0;\n" "\n" "$L__BB8_134:\n" " add.s32 %r478, %r630, %r612;\n" " setp.lt.s32 %p134, %r478, %r131;\n" " cvt.s64.s32 %rd41, %r478;\n" " @%p134 bra $L__BB8_136;\n" " bra.uni $L__BB8_135;\n" "\n" "$L__BB8_136:\n" " mul.lo.s64 %rd108, %rd41, %rd10;\n" " add.s64 %rd109, %rd27, %rd108;\n" " shl.b64 %rd110, %rd109, 1;\n" " add.s64 %rd111, %rd5, %rd110;\n" " ld.global.u16 %rs41, [%rd111];\n" " // begin inline asm\n" " { cvt.f32.f16 %f689, %rs41;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB8_137;\n" "\n" "$L__BB8_135:\n" " mul.lo.s64 %rd102, %rd41, %rd21;\n" " add.s64 %rd103, %rd102, %rd22;\n" " add.s64 %rd104, %rd103, %rd29;\n" " add.s64 %rd105, %rd6, %rd104;\n" " ld.global.u8 %r479, [%rd105];\n" " ld.global.u8 %r480, [%rd105+1];\n" " prmt.b32 %r481, %r480, %r479, 30212;\n" " cvt.u16.u32 %rs40, %r481;\n" " // begin inline asm\n" " { cvt.f32.f16 %f519, %rs40;}\n" "\n" " // end inline asm\n" " add.s64 %rd106, %rd30, %rd103;\n" " add.s64 %rd107, %rd6, %rd106;\n" " ld.global.u8 %r482, [%rd107];\n" " shr.u32 %r483, %r482, %r37;\n" " and.b32 %r484, %r483, 15;\n" " add.s32 %r485, %r484, -8;\n" " cvt.rn.f32.s32 %f520, %r485;\n" " mul.f32 %f689, %f519, %f520;\n" "\n" "$L__BB8_137:\n" " mad.lo.s32 %r486, %r630, %r127, %r7;\n" " shl.b32 %r487, %r486, 2;\n" " add.s32 %r118, %r26, %r487;\n" " setp.eq.s32 %p135, %r630, 0;\n" " @%p135 bra $L__BB8_154;\n" "\n" " @%p2 bra $L__BB8_140;\n" "\n" " ld.shared.f32 %f522, [%r118];\n" " fma.rn.f32 %f711, %f689, %f522, %f711;\n" "\n" "$L__BB8_140:\n" " @%p77 bra $L__BB8_142;\n" "\n" " ld.shared.f32 %f523, [%r118+4];\n" " fma.rn.f32 %f710, %f689, %f523, %f710;\n" "\n" "$L__BB8_142:\n" " @%p83 bra $L__BB8_144;\n" "\n" " ld.shared.f32 %f524, [%r118+8];\n" " fma.rn.f32 %f709, %f689, %f524, %f709;\n" "\n" "$L__BB8_144:\n" " @%p89 bra $L__BB8_146;\n" "\n" " ld.shared.f32 %f525, [%r118+12];\n" " fma.rn.f32 %f708, %f689, %f525, %f708;\n" "\n" "$L__BB8_146:\n" " @%p95 bra $L__BB8_148;\n" "\n" " ld.shared.f32 %f526, [%r118+16];\n" " fma.rn.f32 %f707, %f689, %f526, %f707;\n" "\n" "$L__BB8_148:\n" " @%p101 bra $L__BB8_150;\n" "\n" " ld.shared.f32 %f527, [%r118+20];\n" " fma.rn.f32 %f706, %f689, %f527, %f706;\n" "\n" "$L__BB8_150:\n" " @%p107 bra $L__BB8_152;\n" "\n" " ld.shared.f32 %f528, [%r118+24];\n" " fma.rn.f32 %f705, %f689, %f528, %f705;\n" "\n" "$L__BB8_152:\n" " @%p113 bra $L__BB8_170;\n" "\n" " ld.shared.f32 %f529, [%r118+28];\n" " fma.rn.f32 %f704, %f689, %f529, %f704;\n" " bra.uni $L__BB8_170;\n" "\n" "$L__BB8_154:\n" " @%p2 bra $L__BB8_156;\n" "\n" " ld.shared.f32 %f530, [%r51];\n" " ld.shared.f32 %f531, [%r118];\n" " mul.f32 %f532, %f689, %f531;\n" " fma.rn.f32 %f711, %f530, %f711, %f532;\n" "\n" "$L__BB8_156:\n" " @%p77 bra $L__BB8_158;\n" "\n" " ld.shared.f32 %f533, [%r51+4];\n" " ld.shared.f32 %f534, [%r118+4];\n" " mul.f32 %f535, %f689, %f534;\n" " fma.rn.f32 %f710, %f533, %f710, %f535;\n" "\n" "$L__BB8_158:\n" " @%p83 bra $L__BB8_160;\n" "\n" " ld.shared.f32 %f536, [%r51+8];\n" " ld.shared.f32 %f537, [%r118+8];\n" " mul.f32 %f538, %f689, %f537;\n" " fma.rn.f32 %f709, %f536, %f709, %f538;\n" "\n" "$L__BB8_160:\n" " @%p89 bra $L__BB8_162;\n" "\n" " ld.shared.f32 %f539, [%r51+12];\n" " ld.shared.f32 %f540, [%r118+12];\n" " mul.f32 %f541, %f689, %f540;\n" " fma.rn.f32 %f708, %f539, %f708, %f541;\n" "\n" "$L__BB8_162:\n" " @%p95 bra $L__BB8_164;\n" "\n" " ld.shared.f32 %f542, [%r51+16];\n" " ld.shared.f32 %f543, [%r118+16];\n" " mul.f32 %f544, %f689, %f543;\n" " fma.rn.f32 %f707, %f542, %f707, %f544;\n" "\n" "$L__BB8_164:\n" " @%p101 bra $L__BB8_166;\n" "\n" " ld.shared.f32 %f545, [%r51+20];\n" " ld.shared.f32 %f546, [%r118+20];\n" " mul.f32 %f547, %f689, %f546;\n" " fma.rn.f32 %f706, %f545, %f706, %f547;\n" "\n" "$L__BB8_166:\n" " @%p107 bra $L__BB8_168;\n" "\n" " ld.shared.f32 %f548, [%r51+24];\n" " ld.shared.f32 %f549, [%r118+24];\n" " mul.f32 %f550, %f689, %f549;\n" " fma.rn.f32 %f705, %f548, %f705, %f550;\n" "\n" "$L__BB8_168:\n" " @%p113 bra $L__BB8_170;\n" "\n" " ld.shared.f32 %f551, [%r51+28];\n" " ld.shared.f32 %f552, [%r118+28];\n" " mul.f32 %f553, %f689, %f552;\n" " fma.rn.f32 %f704, %f551, %f704, %f553;\n" "\n" "$L__BB8_170:\n" " add.s32 %r630, %r630, 1;\n" " setp.lt.s32 %p152, %r630, %r60;\n" " @%p152 bra $L__BB8_134;\n" "\n" "$L__BB8_211:\n" " bar.sync 0;\n" "\n" "$L__BB8_212:\n" " add.s32 %r612, %r612, 16;\n" " setp.ge.s32 %p175, %r3, %r612;\n" " add.s32 %r611, %r611, 1;\n" " @%p175 bra $L__BB8_11;\n" "\n" "$L__BB8_213:\n" " shl.b32 %r517, %r127, 4;\n" " cvt.s64.s32 %rd168, %r127;\n" " mul.wide.s32 %rd167, %r517, 2;\n" " add.s64 %rd166, %rd167, %rd168;\n" " cvt.u32.u64 %r516, %rd166;\n" " cvt.s64.s32 %rd165, %r1;\n" " ld.param.u32 %r515, [fc_attn_flash_prefill_q4_kqh_param_9];\n" " mov.u32 %r514, %ctaid.x;\n" " mov.u32 %r513, %nctaid.y;\n" " cvt.u64.u32 %rd164, %r513;\n" " cvt.s64.s32 %rd163, %r514;\n" " mov.u32 %r512, %ctaid.y;\n" " cvt.s64.s32 %rd162, %r512;\n" " mul.lo.s64 %rd161, %rd163, %rd164;\n" " add.s64 %rd160, %rd161, %rd162;\n" " mul.wide.s32 %rd159, %r515, %r127;\n" " mul.lo.s64 %rd158, %rd159, %rd160;\n" " add.s64 %rd42, %rd158, %rd165;\n" " shl.b32 %r499, %r516, 2;\n" " add.s32 %r126, %r26, %r499;\n" " @%p2 bra $L__BB8_215;\n" "\n" " ld.shared.f32 %f590, [%r126];\n" " div.rn.f32 %f591, %f711, %f590;\n" " shl.b64 %rd129, %rd42, 2;\n" " add.s64 %rd130, %rd2, %rd129;\n" " st.global.f32 [%rd130], %f591;\n" "\n" "$L__BB8_215:\n" " setp.lt.s32 %p177, %r127, 2;\n" " @%p177 bra $L__BB8_217;\n" "\n" " ld.shared.f32 %f592, [%r126+4];\n" " div.rn.f32 %f593, %f710, %f592;\n" " add.s64 %rd131, %rd42, %rd10;\n" " shl.b64 %rd132, %rd131, 2;\n" " add.s64 %rd133, %rd2, %rd132;\n" " st.global.f32 [%rd133], %f593;\n" "\n" "$L__BB8_217:\n" " setp.lt.s32 %p178, %r127, 3;\n" " @%p178 bra $L__BB8_219;\n" "\n" " ld.shared.f32 %f594, [%r126+8];\n" " div.rn.f32 %f595, %f709, %f594;\n" " shl.b64 %rd134, %rd10, 1;\n" " add.s64 %rd135, %rd42, %rd134;\n" " shl.b64 %rd136, %rd135, 2;\n" " add.s64 %rd137, %rd2, %rd136;\n" " st.global.f32 [%rd137], %f595;\n" "\n" "$L__BB8_219:\n" " setp.lt.s32 %p179, %r127, 4;\n" " @%p179 bra $L__BB8_221;\n" "\n" " ld.shared.f32 %f596, [%r126+12];\n" " div.rn.f32 %f597, %f708, %f596;\n" " mul.lo.s64 %rd138, %rd10, 3;\n" " add.s64 %rd139, %rd42, %rd138;\n" " shl.b64 %rd140, %rd139, 2;\n" " add.s64 %rd141, %rd2, %rd140;\n" " st.global.f32 [%rd141], %f597;\n" "\n" "$L__BB8_221:\n" " setp.lt.s32 %p180, %r127, 5;\n" " @%p180 bra $L__BB8_223;\n" "\n" " ld.shared.f32 %f598, [%r126+16];\n" " div.rn.f32 %f599, %f707, %f598;\n" " shl.b64 %rd142, %rd10, 2;\n" " add.s64 %rd143, %rd42, %rd142;\n" " shl.b64 %rd144, %rd143, 2;\n" " add.s64 %rd145, %rd2, %rd144;\n" " st.global.f32 [%rd145], %f599;\n" "\n" "$L__BB8_223:\n" " setp.lt.s32 %p181, %r127, 6;\n" " @%p181 bra $L__BB8_225;\n" "\n" " ld.shared.f32 %f600, [%r126+20];\n" " div.rn.f32 %f601, %f706, %f600;\n" " mul.lo.s64 %rd146, %rd10, 5;\n" " add.s64 %rd147, %rd42, %rd146;\n" " shl.b64 %rd148, %rd147, 2;\n" " add.s64 %rd149, %rd2, %rd148;\n" " st.global.f32 [%rd149], %f601;\n" "\n" "$L__BB8_225:\n" " setp.lt.s32 %p182, %r127, 7;\n" " @%p182 bra $L__BB8_227;\n" "\n" " ld.shared.f32 %f602, [%r126+24];\n" " div.rn.f32 %f603, %f705, %f602;\n" " mul.lo.s64 %rd150, %rd10, 6;\n" " add.s64 %rd151, %rd42, %rd150;\n" " shl.b64 %rd152, %rd151, 2;\n" " add.s64 %rd153, %rd2, %rd152;\n" " st.global.f32 [%rd153], %f603;\n" "\n" "$L__BB8_227:\n" " setp.lt.s32 %p183, %r127, 8;\n" " @%p183 bra $L__BB8_229;\n" "\n" " ld.shared.f32 %f604, [%r126+28];\n" " div.rn.f32 %f605, %f704, %f604;\n" " mul.lo.s64 %rd154, %rd10, 7;\n" " add.s64 %rd155, %rd42, %rd154;\n" " shl.b64 %rd156, %rd155, 2;\n" " add.s64 %rd157, %rd2, %rd156;\n" " st.global.f32 [%rd157], %f605;\n" "\n" "$L__BB8_229:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode_combine\n" ".visible .entry fc_attn_flash_decode_combine(\n" " .param .u64 fc_attn_flash_decode_combine_param_0,\n" " .param .u64 fc_attn_flash_decode_combine_param_1,\n" " .param .u64 fc_attn_flash_decode_combine_param_2,\n" " .param .u64 fc_attn_flash_decode_combine_param_3,\n" " .param .u32 fc_attn_flash_decode_combine_param_4,\n" " .param .u32 fc_attn_flash_decode_combine_param_5\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<145>;\n" " .reg .b32 %r<47>;\n" " .reg .b64 %rd<74>;\n" "\n" "\n" " ld.param.u64 %rd35, [fc_attn_flash_decode_combine_param_0];\n" " ld.param.u64 %rd36, [fc_attn_flash_decode_combine_param_1];\n" " ld.param.u64 %rd37, [fc_attn_flash_decode_combine_param_2];\n" " ld.param.u64 %rd34, [fc_attn_flash_decode_combine_param_3];\n" " ld.param.u32 %r20, [fc_attn_flash_decode_combine_param_4];\n" " ld.param.u32 %r21, [fc_attn_flash_decode_combine_param_5];\n" " cvta.to.global.u64 %rd1, %rd37;\n" " cvta.to.global.u64 %rd2, %rd36;\n" " cvta.to.global.u64 %rd3, %rd35;\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r20;\n" " @%p1 bra $L__BB9_16;\n" "\n" " mov.u32 %r22, %ctaid.x;\n" " cvt.s64.s32 %rd4, %r22;\n" " cvt.s64.s32 %rd5, %r21;\n" " cvt.s64.s32 %rd6, %r20;\n" " setp.lt.s32 %p2, %r21, 1;\n" " mov.f32 %f134, 0fFF800000;\n" " @%p2 bra $L__BB9_8;\n" "\n" " add.s32 %r24, %r21, -1;\n" " and.b32 %r42, %r21, 3;\n" " setp.lt.u32 %p3, %r24, 3;\n" " mov.f32 %f134, 0fFF800000;\n" " mov.u32 %r41, 0;\n" " @%p3 bra $L__BB9_5;\n" "\n" " sub.s32 %r40, %r21, %r42;\n" " mul.lo.s64 %rd38, %rd4, %rd5;\n" " shl.b64 %rd39, %rd38, 2;\n" " add.s64 %rd40, %rd3, %rd39;\n" " add.s64 %rd66, %rd40, 8;\n" " mov.f32 %f134, 0fFF800000;\n" " mov.u32 %r41, 0;\n" "\n" "$L__BB9_4:\n" " ld.global.f32 %f26, [%rd66+-8];\n" " max.f32 %f27, %f134, %f26;\n" " ld.global.f32 %f28, [%rd66+-4];\n" " max.f32 %f29, %f27, %f28;\n" " ld.global.f32 %f30, [%rd66];\n" " max.f32 %f31, %f29, %f30;\n" " ld.global.f32 %f32, [%rd66+4];\n" " max.f32 %f134, %f31, %f32;\n" " add.s32 %r41, %r41, 4;\n" " add.s64 %rd66, %rd66, 16;\n" " add.s32 %r40, %r40, -4;\n" " setp.ne.s32 %p4, %r40, 0;\n" " @%p4 bra $L__BB9_4;\n" "\n" "$L__BB9_5:\n" " setp.eq.s32 %p5, %r42, 0;\n" " @%p5 bra $L__BB9_8;\n" "\n" " mul.lo.s64 %rd41, %rd4, %rd5;\n" " cvt.s64.s32 %rd42, %r41;\n" " add.s64 %rd43, %rd41, %rd42;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd67, %rd3, %rd44;\n" "\n" "$L__BB9_7:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f33, [%rd67];\n" " max.f32 %f134, %f134, %f33;\n" " add.s64 %rd67, %rd67, 4;\n" " add.s32 %r42, %r42, -1;\n" " setp.ne.s32 %p6, %r42, 0;\n" " @%p6 bra $L__BB9_7;\n" "\n" "$L__BB9_8:\n" " cvt.s64.s32 %rd13, %r1;\n" " mov.f32 %f143, 0f00000000;\n" " mov.f32 %f144, %f143;\n" " @%p2 bra $L__BB9_15;\n" "\n" " add.s32 %r27, %r21, -1;\n" " and.b32 %r46, %r21, 3;\n" " setp.lt.u32 %p8, %r27, 3;\n" " mov.f32 %f144, 0f00000000;\n" " mov.u32 %r45, 0;\n" " mov.f32 %f143, %f144;\n" " @%p8 bra $L__BB9_12;\n" "\n" " sub.s32 %r44, %r21, %r46;\n" " mul.lo.s64 %rd45, %rd4, %rd5;\n" " shl.b64 %rd46, %rd45, 2;\n" " add.s64 %rd47, %rd46, 8;\n" " add.s64 %rd69, %rd2, %rd47;\n" " add.s64 %rd68, %rd3, %rd47;\n" " mul.lo.s64 %rd48, %rd45, %rd6;\n" " add.s64 %rd49, %rd48, %rd13;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd70, %rd1, %rd50;\n" " shl.b64 %rd17, %rd6, 2;\n" " mov.f32 %f144, 0f00000000;\n" " mov.u32 %r45, 0;\n" " mov.f32 %f43, 0f3F000000;\n" " mov.f32 %f44, 0f3BBB989D;\n" " mov.f32 %f143, %f144;\n" "\n" "$L__BB9_11:\n" " ld.global.f32 %f41, [%rd68+-8];\n" " sub.f32 %f42, %f41, %f134;\n" " fma.rn.f32 %f45, %f42, %f44, %f43;\n" " mov.f32 %f46, 0f3FB8AA3B;\n" " mov.f32 %f47, 0f437C0000;\n" " cvt.sat.f32.f32 %f48, %f45;\n" " mov.f32 %f49, 0f4B400001;\n" " fma.rm.f32 %f50, %f48, %f47, %f49;\n" " add.f32 %f51, %f50, 0fCB40007F;\n" " neg.f32 %f52, %f51;\n" " fma.rn.f32 %f53, %f42, %f46, %f52;\n" " mov.f32 %f54, 0f32A57060;\n" " fma.rn.f32 %f55, %f42, %f54, %f53;\n" " mov.b32 %r29, %f50;\n" " shl.b32 %r30, %r29, 23;\n" " mov.b32 %f56, %r30;\n" " ex2.approx.ftz.f32 %f57, %f55;\n" " mul.f32 %f58, %f57, %f56;\n" " ld.global.f32 %f59, [%rd69+-8];\n" " fma.rn.f32 %f60, %f59, %f58, %f144;\n" " ld.global.f32 %f61, [%rd70];\n" " fma.rn.f32 %f62, %f58, %f61, %f143;\n" " ld.global.f32 %f63, [%rd68+-4];\n" " sub.f32 %f64, %f63, %f134;\n" " fma.rn.f32 %f65, %f64, %f44, %f43;\n" " cvt.sat.f32.f32 %f66, %f65;\n" " fma.rm.f32 %f67, %f66, %f47, %f49;\n" " add.f32 %f68, %f67, 0fCB40007F;\n" " neg.f32 %f69, %f68;\n" " fma.rn.f32 %f70, %f64, %f46, %f69;\n" " fma.rn.f32 %f71, %f64, %f54, %f70;\n" " mov.b32 %r31, %f67;\n" " shl.b32 %r32, %r31, 23;\n" " mov.b32 %f72, %r32;\n" " ex2.approx.ftz.f32 %f73, %f71;\n" " mul.f32 %f74, %f73, %f72;\n" " ld.global.f32 %f75, [%rd69+-4];\n" " fma.rn.f32 %f76, %f75, %f74, %f60;\n" " add.s64 %rd51, %rd70, %rd17;\n" " ld.global.f32 %f77, [%rd51];\n" " fma.rn.f32 %f78, %f74, %f77, %f62;\n" " ld.global.f32 %f79, [%rd68];\n" " sub.f32 %f80, %f79, %f134;\n" " fma.rn.f32 %f81, %f80, %f44, %f43;\n" " cvt.sat.f32.f32 %f82, %f81;\n" " fma.rm.f32 %f83, %f82, %f47, %f49;\n" " add.f32 %f84, %f83, 0fCB40007F;\n" " neg.f32 %f85, %f84;\n" " fma.rn.f32 %f86, %f80, %f46, %f85;\n" " fma.rn.f32 %f87, %f80, %f54, %f86;\n" " mov.b32 %r33, %f83;\n" " shl.b32 %r34, %r33, 23;\n" " mov.b32 %f88, %r34;\n" " ex2.approx.ftz.f32 %f89, %f87;\n" " mul.f32 %f90, %f89, %f88;\n" " ld.global.f32 %f91, [%rd69];\n" " fma.rn.f32 %f92, %f91, %f90, %f76;\n" " add.s64 %rd52, %rd51, %rd17;\n" " ld.global.f32 %f93, [%rd52];\n" " fma.rn.f32 %f94, %f90, %f93, %f78;\n" " ld.global.f32 %f95, [%rd68+4];\n" " sub.f32 %f96, %f95, %f134;\n" " fma.rn.f32 %f97, %f96, %f44, %f43;\n" " cvt.sat.f32.f32 %f98, %f97;\n" " fma.rm.f32 %f99, %f98, %f47, %f49;\n" " add.f32 %f100, %f99, 0fCB40007F;\n" " neg.f32 %f101, %f100;\n" " fma.rn.f32 %f102, %f96, %f46, %f101;\n" " fma.rn.f32 %f103, %f96, %f54, %f102;\n" " mov.b32 %r35, %f99;\n" " shl.b32 %r36, %r35, 23;\n" " mov.b32 %f104, %r36;\n" " ex2.approx.ftz.f32 %f105, %f103;\n" " mul.f32 %f106, %f105, %f104;\n" " ld.global.f32 %f107, [%rd69+4];\n" " fma.rn.f32 %f144, %f107, %f106, %f92;\n" " add.s64 %rd53, %rd52, %rd17;\n" " add.s64 %rd70, %rd53, %rd17;\n" " ld.global.f32 %f108, [%rd53];\n" " fma.rn.f32 %f143, %f106, %f108, %f94;\n" " add.s32 %r45, %r45, 4;\n" " add.s64 %rd69, %rd69, 16;\n" " add.s64 %rd68, %rd68, 16;\n" " add.s32 %r44, %r44, -4;\n" " setp.ne.s32 %p9, %r44, 0;\n" " @%p9 bra $L__BB9_11;\n" "\n" "$L__BB9_12:\n" " setp.eq.s32 %p10, %r46, 0;\n" " @%p10 bra $L__BB9_15;\n" "\n" " mul.lo.s64 %rd54, %rd4, %rd5;\n" " cvt.s64.s32 %rd55, %r45;\n" " add.s64 %rd56, %rd54, %rd55;\n" " mul.lo.s64 %rd57, %rd56, %rd6;\n" " add.s64 %rd58, %rd57, %rd13;\n" " shl.b64 %rd59, %rd58, 2;\n" " add.s64 %rd73, %rd1, %rd59;\n" " shl.b64 %rd25, %rd6, 2;\n" " shl.b64 %rd60, %rd56, 2;\n" " add.s64 %rd72, %rd2, %rd60;\n" " add.s64 %rd71, %rd3, %rd60;\n" "\n" "$L__BB9_14:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f109, [%rd71];\n" " sub.f32 %f110, %f109, %f134;\n" " mov.f32 %f111, 0f3F000000;\n" " mov.f32 %f112, 0f3BBB989D;\n" " fma.rn.f32 %f113, %f110, %f112, %f111;\n" " mov.f32 %f114, 0f3FB8AA3B;\n" " mov.f32 %f115, 0f437C0000;\n" " cvt.sat.f32.f32 %f116, %f113;\n" " mov.f32 %f117, 0f4B400001;\n" " fma.rm.f32 %f118, %f116, %f115, %f117;\n" " add.f32 %f119, %f118, 0fCB40007F;\n" " neg.f32 %f120, %f119;\n" " fma.rn.f32 %f121, %f110, %f114, %f120;\n" " mov.f32 %f122, 0f32A57060;\n" " fma.rn.f32 %f123, %f110, %f122, %f121;\n" " mov.b32 %r37, %f118;\n" " shl.b32 %r38, %r37, 23;\n" " mov.b32 %f124, %r38;\n" " ex2.approx.ftz.f32 %f125, %f123;\n" " mul.f32 %f126, %f125, %f124;\n" " ld.global.f32 %f127, [%rd72];\n" " fma.rn.f32 %f144, %f127, %f126, %f144;\n" " ld.global.f32 %f128, [%rd73];\n" " fma.rn.f32 %f143, %f126, %f128, %f143;\n" " add.s64 %rd73, %rd73, %rd25;\n" " add.s64 %rd72, %rd72, 4;\n" " add.s64 %rd71, %rd71, 4;\n" " add.s32 %r46, %r46, -1;\n" " setp.ne.s32 %p11, %r46, 0;\n" " @%p11 bra $L__BB9_14;\n" "\n" "$L__BB9_15:\n" " mul.lo.s64 %rd61, %rd6, %rd4;\n" " add.s64 %rd62, %rd61, %rd13;\n" " cvta.to.global.u64 %rd63, %rd34;\n" " shl.b64 %rd64, %rd62, 2;\n" " add.s64 %rd65, %rd63, %rd64;\n" " div.rn.f32 %f129, %f143, %f144;\n" " st.global.f32 [%rd65], %f129;\n" "\n" "$L__BB9_16:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode_combine_dev\n" ".visible .entry fc_attn_flash_decode_combine_dev(\n" " .param .u64 fc_attn_flash_decode_combine_dev_param_0,\n" " .param .u64 fc_attn_flash_decode_combine_dev_param_1,\n" " .param .u64 fc_attn_flash_decode_combine_dev_param_2,\n" " .param .u64 fc_attn_flash_decode_combine_dev_param_3,\n" " .param .u32 fc_attn_flash_decode_combine_dev_param_4,\n" " .param .u64 fc_attn_flash_decode_combine_dev_param_5,\n" " .param .u32 fc_attn_flash_decode_combine_dev_param_6,\n" " .param .u32 fc_attn_flash_decode_combine_dev_param_7\n" ")\n" "{\n" " .reg .pred %p<12>;\n" " .reg .f32 %f<145>;\n" " .reg .b32 %r<60>;\n" " .reg .b64 %rd<76>;\n" "\n" "\n" " ld.param.u64 %rd35, [fc_attn_flash_decode_combine_dev_param_0];\n" " ld.param.u64 %rd36, [fc_attn_flash_decode_combine_dev_param_1];\n" " ld.param.u64 %rd37, [fc_attn_flash_decode_combine_dev_param_2];\n" " ld.param.u64 %rd34, [fc_attn_flash_decode_combine_dev_param_3];\n" " ld.param.u32 %r22, [fc_attn_flash_decode_combine_dev_param_4];\n" " ld.param.u64 %rd38, [fc_attn_flash_decode_combine_dev_param_5];\n" " ld.param.u32 %r23, [fc_attn_flash_decode_combine_dev_param_6];\n" " ld.param.u32 %r24, [fc_attn_flash_decode_combine_dev_param_7];\n" " cvta.to.global.u64 %rd1, %rd37;\n" " cvta.to.global.u64 %rd2, %rd36;\n" " cvta.to.global.u64 %rd3, %rd35;\n" " cvta.to.global.u64 %rd39, %rd38;\n" " ld.global.u32 %r25, [%rd39];\n" " add.s32 %r26, %r25, %r23;\n" " div.s32 %r27, %r26, %r23;\n" " max.s32 %r28, %r27, 1;\n" " min.s32 %r29, %r28, %r24;\n" " add.s32 %r30, %r29, %r25;\n" " div.s32 %r31, %r30, %r29;\n" " add.s32 %r32, %r31, 15;\n" " and.b32 %r33, %r32, -16;\n" " add.s32 %r34, %r33, %r25;\n" " div.s32 %r1, %r34, %r33;\n" " mov.u32 %r2, %tid.x;\n" " setp.ge.s32 %p1, %r2, %r22;\n" " @%p1 bra $L__BB10_16;\n" "\n" " mov.u32 %r35, %ctaid.x;\n" " cvt.s64.s32 %rd4, %r35;\n" " cvt.s64.s32 %rd5, %r1;\n" " cvt.s64.s32 %rd6, %r22;\n" " setp.lt.s32 %p2, %r1, 1;\n" " mov.f32 %f134, 0fFF800000;\n" " @%p2 bra $L__BB10_8;\n" "\n" " add.s32 %r37, %r1, -1;\n" " and.b32 %r55, %r1, 3;\n" " setp.lt.u32 %p3, %r37, 3;\n" " mov.f32 %f134, 0fFF800000;\n" " mov.u32 %r54, 0;\n" " @%p3 bra $L__BB10_5;\n" "\n" " sub.s32 %r53, %r1, %r55;\n" " mul.lo.s64 %rd40, %rd4, %rd5;\n" " shl.b64 %rd41, %rd40, 2;\n" " add.s64 %rd42, %rd3, %rd41;\n" " add.s64 %rd68, %rd42, 8;\n" " mov.f32 %f134, 0fFF800000;\n" " mov.u32 %r54, 0;\n" "\n" "$L__BB10_4:\n" " ld.global.f32 %f26, [%rd68+-8];\n" " max.f32 %f27, %f134, %f26;\n" " ld.global.f32 %f28, [%rd68+-4];\n" " max.f32 %f29, %f27, %f28;\n" " ld.global.f32 %f30, [%rd68];\n" " max.f32 %f31, %f29, %f30;\n" " ld.global.f32 %f32, [%rd68+4];\n" " max.f32 %f134, %f31, %f32;\n" " add.s32 %r54, %r54, 4;\n" " add.s64 %rd68, %rd68, 16;\n" " add.s32 %r53, %r53, -4;\n" " setp.ne.s32 %p4, %r53, 0;\n" " @%p4 bra $L__BB10_4;\n" "\n" "$L__BB10_5:\n" " setp.eq.s32 %p5, %r55, 0;\n" " @%p5 bra $L__BB10_8;\n" "\n" " mul.lo.s64 %rd43, %rd4, %rd5;\n" " cvt.s64.s32 %rd44, %r54;\n" " add.s64 %rd45, %rd43, %rd44;\n" " shl.b64 %rd46, %rd45, 2;\n" " add.s64 %rd69, %rd3, %rd46;\n" "\n" "$L__BB10_7:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f33, [%rd69];\n" " max.f32 %f134, %f134, %f33;\n" " add.s64 %rd69, %rd69, 4;\n" " add.s32 %r55, %r55, -1;\n" " setp.ne.s32 %p6, %r55, 0;\n" " @%p6 bra $L__BB10_7;\n" "\n" "$L__BB10_8:\n" " cvt.s64.s32 %rd13, %r2;\n" " mov.f32 %f143, 0f00000000;\n" " mov.f32 %f144, %f143;\n" " @%p2 bra $L__BB10_15;\n" "\n" " add.s32 %r40, %r1, -1;\n" " and.b32 %r59, %r1, 3;\n" " setp.lt.u32 %p8, %r40, 3;\n" " mov.f32 %f144, 0f00000000;\n" " mov.u32 %r58, 0;\n" " mov.f32 %f143, %f144;\n" " @%p8 bra $L__BB10_12;\n" "\n" " sub.s32 %r57, %r1, %r59;\n" " mul.lo.s64 %rd47, %rd4, %rd5;\n" " shl.b64 %rd48, %rd47, 2;\n" " add.s64 %rd49, %rd48, 8;\n" " add.s64 %rd71, %rd2, %rd49;\n" " add.s64 %rd70, %rd3, %rd49;\n" " mul.lo.s64 %rd50, %rd47, %rd6;\n" " add.s64 %rd51, %rd50, %rd13;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd72, %rd1, %rd52;\n" " shl.b64 %rd17, %rd6, 2;\n" " mov.f32 %f144, 0f00000000;\n" " mov.u32 %r58, 0;\n" " mov.f32 %f43, 0f3F000000;\n" " mov.f32 %f44, 0f3BBB989D;\n" " mov.f32 %f143, %f144;\n" "\n" "$L__BB10_11:\n" " ld.global.f32 %f41, [%rd70+-8];\n" " sub.f32 %f42, %f41, %f134;\n" " fma.rn.f32 %f45, %f42, %f44, %f43;\n" " mov.f32 %f46, 0f3FB8AA3B;\n" " mov.f32 %f47, 0f437C0000;\n" " cvt.sat.f32.f32 %f48, %f45;\n" " mov.f32 %f49, 0f4B400001;\n" " fma.rm.f32 %f50, %f48, %f47, %f49;\n" " add.f32 %f51, %f50, 0fCB40007F;\n" " neg.f32 %f52, %f51;\n" " fma.rn.f32 %f53, %f42, %f46, %f52;\n" " mov.f32 %f54, 0f32A57060;\n" " fma.rn.f32 %f55, %f42, %f54, %f53;\n" " mov.b32 %r42, %f50;\n" " shl.b32 %r43, %r42, 23;\n" " mov.b32 %f56, %r43;\n" " ex2.approx.ftz.f32 %f57, %f55;\n" " mul.f32 %f58, %f57, %f56;\n" " ld.global.f32 %f59, [%rd71+-8];\n" " fma.rn.f32 %f60, %f59, %f58, %f144;\n" " ld.global.f32 %f61, [%rd72];\n" " fma.rn.f32 %f62, %f58, %f61, %f143;\n" " ld.global.f32 %f63, [%rd70+-4];\n" " sub.f32 %f64, %f63, %f134;\n" " fma.rn.f32 %f65, %f64, %f44, %f43;\n" " cvt.sat.f32.f32 %f66, %f65;\n" " fma.rm.f32 %f67, %f66, %f47, %f49;\n" " add.f32 %f68, %f67, 0fCB40007F;\n" " neg.f32 %f69, %f68;\n" " fma.rn.f32 %f70, %f64, %f46, %f69;\n" " fma.rn.f32 %f71, %f64, %f54, %f70;\n" " mov.b32 %r44, %f67;\n" " shl.b32 %r45, %r44, 23;\n" " mov.b32 %f72, %r45;\n" " ex2.approx.ftz.f32 %f73, %f71;\n" " mul.f32 %f74, %f73, %f72;\n" " ld.global.f32 %f75, [%rd71+-4];\n" " fma.rn.f32 %f76, %f75, %f74, %f60;\n" " add.s64 %rd53, %rd72, %rd17;\n" " ld.global.f32 %f77, [%rd53];\n" " fma.rn.f32 %f78, %f74, %f77, %f62;\n" " ld.global.f32 %f79, [%rd70];\n" " sub.f32 %f80, %f79, %f134;\n" " fma.rn.f32 %f81, %f80, %f44, %f43;\n" " cvt.sat.f32.f32 %f82, %f81;\n" " fma.rm.f32 %f83, %f82, %f47, %f49;\n" " add.f32 %f84, %f83, 0fCB40007F;\n" " neg.f32 %f85, %f84;\n" " fma.rn.f32 %f86, %f80, %f46, %f85;\n" " fma.rn.f32 %f87, %f80, %f54, %f86;\n" " mov.b32 %r46, %f83;\n" " shl.b32 %r47, %r46, 23;\n" " mov.b32 %f88, %r47;\n" " ex2.approx.ftz.f32 %f89, %f87;\n" " mul.f32 %f90, %f89, %f88;\n" " ld.global.f32 %f91, [%rd71];\n" " fma.rn.f32 %f92, %f91, %f90, %f76;\n" " add.s64 %rd54, %rd53, %rd17;\n" " ld.global.f32 %f93, [%rd54];\n" " fma.rn.f32 %f94, %f90, %f93, %f78;\n" " ld.global.f32 %f95, [%rd70+4];\n" " sub.f32 %f96, %f95, %f134;\n" " fma.rn.f32 %f97, %f96, %f44, %f43;\n" " cvt.sat.f32.f32 %f98, %f97;\n" " fma.rm.f32 %f99, %f98, %f47, %f49;\n" " add.f32 %f100, %f99, 0fCB40007F;\n" " neg.f32 %f101, %f100;\n" " fma.rn.f32 %f102, %f96, %f46, %f101;\n" " fma.rn.f32 %f103, %f96, %f54, %f102;\n" " mov.b32 %r48, %f99;\n" " shl.b32 %r49, %r48, 23;\n" " mov.b32 %f104, %r49;\n" " ex2.approx.ftz.f32 %f105, %f103;\n" " mul.f32 %f106, %f105, %f104;\n" " ld.global.f32 %f107, [%rd71+4];\n" " fma.rn.f32 %f144, %f107, %f106, %f92;\n" " add.s64 %rd55, %rd54, %rd17;\n" " add.s64 %rd72, %rd55, %rd17;\n" " ld.global.f32 %f108, [%rd55];\n" " fma.rn.f32 %f143, %f106, %f108, %f94;\n" " add.s32 %r58, %r58, 4;\n" " add.s64 %rd71, %rd71, 16;\n" " add.s64 %rd70, %rd70, 16;\n" " add.s32 %r57, %r57, -4;\n" " setp.ne.s32 %p9, %r57, 0;\n" " @%p9 bra $L__BB10_11;\n" "\n" "$L__BB10_12:\n" " setp.eq.s32 %p10, %r59, 0;\n" " @%p10 bra $L__BB10_15;\n" "\n" " mul.lo.s64 %rd56, %rd4, %rd5;\n" " cvt.s64.s32 %rd57, %r58;\n" " add.s64 %rd58, %rd56, %rd57;\n" " mul.lo.s64 %rd59, %rd58, %rd6;\n" " add.s64 %rd60, %rd59, %rd13;\n" " shl.b64 %rd61, %rd60, 2;\n" " add.s64 %rd75, %rd1, %rd61;\n" " shl.b64 %rd25, %rd6, 2;\n" " shl.b64 %rd62, %rd58, 2;\n" " add.s64 %rd74, %rd2, %rd62;\n" " add.s64 %rd73, %rd3, %rd62;\n" "\n" "$L__BB10_14:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f109, [%rd73];\n" " sub.f32 %f110, %f109, %f134;\n" " mov.f32 %f111, 0f3F000000;\n" " mov.f32 %f112, 0f3BBB989D;\n" " fma.rn.f32 %f113, %f110, %f112, %f111;\n" " mov.f32 %f114, 0f3FB8AA3B;\n" " mov.f32 %f115, 0f437C0000;\n" " cvt.sat.f32.f32 %f116, %f113;\n" " mov.f32 %f117, 0f4B400001;\n" " fma.rm.f32 %f118, %f116, %f115, %f117;\n" " add.f32 %f119, %f118, 0fCB40007F;\n" " neg.f32 %f120, %f119;\n" " fma.rn.f32 %f121, %f110, %f114, %f120;\n" " mov.f32 %f122, 0f32A57060;\n" " fma.rn.f32 %f123, %f110, %f122, %f121;\n" " mov.b32 %r50, %f118;\n" " shl.b32 %r51, %r50, 23;\n" " mov.b32 %f124, %r51;\n" " ex2.approx.ftz.f32 %f125, %f123;\n" " mul.f32 %f126, %f125, %f124;\n" " ld.global.f32 %f127, [%rd74];\n" " fma.rn.f32 %f144, %f127, %f126, %f144;\n" " ld.global.f32 %f128, [%rd75];\n" " fma.rn.f32 %f143, %f126, %f128, %f143;\n" " add.s64 %rd75, %rd75, %rd25;\n" " add.s64 %rd74, %rd74, 4;\n" " add.s64 %rd73, %rd73, 4;\n" " add.s32 %r59, %r59, -1;\n" " setp.ne.s32 %p11, %r59, 0;\n" " @%p11 bra $L__BB10_14;\n" "\n" "$L__BB10_15:\n" " mul.lo.s64 %rd63, %rd6, %rd4;\n" " add.s64 %rd64, %rd63, %rd13;\n" " cvta.to.global.u64 %rd65, %rd34;\n" " shl.b64 %rd66, %rd64, 2;\n" " add.s64 %rd67, %rd65, %rd66;\n" " div.rn.f32 %f129, %f143, %f144;\n" " st.global.f32 [%rd67], %f129;\n" "\n" "$L__BB10_16:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode\n" ".visible .entry fc_attn_flash_decode(\n" " .param .u64 fc_attn_flash_decode_param_0,\n" " .param .u64 fc_attn_flash_decode_param_1,\n" " .param .u64 fc_attn_flash_decode_param_2,\n" " .param .u64 fc_attn_flash_decode_param_3,\n" " .param .u64 fc_attn_flash_decode_param_4,\n" " .param .u64 fc_attn_flash_decode_param_5,\n" " .param .u32 fc_attn_flash_decode_param_6,\n" " .param .u32 fc_attn_flash_decode_param_7,\n" " .param .u32 fc_attn_flash_decode_param_8,\n" " .param .u32 fc_attn_flash_decode_param_9,\n" " .param .u32 fc_attn_flash_decode_param_10,\n" " .param .u32 fc_attn_flash_decode_param_11,\n" " .param .u32 fc_attn_flash_decode_param_12,\n" " .param .u32 fc_attn_flash_decode_param_13,\n" " .param .f32 fc_attn_flash_decode_param_14\n" ")\n" "{\n" " .reg .pred %p<171>;\n" " .reg .f32 %f<637>;\n" " .reg .b32 %r<556>;\n" " .reg .b64 %rd<158>;\n" "\n" "\n" " ld.param.u64 %rd36, [fc_attn_flash_decode_param_0];\n" " ld.param.u64 %rd34, [fc_attn_flash_decode_param_1];\n" " ld.param.u64 %rd37, [fc_attn_flash_decode_param_3];\n" " ld.param.u64 %rd38, [fc_attn_flash_decode_param_4];\n" " ld.param.u64 %rd39, [fc_attn_flash_decode_param_5];\n" " ld.param.u32 %r102, [fc_attn_flash_decode_param_6];\n" " ld.param.u32 %r103, [fc_attn_flash_decode_param_7];\n" " ld.param.u32 %r104, [fc_attn_flash_decode_param_8];\n" " ld.param.u32 %r105, [fc_attn_flash_decode_param_9];\n" " ld.param.u32 %r107, [fc_attn_flash_decode_param_11];\n" " ld.param.u32 %r109, [fc_attn_flash_decode_param_13];\n" " ld.param.f32 %f179, [fc_attn_flash_decode_param_14];\n" " cvta.to.global.u64 %rd1, %rd36;\n" " cvta.to.global.u64 %rd2, %rd38;\n" " cvta.to.global.u64 %rd3, %rd37;\n" " cvta.to.global.u64 %rd4, %rd39;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %ctaid.x;\n" " mul.lo.s32 %r541, %r2, %r107;\n" " add.s32 %r110, %r541, %r107;\n" " min.s32 %r4, %r110, %r105;\n" " mul.lo.s32 %r5, %r103, %r102;\n" " add.s32 %r6, %r103, 16;\n" " mul.wide.s32 %rd5, %r6, 16;\n" " shl.b32 %r111, %r102, 4;\n" " cvt.s64.s32 %rd40, %r111;\n" " add.s64 %rd6, %rd5, %rd40;\n" " add.s64 %rd7, %rd6, %rd40;\n" " cvt.s64.s32 %rd8, %r102;\n" " add.s64 %rd9, %rd7, %rd8;\n" " mov.u32 %r112, %ctaid.y;\n" " cvt.s64.s32 %rd10, %r112;\n" " mul.wide.s32 %rd11, %r102, %r112;\n" " cvt.s64.s32 %rd12, %r103;\n" " setp.lt.s32 %p2, %r102, 1;\n" " @%p2 bra $L__BB11_7;\n" "\n" " cvt.s64.s32 %rd13, %r1;\n" " and.b32 %r539, %r102, 3;\n" " add.s32 %r114, %r102, -1;\n" " setp.lt.u32 %p3, %r114, 3;\n" " mov.u32 %r537, 0;\n" " @%p3 bra $L__BB11_4;\n" "\n" " sub.s32 %r535, %r102, %r539;\n" " mul.lo.s64 %rd41, %rd10, %rd12;\n" " mul.lo.s64 %rd42, %rd41, %rd8;\n" " add.s64 %rd43, %rd42, %rd13;\n" " shl.b64 %rd44, %rd43, 2;\n" " add.s64 %rd157, %rd1, %rd44;\n" " shl.b64 %rd15, %rd12, 2;\n" " shl.b32 %r116, %r1, 2;\n" " mov.u32 %r117, sh;\n" " add.s32 %r536, %r117, %r116;\n" " shl.b32 %r10, %r103, 2;\n" " mov.u32 %r537, 0;\n" "\n" "$L__BB11_3:\n" " ld.global.f32 %f180, [%rd157];\n" " st.shared.f32 [%r536], %f180;\n" " add.s64 %rd45, %rd157, %rd15;\n" " ld.global.f32 %f181, [%rd45];\n" " add.s32 %r118, %r536, %r10;\n" " st.shared.f32 [%r118], %f181;\n" " add.s64 %rd46, %rd45, %rd15;\n" " ld.global.f32 %f182, [%rd46];\n" " add.s32 %r119, %r118, %r10;\n" " st.shared.f32 [%r119], %f182;\n" " add.s64 %rd47, %rd46, %rd15;\n" " add.s64 %rd157, %rd47, %rd15;\n" " ld.global.f32 %f183, [%rd47];\n" " add.s32 %r120, %r119, %r10;\n" " add.s32 %r536, %r120, %r10;\n" " st.shared.f32 [%r120], %f183;\n" " add.s32 %r537, %r537, 4;\n" " add.s32 %r535, %r535, -4;\n" " setp.ne.s32 %p4, %r535, 0;\n" " @%p4 bra $L__BB11_3;\n" "\n" "$L__BB11_4:\n" " setp.eq.s32 %p5, %r539, 0;\n" " @%p5 bra $L__BB11_7;\n" "\n" " mov.u32 %r123, sh;\n" "\n" "$L__BB11_6:\n" " .pragma \"nounroll\";\n" " cvt.s64.s32 %rd48, %r537;\n" " add.s64 %rd49, %rd11, %rd48;\n" " mul.lo.s64 %rd50, %rd49, %rd12;\n" " add.s64 %rd51, %rd50, %rd13;\n" " shl.b64 %rd52, %rd51, 2;\n" " add.s64 %rd53, %rd1, %rd52;\n" " ld.global.f32 %f184, [%rd53];\n" " mad.lo.s32 %r121, %r537, %r103, %r1;\n" " shl.b32 %r122, %r121, 2;\n" " add.s32 %r124, %r123, %r122;\n" " st.shared.f32 [%r124], %f184;\n" " add.s32 %r537, %r537, 1;\n" " add.s32 %r539, %r539, -1;\n" " setp.ne.s32 %p6, %r539, 0;\n" " @%p6 bra $L__BB11_6;\n" "\n" "$L__BB11_7:\n" " setp.ge.s32 %p7, %r1, %r102;\n" " add.s32 %r125, %r5, %r1;\n" " cvt.u32.u64 %r126, %rd7;\n" " add.s32 %r127, %r125, %r126;\n" " shl.b32 %r128, %r127, 2;\n" " mov.u32 %r129, sh;\n" " add.s32 %r22, %r129, %r128;\n" " cvt.u32.u64 %r130, %rd9;\n" " add.s32 %r131, %r125, %r130;\n" " shl.b32 %r132, %r131, 2;\n" " add.s32 %r23, %r129, %r132;\n" " @%p7 bra $L__BB11_9;\n" "\n" " mov.u32 %r133, -8388608;\n" " st.shared.u32 [%r22], %r133;\n" " mov.u32 %r134, 0;\n" " st.shared.u32 [%r23], %r134;\n" "\n" "$L__BB11_9:\n" " bar.sync 0;\n" " mul.lo.s64 %rd54, %rd12, %rd10;\n" " cvt.s64.s32 %rd55, %r104;\n" " mul.lo.s64 %rd18, %rd54, %rd55;\n" " shr.s32 %r135, %r1, 31;\n" " shr.u32 %r136, %r135, 28;\n" " add.s32 %r137, %r1, %r136;\n" " shr.s32 %r25, %r137, 4;\n" " and.b32 %r138, %r137, -16;\n" " sub.s32 %r26, %r1, %r138;\n" " cvt.s64.s32 %rd19, %r1;\n" " setp.le.s32 %p8, %r4, %r541;\n" " mov.f32 %f605, 0f00000000;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" " @%p8 bra $L__BB11_151;\n" "\n" " shr.s32 %r140, %r103, 31;\n" " shr.u32 %r141, %r140, 28;\n" " add.s32 %r142, %r103, %r141;\n" " shr.s32 %r143, %r142, 4;\n" " sub.s32 %r144, %r105, %r109;\n" " mov.u32 %r540, 0;\n" " max.s32 %r27, %r144, 0;\n" " add.s32 %r145, %r5, %r25;\n" " cvt.u32.u64 %r146, %rd5;\n" " add.s32 %r147, %r145, %r146;\n" " mad.lo.s32 %r28, %r6, %r25, %r5;\n" " shl.b32 %r148, %r1, 4;\n" " add.s32 %r149, %r148, %r146;\n" " add.s32 %r29, %r5, %r149;\n" " add.s64 %rd20, %rd18, %rd19;\n" " add.s32 %r150, %r130, %r102;\n" " add.s32 %r151, %r5, %r150;\n" " add.s32 %r153, %r125, %r150;\n" " shl.b32 %r154, %r153, 2;\n" " add.s32 %r30, %r129, %r154;\n" " cvt.u64.u32 %rd56, %r1;\n" " add.s64 %rd21, %rd6, %rd56;\n" " add.s32 %r156, %r2, 1;\n" " mul.lo.s32 %r157, %r156, %r107;\n" " min.s32 %r158, %r157, %r105;\n" " not.b32 %r159, %r158;\n" " add.s32 %r31, %r541, %r159;\n" " bfe.u32 %r33, %r142, 4, 1;\n" " sub.s32 %r34, %r143, %r33;\n" " shl.b32 %r35, %r103, 1;\n" " mul.lo.s32 %r36, %r103, 3;\n" " shl.b32 %r37, %r103, 2;\n" " mul.lo.s32 %r38, %r103, 5;\n" " mul.lo.s32 %r39, %r103, 6;\n" " mul.lo.s32 %r40, %r103, 7;\n" " shl.b32 %r160, %r147, 2;\n" " add.s32 %r41, %r129, %r160;\n" " shl.b32 %r161, %r151, 2;\n" " add.s32 %r42, %r129, %r161;\n" " shl.b32 %r162, %r125, 2;\n" " add.s32 %r43, %r129, %r162;\n" " mad.lo.s32 %r164, %r102, %r6, %r1;\n" " add.s32 %r165, %r164, %r146;\n" " shl.b32 %r166, %r165, 2;\n" " add.s32 %r167, %r129, %r166;\n" " add.s32 %r46, %r167, -1024;\n" " cvta.to.global.u64 %rd22, %rd34;\n" " mov.f32 %f605, 0f00000000;\n" " setp.lt.s32 %p10, %r109, 1;\n" " setp.ge.s32 %p18, %r1, %r103;\n" " setp.lt.s32 %p21, %r103, 16;\n" " setp.eq.s32 %p51, %r26, 0;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" "\n" "$L__BB11_11:\n" " ld.param.u32 %r435, [fc_attn_flash_decode_param_12];\n" " ld.param.u32 %r434, [fc_attn_flash_decode_param_9];\n" " shl.b32 %r168, %r540, 4;\n" " add.s32 %r169, %r31, %r168;\n" " max.s32 %r170, %r169, -17;\n" " not.b32 %r171, %r170;\n" " max.s32 %r51, %r171, 1;\n" " sub.s32 %r52, %r4, %r541;\n" " min.s32 %r53, %r52, 16;\n" " setp.lt.s32 %p9, %r541, %r435;\n" " or.pred %p11, %p10, %p9;\n" " add.s32 %r172, %r53, %r541;\n" " setp.gt.s32 %p12, %r172, %r27;\n" " setp.lt.s32 %p13, %r541, %r434;\n" " and.pred %p14, %p13, %p12;\n" " or.pred %p15, %p11, %p14;\n" " not.pred %p16, %p15;\n" " @%p16 bra $L__BB11_150;\n" "\n" " setp.lt.s32 %p17, %r52, 1;\n" " @%p17 bra $L__BB11_18;\n" "\n" " mov.u32 %r542, 0;\n" "\n" "$L__BB11_14:\n" " @%p18 bra $L__BB11_17;\n" "\n" " add.s32 %r445, %r103, 16;\n" " add.s32 %r174, %r542, %r541;\n" " cvt.s64.s32 %rd57, %r174;\n" " mul.lo.s64 %rd58, %rd57, %rd12;\n" " add.s64 %rd23, %rd58, %rd18;\n" " mul.lo.s32 %r175, %r542, %r445;\n" " shl.b32 %r176, %r175, 2;\n" " add.s32 %r543, %r43, %r176;\n" " mov.u32 %r544, %r1;\n" "\n" "$L__BB11_16:\n" " mov.u32 %r448, %ntid.x;\n" " mov.u32 %r447, %ntid.x;\n" " shl.b32 %r446, %r447, 2;\n" " cvt.s64.s32 %rd59, %r544;\n" " add.s64 %rd60, %rd23, %rd59;\n" " shl.b64 %rd61, %rd60, 2;\n" " add.s64 %rd62, %rd22, %rd61;\n" " ld.global.f32 %f201, [%rd62];\n" " st.shared.f32 [%r543], %f201;\n" " add.s32 %r543, %r543, %r446;\n" " add.s32 %r544, %r544, %r447;\n" " setp.lt.s32 %p19, %r544, %r103;\n" " @%p19 bra $L__BB11_16;\n" "\n" "$L__BB11_17:\n" " sub.s32 %r444, %r4, %r541;\n" " min.s32 %r443, %r444, 16;\n" " add.s32 %r542, %r542, 1;\n" " setp.lt.s32 %p20, %r542, %r443;\n" " @%p20 bra $L__BB11_14;\n" "\n" "$L__BB11_18:\n" " sub.s32 %r437, %r4, %r541;\n" " min.s32 %r436, %r437, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p22, %r25, %r436;\n" " or.pred %p23, %p21, %p22;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p23 bra $L__BB11_71;\n" "\n" " and.b32 %r438, %r103, -16;\n" " setp.eq.s32 %p24, %r438, 16;\n" " mov.u32 %r547, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p24 bra $L__BB11_54;\n" "\n" " mov.u32 %r547, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " mov.u32 %r546, %r34;\n" "\n" "$L__BB11_21:\n" " shl.b32 %r179, %r547, 4;\n" " add.s32 %r63, %r179, %r26;\n" " add.s32 %r180, %r28, %r63;\n" " shl.b32 %r181, %r180, 2;\n" " add.s32 %r64, %r129, %r181;\n" " ld.shared.f32 %f17, [%r64];\n" " @%p2 bra $L__BB11_23;\n" "\n" " shl.b32 %r528, %r547, 4;\n" " add.s32 %r527, %r528, %r26;\n" " shl.b32 %r526, %r527, 2;\n" " add.s32 %r525, %r129, %r526;\n" " ld.shared.f32 %f226, [%r525];\n" " fma.rn.f32 %f536, %f17, %f226, %f536;\n" "\n" "$L__BB11_23:\n" " setp.lt.s32 %p26, %r102, 2;\n" " @%p26 bra $L__BB11_25;\n" "\n" " shl.b32 %r533, %r547, 4;\n" " add.s32 %r532, %r533, %r26;\n" " add.s32 %r531, %r532, %r103;\n" " shl.b32 %r530, %r531, 2;\n" " add.s32 %r529, %r129, %r530;\n" " ld.shared.f32 %f227, [%r529];\n" " fma.rn.f32 %f537, %f17, %f227, %f537;\n" "\n" "$L__BB11_25:\n" " setp.lt.s32 %p27, %r102, 3;\n" " @%p27 bra $L__BB11_27;\n" "\n" " shl.b32 %r474, %r547, 4;\n" " add.s32 %r473, %r474, %r26;\n" " add.s32 %r472, %r35, %r473;\n" " shl.b32 %r471, %r472, 2;\n" " add.s32 %r470, %r129, %r471;\n" " ld.shared.f32 %f228, [%r470];\n" " fma.rn.f32 %f538, %f17, %f228, %f538;\n" "\n" "$L__BB11_27:\n" " setp.lt.s32 %p28, %r102, 4;\n" " @%p28 bra $L__BB11_29;\n" "\n" " shl.b32 %r484, %r547, 4;\n" " add.s32 %r483, %r484, %r26;\n" " add.s32 %r482, %r36, %r483;\n" " shl.b32 %r481, %r482, 2;\n" " add.s32 %r480, %r129, %r481;\n" " ld.shared.f32 %f229, [%r480];\n" " fma.rn.f32 %f539, %f17, %f229, %f539;\n" "\n" "$L__BB11_29:\n" " setp.lt.s32 %p29, %r102, 5;\n" " @%p29 bra $L__BB11_31;\n" "\n" " shl.b32 %r494, %r547, 4;\n" " add.s32 %r493, %r494, %r26;\n" " add.s32 %r492, %r37, %r493;\n" " shl.b32 %r491, %r492, 2;\n" " add.s32 %r490, %r129, %r491;\n" " ld.shared.f32 %f230, [%r490];\n" " fma.rn.f32 %f540, %f17, %f230, %f540;\n" "\n" "$L__BB11_31:\n" " setp.lt.s32 %p30, %r102, 6;\n" " @%p30 bra $L__BB11_33;\n" "\n" " shl.b32 %r504, %r547, 4;\n" " add.s32 %r503, %r504, %r26;\n" " add.s32 %r502, %r38, %r503;\n" " shl.b32 %r501, %r502, 2;\n" " add.s32 %r500, %r129, %r501;\n" " ld.shared.f32 %f231, [%r500];\n" " fma.rn.f32 %f541, %f17, %f231, %f541;\n" "\n" "$L__BB11_33:\n" " setp.lt.s32 %p31, %r102, 7;\n" " @%p31 bra $L__BB11_35;\n" "\n" " shl.b32 %r514, %r547, 4;\n" " add.s32 %r513, %r514, %r26;\n" " add.s32 %r512, %r39, %r513;\n" " shl.b32 %r511, %r512, 2;\n" " add.s32 %r510, %r129, %r511;\n" " ld.shared.f32 %f232, [%r510];\n" " fma.rn.f32 %f542, %f17, %f232, %f542;\n" "\n" "$L__BB11_35:\n" " setp.lt.s32 %p32, %r102, 8;\n" " @%p32 bra $L__BB11_37;\n" "\n" " shl.b32 %r524, %r547, 4;\n" " add.s32 %r523, %r524, %r26;\n" " add.s32 %r522, %r40, %r523;\n" " shl.b32 %r521, %r522, 2;\n" " add.s32 %r520, %r129, %r521;\n" " ld.shared.f32 %f233, [%r520];\n" " fma.rn.f32 %f543, %f17, %f233, %f543;\n" "\n" "$L__BB11_37:\n" " shl.b32 %r455, %r547, 4;\n" " add.s32 %r454, %r455, %r26;\n" " add.s32 %r453, %r28, %r454;\n" " shl.b32 %r452, %r453, 2;\n" " add.s32 %r451, %r129, %r452;\n" " ld.shared.f32 %f34, [%r451+64];\n" " @%p2 bra $L__BB11_39;\n" "\n" " shl.b32 %r469, %r547, 4;\n" " add.s32 %r468, %r469, %r26;\n" " shl.b32 %r467, %r468, 2;\n" " add.s32 %r466, %r129, %r467;\n" " ld.shared.f32 %f234, [%r466+64];\n" " fma.rn.f32 %f536, %f34, %f234, %f536;\n" "\n" "$L__BB11_39:\n" " setp.lt.s32 %p164, %r102, 2;\n" " @%p164 bra $L__BB11_41;\n" "\n" " shl.b32 %r465, %r547, 4;\n" " add.s32 %r464, %r465, %r26;\n" " add.s32 %r463, %r464, %r103;\n" " shl.b32 %r462, %r463, 2;\n" " add.s32 %r461, %r129, %r462;\n" " ld.shared.f32 %f235, [%r461+64];\n" " fma.rn.f32 %f537, %f34, %f235, %f537;\n" "\n" "$L__BB11_41:\n" " setp.lt.s32 %p165, %r102, 3;\n" " @%p165 bra $L__BB11_43;\n" "\n" " shl.b32 %r460, %r547, 4;\n" " add.s32 %r459, %r460, %r26;\n" " add.s32 %r458, %r35, %r459;\n" " shl.b32 %r457, %r458, 2;\n" " add.s32 %r456, %r129, %r457;\n" " ld.shared.f32 %f236, [%r456+64];\n" " fma.rn.f32 %f538, %f34, %f236, %f538;\n" "\n" "$L__BB11_43:\n" " setp.lt.s32 %p166, %r102, 4;\n" " @%p166 bra $L__BB11_45;\n" "\n" " shl.b32 %r479, %r547, 4;\n" " add.s32 %r478, %r479, %r26;\n" " add.s32 %r477, %r36, %r478;\n" " shl.b32 %r476, %r477, 2;\n" " add.s32 %r475, %r129, %r476;\n" " ld.shared.f32 %f237, [%r475+64];\n" " fma.rn.f32 %f539, %f34, %f237, %f539;\n" "\n" "$L__BB11_45:\n" " setp.lt.s32 %p167, %r102, 5;\n" " @%p167 bra $L__BB11_47;\n" "\n" " shl.b32 %r489, %r547, 4;\n" " add.s32 %r488, %r489, %r26;\n" " add.s32 %r487, %r37, %r488;\n" " shl.b32 %r486, %r487, 2;\n" " add.s32 %r485, %r129, %r486;\n" " ld.shared.f32 %f238, [%r485+64];\n" " fma.rn.f32 %f540, %f34, %f238, %f540;\n" "\n" "$L__BB11_47:\n" " setp.lt.s32 %p168, %r102, 6;\n" " @%p168 bra $L__BB11_49;\n" "\n" " shl.b32 %r499, %r547, 4;\n" " add.s32 %r498, %r499, %r26;\n" " add.s32 %r497, %r38, %r498;\n" " shl.b32 %r496, %r497, 2;\n" " add.s32 %r495, %r129, %r496;\n" " ld.shared.f32 %f239, [%r495+64];\n" " fma.rn.f32 %f541, %f34, %f239, %f541;\n" "\n" "$L__BB11_49:\n" " setp.lt.s32 %p169, %r102, 7;\n" " @%p169 bra $L__BB11_51;\n" "\n" " shl.b32 %r509, %r547, 4;\n" " add.s32 %r508, %r509, %r26;\n" " add.s32 %r507, %r39, %r508;\n" " shl.b32 %r506, %r507, 2;\n" " add.s32 %r505, %r129, %r506;\n" " ld.shared.f32 %f240, [%r505+64];\n" " fma.rn.f32 %f542, %f34, %f240, %f542;\n" "\n" "$L__BB11_51:\n" " setp.lt.s32 %p170, %r102, 8;\n" " @%p170 bra $L__BB11_53;\n" "\n" " shl.b32 %r519, %r547, 4;\n" " add.s32 %r518, %r519, %r26;\n" " add.s32 %r517, %r40, %r518;\n" " shl.b32 %r516, %r517, 2;\n" " add.s32 %r515, %r129, %r516;\n" " ld.shared.f32 %f241, [%r515+64];\n" " fma.rn.f32 %f543, %f34, %f241, %f543;\n" "\n" "$L__BB11_53:\n" " add.s32 %r547, %r547, 2;\n" " add.s32 %r546, %r546, -2;\n" " setp.ne.s32 %p41, %r546, 0;\n" " @%p41 bra $L__BB11_21;\n" "\n" "$L__BB11_54:\n" " setp.eq.s32 %p42, %r33, 0;\n" " @%p42 bra $L__BB11_71;\n" "\n" " shl.b32 %r205, %r547, 4;\n" " add.s32 %r76, %r205, %r26;\n" " add.s32 %r206, %r28, %r76;\n" " shl.b32 %r207, %r206, 2;\n" " add.s32 %r209, %r129, %r207;\n" " ld.shared.f32 %f59, [%r209];\n" " @%p2 bra $L__BB11_57;\n" "\n" " shl.b32 %r210, %r76, 2;\n" " add.s32 %r212, %r129, %r210;\n" " ld.shared.f32 %f242, [%r212];\n" " fma.rn.f32 %f536, %f59, %f242, %f536;\n" "\n" "$L__BB11_57:\n" " setp.lt.s32 %p44, %r102, 2;\n" " @%p44 bra $L__BB11_59;\n" "\n" " add.s32 %r213, %r76, %r103;\n" " shl.b32 %r214, %r213, 2;\n" " add.s32 %r216, %r129, %r214;\n" " ld.shared.f32 %f243, [%r216];\n" " fma.rn.f32 %f537, %f59, %f243, %f537;\n" "\n" "$L__BB11_59:\n" " setp.lt.s32 %p45, %r102, 3;\n" " @%p45 bra $L__BB11_61;\n" "\n" " add.s32 %r217, %r35, %r76;\n" " shl.b32 %r218, %r217, 2;\n" " add.s32 %r220, %r129, %r218;\n" " ld.shared.f32 %f244, [%r220];\n" " fma.rn.f32 %f538, %f59, %f244, %f538;\n" "\n" "$L__BB11_61:\n" " setp.lt.s32 %p46, %r102, 4;\n" " @%p46 bra $L__BB11_63;\n" "\n" " add.s32 %r221, %r36, %r76;\n" " shl.b32 %r222, %r221, 2;\n" " add.s32 %r224, %r129, %r222;\n" " ld.shared.f32 %f245, [%r224];\n" " fma.rn.f32 %f539, %f59, %f245, %f539;\n" "\n" "$L__BB11_63:\n" " setp.lt.s32 %p47, %r102, 5;\n" " @%p47 bra $L__BB11_65;\n" "\n" " add.s32 %r225, %r37, %r76;\n" " shl.b32 %r226, %r225, 2;\n" " add.s32 %r228, %r129, %r226;\n" " ld.shared.f32 %f246, [%r228];\n" " fma.rn.f32 %f540, %f59, %f246, %f540;\n" "\n" "$L__BB11_65:\n" " setp.lt.s32 %p48, %r102, 6;\n" " @%p48 bra $L__BB11_67;\n" "\n" " add.s32 %r229, %r38, %r76;\n" " shl.b32 %r230, %r229, 2;\n" " add.s32 %r232, %r129, %r230;\n" " ld.shared.f32 %f247, [%r232];\n" " fma.rn.f32 %f541, %f59, %f247, %f541;\n" "\n" "$L__BB11_67:\n" " setp.lt.s32 %p49, %r102, 7;\n" " @%p49 bra $L__BB11_69;\n" "\n" " add.s32 %r233, %r39, %r76;\n" " shl.b32 %r234, %r233, 2;\n" " add.s32 %r236, %r129, %r234;\n" " ld.shared.f32 %f248, [%r236];\n" " fma.rn.f32 %f542, %f59, %f248, %f542;\n" "\n" "$L__BB11_69:\n" " setp.lt.s32 %p50, %r102, 8;\n" " @%p50 bra $L__BB11_71;\n" "\n" " add.s32 %r237, %r40, %r76;\n" " shl.b32 %r238, %r237, 2;\n" " add.s32 %r240, %r129, %r238;\n" " ld.shared.f32 %f249, [%r240];\n" " fma.rn.f32 %f543, %f59, %f249, %f543;\n" "\n" "$L__BB11_71:\n" " sub.s32 %r440, %r4, %r541;\n" " min.s32 %r439, %r440, 16;\n" " setp.lt.s32 %p52, %r25, %r439;\n" " and.pred %p1, %p51, %p52;\n" " @%p2 bra $L__BB11_74;\n" "\n" " mov.b32 %r241, %f536;\n" " mov.u32 %r242, 2;\n" " mov.u32 %r243, 4127;\n" " mov.u32 %r244, 8;\n" " mov.u32 %r245, -1;\n" " shfl.sync.down.b32 %r246|%p54, %r241, %r244, %r243, %r245;\n" " mov.b32 %f250, %r246;\n" " add.f32 %f251, %f536, %f250;\n" " mov.b32 %r247, %f251;\n" " mov.u32 %r248, 4;\n" " shfl.sync.down.b32 %r249|%p55, %r247, %r248, %r243, %r245;\n" " mov.b32 %f252, %r249;\n" " add.f32 %f253, %f251, %f252;\n" " mov.b32 %r250, %f253;\n" " shfl.sync.down.b32 %r251|%p56, %r250, %r242, %r243, %r245;\n" " mov.b32 %f254, %r251;\n" " add.f32 %f255, %f253, %f254;\n" " mov.b32 %r252, %f255;\n" " mov.u32 %r253, 1;\n" " shfl.sync.down.b32 %r254|%p57, %r252, %r253, %r243, %r245;\n" " mov.b32 %f256, %r254;\n" " add.f32 %f83, %f255, %f256;\n" " not.pred %p58, %p1;\n" " @%p58 bra $L__BB11_74;\n" "\n" " mul.f32 %f257, %f83, %f179;\n" " st.shared.f32 [%r41], %f257;\n" "\n" "$L__BB11_74:\n" " setp.lt.s32 %p59, %r102, 2;\n" " @%p59 bra $L__BB11_77;\n" "\n" " mov.b32 %r255, %f537;\n" " mov.u32 %r256, 2;\n" " mov.u32 %r257, 4127;\n" " mov.u32 %r258, 8;\n" " mov.u32 %r259, -1;\n" " shfl.sync.down.b32 %r260|%p60, %r255, %r258, %r257, %r259;\n" " mov.b32 %f258, %r260;\n" " add.f32 %f259, %f537, %f258;\n" " mov.b32 %r261, %f259;\n" " mov.u32 %r262, 4;\n" " shfl.sync.down.b32 %r263|%p61, %r261, %r262, %r257, %r259;\n" " mov.b32 %f260, %r263;\n" " add.f32 %f261, %f259, %f260;\n" " mov.b32 %r264, %f261;\n" " shfl.sync.down.b32 %r265|%p62, %r264, %r256, %r257, %r259;\n" " mov.b32 %f262, %r265;\n" " add.f32 %f263, %f261, %f262;\n" " mov.b32 %r266, %f263;\n" " mov.u32 %r267, 1;\n" " shfl.sync.down.b32 %r268|%p63, %r266, %r267, %r257, %r259;\n" " mov.b32 %f264, %r268;\n" " add.f32 %f84, %f263, %f264;\n" " not.pred %p64, %p1;\n" " @%p64 bra $L__BB11_77;\n" "\n" " mul.f32 %f265, %f84, %f179;\n" " st.shared.f32 [%r41+64], %f265;\n" "\n" "$L__BB11_77:\n" " setp.lt.s32 %p65, %r102, 3;\n" " @%p65 bra $L__BB11_80;\n" "\n" " mov.b32 %r269, %f538;\n" " mov.u32 %r270, 2;\n" " mov.u32 %r271, 4127;\n" " mov.u32 %r272, 8;\n" " mov.u32 %r273, -1;\n" " shfl.sync.down.b32 %r274|%p66, %r269, %r272, %r271, %r273;\n" " mov.b32 %f266, %r274;\n" " add.f32 %f267, %f538, %f266;\n" " mov.b32 %r275, %f267;\n" " mov.u32 %r276, 4;\n" " shfl.sync.down.b32 %r277|%p67, %r275, %r276, %r271, %r273;\n" " mov.b32 %f268, %r277;\n" " add.f32 %f269, %f267, %f268;\n" " mov.b32 %r278, %f269;\n" " shfl.sync.down.b32 %r279|%p68, %r278, %r270, %r271, %r273;\n" " mov.b32 %f270, %r279;\n" " add.f32 %f271, %f269, %f270;\n" " mov.b32 %r280, %f271;\n" " mov.u32 %r281, 1;\n" " shfl.sync.down.b32 %r282|%p69, %r280, %r281, %r271, %r273;\n" " mov.b32 %f272, %r282;\n" " add.f32 %f85, %f271, %f272;\n" " not.pred %p70, %p1;\n" " @%p70 bra $L__BB11_80;\n" "\n" " mul.f32 %f273, %f85, %f179;\n" " st.shared.f32 [%r41+128], %f273;\n" "\n" "$L__BB11_80:\n" " setp.lt.s32 %p71, %r102, 4;\n" " @%p71 bra $L__BB11_83;\n" "\n" " mov.b32 %r283, %f539;\n" " mov.u32 %r284, 2;\n" " mov.u32 %r285, 4127;\n" " mov.u32 %r286, 8;\n" " mov.u32 %r287, -1;\n" " shfl.sync.down.b32 %r288|%p72, %r283, %r286, %r285, %r287;\n" " mov.b32 %f274, %r288;\n" " add.f32 %f275, %f539, %f274;\n" " mov.b32 %r289, %f275;\n" " mov.u32 %r290, 4;\n" " shfl.sync.down.b32 %r291|%p73, %r289, %r290, %r285, %r287;\n" " mov.b32 %f276, %r291;\n" " add.f32 %f277, %f275, %f276;\n" " mov.b32 %r292, %f277;\n" " shfl.sync.down.b32 %r293|%p74, %r292, %r284, %r285, %r287;\n" " mov.b32 %f278, %r293;\n" " add.f32 %f279, %f277, %f278;\n" " mov.b32 %r294, %f279;\n" " mov.u32 %r295, 1;\n" " shfl.sync.down.b32 %r296|%p75, %r294, %r295, %r285, %r287;\n" " mov.b32 %f280, %r296;\n" " add.f32 %f86, %f279, %f280;\n" " not.pred %p76, %p1;\n" " @%p76 bra $L__BB11_83;\n" "\n" " mul.f32 %f281, %f86, %f179;\n" " st.shared.f32 [%r41+192], %f281;\n" "\n" "$L__BB11_83:\n" " setp.lt.s32 %p77, %r102, 5;\n" " @%p77 bra $L__BB11_86;\n" "\n" " mov.b32 %r297, %f540;\n" " mov.u32 %r298, 2;\n" " mov.u32 %r299, 4127;\n" " mov.u32 %r300, 8;\n" " mov.u32 %r301, -1;\n" " shfl.sync.down.b32 %r302|%p78, %r297, %r300, %r299, %r301;\n" " mov.b32 %f282, %r302;\n" " add.f32 %f283, %f540, %f282;\n" " mov.b32 %r303, %f283;\n" " mov.u32 %r304, 4;\n" " shfl.sync.down.b32 %r305|%p79, %r303, %r304, %r299, %r301;\n" " mov.b32 %f284, %r305;\n" " add.f32 %f285, %f283, %f284;\n" " mov.b32 %r306, %f285;\n" " shfl.sync.down.b32 %r307|%p80, %r306, %r298, %r299, %r301;\n" " mov.b32 %f286, %r307;\n" " add.f32 %f287, %f285, %f286;\n" " mov.b32 %r308, %f287;\n" " mov.u32 %r309, 1;\n" " shfl.sync.down.b32 %r310|%p81, %r308, %r309, %r299, %r301;\n" " mov.b32 %f288, %r310;\n" " add.f32 %f87, %f287, %f288;\n" " not.pred %p82, %p1;\n" " @%p82 bra $L__BB11_86;\n" "\n" " mul.f32 %f289, %f87, %f179;\n" " st.shared.f32 [%r41+256], %f289;\n" "\n" "$L__BB11_86:\n" " setp.lt.s32 %p83, %r102, 6;\n" " @%p83 bra $L__BB11_89;\n" "\n" " mov.b32 %r311, %f541;\n" " mov.u32 %r312, 2;\n" " mov.u32 %r313, 4127;\n" " mov.u32 %r314, 8;\n" " mov.u32 %r315, -1;\n" " shfl.sync.down.b32 %r316|%p84, %r311, %r314, %r313, %r315;\n" " mov.b32 %f290, %r316;\n" " add.f32 %f291, %f541, %f290;\n" " mov.b32 %r317, %f291;\n" " mov.u32 %r318, 4;\n" " shfl.sync.down.b32 %r319|%p85, %r317, %r318, %r313, %r315;\n" " mov.b32 %f292, %r319;\n" " add.f32 %f293, %f291, %f292;\n" " mov.b32 %r320, %f293;\n" " shfl.sync.down.b32 %r321|%p86, %r320, %r312, %r313, %r315;\n" " mov.b32 %f294, %r321;\n" " add.f32 %f295, %f293, %f294;\n" " mov.b32 %r322, %f295;\n" " mov.u32 %r323, 1;\n" " shfl.sync.down.b32 %r324|%p87, %r322, %r323, %r313, %r315;\n" " mov.b32 %f296, %r324;\n" " add.f32 %f88, %f295, %f296;\n" " not.pred %p88, %p1;\n" " @%p88 bra $L__BB11_89;\n" "\n" " mul.f32 %f297, %f88, %f179;\n" " st.shared.f32 [%r41+320], %f297;\n" "\n" "$L__BB11_89:\n" " setp.lt.s32 %p89, %r102, 7;\n" " @%p89 bra $L__BB11_92;\n" "\n" " mov.b32 %r325, %f542;\n" " mov.u32 %r326, 2;\n" " mov.u32 %r327, 4127;\n" " mov.u32 %r328, 8;\n" " mov.u32 %r329, -1;\n" " shfl.sync.down.b32 %r330|%p90, %r325, %r328, %r327, %r329;\n" " mov.b32 %f298, %r330;\n" " add.f32 %f299, %f542, %f298;\n" " mov.b32 %r331, %f299;\n" " mov.u32 %r332, 4;\n" " shfl.sync.down.b32 %r333|%p91, %r331, %r332, %r327, %r329;\n" " mov.b32 %f300, %r333;\n" " add.f32 %f301, %f299, %f300;\n" " mov.b32 %r334, %f301;\n" " shfl.sync.down.b32 %r335|%p92, %r334, %r326, %r327, %r329;\n" " mov.b32 %f302, %r335;\n" " add.f32 %f303, %f301, %f302;\n" " mov.b32 %r336, %f303;\n" " mov.u32 %r337, 1;\n" " shfl.sync.down.b32 %r338|%p93, %r336, %r337, %r327, %r329;\n" " mov.b32 %f304, %r338;\n" " add.f32 %f89, %f303, %f304;\n" " not.pred %p94, %p1;\n" " @%p94 bra $L__BB11_92;\n" "\n" " mul.f32 %f305, %f89, %f179;\n" " st.shared.f32 [%r41+384], %f305;\n" "\n" "$L__BB11_92:\n" " setp.lt.s32 %p95, %r102, 8;\n" " @%p95 bra $L__BB11_95;\n" "\n" " mov.b32 %r339, %f543;\n" " mov.u32 %r340, 2;\n" " mov.u32 %r341, 4127;\n" " mov.u32 %r342, 8;\n" " mov.u32 %r343, -1;\n" " shfl.sync.down.b32 %r344|%p96, %r339, %r342, %r341, %r343;\n" " mov.b32 %f306, %r344;\n" " add.f32 %f307, %f543, %f306;\n" " mov.b32 %r345, %f307;\n" " mov.u32 %r346, 4;\n" " shfl.sync.down.b32 %r347|%p97, %r345, %r346, %r341, %r343;\n" " mov.b32 %f308, %r347;\n" " add.f32 %f309, %f307, %f308;\n" " mov.b32 %r348, %f309;\n" " shfl.sync.down.b32 %r349|%p98, %r348, %r340, %r341, %r343;\n" " mov.b32 %f310, %r349;\n" " add.f32 %f311, %f309, %f310;\n" " mov.b32 %r350, %f311;\n" " mov.u32 %r351, 1;\n" " shfl.sync.down.b32 %r352|%p99, %r350, %r351, %r341, %r343;\n" " mov.b32 %f312, %r352;\n" " add.f32 %f90, %f311, %f312;\n" " not.pred %p100, %p1;\n" " @%p100 bra $L__BB11_95;\n" "\n" " mul.f32 %f313, %f90, %f179;\n" " st.shared.f32 [%r41+448], %f313;\n" "\n" "$L__BB11_95:\n" " bar.sync 0;\n" " @%p7 bra $L__BB11_113;\n" "\n" " ld.shared.f32 %f91, [%r22];\n" " mov.f32 %f578, %f91;\n" " @%p17 bra $L__BB11_104;\n" "\n" " and.b32 %r77, %r51, 3;\n" " add.s32 %r354, %r51, -1;\n" " setp.lt.u32 %p103, %r354, 3;\n" " mov.u32 %r550, 0;\n" " mov.f32 %f578, %f91;\n" " @%p103 bra $L__BB11_100;\n" "\n" " sub.s32 %r549, %r51, %r77;\n" " mov.u32 %r550, 0;\n" " mov.f32 %f578, %f91;\n" "\n" "$L__BB11_99:\n" " add.s32 %r356, %r29, %r550;\n" " shl.b32 %r357, %r356, 2;\n" " add.s32 %r359, %r129, %r357;\n" " ld.shared.f32 %f315, [%r359];\n" " max.f32 %f316, %f578, %f315;\n" " ld.shared.f32 %f317, [%r359+4];\n" " max.f32 %f318, %f316, %f317;\n" " ld.shared.f32 %f319, [%r359+8];\n" " max.f32 %f320, %f318, %f319;\n" " ld.shared.f32 %f321, [%r359+12];\n" " max.f32 %f578, %f320, %f321;\n" " add.s32 %r550, %r550, 4;\n" " add.s32 %r549, %r549, -4;\n" " setp.ne.s32 %p104, %r549, 0;\n" " @%p104 bra $L__BB11_99;\n" "\n" "$L__BB11_100:\n" " setp.eq.s32 %p105, %r77, 0;\n" " @%p105 bra $L__BB11_104;\n" "\n" " add.s32 %r360, %r29, %r550;\n" " shl.b32 %r361, %r360, 2;\n" " add.s32 %r84, %r129, %r361;\n" " ld.shared.f32 %f322, [%r84];\n" " max.f32 %f578, %f578, %f322;\n" " setp.eq.s32 %p106, %r77, 1;\n" " @%p106 bra $L__BB11_104;\n" "\n" " ld.shared.f32 %f323, [%r84+4];\n" " max.f32 %f578, %f578, %f323;\n" " setp.eq.s32 %p107, %r77, 2;\n" " @%p107 bra $L__BB11_104;\n" "\n" " ld.shared.f32 %f324, [%r84+8];\n" " max.f32 %f578, %f578, %f324;\n" "\n" "$L__BB11_104:\n" " mov.f32 %f582, 0f00000000;\n" " @%p17 bra $L__BB11_112;\n" "\n" " and.b32 %r85, %r51, 3;\n" " add.s32 %r364, %r51, -1;\n" " setp.lt.u32 %p109, %r364, 3;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r554, 0;\n" " @%p109 bra $L__BB11_108;\n" "\n" " sub.s32 %r552, %r51, %r85;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r554, 0;\n" " mov.u32 %r553, %r46;\n" "\n" "$L__BB11_107:\n" " shl.b32 %r442, %r102, 2;\n" " add.s32 %r441, %r442, 1024;\n" " add.s32 %r366, %r29, %r554;\n" " shl.b32 %r367, %r366, 2;\n" " add.s32 %r369, %r129, %r367;\n" " ld.shared.f32 %f329, [%r369];\n" " sub.f32 %f330, %f329, %f578;\n" " mov.f32 %f331, 0f3F000000;\n" " mov.f32 %f332, 0f3BBB989D;\n" " fma.rn.f32 %f333, %f330, %f332, %f331;\n" " mov.f32 %f334, 0f3FB8AA3B;\n" " mov.f32 %f335, 0f437C0000;\n" " cvt.sat.f32.f32 %f336, %f333;\n" " mov.f32 %f337, 0f4B400001;\n" " fma.rm.f32 %f338, %f336, %f335, %f337;\n" " add.f32 %f339, %f338, 0fCB40007F;\n" " neg.f32 %f340, %f339;\n" " fma.rn.f32 %f341, %f330, %f334, %f340;\n" " mov.f32 %f342, 0f32A57060;\n" " fma.rn.f32 %f343, %f330, %f342, %f341;\n" " mov.b32 %r370, %f338;\n" " shl.b32 %r371, %r370, 23;\n" " mov.b32 %f344, %r371;\n" " ex2.approx.ftz.f32 %f345, %f343;\n" " mul.f32 %f346, %f345, %f344;\n" " st.shared.f32 [%r553+1024], %f346;\n" " add.f32 %f347, %f582, %f346;\n" " ld.shared.f32 %f348, [%r369+4];\n" " sub.f32 %f349, %f348, %f578;\n" " fma.rn.f32 %f350, %f349, %f332, %f331;\n" " cvt.sat.f32.f32 %f351, %f350;\n" " fma.rm.f32 %f352, %f351, %f335, %f337;\n" " add.f32 %f353, %f352, 0fCB40007F;\n" " neg.f32 %f354, %f353;\n" " fma.rn.f32 %f355, %f349, %f334, %f354;\n" " fma.rn.f32 %f356, %f349, %f342, %f355;\n" " mov.b32 %r372, %f352;\n" " shl.b32 %r373, %r372, 23;\n" " mov.b32 %f357, %r373;\n" " ex2.approx.ftz.f32 %f358, %f356;\n" " mul.f32 %f359, %f358, %f357;\n" " add.s32 %r374, %r553, %r442;\n" " st.shared.f32 [%r374+1024], %f359;\n" " add.f32 %f360, %f347, %f359;\n" " ld.shared.f32 %f361, [%r369+8];\n" " sub.f32 %f362, %f361, %f578;\n" " fma.rn.f32 %f363, %f362, %f332, %f331;\n" " cvt.sat.f32.f32 %f364, %f363;\n" " fma.rm.f32 %f365, %f364, %f335, %f337;\n" " add.f32 %f366, %f365, 0fCB40007F;\n" " neg.f32 %f367, %f366;\n" " fma.rn.f32 %f368, %f362, %f334, %f367;\n" " fma.rn.f32 %f369, %f362, %f342, %f368;\n" " mov.b32 %r375, %f365;\n" " shl.b32 %r376, %r375, 23;\n" " mov.b32 %f370, %r376;\n" " ex2.approx.ftz.f32 %f371, %f369;\n" " mul.f32 %f372, %f371, %f370;\n" " add.s32 %r377, %r374, %r442;\n" " add.s32 %r378, %r374, %r441;\n" " st.shared.f32 [%r378], %f372;\n" " add.f32 %f373, %f360, %f372;\n" " ld.shared.f32 %f374, [%r369+12];\n" " sub.f32 %f375, %f374, %f578;\n" " fma.rn.f32 %f376, %f375, %f332, %f331;\n" " cvt.sat.f32.f32 %f377, %f376;\n" " fma.rm.f32 %f378, %f377, %f335, %f337;\n" " add.f32 %f379, %f378, 0fCB40007F;\n" " neg.f32 %f380, %f379;\n" " fma.rn.f32 %f381, %f375, %f334, %f380;\n" " fma.rn.f32 %f382, %f375, %f342, %f381;\n" " mov.b32 %r379, %f378;\n" " shl.b32 %r380, %r379, 23;\n" " mov.b32 %f383, %r380;\n" " ex2.approx.ftz.f32 %f384, %f382;\n" " mul.f32 %f385, %f384, %f383;\n" " add.s32 %r381, %r377, %r442;\n" " add.s32 %r382, %r377, %r441;\n" " add.s32 %r553, %r381, %r442;\n" " st.shared.f32 [%r382], %f385;\n" " add.f32 %f582, %f373, %f385;\n" " add.s32 %r554, %r554, 4;\n" " add.s32 %r552, %r552, -4;\n" " setp.ne.s32 %p110, %r552, 0;\n" " @%p110 bra $L__BB11_107;\n" "\n" "$L__BB11_108:\n" " setp.eq.s32 %p111, %r85, 0;\n" " @%p111 bra $L__BB11_112;\n" "\n" " add.s32 %r383, %r29, %r554;\n" " shl.b32 %r384, %r383, 2;\n" " add.s32 %r94, %r129, %r384;\n" " ld.shared.f32 %f386, [%r94];\n" " sub.f32 %f387, %f386, %f578;\n" " mov.f32 %f388, 0f3F000000;\n" " mov.f32 %f389, 0f3BBB989D;\n" " fma.rn.f32 %f390, %f387, %f389, %f388;\n" " mov.f32 %f391, 0f3FB8AA3B;\n" " mov.f32 %f392, 0f437C0000;\n" " cvt.sat.f32.f32 %f393, %f390;\n" " mov.f32 %f394, 0f4B400001;\n" " fma.rm.f32 %f395, %f393, %f392, %f394;\n" " add.f32 %f396, %f395, 0fCB40007F;\n" " neg.f32 %f397, %f396;\n" " fma.rn.f32 %f398, %f387, %f391, %f397;\n" " mov.f32 %f399, 0f32A57060;\n" " fma.rn.f32 %f400, %f387, %f399, %f398;\n" " mov.b32 %r386, %f395;\n" " shl.b32 %r387, %r386, 23;\n" " mov.b32 %f401, %r387;\n" " ex2.approx.ftz.f32 %f402, %f400;\n" " mul.f32 %f403, %f402, %f401;\n" " cvt.u64.u32 %rd63, %r554;\n" " mul.lo.s64 %rd64, %rd63, %rd8;\n" " add.s64 %rd65, %rd21, %rd64;\n" " cvt.u32.u64 %r388, %rd65;\n" " add.s32 %r389, %r5, %r388;\n" " shl.b32 %r390, %r389, 2;\n" " add.s32 %r391, %r129, %r390;\n" " st.shared.f32 [%r391], %f403;\n" " add.f32 %f582, %f582, %f403;\n" " setp.eq.s32 %p112, %r85, 1;\n" " @%p112 bra $L__BB11_112;\n" "\n" " mov.f32 %f513, 0f32A57060;\n" " mov.f32 %f512, 0f3FB8AA3B;\n" " mov.f32 %f511, 0f4B400001;\n" " mov.f32 %f510, 0f437C0000;\n" " mov.f32 %f509, 0f3F000000;\n" " mov.f32 %f508, 0f3BBB989D;\n" " add.s32 %r424, %r29, %r554;\n" " shl.b32 %r423, %r424, 2;\n" " add.s32 %r422, %r129, %r423;\n" " ld.shared.f32 %f404, [%r422+4];\n" " sub.f32 %f405, %f404, %f578;\n" " fma.rn.f32 %f408, %f405, %f508, %f509;\n" " cvt.sat.f32.f32 %f411, %f408;\n" " fma.rm.f32 %f413, %f411, %f510, %f511;\n" " add.f32 %f414, %f413, 0fCB40007F;\n" " neg.f32 %f415, %f414;\n" " fma.rn.f32 %f416, %f405, %f512, %f415;\n" " fma.rn.f32 %f418, %f405, %f513, %f416;\n" " mov.b32 %r392, %f413;\n" " shl.b32 %r393, %r392, 23;\n" " mov.b32 %f419, %r393;\n" " ex2.approx.ftz.f32 %f420, %f418;\n" " mul.f32 %f421, %f420, %f419;\n" " add.s32 %r394, %r554, 1;\n" " cvt.u64.u32 %rd66, %r394;\n" " mul.lo.s64 %rd67, %rd66, %rd8;\n" " add.s64 %rd68, %rd21, %rd67;\n" " cvt.u32.u64 %r395, %rd68;\n" " add.s32 %r396, %r5, %r395;\n" " shl.b32 %r397, %r396, 2;\n" " add.s32 %r399, %r129, %r397;\n" " st.shared.f32 [%r399], %f421;\n" " add.f32 %f582, %f582, %f421;\n" " setp.eq.s32 %p113, %r85, 2;\n" " @%p113 bra $L__BB11_112;\n" "\n" " mov.f32 %f519, 0f32A57060;\n" " mov.f32 %f518, 0f3FB8AA3B;\n" " mov.f32 %f517, 0f4B400001;\n" " mov.f32 %f516, 0f437C0000;\n" " mov.f32 %f515, 0f3F000000;\n" " mov.f32 %f514, 0f3BBB989D;\n" " add.s32 %r427, %r29, %r554;\n" " shl.b32 %r426, %r427, 2;\n" " add.s32 %r425, %r129, %r426;\n" " ld.shared.f32 %f422, [%r425+8];\n" " sub.f32 %f423, %f422, %f578;\n" " fma.rn.f32 %f426, %f423, %f514, %f515;\n" " cvt.sat.f32.f32 %f429, %f426;\n" " fma.rm.f32 %f431, %f429, %f516, %f517;\n" " add.f32 %f432, %f431, 0fCB40007F;\n" " neg.f32 %f433, %f432;\n" " fma.rn.f32 %f434, %f423, %f518, %f433;\n" " fma.rn.f32 %f436, %f423, %f519, %f434;\n" " mov.b32 %r400, %f431;\n" " shl.b32 %r401, %r400, 23;\n" " mov.b32 %f437, %r401;\n" " ex2.approx.ftz.f32 %f438, %f436;\n" " mul.f32 %f439, %f438, %f437;\n" " add.s32 %r402, %r554, 2;\n" " cvt.u64.u32 %rd69, %r402;\n" " mul.lo.s64 %rd70, %rd69, %rd8;\n" " add.s64 %rd71, %rd21, %rd70;\n" " cvt.u32.u64 %r403, %rd71;\n" " add.s32 %r404, %r5, %r403;\n" " shl.b32 %r405, %r404, 2;\n" " add.s32 %r407, %r129, %r405;\n" " st.shared.f32 [%r407], %f439;\n" " add.f32 %f582, %f582, %f439;\n" "\n" "$L__BB11_112:\n" " sub.f32 %f440, %f91, %f578;\n" " mov.f32 %f441, 0f3F000000;\n" " mov.f32 %f442, 0f3BBB989D;\n" " fma.rn.f32 %f443, %f440, %f442, %f441;\n" " mov.f32 %f444, 0f3FB8AA3B;\n" " mov.f32 %f445, 0f437C0000;\n" " cvt.sat.f32.f32 %f446, %f443;\n" " mov.f32 %f447, 0f4B400001;\n" " fma.rm.f32 %f448, %f446, %f445, %f447;\n" " add.f32 %f449, %f448, 0fCB40007F;\n" " neg.f32 %f450, %f449;\n" " fma.rn.f32 %f451, %f440, %f444, %f450;\n" " mov.f32 %f452, 0f32A57060;\n" " fma.rn.f32 %f453, %f440, %f452, %f451;\n" " ex2.approx.ftz.f32 %f454, %f453;\n" " mov.b32 %r408, %f448;\n" " shl.b32 %r409, %r408, 23;\n" " mov.b32 %f455, %r409;\n" " mul.f32 %f456, %f454, %f455;\n" " ld.shared.f32 %f457, [%r23];\n" " fma.rn.f32 %f458, %f456, %f457, %f582;\n" " st.shared.f32 [%r23], %f458;\n" " st.shared.f32 [%r22], %f578;\n" " st.shared.f32 [%r30], %f456;\n" "\n" "$L__BB11_113:\n" " bar.sync 0;\n" " @%p17 bra $L__BB11_149;\n" "\n" " mov.u32 %r555, 0;\n" "\n" "$L__BB11_115:\n" " ld.param.u64 %rd156, [fc_attn_flash_decode_param_2];\n" " add.s32 %r411, %r555, %r541;\n" " cvt.s64.s32 %rd73, %r411;\n" " mul.lo.s64 %rd74, %rd73, %rd12;\n" " add.s64 %rd75, %rd20, %rd74;\n" " shl.b64 %rd76, %rd75, 2;\n" " add.s64 %rd72, %rd156, %rd76;\n" " // begin inline asm\n" " ld.global.nc.f32 %f459, [%rd72];\n" " // end inline asm\n" " cvt.u64.u32 %rd77, %r555;\n" " mul.lo.s64 %rd78, %rd77, %rd8;\n" " add.s64 %rd79, %rd78, %rd6;\n" " cvt.u32.u64 %r412, %rd79;\n" " add.s32 %r413, %r5, %r412;\n" " shl.b32 %r414, %r413, 2;\n" " add.s32 %r96, %r129, %r414;\n" " setp.eq.s32 %p115, %r555, 0;\n" " @%p115 bra $L__BB11_132;\n" "\n" " @%p2 bra $L__BB11_118;\n" "\n" " ld.shared.f32 %f460, [%r96];\n" " fma.rn.f32 %f612, %f459, %f460, %f612;\n" "\n" "$L__BB11_118:\n" " setp.lt.s32 %p150, %r102, 2;\n" " @%p150 bra $L__BB11_120;\n" "\n" " ld.shared.f32 %f461, [%r96+4];\n" " fma.rn.f32 %f611, %f459, %f461, %f611;\n" "\n" "$L__BB11_120:\n" " setp.lt.s32 %p151, %r102, 3;\n" " @%p151 bra $L__BB11_122;\n" "\n" " ld.shared.f32 %f462, [%r96+8];\n" " fma.rn.f32 %f610, %f459, %f462, %f610;\n" "\n" "$L__BB11_122:\n" " setp.lt.s32 %p152, %r102, 4;\n" " @%p152 bra $L__BB11_124;\n" "\n" " ld.shared.f32 %f463, [%r96+12];\n" " fma.rn.f32 %f609, %f459, %f463, %f609;\n" "\n" "$L__BB11_124:\n" " setp.lt.s32 %p153, %r102, 5;\n" " @%p153 bra $L__BB11_126;\n" "\n" " ld.shared.f32 %f464, [%r96+16];\n" " fma.rn.f32 %f608, %f459, %f464, %f608;\n" "\n" "$L__BB11_126:\n" " setp.lt.s32 %p154, %r102, 6;\n" " @%p154 bra $L__BB11_128;\n" "\n" " ld.shared.f32 %f465, [%r96+20];\n" " fma.rn.f32 %f607, %f459, %f465, %f607;\n" "\n" "$L__BB11_128:\n" " setp.lt.s32 %p155, %r102, 7;\n" " @%p155 bra $L__BB11_130;\n" "\n" " ld.shared.f32 %f466, [%r96+24];\n" " fma.rn.f32 %f606, %f459, %f466, %f606;\n" "\n" "$L__BB11_130:\n" " setp.lt.s32 %p156, %r102, 8;\n" " @%p156 bra $L__BB11_148;\n" "\n" " ld.shared.f32 %f467, [%r96+28];\n" " fma.rn.f32 %f605, %f459, %f467, %f605;\n" " bra.uni $L__BB11_148;\n" "\n" "$L__BB11_132:\n" " @%p2 bra $L__BB11_134;\n" "\n" " ld.shared.f32 %f468, [%r42];\n" " ld.shared.f32 %f469, [%r96];\n" " mul.f32 %f470, %f459, %f469;\n" " fma.rn.f32 %f612, %f468, %f612, %f470;\n" "\n" "$L__BB11_134:\n" " setp.lt.s32 %p157, %r102, 2;\n" " @%p157 bra $L__BB11_136;\n" "\n" " ld.shared.f32 %f471, [%r42+4];\n" " ld.shared.f32 %f472, [%r96+4];\n" " mul.f32 %f473, %f459, %f472;\n" " fma.rn.f32 %f611, %f471, %f611, %f473;\n" "\n" "$L__BB11_136:\n" " setp.lt.s32 %p158, %r102, 3;\n" " @%p158 bra $L__BB11_138;\n" "\n" " ld.shared.f32 %f474, [%r42+8];\n" " ld.shared.f32 %f475, [%r96+8];\n" " mul.f32 %f476, %f459, %f475;\n" " fma.rn.f32 %f610, %f474, %f610, %f476;\n" "\n" "$L__BB11_138:\n" " setp.lt.s32 %p159, %r102, 4;\n" " @%p159 bra $L__BB11_140;\n" "\n" " ld.shared.f32 %f477, [%r42+12];\n" " ld.shared.f32 %f478, [%r96+12];\n" " mul.f32 %f479, %f459, %f478;\n" " fma.rn.f32 %f609, %f477, %f609, %f479;\n" "\n" "$L__BB11_140:\n" " setp.lt.s32 %p160, %r102, 5;\n" " @%p160 bra $L__BB11_142;\n" "\n" " ld.shared.f32 %f480, [%r42+16];\n" " ld.shared.f32 %f481, [%r96+16];\n" " mul.f32 %f482, %f459, %f481;\n" " fma.rn.f32 %f608, %f480, %f608, %f482;\n" "\n" "$L__BB11_142:\n" " setp.lt.s32 %p161, %r102, 6;\n" " @%p161 bra $L__BB11_144;\n" "\n" " ld.shared.f32 %f483, [%r42+20];\n" " ld.shared.f32 %f484, [%r96+20];\n" " mul.f32 %f485, %f459, %f484;\n" " fma.rn.f32 %f607, %f483, %f607, %f485;\n" "\n" "$L__BB11_144:\n" " setp.lt.s32 %p162, %r102, 7;\n" " @%p162 bra $L__BB11_146;\n" "\n" " ld.shared.f32 %f486, [%r42+24];\n" " ld.shared.f32 %f487, [%r96+24];\n" " mul.f32 %f488, %f459, %f487;\n" " fma.rn.f32 %f606, %f486, %f606, %f488;\n" "\n" "$L__BB11_146:\n" " setp.lt.s32 %p163, %r102, 8;\n" " @%p163 bra $L__BB11_148;\n" "\n" " ld.shared.f32 %f489, [%r42+28];\n" " ld.shared.f32 %f490, [%r96+28];\n" " mul.f32 %f491, %f459, %f490;\n" " fma.rn.f32 %f605, %f489, %f605, %f491;\n" "\n" "$L__BB11_148:\n" " sub.s32 %r450, %r4, %r541;\n" " min.s32 %r449, %r450, 16;\n" " add.s32 %r555, %r555, 1;\n" " setp.lt.s32 %p132, %r555, %r449;\n" " @%p132 bra $L__BB11_115;\n" "\n" "$L__BB11_149:\n" " bar.sync 0;\n" "\n" "$L__BB11_150:\n" " add.s32 %r541, %r541, 16;\n" " setp.gt.s32 %p133, %r4, %r541;\n" " add.s32 %r540, %r540, 1;\n" " @%p133 bra $L__BB11_11;\n" "\n" "$L__BB11_151:\n" " add.s32 %r433, %r103, 16;\n" " mul.wide.s32 %rd155, %r433, 16;\n" " shl.b32 %r432, %r102, 4;\n" " cvt.s64.s32 %rd154, %r432;\n" " add.s64 %rd153, %rd155, %rd154;\n" " add.s64 %rd152, %rd153, %rd154;\n" " cvt.u32.u64 %r431, %rd152;\n" " ld.param.u32 %r430, [fc_attn_flash_decode_param_10];\n" " mov.u32 %r429, %ctaid.x;\n" " add.s64 %rd151, %rd152, %rd8;\n" " cvt.u32.u64 %r428, %rd151;\n" " cvt.s64.s32 %rd24, %r430;\n" " cvt.s64.s32 %rd25, %r429;\n" " add.s32 %r417, %r5, %r431;\n" " shl.b32 %r418, %r417, 2;\n" " add.s32 %r100, %r129, %r418;\n" " add.s32 %r420, %r5, %r428;\n" " shl.b32 %r421, %r420, 2;\n" " add.s32 %r101, %r129, %r421;\n" " @%p2 bra $L__BB11_154;\n" "\n" " setp.ne.s32 %p135, %r1, 0;\n" " mul.lo.s64 %rd80, %rd11, %rd24;\n" " add.s64 %rd26, %rd80, %rd25;\n" " mul.lo.s64 %rd81, %rd26, %rd12;\n" " add.s64 %rd82, %rd81, %rd19;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd84, %rd4, %rd83;\n" " st.global.f32 [%rd84], %f612;\n" " @%p135 bra $L__BB11_154;\n" "\n" " ld.shared.f32 %f492, [%r100];\n" " shl.b64 %rd85, %rd26, 2;\n" " add.s64 %rd86, %rd3, %rd85;\n" " st.global.f32 [%rd86], %f492;\n" " ld.shared.f32 %f493, [%r101];\n" " add.s64 %rd87, %rd2, %rd85;\n" " st.global.f32 [%rd87], %f493;\n" "\n" "$L__BB11_154:\n" " setp.lt.s32 %p136, %r102, 2;\n" " @%p136 bra $L__BB11_157;\n" "\n" " setp.ne.s32 %p137, %r1, 0;\n" " add.s64 %rd88, %rd11, 1;\n" " mul.lo.s64 %rd89, %rd88, %rd24;\n" " add.s64 %rd27, %rd89, %rd25;\n" " mul.lo.s64 %rd90, %rd27, %rd12;\n" " add.s64 %rd91, %rd90, %rd19;\n" " shl.b64 %rd92, %rd91, 2;\n" " add.s64 %rd93, %rd4, %rd92;\n" " st.global.f32 [%rd93], %f611;\n" " @%p137 bra $L__BB11_157;\n" "\n" " ld.shared.f32 %f494, [%r100+4];\n" " shl.b64 %rd94, %rd27, 2;\n" " add.s64 %rd95, %rd3, %rd94;\n" " st.global.f32 [%rd95], %f494;\n" " ld.shared.f32 %f495, [%r101+4];\n" " add.s64 %rd96, %rd2, %rd94;\n" " st.global.f32 [%rd96], %f495;\n" "\n" "$L__BB11_157:\n" " setp.lt.s32 %p138, %r102, 3;\n" " @%p138 bra $L__BB11_160;\n" "\n" " setp.ne.s32 %p139, %r1, 0;\n" " add.s64 %rd97, %rd11, 2;\n" " mul.lo.s64 %rd98, %rd97, %rd24;\n" " add.s64 %rd28, %rd98, %rd25;\n" " mul.lo.s64 %rd99, %rd28, %rd12;\n" " add.s64 %rd100, %rd99, %rd19;\n" " shl.b64 %rd101, %rd100, 2;\n" " add.s64 %rd102, %rd4, %rd101;\n" " st.global.f32 [%rd102], %f610;\n" " @%p139 bra $L__BB11_160;\n" "\n" " ld.shared.f32 %f496, [%r100+8];\n" " shl.b64 %rd103, %rd28, 2;\n" " add.s64 %rd104, %rd3, %rd103;\n" " st.global.f32 [%rd104], %f496;\n" " ld.shared.f32 %f497, [%r101+8];\n" " add.s64 %rd105, %rd2, %rd103;\n" " st.global.f32 [%rd105], %f497;\n" "\n" "$L__BB11_160:\n" " setp.lt.s32 %p140, %r102, 4;\n" " @%p140 bra $L__BB11_163;\n" "\n" " setp.ne.s32 %p141, %r1, 0;\n" " add.s64 %rd106, %rd11, 3;\n" " mul.lo.s64 %rd107, %rd106, %rd24;\n" " add.s64 %rd29, %rd107, %rd25;\n" " mul.lo.s64 %rd108, %rd29, %rd12;\n" " add.s64 %rd109, %rd108, %rd19;\n" " shl.b64 %rd110, %rd109, 2;\n" " add.s64 %rd111, %rd4, %rd110;\n" " st.global.f32 [%rd111], %f609;\n" " @%p141 bra $L__BB11_163;\n" "\n" " ld.shared.f32 %f498, [%r100+12];\n" " shl.b64 %rd112, %rd29, 2;\n" " add.s64 %rd113, %rd3, %rd112;\n" " st.global.f32 [%rd113], %f498;\n" " ld.shared.f32 %f499, [%r101+12];\n" " add.s64 %rd114, %rd2, %rd112;\n" " st.global.f32 [%rd114], %f499;\n" "\n" "$L__BB11_163:\n" " setp.lt.s32 %p142, %r102, 5;\n" " @%p142 bra $L__BB11_166;\n" "\n" " setp.ne.s32 %p143, %r1, 0;\n" " add.s64 %rd115, %rd11, 4;\n" " mul.lo.s64 %rd116, %rd115, %rd24;\n" " add.s64 %rd30, %rd116, %rd25;\n" " mul.lo.s64 %rd117, %rd30, %rd12;\n" " add.s64 %rd118, %rd117, %rd19;\n" " shl.b64 %rd119, %rd118, 2;\n" " add.s64 %rd120, %rd4, %rd119;\n" " st.global.f32 [%rd120], %f608;\n" " @%p143 bra $L__BB11_166;\n" "\n" " ld.shared.f32 %f500, [%r100+16];\n" " shl.b64 %rd121, %rd30, 2;\n" " add.s64 %rd122, %rd3, %rd121;\n" " st.global.f32 [%rd122], %f500;\n" " ld.shared.f32 %f501, [%r101+16];\n" " add.s64 %rd123, %rd2, %rd121;\n" " st.global.f32 [%rd123], %f501;\n" "\n" "$L__BB11_166:\n" " setp.lt.s32 %p144, %r102, 6;\n" " @%p144 bra $L__BB11_169;\n" "\n" " setp.ne.s32 %p145, %r1, 0;\n" " add.s64 %rd124, %rd11, 5;\n" " mul.lo.s64 %rd125, %rd124, %rd24;\n" " add.s64 %rd31, %rd125, %rd25;\n" " mul.lo.s64 %rd126, %rd31, %rd12;\n" " add.s64 %rd127, %rd126, %rd19;\n" " shl.b64 %rd128, %rd127, 2;\n" " add.s64 %rd129, %rd4, %rd128;\n" " st.global.f32 [%rd129], %f607;\n" " @%p145 bra $L__BB11_169;\n" "\n" " ld.shared.f32 %f502, [%r100+20];\n" " shl.b64 %rd130, %rd31, 2;\n" " add.s64 %rd131, %rd3, %rd130;\n" " st.global.f32 [%rd131], %f502;\n" " ld.shared.f32 %f503, [%r101+20];\n" " add.s64 %rd132, %rd2, %rd130;\n" " st.global.f32 [%rd132], %f503;\n" "\n" "$L__BB11_169:\n" " setp.lt.s32 %p146, %r102, 7;\n" " @%p146 bra $L__BB11_172;\n" "\n" " setp.ne.s32 %p147, %r1, 0;\n" " add.s64 %rd133, %rd11, 6;\n" " mul.lo.s64 %rd134, %rd133, %rd24;\n" " add.s64 %rd32, %rd134, %rd25;\n" " mul.lo.s64 %rd135, %rd32, %rd12;\n" " add.s64 %rd136, %rd135, %rd19;\n" " shl.b64 %rd137, %rd136, 2;\n" " add.s64 %rd138, %rd4, %rd137;\n" " st.global.f32 [%rd138], %f606;\n" " @%p147 bra $L__BB11_172;\n" "\n" " ld.shared.f32 %f504, [%r100+24];\n" " shl.b64 %rd139, %rd32, 2;\n" " add.s64 %rd140, %rd3, %rd139;\n" " st.global.f32 [%rd140], %f504;\n" " ld.shared.f32 %f505, [%r101+24];\n" " add.s64 %rd141, %rd2, %rd139;\n" " st.global.f32 [%rd141], %f505;\n" "\n" "$L__BB11_172:\n" " setp.lt.s32 %p148, %r102, 8;\n" " @%p148 bra $L__BB11_175;\n" "\n" " setp.ne.s32 %p149, %r1, 0;\n" " add.s64 %rd142, %rd11, 7;\n" " mul.lo.s64 %rd143, %rd142, %rd24;\n" " add.s64 %rd33, %rd143, %rd25;\n" " mul.lo.s64 %rd144, %rd33, %rd12;\n" " add.s64 %rd145, %rd144, %rd19;\n" " shl.b64 %rd146, %rd145, 2;\n" " add.s64 %rd147, %rd4, %rd146;\n" " st.global.f32 [%rd147], %f605;\n" " @%p149 bra $L__BB11_175;\n" "\n" " ld.shared.f32 %f506, [%r100+28];\n" " shl.b64 %rd148, %rd33, 2;\n" " add.s64 %rd149, %rd3, %rd148;\n" " st.global.f32 [%rd149], %f506;\n" " ld.shared.f32 %f507, [%r101+28];\n" " add.s64 %rd150, %rd2, %rd148;\n" " st.global.f32 [%rd150], %f507;\n" "\n" "$L__BB11_175:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode_dev\n" ".visible .entry fc_attn_flash_decode_dev(\n" " .param .u64 fc_attn_flash_decode_dev_param_0,\n" " .param .u64 fc_attn_flash_decode_dev_param_1,\n" " .param .u64 fc_attn_flash_decode_dev_param_2,\n" " .param .u64 fc_attn_flash_decode_dev_param_3,\n" " .param .u64 fc_attn_flash_decode_dev_param_4,\n" " .param .u64 fc_attn_flash_decode_dev_param_5,\n" " .param .u32 fc_attn_flash_decode_dev_param_6,\n" " .param .u32 fc_attn_flash_decode_dev_param_7,\n" " .param .u32 fc_attn_flash_decode_dev_param_8,\n" " .param .u64 fc_attn_flash_decode_dev_param_9,\n" " .param .u32 fc_attn_flash_decode_dev_param_10,\n" " .param .u32 fc_attn_flash_decode_dev_param_11,\n" " .param .u32 fc_attn_flash_decode_dev_param_12,\n" " .param .u32 fc_attn_flash_decode_dev_param_13,\n" " .param .f32 fc_attn_flash_decode_dev_param_14\n" ")\n" "{\n" " .reg .pred %p<172>;\n" " .reg .f32 %f<637>;\n" " .reg .b32 %r<567>;\n" " .reg .b64 %rd<164>;\n" "\n" "\n" " ld.param.u64 %rd40, [fc_attn_flash_decode_dev_param_0];\n" " ld.param.u64 %rd38, [fc_attn_flash_decode_dev_param_1];\n" " ld.param.u64 %rd41, [fc_attn_flash_decode_dev_param_3];\n" " ld.param.u64 %rd42, [fc_attn_flash_decode_dev_param_4];\n" " ld.param.u64 %rd43, [fc_attn_flash_decode_dev_param_5];\n" " ld.param.u32 %r104, [fc_attn_flash_decode_dev_param_6];\n" " ld.param.u32 %r105, [fc_attn_flash_decode_dev_param_7];\n" " ld.param.u32 %r106, [fc_attn_flash_decode_dev_param_8];\n" " ld.param.u64 %rd44, [fc_attn_flash_decode_dev_param_9];\n" " ld.param.u32 %r109, [fc_attn_flash_decode_dev_param_10];\n" " ld.param.u32 %r110, [fc_attn_flash_decode_dev_param_11];\n" " ld.param.u32 %r108, [fc_attn_flash_decode_dev_param_13];\n" " ld.param.f32 %f179, [fc_attn_flash_decode_dev_param_14];\n" " cvta.to.global.u64 %rd1, %rd40;\n" " cvta.to.global.u64 %rd2, %rd42;\n" " cvta.to.global.u64 %rd3, %rd41;\n" " cvta.to.global.u64 %rd4, %rd43;\n" " cvta.to.global.u64 %rd45, %rd44;\n" " ld.global.u32 %r1, [%rd45];\n" " add.s32 %r111, %r1, %r109;\n" " div.s32 %r112, %r111, %r109;\n" " max.s32 %r113, %r112, 1;\n" " min.s32 %r114, %r113, %r110;\n" " add.s32 %r115, %r114, %r1;\n" " div.s32 %r116, %r115, %r114;\n" " add.s32 %r2, %r116, 15;\n" " and.b32 %r3, %r2, -16;\n" " add.s32 %r117, %r3, %r1;\n" " div.s32 %r4, %r117, %r3;\n" " mov.u32 %r5, %ctaid.x;\n" " setp.ge.s32 %p2, %r5, %r4;\n" " @%p2 bra $L__BB12_176;\n" "\n" " mov.u32 %r118, %ctaid.y;\n" " mov.u32 %r6, %tid.x;\n" " mul.lo.s32 %r553, %r3, %r5;\n" " add.s32 %r119, %r553, %r3;\n" " add.s32 %r120, %r1, 1;\n" " min.s32 %r8, %r119, %r120;\n" " mul.lo.s32 %r9, %r105, %r104;\n" " add.s32 %r10, %r105, 16;\n" " mul.wide.s32 %rd5, %r10, 16;\n" " shl.b32 %r121, %r104, 4;\n" " cvt.s64.s32 %rd46, %r121;\n" " add.s64 %rd6, %rd5, %rd46;\n" " add.s64 %rd7, %rd6, %rd46;\n" " cvt.s64.s32 %rd8, %r104;\n" " add.s64 %rd9, %rd7, %rd8;\n" " cvt.s64.s32 %rd10, %r118;\n" " mul.wide.s32 %rd11, %r104, %r118;\n" " cvt.s64.s32 %rd12, %r105;\n" " setp.lt.s32 %p3, %r104, 1;\n" " @%p3 bra $L__BB12_8;\n" "\n" " cvt.s64.s32 %rd13, %r6;\n" " and.b32 %r551, %r104, 3;\n" " add.s32 %r123, %r104, -1;\n" " setp.lt.u32 %p4, %r123, 3;\n" " mov.u32 %r549, 0;\n" " @%p4 bra $L__BB12_5;\n" "\n" " sub.s32 %r547, %r104, %r551;\n" " mul.lo.s64 %rd47, %rd10, %rd12;\n" " mul.lo.s64 %rd48, %rd47, %rd8;\n" " add.s64 %rd49, %rd48, %rd13;\n" " shl.b64 %rd50, %rd49, 2;\n" " add.s64 %rd162, %rd1, %rd50;\n" " shl.b64 %rd15, %rd12, 2;\n" " shl.b32 %r125, %r6, 2;\n" " mov.u32 %r126, sh;\n" " add.s32 %r548, %r126, %r125;\n" " shl.b32 %r14, %r105, 2;\n" " mov.u32 %r549, 0;\n" "\n" "$L__BB12_4:\n" " ld.global.f32 %f180, [%rd162];\n" " st.shared.f32 [%r548], %f180;\n" " add.s64 %rd51, %rd162, %rd15;\n" " ld.global.f32 %f181, [%rd51];\n" " add.s32 %r127, %r548, %r14;\n" " st.shared.f32 [%r127], %f181;\n" " add.s64 %rd52, %rd51, %rd15;\n" " ld.global.f32 %f182, [%rd52];\n" " add.s32 %r128, %r127, %r14;\n" " st.shared.f32 [%r128], %f182;\n" " add.s64 %rd53, %rd52, %rd15;\n" " add.s64 %rd162, %rd53, %rd15;\n" " ld.global.f32 %f183, [%rd53];\n" " add.s32 %r129, %r128, %r14;\n" " add.s32 %r548, %r129, %r14;\n" " st.shared.f32 [%r129], %f183;\n" " add.s32 %r549, %r549, 4;\n" " add.s32 %r547, %r547, -4;\n" " setp.ne.s32 %p5, %r547, 0;\n" " @%p5 bra $L__BB12_4;\n" "\n" "$L__BB12_5:\n" " setp.eq.s32 %p6, %r551, 0;\n" " @%p6 bra $L__BB12_8;\n" "\n" " mad.lo.s32 %r130, %r549, %r105, %r6;\n" " shl.b32 %r131, %r130, 2;\n" " mov.u32 %r132, sh;\n" " add.s32 %r550, %r132, %r131;\n" " shl.b32 %r23, %r105, 2;\n" " cvt.s64.s32 %rd54, %r549;\n" " add.s64 %rd55, %rd11, %rd54;\n" " mul.lo.s64 %rd56, %rd55, %rd12;\n" " add.s64 %rd57, %rd56, %rd13;\n" " shl.b64 %rd58, %rd57, 2;\n" " add.s64 %rd163, %rd1, %rd58;\n" " shl.b64 %rd19, %rd12, 2;\n" "\n" "$L__BB12_7:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f184, [%rd163];\n" " st.shared.f32 [%r550], %f184;\n" " add.s32 %r550, %r550, %r23;\n" " add.s64 %rd163, %rd163, %rd19;\n" " add.s32 %r551, %r551, -1;\n" " setp.ne.s32 %p7, %r551, 0;\n" " @%p7 bra $L__BB12_7;\n" "\n" "$L__BB12_8:\n" " setp.ge.s32 %p8, %r6, %r104;\n" " add.s32 %r133, %r9, %r6;\n" " cvt.u32.u64 %r134, %rd7;\n" " add.s32 %r135, %r133, %r134;\n" " shl.b32 %r136, %r135, 2;\n" " mov.u32 %r137, sh;\n" " add.s32 %r28, %r137, %r136;\n" " cvt.u32.u64 %r138, %rd9;\n" " add.s32 %r139, %r133, %r138;\n" " shl.b32 %r140, %r139, 2;\n" " add.s32 %r29, %r137, %r140;\n" " @%p8 bra $L__BB12_10;\n" "\n" " mov.u32 %r141, -8388608;\n" " st.shared.u32 [%r28], %r141;\n" " mov.u32 %r142, 0;\n" " st.shared.u32 [%r29], %r142;\n" "\n" "$L__BB12_10:\n" " bar.sync 0;\n" " mul.lo.s64 %rd59, %rd12, %rd10;\n" " cvt.s64.s32 %rd60, %r106;\n" " mul.lo.s64 %rd22, %rd59, %rd60;\n" " shr.s32 %r143, %r6, 31;\n" " shr.u32 %r144, %r143, 28;\n" " add.s32 %r145, %r6, %r144;\n" " shr.s32 %r31, %r145, 4;\n" " and.b32 %r146, %r145, -16;\n" " sub.s32 %r32, %r6, %r146;\n" " cvt.s64.s32 %rd23, %r6;\n" " setp.le.s32 %p9, %r8, %r553;\n" " mov.f32 %f605, 0f00000000;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" " @%p9 bra $L__BB12_152;\n" "\n" " shr.s32 %r148, %r105, 31;\n" " shr.u32 %r149, %r148, 28;\n" " add.s32 %r150, %r105, %r149;\n" " shr.s32 %r151, %r150, 4;\n" " sub.s32 %r152, %r1, %r108;\n" " add.s32 %r153, %r152, 1;\n" " mov.u32 %r552, 0;\n" " max.s32 %r33, %r153, 0;\n" " add.s32 %r154, %r9, %r31;\n" " cvt.u32.u64 %r155, %rd5;\n" " add.s32 %r156, %r154, %r155;\n" " mad.lo.s32 %r35, %r10, %r31, %r9;\n" " shl.b32 %r157, %r6, 4;\n" " add.s32 %r158, %r157, %r155;\n" " add.s32 %r36, %r9, %r158;\n" " add.s64 %rd24, %rd22, %rd23;\n" " add.s32 %r159, %r138, %r104;\n" " add.s32 %r160, %r9, %r159;\n" " add.s32 %r162, %r133, %r159;\n" " shl.b32 %r163, %r162, 2;\n" " add.s32 %r37, %r137, %r163;\n" " cvt.u64.u32 %rd61, %r6;\n" " add.s64 %rd25, %rd6, %rd61;\n" " shl.b32 %r165, %r5, 4;\n" " add.s32 %r166, %r165, 16;\n" " shr.u32 %r167, %r2, 4;\n" " mul.lo.s32 %r168, %r167, %r166;\n" " not.b32 %r169, %r168;\n" " mov.u32 %r170, -2;\n" " sub.s32 %r171, %r170, %r1;\n" " max.s32 %r172, %r171, %r169;\n" " mul.lo.s32 %r173, %r5, %r167;\n" " shl.b32 %r174, %r173, 4;\n" " add.s32 %r38, %r172, %r174;\n" " bfe.u32 %r40, %r150, 4, 1;\n" " sub.s32 %r41, %r151, %r40;\n" " shl.b32 %r42, %r105, 1;\n" " mul.lo.s32 %r43, %r105, 3;\n" " shl.b32 %r44, %r105, 2;\n" " mul.lo.s32 %r45, %r105, 5;\n" " mul.lo.s32 %r46, %r105, 6;\n" " mul.lo.s32 %r47, %r105, 7;\n" " shl.b32 %r175, %r156, 2;\n" " add.s32 %r48, %r137, %r175;\n" " shl.b32 %r176, %r160, 2;\n" " add.s32 %r49, %r137, %r176;\n" " mad.lo.s32 %r178, %r104, %r10, %r6;\n" " add.s32 %r179, %r178, %r155;\n" " shl.b32 %r180, %r179, 2;\n" " add.s32 %r181, %r137, %r180;\n" " add.s32 %r50, %r181, -1024;\n" " cvta.to.global.u64 %rd26, %rd38;\n" " mov.f32 %f605, 0f00000000;\n" " setp.lt.s32 %p11, %r108, 1;\n" " setp.ge.s32 %p19, %r6, %r105;\n" " setp.lt.s32 %p22, %r105, 16;\n" " setp.eq.s32 %p52, %r32, 0;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" "\n" "$L__BB12_12:\n" " ld.param.u32 %r449, [fc_attn_flash_decode_dev_param_12];\n" " shl.b32 %r182, %r552, 4;\n" " add.s32 %r183, %r38, %r182;\n" " max.s32 %r184, %r183, -17;\n" " not.b32 %r185, %r184;\n" " max.s32 %r55, %r185, 1;\n" " sub.s32 %r56, %r8, %r553;\n" " min.s32 %r57, %r56, 16;\n" " setp.lt.s32 %p10, %r553, %r449;\n" " or.pred %p12, %p11, %p10;\n" " add.s32 %r186, %r57, %r553;\n" " setp.gt.s32 %p13, %r186, %r33;\n" " setp.le.s32 %p14, %r553, %r1;\n" " and.pred %p15, %p14, %p13;\n" " or.pred %p16, %p12, %p15;\n" " not.pred %p17, %p16;\n" " @%p17 bra $L__BB12_151;\n" "\n" " setp.lt.s32 %p18, %r56, 1;\n" " @%p18 bra $L__BB12_19;\n" "\n" " mov.u32 %r554, 0;\n" "\n" "$L__BB12_15:\n" " @%p19 bra $L__BB12_18;\n" "\n" " add.s32 %r459, %r105, 16;\n" " add.s32 %r188, %r554, %r553;\n" " cvt.s64.s32 %rd62, %r188;\n" " mul.lo.s64 %rd63, %rd62, %rd12;\n" " add.s64 %rd27, %rd63, %rd22;\n" " mad.lo.s32 %r59, %r554, %r459, %r9;\n" " mov.u32 %r555, %r6;\n" "\n" "$L__BB12_17:\n" " mov.u32 %r460, %ntid.x;\n" " cvt.s64.s32 %rd64, %r555;\n" " add.s64 %rd65, %rd27, %rd64;\n" " shl.b64 %rd66, %rd65, 2;\n" " add.s64 %rd67, %rd26, %rd66;\n" " ld.global.f32 %f201, [%rd67];\n" " add.s32 %r189, %r59, %r555;\n" " shl.b32 %r190, %r189, 2;\n" " add.s32 %r192, %r137, %r190;\n" " st.shared.f32 [%r192], %f201;\n" " add.s32 %r555, %r555, %r460;\n" " setp.lt.s32 %p20, %r555, %r105;\n" " @%p20 bra $L__BB12_17;\n" "\n" "$L__BB12_18:\n" " sub.s32 %r458, %r8, %r553;\n" " min.s32 %r457, %r458, 16;\n" " add.s32 %r554, %r554, 1;\n" " setp.lt.s32 %p21, %r554, %r457;\n" " @%p21 bra $L__BB12_15;\n" "\n" "$L__BB12_19:\n" " sub.s32 %r451, %r8, %r553;\n" " min.s32 %r450, %r451, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p23, %r31, %r450;\n" " or.pred %p24, %p22, %p23;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p24 bra $L__BB12_72;\n" "\n" " and.b32 %r452, %r105, -16;\n" " setp.eq.s32 %p25, %r452, 16;\n" " mov.u32 %r558, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p25 bra $L__BB12_55;\n" "\n" " mov.u32 %r558, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " mov.u32 %r557, %r41;\n" "\n" "$L__BB12_22:\n" " shl.b32 %r195, %r558, 4;\n" " add.s32 %r65, %r195, %r32;\n" " add.s32 %r196, %r35, %r65;\n" " shl.b32 %r197, %r196, 2;\n" " add.s32 %r66, %r137, %r197;\n" " ld.shared.f32 %f17, [%r66];\n" " @%p3 bra $L__BB12_24;\n" "\n" " shl.b32 %r540, %r558, 4;\n" " add.s32 %r539, %r540, %r32;\n" " shl.b32 %r538, %r539, 2;\n" " add.s32 %r537, %r137, %r538;\n" " ld.shared.f32 %f226, [%r537];\n" " fma.rn.f32 %f536, %f17, %f226, %f536;\n" "\n" "$L__BB12_24:\n" " setp.lt.s32 %p27, %r104, 2;\n" " @%p27 bra $L__BB12_26;\n" "\n" " shl.b32 %r545, %r558, 4;\n" " add.s32 %r544, %r545, %r32;\n" " add.s32 %r543, %r544, %r105;\n" " shl.b32 %r542, %r543, 2;\n" " add.s32 %r541, %r137, %r542;\n" " ld.shared.f32 %f227, [%r541];\n" " fma.rn.f32 %f537, %f17, %f227, %f537;\n" "\n" "$L__BB12_26:\n" " setp.lt.s32 %p28, %r104, 3;\n" " @%p28 bra $L__BB12_28;\n" "\n" " shl.b32 %r486, %r558, 4;\n" " add.s32 %r485, %r486, %r32;\n" " add.s32 %r484, %r42, %r485;\n" " shl.b32 %r483, %r484, 2;\n" " add.s32 %r482, %r137, %r483;\n" " ld.shared.f32 %f228, [%r482];\n" " fma.rn.f32 %f538, %f17, %f228, %f538;\n" "\n" "$L__BB12_28:\n" " setp.lt.s32 %p29, %r104, 4;\n" " @%p29 bra $L__BB12_30;\n" "\n" " shl.b32 %r496, %r558, 4;\n" " add.s32 %r495, %r496, %r32;\n" " add.s32 %r494, %r43, %r495;\n" " shl.b32 %r493, %r494, 2;\n" " add.s32 %r492, %r137, %r493;\n" " ld.shared.f32 %f229, [%r492];\n" " fma.rn.f32 %f539, %f17, %f229, %f539;\n" "\n" "$L__BB12_30:\n" " setp.lt.s32 %p30, %r104, 5;\n" " @%p30 bra $L__BB12_32;\n" "\n" " shl.b32 %r506, %r558, 4;\n" " add.s32 %r505, %r506, %r32;\n" " add.s32 %r504, %r44, %r505;\n" " shl.b32 %r503, %r504, 2;\n" " add.s32 %r502, %r137, %r503;\n" " ld.shared.f32 %f230, [%r502];\n" " fma.rn.f32 %f540, %f17, %f230, %f540;\n" "\n" "$L__BB12_32:\n" " setp.lt.s32 %p31, %r104, 6;\n" " @%p31 bra $L__BB12_34;\n" "\n" " shl.b32 %r516, %r558, 4;\n" " add.s32 %r515, %r516, %r32;\n" " add.s32 %r514, %r45, %r515;\n" " shl.b32 %r513, %r514, 2;\n" " add.s32 %r512, %r137, %r513;\n" " ld.shared.f32 %f231, [%r512];\n" " fma.rn.f32 %f541, %f17, %f231, %f541;\n" "\n" "$L__BB12_34:\n" " setp.lt.s32 %p32, %r104, 7;\n" " @%p32 bra $L__BB12_36;\n" "\n" " shl.b32 %r526, %r558, 4;\n" " add.s32 %r525, %r526, %r32;\n" " add.s32 %r524, %r46, %r525;\n" " shl.b32 %r523, %r524, 2;\n" " add.s32 %r522, %r137, %r523;\n" " ld.shared.f32 %f232, [%r522];\n" " fma.rn.f32 %f542, %f17, %f232, %f542;\n" "\n" "$L__BB12_36:\n" " setp.lt.s32 %p33, %r104, 8;\n" " @%p33 bra $L__BB12_38;\n" "\n" " shl.b32 %r536, %r558, 4;\n" " add.s32 %r535, %r536, %r32;\n" " add.s32 %r534, %r47, %r535;\n" " shl.b32 %r533, %r534, 2;\n" " add.s32 %r532, %r137, %r533;\n" " ld.shared.f32 %f233, [%r532];\n" " fma.rn.f32 %f543, %f17, %f233, %f543;\n" "\n" "$L__BB12_38:\n" " shl.b32 %r467, %r558, 4;\n" " add.s32 %r466, %r467, %r32;\n" " add.s32 %r465, %r35, %r466;\n" " shl.b32 %r464, %r465, 2;\n" " add.s32 %r463, %r137, %r464;\n" " ld.shared.f32 %f34, [%r463+64];\n" " @%p3 bra $L__BB12_40;\n" "\n" " shl.b32 %r481, %r558, 4;\n" " add.s32 %r480, %r481, %r32;\n" " shl.b32 %r479, %r480, 2;\n" " add.s32 %r478, %r137, %r479;\n" " ld.shared.f32 %f234, [%r478+64];\n" " fma.rn.f32 %f536, %f34, %f234, %f536;\n" "\n" "$L__BB12_40:\n" " setp.lt.s32 %p165, %r104, 2;\n" " @%p165 bra $L__BB12_42;\n" "\n" " shl.b32 %r477, %r558, 4;\n" " add.s32 %r476, %r477, %r32;\n" " add.s32 %r475, %r476, %r105;\n" " shl.b32 %r474, %r475, 2;\n" " add.s32 %r473, %r137, %r474;\n" " ld.shared.f32 %f235, [%r473+64];\n" " fma.rn.f32 %f537, %f34, %f235, %f537;\n" "\n" "$L__BB12_42:\n" " setp.lt.s32 %p166, %r104, 3;\n" " @%p166 bra $L__BB12_44;\n" "\n" " shl.b32 %r472, %r558, 4;\n" " add.s32 %r471, %r472, %r32;\n" " add.s32 %r470, %r42, %r471;\n" " shl.b32 %r469, %r470, 2;\n" " add.s32 %r468, %r137, %r469;\n" " ld.shared.f32 %f236, [%r468+64];\n" " fma.rn.f32 %f538, %f34, %f236, %f538;\n" "\n" "$L__BB12_44:\n" " setp.lt.s32 %p167, %r104, 4;\n" " @%p167 bra $L__BB12_46;\n" "\n" " shl.b32 %r491, %r558, 4;\n" " add.s32 %r490, %r491, %r32;\n" " add.s32 %r489, %r43, %r490;\n" " shl.b32 %r488, %r489, 2;\n" " add.s32 %r487, %r137, %r488;\n" " ld.shared.f32 %f237, [%r487+64];\n" " fma.rn.f32 %f539, %f34, %f237, %f539;\n" "\n" "$L__BB12_46:\n" " setp.lt.s32 %p168, %r104, 5;\n" " @%p168 bra $L__BB12_48;\n" "\n" " shl.b32 %r501, %r558, 4;\n" " add.s32 %r500, %r501, %r32;\n" " add.s32 %r499, %r44, %r500;\n" " shl.b32 %r498, %r499, 2;\n" " add.s32 %r497, %r137, %r498;\n" " ld.shared.f32 %f238, [%r497+64];\n" " fma.rn.f32 %f540, %f34, %f238, %f540;\n" "\n" "$L__BB12_48:\n" " setp.lt.s32 %p169, %r104, 6;\n" " @%p169 bra $L__BB12_50;\n" "\n" " shl.b32 %r511, %r558, 4;\n" " add.s32 %r510, %r511, %r32;\n" " add.s32 %r509, %r45, %r510;\n" " shl.b32 %r508, %r509, 2;\n" " add.s32 %r507, %r137, %r508;\n" " ld.shared.f32 %f239, [%r507+64];\n" " fma.rn.f32 %f541, %f34, %f239, %f541;\n" "\n" "$L__BB12_50:\n" " setp.lt.s32 %p170, %r104, 7;\n" " @%p170 bra $L__BB12_52;\n" "\n" " shl.b32 %r521, %r558, 4;\n" " add.s32 %r520, %r521, %r32;\n" " add.s32 %r519, %r46, %r520;\n" " shl.b32 %r518, %r519, 2;\n" " add.s32 %r517, %r137, %r518;\n" " ld.shared.f32 %f240, [%r517+64];\n" " fma.rn.f32 %f542, %f34, %f240, %f542;\n" "\n" "$L__BB12_52:\n" " setp.lt.s32 %p171, %r104, 8;\n" " @%p171 bra $L__BB12_54;\n" "\n" " shl.b32 %r531, %r558, 4;\n" " add.s32 %r530, %r531, %r32;\n" " add.s32 %r529, %r47, %r530;\n" " shl.b32 %r528, %r529, 2;\n" " add.s32 %r527, %r137, %r528;\n" " ld.shared.f32 %f241, [%r527+64];\n" " fma.rn.f32 %f543, %f34, %f241, %f543;\n" "\n" "$L__BB12_54:\n" " add.s32 %r558, %r558, 2;\n" " add.s32 %r557, %r557, -2;\n" " setp.ne.s32 %p42, %r557, 0;\n" " @%p42 bra $L__BB12_22;\n" "\n" "$L__BB12_55:\n" " setp.eq.s32 %p43, %r40, 0;\n" " @%p43 bra $L__BB12_72;\n" "\n" " shl.b32 %r221, %r558, 4;\n" " add.s32 %r78, %r221, %r32;\n" " add.s32 %r222, %r35, %r78;\n" " shl.b32 %r223, %r222, 2;\n" " add.s32 %r225, %r137, %r223;\n" " ld.shared.f32 %f59, [%r225];\n" " @%p3 bra $L__BB12_58;\n" "\n" " shl.b32 %r226, %r78, 2;\n" " add.s32 %r228, %r137, %r226;\n" " ld.shared.f32 %f242, [%r228];\n" " fma.rn.f32 %f536, %f59, %f242, %f536;\n" "\n" "$L__BB12_58:\n" " setp.lt.s32 %p45, %r104, 2;\n" " @%p45 bra $L__BB12_60;\n" "\n" " add.s32 %r229, %r78, %r105;\n" " shl.b32 %r230, %r229, 2;\n" " add.s32 %r232, %r137, %r230;\n" " ld.shared.f32 %f243, [%r232];\n" " fma.rn.f32 %f537, %f59, %f243, %f537;\n" "\n" "$L__BB12_60:\n" " setp.lt.s32 %p46, %r104, 3;\n" " @%p46 bra $L__BB12_62;\n" "\n" " add.s32 %r233, %r42, %r78;\n" " shl.b32 %r234, %r233, 2;\n" " add.s32 %r236, %r137, %r234;\n" " ld.shared.f32 %f244, [%r236];\n" " fma.rn.f32 %f538, %f59, %f244, %f538;\n" "\n" "$L__BB12_62:\n" " setp.lt.s32 %p47, %r104, 4;\n" " @%p47 bra $L__BB12_64;\n" "\n" " add.s32 %r237, %r43, %r78;\n" " shl.b32 %r238, %r237, 2;\n" " add.s32 %r240, %r137, %r238;\n" " ld.shared.f32 %f245, [%r240];\n" " fma.rn.f32 %f539, %f59, %f245, %f539;\n" "\n" "$L__BB12_64:\n" " setp.lt.s32 %p48, %r104, 5;\n" " @%p48 bra $L__BB12_66;\n" "\n" " add.s32 %r241, %r44, %r78;\n" " shl.b32 %r242, %r241, 2;\n" " add.s32 %r244, %r137, %r242;\n" " ld.shared.f32 %f246, [%r244];\n" " fma.rn.f32 %f540, %f59, %f246, %f540;\n" "\n" "$L__BB12_66:\n" " setp.lt.s32 %p49, %r104, 6;\n" " @%p49 bra $L__BB12_68;\n" "\n" " add.s32 %r245, %r45, %r78;\n" " shl.b32 %r246, %r245, 2;\n" " add.s32 %r248, %r137, %r246;\n" " ld.shared.f32 %f247, [%r248];\n" " fma.rn.f32 %f541, %f59, %f247, %f541;\n" "\n" "$L__BB12_68:\n" " setp.lt.s32 %p50, %r104, 7;\n" " @%p50 bra $L__BB12_70;\n" "\n" " add.s32 %r249, %r46, %r78;\n" " shl.b32 %r250, %r249, 2;\n" " add.s32 %r252, %r137, %r250;\n" " ld.shared.f32 %f248, [%r252];\n" " fma.rn.f32 %f542, %f59, %f248, %f542;\n" "\n" "$L__BB12_70:\n" " setp.lt.s32 %p51, %r104, 8;\n" " @%p51 bra $L__BB12_72;\n" "\n" " add.s32 %r253, %r47, %r78;\n" " shl.b32 %r254, %r253, 2;\n" " add.s32 %r256, %r137, %r254;\n" " ld.shared.f32 %f249, [%r256];\n" " fma.rn.f32 %f543, %f59, %f249, %f543;\n" "\n" "$L__BB12_72:\n" " sub.s32 %r454, %r8, %r553;\n" " min.s32 %r453, %r454, 16;\n" " setp.lt.s32 %p53, %r31, %r453;\n" " and.pred %p1, %p52, %p53;\n" " @%p3 bra $L__BB12_75;\n" "\n" " mov.b32 %r257, %f536;\n" " mov.u32 %r258, 2;\n" " mov.u32 %r259, 4127;\n" " mov.u32 %r260, 8;\n" " mov.u32 %r261, -1;\n" " shfl.sync.down.b32 %r262|%p55, %r257, %r260, %r259, %r261;\n" " mov.b32 %f250, %r262;\n" " add.f32 %f251, %f536, %f250;\n" " mov.b32 %r263, %f251;\n" " mov.u32 %r264, 4;\n" " shfl.sync.down.b32 %r265|%p56, %r263, %r264, %r259, %r261;\n" " mov.b32 %f252, %r265;\n" " add.f32 %f253, %f251, %f252;\n" " mov.b32 %r266, %f253;\n" " shfl.sync.down.b32 %r267|%p57, %r266, %r258, %r259, %r261;\n" " mov.b32 %f254, %r267;\n" " add.f32 %f255, %f253, %f254;\n" " mov.b32 %r268, %f255;\n" " mov.u32 %r269, 1;\n" " shfl.sync.down.b32 %r270|%p58, %r268, %r269, %r259, %r261;\n" " mov.b32 %f256, %r270;\n" " add.f32 %f83, %f255, %f256;\n" " not.pred %p59, %p1;\n" " @%p59 bra $L__BB12_75;\n" "\n" " mul.f32 %f257, %f83, %f179;\n" " st.shared.f32 [%r48], %f257;\n" "\n" "$L__BB12_75:\n" " setp.lt.s32 %p60, %r104, 2;\n" " @%p60 bra $L__BB12_78;\n" "\n" " mov.b32 %r271, %f537;\n" " mov.u32 %r272, 2;\n" " mov.u32 %r273, 4127;\n" " mov.u32 %r274, 8;\n" " mov.u32 %r275, -1;\n" " shfl.sync.down.b32 %r276|%p61, %r271, %r274, %r273, %r275;\n" " mov.b32 %f258, %r276;\n" " add.f32 %f259, %f537, %f258;\n" " mov.b32 %r277, %f259;\n" " mov.u32 %r278, 4;\n" " shfl.sync.down.b32 %r279|%p62, %r277, %r278, %r273, %r275;\n" " mov.b32 %f260, %r279;\n" " add.f32 %f261, %f259, %f260;\n" " mov.b32 %r280, %f261;\n" " shfl.sync.down.b32 %r281|%p63, %r280, %r272, %r273, %r275;\n" " mov.b32 %f262, %r281;\n" " add.f32 %f263, %f261, %f262;\n" " mov.b32 %r282, %f263;\n" " mov.u32 %r283, 1;\n" " shfl.sync.down.b32 %r284|%p64, %r282, %r283, %r273, %r275;\n" " mov.b32 %f264, %r284;\n" " add.f32 %f84, %f263, %f264;\n" " not.pred %p65, %p1;\n" " @%p65 bra $L__BB12_78;\n" "\n" " mul.f32 %f265, %f84, %f179;\n" " st.shared.f32 [%r48+64], %f265;\n" "\n" "$L__BB12_78:\n" " setp.lt.s32 %p66, %r104, 3;\n" " @%p66 bra $L__BB12_81;\n" "\n" " mov.b32 %r285, %f538;\n" " mov.u32 %r286, 2;\n" " mov.u32 %r287, 4127;\n" " mov.u32 %r288, 8;\n" " mov.u32 %r289, -1;\n" " shfl.sync.down.b32 %r290|%p67, %r285, %r288, %r287, %r289;\n" " mov.b32 %f266, %r290;\n" " add.f32 %f267, %f538, %f266;\n" " mov.b32 %r291, %f267;\n" " mov.u32 %r292, 4;\n" " shfl.sync.down.b32 %r293|%p68, %r291, %r292, %r287, %r289;\n" " mov.b32 %f268, %r293;\n" " add.f32 %f269, %f267, %f268;\n" " mov.b32 %r294, %f269;\n" " shfl.sync.down.b32 %r295|%p69, %r294, %r286, %r287, %r289;\n" " mov.b32 %f270, %r295;\n" " add.f32 %f271, %f269, %f270;\n" " mov.b32 %r296, %f271;\n" " mov.u32 %r297, 1;\n" " shfl.sync.down.b32 %r298|%p70, %r296, %r297, %r287, %r289;\n" " mov.b32 %f272, %r298;\n" " add.f32 %f85, %f271, %f272;\n" " not.pred %p71, %p1;\n" " @%p71 bra $L__BB12_81;\n" "\n" " mul.f32 %f273, %f85, %f179;\n" " st.shared.f32 [%r48+128], %f273;\n" "\n" "$L__BB12_81:\n" " setp.lt.s32 %p72, %r104, 4;\n" " @%p72 bra $L__BB12_84;\n" "\n" " mov.b32 %r299, %f539;\n" " mov.u32 %r300, 2;\n" " mov.u32 %r301, 4127;\n" " mov.u32 %r302, 8;\n" " mov.u32 %r303, -1;\n" " shfl.sync.down.b32 %r304|%p73, %r299, %r302, %r301, %r303;\n" " mov.b32 %f274, %r304;\n" " add.f32 %f275, %f539, %f274;\n" " mov.b32 %r305, %f275;\n" " mov.u32 %r306, 4;\n" " shfl.sync.down.b32 %r307|%p74, %r305, %r306, %r301, %r303;\n" " mov.b32 %f276, %r307;\n" " add.f32 %f277, %f275, %f276;\n" " mov.b32 %r308, %f277;\n" " shfl.sync.down.b32 %r309|%p75, %r308, %r300, %r301, %r303;\n" " mov.b32 %f278, %r309;\n" " add.f32 %f279, %f277, %f278;\n" " mov.b32 %r310, %f279;\n" " mov.u32 %r311, 1;\n" " shfl.sync.down.b32 %r312|%p76, %r310, %r311, %r301, %r303;\n" " mov.b32 %f280, %r312;\n" " add.f32 %f86, %f279, %f280;\n" " not.pred %p77, %p1;\n" " @%p77 bra $L__BB12_84;\n" "\n" " mul.f32 %f281, %f86, %f179;\n" " st.shared.f32 [%r48+192], %f281;\n" "\n" "$L__BB12_84:\n" " setp.lt.s32 %p78, %r104, 5;\n" " @%p78 bra $L__BB12_87;\n" "\n" " mov.b32 %r313, %f540;\n" " mov.u32 %r314, 2;\n" " mov.u32 %r315, 4127;\n" " mov.u32 %r316, 8;\n" " mov.u32 %r317, -1;\n" " shfl.sync.down.b32 %r318|%p79, %r313, %r316, %r315, %r317;\n" " mov.b32 %f282, %r318;\n" " add.f32 %f283, %f540, %f282;\n" " mov.b32 %r319, %f283;\n" " mov.u32 %r320, 4;\n" " shfl.sync.down.b32 %r321|%p80, %r319, %r320, %r315, %r317;\n" " mov.b32 %f284, %r321;\n" " add.f32 %f285, %f283, %f284;\n" " mov.b32 %r322, %f285;\n" " shfl.sync.down.b32 %r323|%p81, %r322, %r314, %r315, %r317;\n" " mov.b32 %f286, %r323;\n" " add.f32 %f287, %f285, %f286;\n" " mov.b32 %r324, %f287;\n" " mov.u32 %r325, 1;\n" " shfl.sync.down.b32 %r326|%p82, %r324, %r325, %r315, %r317;\n" " mov.b32 %f288, %r326;\n" " add.f32 %f87, %f287, %f288;\n" " not.pred %p83, %p1;\n" " @%p83 bra $L__BB12_87;\n" "\n" " mul.f32 %f289, %f87, %f179;\n" " st.shared.f32 [%r48+256], %f289;\n" "\n" "$L__BB12_87:\n" " setp.lt.s32 %p84, %r104, 6;\n" " @%p84 bra $L__BB12_90;\n" "\n" " mov.b32 %r327, %f541;\n" " mov.u32 %r328, 2;\n" " mov.u32 %r329, 4127;\n" " mov.u32 %r330, 8;\n" " mov.u32 %r331, -1;\n" " shfl.sync.down.b32 %r332|%p85, %r327, %r330, %r329, %r331;\n" " mov.b32 %f290, %r332;\n" " add.f32 %f291, %f541, %f290;\n" " mov.b32 %r333, %f291;\n" " mov.u32 %r334, 4;\n" " shfl.sync.down.b32 %r335|%p86, %r333, %r334, %r329, %r331;\n" " mov.b32 %f292, %r335;\n" " add.f32 %f293, %f291, %f292;\n" " mov.b32 %r336, %f293;\n" " shfl.sync.down.b32 %r337|%p87, %r336, %r328, %r329, %r331;\n" " mov.b32 %f294, %r337;\n" " add.f32 %f295, %f293, %f294;\n" " mov.b32 %r338, %f295;\n" " mov.u32 %r339, 1;\n" " shfl.sync.down.b32 %r340|%p88, %r338, %r339, %r329, %r331;\n" " mov.b32 %f296, %r340;\n" " add.f32 %f88, %f295, %f296;\n" " not.pred %p89, %p1;\n" " @%p89 bra $L__BB12_90;\n" "\n" " mul.f32 %f297, %f88, %f179;\n" " st.shared.f32 [%r48+320], %f297;\n" "\n" "$L__BB12_90:\n" " setp.lt.s32 %p90, %r104, 7;\n" " @%p90 bra $L__BB12_93;\n" "\n" " mov.b32 %r341, %f542;\n" " mov.u32 %r342, 2;\n" " mov.u32 %r343, 4127;\n" " mov.u32 %r344, 8;\n" " mov.u32 %r345, -1;\n" " shfl.sync.down.b32 %r346|%p91, %r341, %r344, %r343, %r345;\n" " mov.b32 %f298, %r346;\n" " add.f32 %f299, %f542, %f298;\n" " mov.b32 %r347, %f299;\n" " mov.u32 %r348, 4;\n" " shfl.sync.down.b32 %r349|%p92, %r347, %r348, %r343, %r345;\n" " mov.b32 %f300, %r349;\n" " add.f32 %f301, %f299, %f300;\n" " mov.b32 %r350, %f301;\n" " shfl.sync.down.b32 %r351|%p93, %r350, %r342, %r343, %r345;\n" " mov.b32 %f302, %r351;\n" " add.f32 %f303, %f301, %f302;\n" " mov.b32 %r352, %f303;\n" " mov.u32 %r353, 1;\n" " shfl.sync.down.b32 %r354|%p94, %r352, %r353, %r343, %r345;\n" " mov.b32 %f304, %r354;\n" " add.f32 %f89, %f303, %f304;\n" " not.pred %p95, %p1;\n" " @%p95 bra $L__BB12_93;\n" "\n" " mul.f32 %f305, %f89, %f179;\n" " st.shared.f32 [%r48+384], %f305;\n" "\n" "$L__BB12_93:\n" " setp.lt.s32 %p96, %r104, 8;\n" " @%p96 bra $L__BB12_96;\n" "\n" " mov.b32 %r355, %f543;\n" " mov.u32 %r356, 2;\n" " mov.u32 %r357, 4127;\n" " mov.u32 %r358, 8;\n" " mov.u32 %r359, -1;\n" " shfl.sync.down.b32 %r360|%p97, %r355, %r358, %r357, %r359;\n" " mov.b32 %f306, %r360;\n" " add.f32 %f307, %f543, %f306;\n" " mov.b32 %r361, %f307;\n" " mov.u32 %r362, 4;\n" " shfl.sync.down.b32 %r363|%p98, %r361, %r362, %r357, %r359;\n" " mov.b32 %f308, %r363;\n" " add.f32 %f309, %f307, %f308;\n" " mov.b32 %r364, %f309;\n" " shfl.sync.down.b32 %r365|%p99, %r364, %r356, %r357, %r359;\n" " mov.b32 %f310, %r365;\n" " add.f32 %f311, %f309, %f310;\n" " mov.b32 %r366, %f311;\n" " mov.u32 %r367, 1;\n" " shfl.sync.down.b32 %r368|%p100, %r366, %r367, %r357, %r359;\n" " mov.b32 %f312, %r368;\n" " add.f32 %f90, %f311, %f312;\n" " not.pred %p101, %p1;\n" " @%p101 bra $L__BB12_96;\n" "\n" " mul.f32 %f313, %f90, %f179;\n" " st.shared.f32 [%r48+448], %f313;\n" "\n" "$L__BB12_96:\n" " bar.sync 0;\n" " @%p8 bra $L__BB12_114;\n" "\n" " ld.shared.f32 %f91, [%r28];\n" " mov.f32 %f578, %f91;\n" " @%p18 bra $L__BB12_105;\n" "\n" " and.b32 %r79, %r55, 3;\n" " add.s32 %r370, %r55, -1;\n" " setp.lt.u32 %p104, %r370, 3;\n" " mov.u32 %r561, 0;\n" " mov.f32 %f578, %f91;\n" " @%p104 bra $L__BB12_101;\n" "\n" " sub.s32 %r560, %r55, %r79;\n" " mov.u32 %r561, 0;\n" " mov.f32 %f578, %f91;\n" "\n" "$L__BB12_100:\n" " add.s32 %r372, %r36, %r561;\n" " shl.b32 %r373, %r372, 2;\n" " add.s32 %r375, %r137, %r373;\n" " ld.shared.f32 %f315, [%r375];\n" " max.f32 %f316, %f578, %f315;\n" " ld.shared.f32 %f317, [%r375+4];\n" " max.f32 %f318, %f316, %f317;\n" " ld.shared.f32 %f319, [%r375+8];\n" " max.f32 %f320, %f318, %f319;\n" " ld.shared.f32 %f321, [%r375+12];\n" " max.f32 %f578, %f320, %f321;\n" " add.s32 %r561, %r561, 4;\n" " add.s32 %r560, %r560, -4;\n" " setp.ne.s32 %p105, %r560, 0;\n" " @%p105 bra $L__BB12_100;\n" "\n" "$L__BB12_101:\n" " setp.eq.s32 %p106, %r79, 0;\n" " @%p106 bra $L__BB12_105;\n" "\n" " add.s32 %r376, %r36, %r561;\n" " shl.b32 %r377, %r376, 2;\n" " add.s32 %r86, %r137, %r377;\n" " ld.shared.f32 %f322, [%r86];\n" " max.f32 %f578, %f578, %f322;\n" " setp.eq.s32 %p107, %r79, 1;\n" " @%p107 bra $L__BB12_105;\n" "\n" " ld.shared.f32 %f323, [%r86+4];\n" " max.f32 %f578, %f578, %f323;\n" " setp.eq.s32 %p108, %r79, 2;\n" " @%p108 bra $L__BB12_105;\n" "\n" " ld.shared.f32 %f324, [%r86+8];\n" " max.f32 %f578, %f578, %f324;\n" "\n" "$L__BB12_105:\n" " mov.f32 %f582, 0f00000000;\n" " @%p18 bra $L__BB12_113;\n" "\n" " and.b32 %r87, %r55, 3;\n" " add.s32 %r380, %r55, -1;\n" " setp.lt.u32 %p110, %r380, 3;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r565, 0;\n" " @%p110 bra $L__BB12_109;\n" "\n" " sub.s32 %r563, %r55, %r87;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r565, 0;\n" " mov.u32 %r564, %r50;\n" "\n" "$L__BB12_108:\n" " shl.b32 %r456, %r104, 2;\n" " add.s32 %r455, %r456, 1024;\n" " add.s32 %r382, %r36, %r565;\n" " shl.b32 %r383, %r382, 2;\n" " add.s32 %r385, %r137, %r383;\n" " ld.shared.f32 %f329, [%r385];\n" " sub.f32 %f330, %f329, %f578;\n" " mov.f32 %f331, 0f3F000000;\n" " mov.f32 %f332, 0f3BBB989D;\n" " fma.rn.f32 %f333, %f330, %f332, %f331;\n" " mov.f32 %f334, 0f3FB8AA3B;\n" " mov.f32 %f335, 0f437C0000;\n" " cvt.sat.f32.f32 %f336, %f333;\n" " mov.f32 %f337, 0f4B400001;\n" " fma.rm.f32 %f338, %f336, %f335, %f337;\n" " add.f32 %f339, %f338, 0fCB40007F;\n" " neg.f32 %f340, %f339;\n" " fma.rn.f32 %f341, %f330, %f334, %f340;\n" " mov.f32 %f342, 0f32A57060;\n" " fma.rn.f32 %f343, %f330, %f342, %f341;\n" " mov.b32 %r386, %f338;\n" " shl.b32 %r387, %r386, 23;\n" " mov.b32 %f344, %r387;\n" " ex2.approx.ftz.f32 %f345, %f343;\n" " mul.f32 %f346, %f345, %f344;\n" " st.shared.f32 [%r564+1024], %f346;\n" " add.f32 %f347, %f582, %f346;\n" " ld.shared.f32 %f348, [%r385+4];\n" " sub.f32 %f349, %f348, %f578;\n" " fma.rn.f32 %f350, %f349, %f332, %f331;\n" " cvt.sat.f32.f32 %f351, %f350;\n" " fma.rm.f32 %f352, %f351, %f335, %f337;\n" " add.f32 %f353, %f352, 0fCB40007F;\n" " neg.f32 %f354, %f353;\n" " fma.rn.f32 %f355, %f349, %f334, %f354;\n" " fma.rn.f32 %f356, %f349, %f342, %f355;\n" " mov.b32 %r388, %f352;\n" " shl.b32 %r389, %r388, 23;\n" " mov.b32 %f357, %r389;\n" " ex2.approx.ftz.f32 %f358, %f356;\n" " mul.f32 %f359, %f358, %f357;\n" " add.s32 %r390, %r564, %r456;\n" " st.shared.f32 [%r390+1024], %f359;\n" " add.f32 %f360, %f347, %f359;\n" " ld.shared.f32 %f361, [%r385+8];\n" " sub.f32 %f362, %f361, %f578;\n" " fma.rn.f32 %f363, %f362, %f332, %f331;\n" " cvt.sat.f32.f32 %f364, %f363;\n" " fma.rm.f32 %f365, %f364, %f335, %f337;\n" " add.f32 %f366, %f365, 0fCB40007F;\n" " neg.f32 %f367, %f366;\n" " fma.rn.f32 %f368, %f362, %f334, %f367;\n" " fma.rn.f32 %f369, %f362, %f342, %f368;\n" " mov.b32 %r391, %f365;\n" " shl.b32 %r392, %r391, 23;\n" " mov.b32 %f370, %r392;\n" " ex2.approx.ftz.f32 %f371, %f369;\n" " mul.f32 %f372, %f371, %f370;\n" " add.s32 %r393, %r390, %r456;\n" " add.s32 %r394, %r390, %r455;\n" " st.shared.f32 [%r394], %f372;\n" " add.f32 %f373, %f360, %f372;\n" " ld.shared.f32 %f374, [%r385+12];\n" " sub.f32 %f375, %f374, %f578;\n" " fma.rn.f32 %f376, %f375, %f332, %f331;\n" " cvt.sat.f32.f32 %f377, %f376;\n" " fma.rm.f32 %f378, %f377, %f335, %f337;\n" " add.f32 %f379, %f378, 0fCB40007F;\n" " neg.f32 %f380, %f379;\n" " fma.rn.f32 %f381, %f375, %f334, %f380;\n" " fma.rn.f32 %f382, %f375, %f342, %f381;\n" " mov.b32 %r395, %f378;\n" " shl.b32 %r396, %r395, 23;\n" " mov.b32 %f383, %r396;\n" " ex2.approx.ftz.f32 %f384, %f382;\n" " mul.f32 %f385, %f384, %f383;\n" " add.s32 %r397, %r393, %r456;\n" " add.s32 %r398, %r393, %r455;\n" " add.s32 %r564, %r397, %r456;\n" " st.shared.f32 [%r398], %f385;\n" " add.f32 %f582, %f373, %f385;\n" " add.s32 %r565, %r565, 4;\n" " add.s32 %r563, %r563, -4;\n" " setp.ne.s32 %p111, %r563, 0;\n" " @%p111 bra $L__BB12_108;\n" "\n" "$L__BB12_109:\n" " setp.eq.s32 %p112, %r87, 0;\n" " @%p112 bra $L__BB12_113;\n" "\n" " add.s32 %r399, %r36, %r565;\n" " shl.b32 %r400, %r399, 2;\n" " add.s32 %r96, %r137, %r400;\n" " ld.shared.f32 %f386, [%r96];\n" " sub.f32 %f387, %f386, %f578;\n" " mov.f32 %f388, 0f3F000000;\n" " mov.f32 %f389, 0f3BBB989D;\n" " fma.rn.f32 %f390, %f387, %f389, %f388;\n" " mov.f32 %f391, 0f3FB8AA3B;\n" " mov.f32 %f392, 0f437C0000;\n" " cvt.sat.f32.f32 %f393, %f390;\n" " mov.f32 %f394, 0f4B400001;\n" " fma.rm.f32 %f395, %f393, %f392, %f394;\n" " add.f32 %f396, %f395, 0fCB40007F;\n" " neg.f32 %f397, %f396;\n" " fma.rn.f32 %f398, %f387, %f391, %f397;\n" " mov.f32 %f399, 0f32A57060;\n" " fma.rn.f32 %f400, %f387, %f399, %f398;\n" " mov.b32 %r402, %f395;\n" " shl.b32 %r403, %r402, 23;\n" " mov.b32 %f401, %r403;\n" " ex2.approx.ftz.f32 %f402, %f400;\n" " mul.f32 %f403, %f402, %f401;\n" " cvt.u64.u32 %rd68, %r565;\n" " mul.lo.s64 %rd69, %rd68, %rd8;\n" " add.s64 %rd70, %rd25, %rd69;\n" " cvt.u32.u64 %r404, %rd70;\n" " add.s32 %r405, %r9, %r404;\n" " shl.b32 %r406, %r405, 2;\n" " add.s32 %r407, %r137, %r406;\n" " st.shared.f32 [%r407], %f403;\n" " add.f32 %f582, %f582, %f403;\n" " setp.eq.s32 %p113, %r87, 1;\n" " @%p113 bra $L__BB12_113;\n" "\n" " mov.f32 %f513, 0f32A57060;\n" " mov.f32 %f512, 0f3FB8AA3B;\n" " mov.f32 %f511, 0f4B400001;\n" " mov.f32 %f510, 0f437C0000;\n" " mov.f32 %f509, 0f3F000000;\n" " mov.f32 %f508, 0f3BBB989D;\n" " add.s32 %r440, %r36, %r565;\n" " shl.b32 %r439, %r440, 2;\n" " add.s32 %r438, %r137, %r439;\n" " ld.shared.f32 %f404, [%r438+4];\n" " sub.f32 %f405, %f404, %f578;\n" " fma.rn.f32 %f408, %f405, %f508, %f509;\n" " cvt.sat.f32.f32 %f411, %f408;\n" " fma.rm.f32 %f413, %f411, %f510, %f511;\n" " add.f32 %f414, %f413, 0fCB40007F;\n" " neg.f32 %f415, %f414;\n" " fma.rn.f32 %f416, %f405, %f512, %f415;\n" " fma.rn.f32 %f418, %f405, %f513, %f416;\n" " mov.b32 %r408, %f413;\n" " shl.b32 %r409, %r408, 23;\n" " mov.b32 %f419, %r409;\n" " ex2.approx.ftz.f32 %f420, %f418;\n" " mul.f32 %f421, %f420, %f419;\n" " add.s32 %r410, %r565, 1;\n" " cvt.u64.u32 %rd71, %r410;\n" " mul.lo.s64 %rd72, %rd71, %rd8;\n" " add.s64 %rd73, %rd25, %rd72;\n" " cvt.u32.u64 %r411, %rd73;\n" " add.s32 %r412, %r9, %r411;\n" " shl.b32 %r413, %r412, 2;\n" " add.s32 %r415, %r137, %r413;\n" " st.shared.f32 [%r415], %f421;\n" " add.f32 %f582, %f582, %f421;\n" " setp.eq.s32 %p114, %r87, 2;\n" " @%p114 bra $L__BB12_113;\n" "\n" " mov.f32 %f519, 0f32A57060;\n" " mov.f32 %f518, 0f3FB8AA3B;\n" " mov.f32 %f517, 0f4B400001;\n" " mov.f32 %f516, 0f437C0000;\n" " mov.f32 %f515, 0f3F000000;\n" " mov.f32 %f514, 0f3BBB989D;\n" " add.s32 %r443, %r36, %r565;\n" " shl.b32 %r442, %r443, 2;\n" " add.s32 %r441, %r137, %r442;\n" " ld.shared.f32 %f422, [%r441+8];\n" " sub.f32 %f423, %f422, %f578;\n" " fma.rn.f32 %f426, %f423, %f514, %f515;\n" " cvt.sat.f32.f32 %f429, %f426;\n" " fma.rm.f32 %f431, %f429, %f516, %f517;\n" " add.f32 %f432, %f431, 0fCB40007F;\n" " neg.f32 %f433, %f432;\n" " fma.rn.f32 %f434, %f423, %f518, %f433;\n" " fma.rn.f32 %f436, %f423, %f519, %f434;\n" " mov.b32 %r416, %f431;\n" " shl.b32 %r417, %r416, 23;\n" " mov.b32 %f437, %r417;\n" " ex2.approx.ftz.f32 %f438, %f436;\n" " mul.f32 %f439, %f438, %f437;\n" " add.s32 %r418, %r565, 2;\n" " cvt.u64.u32 %rd74, %r418;\n" " mul.lo.s64 %rd75, %rd74, %rd8;\n" " add.s64 %rd76, %rd25, %rd75;\n" " cvt.u32.u64 %r419, %rd76;\n" " add.s32 %r420, %r9, %r419;\n" " shl.b32 %r421, %r420, 2;\n" " add.s32 %r423, %r137, %r421;\n" " st.shared.f32 [%r423], %f439;\n" " add.f32 %f582, %f582, %f439;\n" "\n" "$L__BB12_113:\n" " sub.f32 %f440, %f91, %f578;\n" " mov.f32 %f441, 0f3F000000;\n" " mov.f32 %f442, 0f3BBB989D;\n" " fma.rn.f32 %f443, %f440, %f442, %f441;\n" " mov.f32 %f444, 0f3FB8AA3B;\n" " mov.f32 %f445, 0f437C0000;\n" " cvt.sat.f32.f32 %f446, %f443;\n" " mov.f32 %f447, 0f4B400001;\n" " fma.rm.f32 %f448, %f446, %f445, %f447;\n" " add.f32 %f449, %f448, 0fCB40007F;\n" " neg.f32 %f450, %f449;\n" " fma.rn.f32 %f451, %f440, %f444, %f450;\n" " mov.f32 %f452, 0f32A57060;\n" " fma.rn.f32 %f453, %f440, %f452, %f451;\n" " ex2.approx.ftz.f32 %f454, %f453;\n" " mov.b32 %r424, %f448;\n" " shl.b32 %r425, %r424, 23;\n" " mov.b32 %f455, %r425;\n" " mul.f32 %f456, %f454, %f455;\n" " ld.shared.f32 %f457, [%r29];\n" " fma.rn.f32 %f458, %f456, %f457, %f582;\n" " st.shared.f32 [%r29], %f458;\n" " st.shared.f32 [%r28], %f578;\n" " st.shared.f32 [%r37], %f456;\n" "\n" "$L__BB12_114:\n" " bar.sync 0;\n" " @%p18 bra $L__BB12_150;\n" "\n" " mov.u32 %r566, 0;\n" "\n" "$L__BB12_116:\n" " ld.param.u64 %rd161, [fc_attn_flash_decode_dev_param_2];\n" " add.s32 %r427, %r566, %r553;\n" " cvt.s64.s32 %rd78, %r427;\n" " mul.lo.s64 %rd79, %rd78, %rd12;\n" " add.s64 %rd80, %rd24, %rd79;\n" " shl.b64 %rd81, %rd80, 2;\n" " add.s64 %rd77, %rd161, %rd81;\n" " // begin inline asm\n" " ld.global.nc.f32 %f459, [%rd77];\n" " // end inline asm\n" " cvt.u64.u32 %rd82, %r566;\n" " mul.lo.s64 %rd83, %rd82, %rd8;\n" " add.s64 %rd84, %rd83, %rd6;\n" " cvt.u32.u64 %r428, %rd84;\n" " add.s32 %r429, %r9, %r428;\n" " shl.b32 %r430, %r429, 2;\n" " add.s32 %r98, %r137, %r430;\n" " setp.eq.s32 %p116, %r566, 0;\n" " @%p116 bra $L__BB12_133;\n" "\n" " @%p3 bra $L__BB12_119;\n" "\n" " ld.shared.f32 %f460, [%r98];\n" " fma.rn.f32 %f612, %f459, %f460, %f612;\n" "\n" "$L__BB12_119:\n" " setp.lt.s32 %p151, %r104, 2;\n" " @%p151 bra $L__BB12_121;\n" "\n" " ld.shared.f32 %f461, [%r98+4];\n" " fma.rn.f32 %f611, %f459, %f461, %f611;\n" "\n" "$L__BB12_121:\n" " setp.lt.s32 %p152, %r104, 3;\n" " @%p152 bra $L__BB12_123;\n" "\n" " ld.shared.f32 %f462, [%r98+8];\n" " fma.rn.f32 %f610, %f459, %f462, %f610;\n" "\n" "$L__BB12_123:\n" " setp.lt.s32 %p153, %r104, 4;\n" " @%p153 bra $L__BB12_125;\n" "\n" " ld.shared.f32 %f463, [%r98+12];\n" " fma.rn.f32 %f609, %f459, %f463, %f609;\n" "\n" "$L__BB12_125:\n" " setp.lt.s32 %p154, %r104, 5;\n" " @%p154 bra $L__BB12_127;\n" "\n" " ld.shared.f32 %f464, [%r98+16];\n" " fma.rn.f32 %f608, %f459, %f464, %f608;\n" "\n" "$L__BB12_127:\n" " setp.lt.s32 %p155, %r104, 6;\n" " @%p155 bra $L__BB12_129;\n" "\n" " ld.shared.f32 %f465, [%r98+20];\n" " fma.rn.f32 %f607, %f459, %f465, %f607;\n" "\n" "$L__BB12_129:\n" " setp.lt.s32 %p156, %r104, 7;\n" " @%p156 bra $L__BB12_131;\n" "\n" " ld.shared.f32 %f466, [%r98+24];\n" " fma.rn.f32 %f606, %f459, %f466, %f606;\n" "\n" "$L__BB12_131:\n" " setp.lt.s32 %p157, %r104, 8;\n" " @%p157 bra $L__BB12_149;\n" "\n" " ld.shared.f32 %f467, [%r98+28];\n" " fma.rn.f32 %f605, %f459, %f467, %f605;\n" " bra.uni $L__BB12_149;\n" "\n" "$L__BB12_133:\n" " @%p3 bra $L__BB12_135;\n" "\n" " ld.shared.f32 %f468, [%r49];\n" " ld.shared.f32 %f469, [%r98];\n" " mul.f32 %f470, %f459, %f469;\n" " fma.rn.f32 %f612, %f468, %f612, %f470;\n" "\n" "$L__BB12_135:\n" " setp.lt.s32 %p158, %r104, 2;\n" " @%p158 bra $L__BB12_137;\n" "\n" " ld.shared.f32 %f471, [%r49+4];\n" " ld.shared.f32 %f472, [%r98+4];\n" " mul.f32 %f473, %f459, %f472;\n" " fma.rn.f32 %f611, %f471, %f611, %f473;\n" "\n" "$L__BB12_137:\n" " setp.lt.s32 %p159, %r104, 3;\n" " @%p159 bra $L__BB12_139;\n" "\n" " ld.shared.f32 %f474, [%r49+8];\n" " ld.shared.f32 %f475, [%r98+8];\n" " mul.f32 %f476, %f459, %f475;\n" " fma.rn.f32 %f610, %f474, %f610, %f476;\n" "\n" "$L__BB12_139:\n" " setp.lt.s32 %p160, %r104, 4;\n" " @%p160 bra $L__BB12_141;\n" "\n" " ld.shared.f32 %f477, [%r49+12];\n" " ld.shared.f32 %f478, [%r98+12];\n" " mul.f32 %f479, %f459, %f478;\n" " fma.rn.f32 %f609, %f477, %f609, %f479;\n" "\n" "$L__BB12_141:\n" " setp.lt.s32 %p161, %r104, 5;\n" " @%p161 bra $L__BB12_143;\n" "\n" " ld.shared.f32 %f480, [%r49+16];\n" " ld.shared.f32 %f481, [%r98+16];\n" " mul.f32 %f482, %f459, %f481;\n" " fma.rn.f32 %f608, %f480, %f608, %f482;\n" "\n" "$L__BB12_143:\n" " setp.lt.s32 %p162, %r104, 6;\n" " @%p162 bra $L__BB12_145;\n" "\n" " ld.shared.f32 %f483, [%r49+20];\n" " ld.shared.f32 %f484, [%r98+20];\n" " mul.f32 %f485, %f459, %f484;\n" " fma.rn.f32 %f607, %f483, %f607, %f485;\n" "\n" "$L__BB12_145:\n" " setp.lt.s32 %p163, %r104, 7;\n" " @%p163 bra $L__BB12_147;\n" "\n" " ld.shared.f32 %f486, [%r49+24];\n" " ld.shared.f32 %f487, [%r98+24];\n" " mul.f32 %f488, %f459, %f487;\n" " fma.rn.f32 %f606, %f486, %f606, %f488;\n" "\n" "$L__BB12_147:\n" " setp.lt.s32 %p164, %r104, 8;\n" " @%p164 bra $L__BB12_149;\n" "\n" " ld.shared.f32 %f489, [%r49+28];\n" " ld.shared.f32 %f490, [%r98+28];\n" " mul.f32 %f491, %f459, %f490;\n" " fma.rn.f32 %f605, %f489, %f605, %f491;\n" "\n" "$L__BB12_149:\n" " sub.s32 %r462, %r8, %r553;\n" " min.s32 %r461, %r462, 16;\n" " add.s32 %r566, %r566, 1;\n" " setp.lt.s32 %p133, %r566, %r461;\n" " @%p133 bra $L__BB12_116;\n" "\n" "$L__BB12_150:\n" " bar.sync 0;\n" "\n" "$L__BB12_151:\n" " add.s32 %r553, %r553, 16;\n" " setp.gt.s32 %p134, %r8, %r553;\n" " add.s32 %r552, %r552, 1;\n" " @%p134 bra $L__BB12_12;\n" "\n" "$L__BB12_152:\n" " shl.b32 %r448, %r104, 4;\n" " cvt.s64.s32 %rd160, %r448;\n" " add.s32 %r447, %r105, 16;\n" " mul.wide.s32 %rd159, %r447, 16;\n" " add.s64 %rd158, %rd159, %rd160;\n" " add.s64 %rd157, %rd158, %rd160;\n" " cvt.u32.u64 %r446, %rd157;\n" " add.s64 %rd156, %rd157, %rd8;\n" " cvt.u32.u64 %r445, %rd156;\n" " mov.u32 %r444, %ctaid.x;\n" " cvt.s64.s32 %rd28, %r4;\n" " cvt.s64.s32 %rd29, %r444;\n" " add.s32 %r433, %r9, %r446;\n" " shl.b32 %r434, %r433, 2;\n" " add.s32 %r102, %r137, %r434;\n" " add.s32 %r436, %r9, %r445;\n" " shl.b32 %r437, %r436, 2;\n" " add.s32 %r103, %r137, %r437;\n" " @%p3 bra $L__BB12_155;\n" "\n" " setp.ne.s32 %p136, %r6, 0;\n" " mul.lo.s64 %rd85, %rd11, %rd28;\n" " add.s64 %rd30, %rd85, %rd29;\n" " mul.lo.s64 %rd86, %rd30, %rd12;\n" " add.s64 %rd87, %rd86, %rd23;\n" " shl.b64 %rd88, %rd87, 2;\n" " add.s64 %rd89, %rd4, %rd88;\n" " st.global.f32 [%rd89], %f612;\n" " @%p136 bra $L__BB12_155;\n" "\n" " ld.shared.f32 %f492, [%r102];\n" " shl.b64 %rd90, %rd30, 2;\n" " add.s64 %rd91, %rd3, %rd90;\n" " st.global.f32 [%rd91], %f492;\n" " ld.shared.f32 %f493, [%r103];\n" " add.s64 %rd92, %rd2, %rd90;\n" " st.global.f32 [%rd92], %f493;\n" "\n" "$L__BB12_155:\n" " setp.lt.s32 %p137, %r104, 2;\n" " @%p137 bra $L__BB12_158;\n" "\n" " setp.ne.s32 %p138, %r6, 0;\n" " add.s64 %rd93, %rd11, 1;\n" " mul.lo.s64 %rd94, %rd93, %rd28;\n" " add.s64 %rd31, %rd94, %rd29;\n" " mul.lo.s64 %rd95, %rd31, %rd12;\n" " add.s64 %rd96, %rd95, %rd23;\n" " shl.b64 %rd97, %rd96, 2;\n" " add.s64 %rd98, %rd4, %rd97;\n" " st.global.f32 [%rd98], %f611;\n" " @%p138 bra $L__BB12_158;\n" "\n" " ld.shared.f32 %f494, [%r102+4];\n" " shl.b64 %rd99, %rd31, 2;\n" " add.s64 %rd100, %rd3, %rd99;\n" " st.global.f32 [%rd100], %f494;\n" " ld.shared.f32 %f495, [%r103+4];\n" " add.s64 %rd101, %rd2, %rd99;\n" " st.global.f32 [%rd101], %f495;\n" "\n" "$L__BB12_158:\n" " setp.lt.s32 %p139, %r104, 3;\n" " @%p139 bra $L__BB12_161;\n" "\n" " setp.ne.s32 %p140, %r6, 0;\n" " add.s64 %rd102, %rd11, 2;\n" " mul.lo.s64 %rd103, %rd102, %rd28;\n" " add.s64 %rd32, %rd103, %rd29;\n" " mul.lo.s64 %rd104, %rd32, %rd12;\n" " add.s64 %rd105, %rd104, %rd23;\n" " shl.b64 %rd106, %rd105, 2;\n" " add.s64 %rd107, %rd4, %rd106;\n" " st.global.f32 [%rd107], %f610;\n" " @%p140 bra $L__BB12_161;\n" "\n" " ld.shared.f32 %f496, [%r102+8];\n" " shl.b64 %rd108, %rd32, 2;\n" " add.s64 %rd109, %rd3, %rd108;\n" " st.global.f32 [%rd109], %f496;\n" " ld.shared.f32 %f497, [%r103+8];\n" " add.s64 %rd110, %rd2, %rd108;\n" " st.global.f32 [%rd110], %f497;\n" "\n" "$L__BB12_161:\n" " setp.lt.s32 %p141, %r104, 4;\n" " @%p141 bra $L__BB12_164;\n" "\n" " setp.ne.s32 %p142, %r6, 0;\n" " add.s64 %rd111, %rd11, 3;\n" " mul.lo.s64 %rd112, %rd111, %rd28;\n" " add.s64 %rd33, %rd112, %rd29;\n" " mul.lo.s64 %rd113, %rd33, %rd12;\n" " add.s64 %rd114, %rd113, %rd23;\n" " shl.b64 %rd115, %rd114, 2;\n" " add.s64 %rd116, %rd4, %rd115;\n" " st.global.f32 [%rd116], %f609;\n" " @%p142 bra $L__BB12_164;\n" "\n" " ld.shared.f32 %f498, [%r102+12];\n" " shl.b64 %rd117, %rd33, 2;\n" " add.s64 %rd118, %rd3, %rd117;\n" " st.global.f32 [%rd118], %f498;\n" " ld.shared.f32 %f499, [%r103+12];\n" " add.s64 %rd119, %rd2, %rd117;\n" " st.global.f32 [%rd119], %f499;\n" "\n" "$L__BB12_164:\n" " setp.lt.s32 %p143, %r104, 5;\n" " @%p143 bra $L__BB12_167;\n" "\n" " setp.ne.s32 %p144, %r6, 0;\n" " add.s64 %rd120, %rd11, 4;\n" " mul.lo.s64 %rd121, %rd120, %rd28;\n" " add.s64 %rd34, %rd121, %rd29;\n" " mul.lo.s64 %rd122, %rd34, %rd12;\n" " add.s64 %rd123, %rd122, %rd23;\n" " shl.b64 %rd124, %rd123, 2;\n" " add.s64 %rd125, %rd4, %rd124;\n" " st.global.f32 [%rd125], %f608;\n" " @%p144 bra $L__BB12_167;\n" "\n" " ld.shared.f32 %f500, [%r102+16];\n" " shl.b64 %rd126, %rd34, 2;\n" " add.s64 %rd127, %rd3, %rd126;\n" " st.global.f32 [%rd127], %f500;\n" " ld.shared.f32 %f501, [%r103+16];\n" " add.s64 %rd128, %rd2, %rd126;\n" " st.global.f32 [%rd128], %f501;\n" "\n" "$L__BB12_167:\n" " setp.lt.s32 %p145, %r104, 6;\n" " @%p145 bra $L__BB12_170;\n" "\n" " setp.ne.s32 %p146, %r6, 0;\n" " add.s64 %rd129, %rd11, 5;\n" " mul.lo.s64 %rd130, %rd129, %rd28;\n" " add.s64 %rd35, %rd130, %rd29;\n" " mul.lo.s64 %rd131, %rd35, %rd12;\n" " add.s64 %rd132, %rd131, %rd23;\n" " shl.b64 %rd133, %rd132, 2;\n" " add.s64 %rd134, %rd4, %rd133;\n" " st.global.f32 [%rd134], %f607;\n" " @%p146 bra $L__BB12_170;\n" "\n" " ld.shared.f32 %f502, [%r102+20];\n" " shl.b64 %rd135, %rd35, 2;\n" " add.s64 %rd136, %rd3, %rd135;\n" " st.global.f32 [%rd136], %f502;\n" " ld.shared.f32 %f503, [%r103+20];\n" " add.s64 %rd137, %rd2, %rd135;\n" " st.global.f32 [%rd137], %f503;\n" "\n" "$L__BB12_170:\n" " setp.lt.s32 %p147, %r104, 7;\n" " @%p147 bra $L__BB12_173;\n" "\n" " setp.ne.s32 %p148, %r6, 0;\n" " add.s64 %rd138, %rd11, 6;\n" " mul.lo.s64 %rd139, %rd138, %rd28;\n" " add.s64 %rd36, %rd139, %rd29;\n" " mul.lo.s64 %rd140, %rd36, %rd12;\n" " add.s64 %rd141, %rd140, %rd23;\n" " shl.b64 %rd142, %rd141, 2;\n" " add.s64 %rd143, %rd4, %rd142;\n" " st.global.f32 [%rd143], %f606;\n" " @%p148 bra $L__BB12_173;\n" "\n" " ld.shared.f32 %f504, [%r102+24];\n" " shl.b64 %rd144, %rd36, 2;\n" " add.s64 %rd145, %rd3, %rd144;\n" " st.global.f32 [%rd145], %f504;\n" " ld.shared.f32 %f505, [%r103+24];\n" " add.s64 %rd146, %rd2, %rd144;\n" " st.global.f32 [%rd146], %f505;\n" "\n" "$L__BB12_173:\n" " setp.lt.s32 %p149, %r104, 8;\n" " @%p149 bra $L__BB12_176;\n" "\n" " setp.ne.s32 %p150, %r6, 0;\n" " add.s64 %rd147, %rd11, 7;\n" " mul.lo.s64 %rd148, %rd147, %rd28;\n" " add.s64 %rd37, %rd148, %rd29;\n" " mul.lo.s64 %rd149, %rd37, %rd12;\n" " add.s64 %rd150, %rd149, %rd23;\n" " shl.b64 %rd151, %rd150, 2;\n" " add.s64 %rd152, %rd4, %rd151;\n" " st.global.f32 [%rd152], %f605;\n" " @%p150 bra $L__BB12_176;\n" "\n" " ld.shared.f32 %f506, [%r102+28];\n" " shl.b64 %rd153, %rd37, 2;\n" " add.s64 %rd154, %rd3, %rd153;\n" " st.global.f32 [%rd154], %f506;\n" " ld.shared.f32 %f507, [%r103+28];\n" " add.s64 %rd155, %rd2, %rd153;\n" " st.global.f32 [%rd155], %f507;\n" "\n" "$L__BB12_176:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode_q4\n" ".visible .entry fc_attn_flash_decode_q4(\n" " .param .u64 fc_attn_flash_decode_q4_param_0,\n" " .param .u64 fc_attn_flash_decode_q4_param_1,\n" " .param .u64 fc_attn_flash_decode_q4_param_2,\n" " .param .u64 fc_attn_flash_decode_q4_param_3,\n" " .param .u64 fc_attn_flash_decode_q4_param_4,\n" " .param .u64 fc_attn_flash_decode_q4_param_5,\n" " .param .u64 fc_attn_flash_decode_q4_param_6,\n" " .param .u64 fc_attn_flash_decode_q4_param_7,\n" " .param .u64 fc_attn_flash_decode_q4_param_8,\n" " .param .u64 fc_attn_flash_decode_q4_param_9,\n" " .param .u32 fc_attn_flash_decode_q4_param_10,\n" " .param .u32 fc_attn_flash_decode_q4_param_11,\n" " .param .u32 fc_attn_flash_decode_q4_param_12,\n" " .param .u32 fc_attn_flash_decode_q4_param_13,\n" " .param .u32 fc_attn_flash_decode_q4_param_14,\n" " .param .u32 fc_attn_flash_decode_q4_param_15,\n" " .param .u32 fc_attn_flash_decode_q4_param_16,\n" " .param .u32 fc_attn_flash_decode_q4_param_17,\n" " .param .u32 fc_attn_flash_decode_q4_param_18,\n" " .param .u32 fc_attn_flash_decode_q4_param_19,\n" " .param .f32 fc_attn_flash_decode_q4_param_20\n" ")\n" "{\n" " .reg .pred %p<195>;\n" " .reg .b16 %rs<11>;\n" " .reg .f32 %f<767>;\n" " .reg .b32 %r<675>;\n" " .reg .b64 %rd<238>;\n" "\n" "\n" " ld.param.u64 %rd60, [fc_attn_flash_decode_q4_param_0];\n" " ld.param.u64 %rd61, [fc_attn_flash_decode_q4_param_1];\n" " ld.param.u64 %rd62, [fc_attn_flash_decode_q4_param_2];\n" " ld.param.u64 %rd63, [fc_attn_flash_decode_q4_param_3];\n" " ld.param.u64 %rd64, [fc_attn_flash_decode_q4_param_4];\n" " ld.param.u64 %rd58, [fc_attn_flash_decode_q4_param_5];\n" " ld.param.u64 %rd59, [fc_attn_flash_decode_q4_param_6];\n" " ld.param.u64 %rd65, [fc_attn_flash_decode_q4_param_7];\n" " ld.param.u64 %rd66, [fc_attn_flash_decode_q4_param_8];\n" " ld.param.u64 %rd67, [fc_attn_flash_decode_q4_param_9];\n" " ld.param.u32 %r120, [fc_attn_flash_decode_q4_param_10];\n" " ld.param.u32 %r121, [fc_attn_flash_decode_q4_param_11];\n" " ld.param.u32 %r122, [fc_attn_flash_decode_q4_param_12];\n" " ld.param.u32 %r123, [fc_attn_flash_decode_q4_param_13];\n" " ld.param.u32 %r125, [fc_attn_flash_decode_q4_param_15];\n" " ld.param.u32 %r126, [fc_attn_flash_decode_q4_param_16];\n" " ld.param.u32 %r127, [fc_attn_flash_decode_q4_param_17];\n" " ld.param.u32 %r129, [fc_attn_flash_decode_q4_param_19];\n" " ld.param.f32 %f231, [fc_attn_flash_decode_q4_param_20];\n" " cvta.to.global.u64 %rd1, %rd60;\n" " cvta.to.global.u64 %rd2, %rd63;\n" " cvta.to.global.u64 %rd3, %rd61;\n" " cvta.to.global.u64 %rd4, %rd64;\n" " cvta.to.global.u64 %rd5, %rd62;\n" " cvta.to.global.u64 %rd6, %rd66;\n" " cvta.to.global.u64 %rd7, %rd65;\n" " cvta.to.global.u64 %rd8, %rd67;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %ctaid.x;\n" " mul.lo.s32 %r655, %r2, %r125;\n" " add.s32 %r130, %r655, %r125;\n" " min.s32 %r4, %r130, %r123;\n" " mul.lo.s32 %r5, %r121, %r120;\n" " add.s32 %r131, %r121, 16;\n" " cvt.s64.s32 %rd9, %r131;\n" " mul.wide.s32 %rd10, %r131, 16;\n" " shl.b32 %r132, %r120, 4;\n" " cvt.s64.s32 %rd68, %r132;\n" " add.s64 %rd11, %rd10, %rd68;\n" " add.s64 %rd12, %rd11, %rd68;\n" " cvt.s64.s32 %rd13, %r120;\n" " add.s64 %rd14, %rd12, %rd13;\n" " mov.u32 %r133, %ctaid.y;\n" " cvt.s64.s32 %rd15, %r133;\n" " mul.wide.s32 %rd16, %r120, %r133;\n" " cvt.s64.s32 %rd17, %r121;\n" " setp.lt.s32 %p2, %r120, 1;\n" " @%p2 bra $L__BB13_7;\n" "\n" " cvt.s64.s32 %rd18, %r1;\n" " and.b32 %r653, %r120, 3;\n" " add.s32 %r135, %r120, -1;\n" " setp.lt.u32 %p3, %r135, 3;\n" " mov.u32 %r651, 0;\n" " @%p3 bra $L__BB13_4;\n" "\n" " sub.s32 %r649, %r120, %r653;\n" " mul.lo.s64 %rd69, %rd15, %rd17;\n" " mul.lo.s64 %rd70, %rd69, %rd13;\n" " add.s64 %rd71, %rd70, %rd18;\n" " shl.b64 %rd72, %rd71, 2;\n" " add.s64 %rd236, %rd1, %rd72;\n" " shl.b64 %rd20, %rd17, 2;\n" " shl.b32 %r137, %r1, 2;\n" " mov.u32 %r138, sh;\n" " add.s32 %r650, %r138, %r137;\n" " shl.b32 %r9, %r121, 2;\n" " mov.u32 %r651, 0;\n" "\n" "$L__BB13_3:\n" " ld.global.f32 %f232, [%rd236];\n" " st.shared.f32 [%r650], %f232;\n" " add.s64 %rd73, %rd236, %rd20;\n" " ld.global.f32 %f233, [%rd73];\n" " add.s32 %r139, %r650, %r9;\n" " st.shared.f32 [%r139], %f233;\n" " add.s64 %rd74, %rd73, %rd20;\n" " ld.global.f32 %f234, [%rd74];\n" " add.s32 %r140, %r139, %r9;\n" " st.shared.f32 [%r140], %f234;\n" " add.s64 %rd75, %rd74, %rd20;\n" " add.s64 %rd236, %rd75, %rd20;\n" " ld.global.f32 %f235, [%rd75];\n" " add.s32 %r141, %r140, %r9;\n" " add.s32 %r650, %r141, %r9;\n" " st.shared.f32 [%r141], %f235;\n" " add.s32 %r651, %r651, 4;\n" " add.s32 %r649, %r649, -4;\n" " setp.ne.s32 %p4, %r649, 0;\n" " @%p4 bra $L__BB13_3;\n" "\n" "$L__BB13_4:\n" " setp.eq.s32 %p5, %r653, 0;\n" " @%p5 bra $L__BB13_7;\n" "\n" " mad.lo.s32 %r142, %r651, %r121, %r1;\n" " shl.b32 %r143, %r142, 2;\n" " mov.u32 %r144, sh;\n" " add.s32 %r652, %r144, %r143;\n" " shl.b32 %r18, %r121, 2;\n" " cvt.s64.s32 %rd76, %r651;\n" " add.s64 %rd77, %rd16, %rd76;\n" " mul.lo.s64 %rd78, %rd77, %rd17;\n" " add.s64 %rd79, %rd78, %rd18;\n" " shl.b64 %rd80, %rd79, 2;\n" " add.s64 %rd237, %rd1, %rd80;\n" " shl.b64 %rd24, %rd17, 2;\n" "\n" "$L__BB13_6:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f236, [%rd237];\n" " st.shared.f32 [%r652], %f236;\n" " add.s32 %r652, %r652, %r18;\n" " add.s64 %rd237, %rd237, %rd24;\n" " add.s32 %r653, %r653, -1;\n" " setp.ne.s32 %p6, %r653, 0;\n" " @%p6 bra $L__BB13_6;\n" "\n" "$L__BB13_7:\n" " setp.ge.s32 %p7, %r1, %r120;\n" " add.s32 %r145, %r5, %r1;\n" " cvt.u32.u64 %r146, %rd12;\n" " add.s32 %r147, %r145, %r146;\n" " shl.b32 %r148, %r147, 2;\n" " mov.u32 %r149, sh;\n" " add.s32 %r23, %r149, %r148;\n" " cvt.u32.u64 %r150, %rd14;\n" " add.s32 %r151, %r145, %r150;\n" " shl.b32 %r152, %r151, 2;\n" " add.s32 %r24, %r149, %r152;\n" " @%p7 bra $L__BB13_9;\n" "\n" " mov.u32 %r153, -8388608;\n" " st.shared.u32 [%r23], %r153;\n" " mov.u32 %r154, 0;\n" " st.shared.u32 [%r24], %r154;\n" "\n" "$L__BB13_9:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd81, %r122;\n" " mul.lo.s64 %rd82, %rd81, %rd15;\n" " shr.s32 %r155, %r121, 31;\n" " shr.u32 %r156, %r155, 27;\n" " add.s32 %r157, %r121, %r156;\n" " shr.s32 %r158, %r157, 5;\n" " mul.lo.s32 %r159, %r158, 18;\n" " cvt.s64.s32 %rd27, %r159;\n" " mul.lo.s64 %rd28, %rd82, %rd27;\n" " mul.lo.s64 %rd83, %rd15, %rd17;\n" " shl.b64 %rd29, %rd83, 8;\n" " shl.b64 %rd30, %rd83, 3;\n" " shr.s32 %r160, %r1, 31;\n" " shr.u32 %r161, %r160, 28;\n" " add.s32 %r162, %r1, %r161;\n" " shr.s32 %r26, %r162, 4;\n" " and.b32 %r163, %r162, -16;\n" " sub.s32 %r27, %r1, %r163;\n" " cvt.s64.s32 %rd31, %r1;\n" " setp.le.s32 %p8, %r4, %r655;\n" " mov.f32 %f704, 0f00000000;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " @%p8 bra $L__BB13_213;\n" "\n" " add.s32 %r165, %r123, -1;\n" " shr.u32 %r167, %r155, 28;\n" " add.s32 %r168, %r121, %r167;\n" " shr.s32 %r169, %r168, 4;\n" " cvt.u32.u64 %r170, %rd9;\n" " sub.s32 %r171, %r165, %r129;\n" " add.s32 %r172, %r171, 1;\n" " mov.u32 %r654, 0;\n" " max.s32 %r28, %r172, 0;\n" " sub.s32 %r29, %r165, %r127;\n" " mov.u32 %r30, %ntid.x;\n" " add.s32 %r173, %r5, %r26;\n" " cvt.u32.u64 %r174, %rd10;\n" " add.s32 %r175, %r173, %r174;\n" " mad.lo.s32 %r31, %r170, %r26, %r5;\n" " shl.b32 %r176, %r1, 4;\n" " add.s32 %r177, %r176, %r174;\n" " add.s32 %r32, %r5, %r177;\n" " add.s64 %rd32, %rd30, %rd31;\n" " add.s32 %r178, %r150, %r120;\n" " add.s32 %r179, %r5, %r178;\n" " add.s64 %rd33, %rd29, %rd31;\n" " shr.u32 %r181, %r160, 27;\n" " add.s32 %r182, %r1, %r181;\n" " shr.s32 %r183, %r182, 5;\n" " mul.lo.s32 %r184, %r183, 18;\n" " cvt.s64.s32 %rd34, %r184;\n" " and.b32 %r185, %r1, 31;\n" " setp.lt.u32 %p9, %r185, 16;\n" " add.s32 %r186, %r185, -16;\n" " selp.b32 %r187, %r185, %r186, %p9;\n" " selp.b32 %r33, 0, 4, %p9;\n" " add.s32 %r188, %r187, 2;\n" " cvt.s64.s32 %rd84, %r188;\n" " add.s64 %rd35, %rd84, %rd34;\n" " add.s32 %r190, %r145, %r178;\n" " shl.b32 %r191, %r190, 2;\n" " add.s32 %r34, %r149, %r191;\n" " cvt.u64.u32 %rd85, %r1;\n" " add.s64 %rd36, %rd11, %rd85;\n" " add.s32 %r193, %r2, 1;\n" " mul.lo.s32 %r194, %r193, %r125;\n" " min.s32 %r195, %r194, %r123;\n" " not.b32 %r196, %r195;\n" " add.s32 %r35, %r655, %r196;\n" " bfe.u32 %r37, %r168, 4, 1;\n" " sub.s32 %r38, %r169, %r37;\n" " shl.b32 %r39, %r121, 1;\n" " mul.lo.s32 %r40, %r121, 3;\n" " shl.b32 %r41, %r121, 2;\n" " mul.lo.s32 %r42, %r121, 5;\n" " mul.lo.s32 %r43, %r121, 6;\n" " mul.lo.s32 %r44, %r121, 7;\n" " shl.b32 %r197, %r175, 2;\n" " add.s32 %r45, %r149, %r197;\n" " shl.b32 %r198, %r179, 2;\n" " add.s32 %r46, %r149, %r198;\n" " mad.lo.s32 %r199, %r120, %r170, %r1;\n" " add.s32 %r200, %r199, %r174;\n" " shl.b32 %r201, %r200, 2;\n" " add.s32 %r202, %r149, %r201;\n" " add.s32 %r47, %r202, -1024;\n" " cvta.to.global.u64 %rd37, %rd59;\n" " cvta.to.global.u64 %rd38, %rd58;\n" " mov.f32 %f704, 0f00000000;\n" " setp.lt.s32 %p11, %r129, 1;\n" " setp.gt.s32 %p19, %r127, 0;\n" " setp.ge.s32 %p34, %r1, %r121;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" "\n" "$L__BB13_11:\n" " ld.param.u32 %r557, [fc_attn_flash_decode_q4_param_18];\n" " ld.param.u32 %r556, [fc_attn_flash_decode_q4_param_13];\n" " shl.b32 %r203, %r654, 4;\n" " add.s32 %r204, %r35, %r203;\n" " max.s32 %r205, %r204, -17;\n" " not.b32 %r206, %r205;\n" " max.s32 %r52, %r206, 1;\n" " sub.s32 %r53, %r4, %r655;\n" " min.s32 %r54, %r53, 16;\n" " setp.lt.s32 %p10, %r655, %r557;\n" " or.pred %p12, %p11, %p10;\n" " add.s32 %r207, %r54, %r655;\n" " setp.gt.s32 %p13, %r207, %r28;\n" " setp.lt.s32 %p14, %r655, %r556;\n" " and.pred %p15, %p14, %p13;\n" " or.pred %p16, %p12, %p15;\n" " not.pred %p17, %p16;\n" " @%p17 bra $L__BB13_212;\n" "\n" " setp.lt.s32 %p18, %r53, 1;\n" " @%p18 bra $L__BB13_36;\n" "\n" " @%p19 bra $L__BB13_23;\n" " bra.uni $L__BB13_14;\n" "\n" "$L__BB13_23:\n" " mov.u32 %r659, 0;\n" "\n" "$L__BB13_24:\n" " cvt.s64.s32 %rd98, %r659;\n" " mul.lo.s64 %rd43, %rd98, %rd9;\n" " add.s32 %r64, %r659, %r655;\n" " setp.lt.s32 %p27, %r64, %r126;\n" " @%p27 bra $L__BB13_32;\n" " bra.uni $L__BB13_25;\n" "\n" "$L__BB13_32:\n" " @%p34 bra $L__BB13_35;\n" "\n" " cvt.u32.u64 %r270, %rd43;\n" " add.s32 %r71, %r5, %r270;\n" " cvt.s64.s32 %rd112, %r64;\n" " mul.lo.s64 %rd113, %rd112, %rd17;\n" " add.s64 %rd46, %rd113, %rd30;\n" " mov.u32 %r662, %r1;\n" "\n" "$L__BB13_34:\n" " cvt.s64.s32 %rd114, %r662;\n" " add.s64 %rd115, %rd46, %rd114;\n" " shl.b64 %rd116, %rd115, 1;\n" " add.s64 %rd117, %rd2, %rd116;\n" " ld.global.u16 %rs5, [%rd117];\n" " // begin inline asm\n" " { cvt.f32.f16 %f261, %rs5;}\n" "\n" " // end inline asm\n" " add.s32 %r271, %r71, %r662;\n" " shl.b32 %r272, %r271, 2;\n" " add.s32 %r274, %r149, %r272;\n" " st.shared.f32 [%r274], %f261;\n" " add.s32 %r662, %r662, %r30;\n" " setp.lt.s32 %p35, %r662, %r121;\n" " @%p35 bra $L__BB13_34;\n" " bra.uni $L__BB13_35;\n" "\n" "$L__BB13_25:\n" " setp.gt.s32 %p28, %r64, %r29;\n" " @%p28 bra $L__BB13_29;\n" " bra.uni $L__BB13_26;\n" "\n" "$L__BB13_29:\n" " @%p34 bra $L__BB13_35;\n" "\n" " cvt.u32.u64 %r260, %rd43;\n" " add.s32 %r68, %r5, %r260;\n" " shr.s32 %r261, %r64, 31;\n" " shr.u32 %r262, %r261, 24;\n" " add.s32 %r263, %r64, %r262;\n" " and.b32 %r264, %r263, -256;\n" " sub.s32 %r265, %r64, %r264;\n" " cvt.s64.s32 %rd106, %r265;\n" " mul.lo.s64 %rd107, %rd106, %rd17;\n" " add.s64 %rd45, %rd107, %rd29;\n" " mov.u32 %r661, %r1;\n" "\n" "$L__BB13_31:\n" " cvt.s64.s32 %rd108, %r661;\n" " add.s64 %rd109, %rd45, %rd108;\n" " shl.b64 %rd110, %rd109, 1;\n" " add.s64 %rd111, %rd38, %rd110;\n" " ld.global.u16 %rs4, [%rd111];\n" " // begin inline asm\n" " { cvt.f32.f16 %f260, %rs4;}\n" "\n" " // end inline asm\n" " add.s32 %r266, %r68, %r661;\n" " shl.b32 %r267, %r266, 2;\n" " add.s32 %r269, %r149, %r267;\n" " st.shared.f32 [%r269], %f260;\n" " add.s32 %r661, %r661, %r30;\n" " setp.lt.s32 %p33, %r661, %r121;\n" " @%p33 bra $L__BB13_31;\n" " bra.uni $L__BB13_35;\n" "\n" "$L__BB13_26:\n" " @%p34 bra $L__BB13_35;\n" "\n" " cvt.u32.u64 %r238, %rd43;\n" " add.s32 %r65, %r5, %r238;\n" " cvt.s64.s32 %rd99, %r64;\n" " mul.lo.s64 %rd100, %rd99, %rd27;\n" " add.s64 %rd44, %rd100, %rd28;\n" " mov.u32 %r660, %r1;\n" "\n" "$L__BB13_28:\n" " shr.s32 %r239, %r660, 31;\n" " shr.u32 %r240, %r239, 27;\n" " add.s32 %r241, %r660, %r240;\n" " shr.s32 %r242, %r241, 5;\n" " mul.lo.s32 %r243, %r242, 18;\n" " cvt.s64.s32 %rd101, %r243;\n" " add.s64 %rd102, %rd44, %rd101;\n" " add.s64 %rd103, %rd3, %rd102;\n" " ld.global.u8 %r244, [%rd103];\n" " ld.global.u8 %r245, [%rd103+1];\n" " prmt.b32 %r246, %r245, %r244, 30212;\n" " cvt.u16.u32 %rs3, %r246;\n" " // begin inline asm\n" " { cvt.f32.f16 %f257, %rs3;}\n" "\n" " // end inline asm\n" " and.b32 %r247, %r660, 31;\n" " setp.lt.u32 %p30, %r247, 16;\n" " add.s32 %r248, %r247, -16;\n" " selp.b32 %r249, %r247, %r248, %p30;\n" " selp.b32 %r250, 0, 4, %p30;\n" " add.s32 %r251, %r249, 2;\n" " cvt.s64.s32 %rd104, %r251;\n" " add.s64 %rd105, %rd103, %rd104;\n" " ld.global.u8 %r252, [%rd105];\n" " shr.u32 %r253, %r252, %r250;\n" " and.b32 %r254, %r253, 15;\n" " add.s32 %r255, %r254, -8;\n" " cvt.rn.f32.s32 %f258, %r255;\n" " mul.f32 %f259, %f257, %f258;\n" " add.s32 %r256, %r65, %r660;\n" " shl.b32 %r257, %r256, 2;\n" " add.s32 %r259, %r149, %r257;\n" " st.shared.f32 [%r259], %f259;\n" " add.s32 %r660, %r660, %r30;\n" " setp.lt.s32 %p31, %r660, %r121;\n" " @%p31 bra $L__BB13_28;\n" "\n" "$L__BB13_35:\n" " add.s32 %r659, %r659, 1;\n" " setp.lt.s32 %p36, %r659, %r54;\n" " @%p36 bra $L__BB13_24;\n" " bra.uni $L__BB13_36;\n" "\n" "$L__BB13_14:\n" " mov.u32 %r656, 0;\n" "\n" "$L__BB13_15:\n" " cvt.s64.s32 %rd86, %r656;\n" " mul.lo.s64 %rd39, %rd86, %rd9;\n" " add.s32 %r209, %r656, %r655;\n" " setp.lt.s32 %p20, %r209, %r126;\n" " cvt.s64.s32 %rd40, %r209;\n" " @%p20 bra $L__BB13_19;\n" " bra.uni $L__BB13_16;\n" "\n" "$L__BB13_19:\n" " @%p34 bra $L__BB13_22;\n" "\n" " cvt.u32.u64 %r232, %rd39;\n" " add.s32 %r59, %r5, %r232;\n" " mul.lo.s64 %rd93, %rd40, %rd17;\n" " add.s64 %rd42, %rd93, %rd30;\n" " mov.u32 %r658, %r1;\n" "\n" "$L__BB13_21:\n" " cvt.s64.s32 %rd94, %r658;\n" " add.s64 %rd95, %rd42, %rd94;\n" " shl.b64 %rd96, %rd95, 1;\n" " add.s64 %rd97, %rd2, %rd96;\n" " ld.global.u16 %rs2, [%rd97];\n" " // begin inline asm\n" " { cvt.f32.f16 %f256, %rs2;}\n" "\n" " // end inline asm\n" " add.s32 %r233, %r59, %r658;\n" " shl.b32 %r234, %r233, 2;\n" " add.s32 %r236, %r149, %r234;\n" " st.shared.f32 [%r236], %f256;\n" " add.s32 %r658, %r658, %r30;\n" " setp.lt.s32 %p25, %r658, %r121;\n" " @%p25 bra $L__BB13_21;\n" " bra.uni $L__BB13_22;\n" "\n" "$L__BB13_16:\n" " @%p34 bra $L__BB13_22;\n" "\n" " cvt.u32.u64 %r210, %rd39;\n" " add.s32 %r56, %r5, %r210;\n" " mul.lo.s64 %rd87, %rd40, %rd27;\n" " add.s64 %rd41, %rd87, %rd28;\n" " mov.u32 %r657, %r1;\n" "\n" "$L__BB13_18:\n" " shr.s32 %r211, %r657, 31;\n" " shr.u32 %r212, %r211, 27;\n" " add.s32 %r213, %r657, %r212;\n" " shr.s32 %r214, %r213, 5;\n" " mul.lo.s32 %r215, %r214, 18;\n" " cvt.s64.s32 %rd88, %r215;\n" " add.s64 %rd89, %rd41, %rd88;\n" " add.s64 %rd90, %rd3, %rd89;\n" " ld.global.u8 %r216, [%rd90];\n" " ld.global.u8 %r217, [%rd90+1];\n" " prmt.b32 %r218, %r217, %r216, 30212;\n" " cvt.u16.u32 %rs1, %r218;\n" " // begin inline asm\n" " { cvt.f32.f16 %f253, %rs1;}\n" "\n" " // end inline asm\n" " and.b32 %r219, %r657, 31;\n" " setp.lt.u32 %p22, %r219, 16;\n" " add.s32 %r220, %r219, -16;\n" " selp.b32 %r221, %r219, %r220, %p22;\n" " selp.b32 %r222, 0, 4, %p22;\n" " add.s32 %r223, %r221, 2;\n" " cvt.s64.s32 %rd91, %r223;\n" " add.s64 %rd92, %rd90, %rd91;\n" " ld.global.u8 %r224, [%rd92];\n" " shr.u32 %r225, %r224, %r222;\n" " and.b32 %r226, %r225, 15;\n" " add.s32 %r227, %r226, -8;\n" " cvt.rn.f32.s32 %f254, %r227;\n" " mul.f32 %f255, %f253, %f254;\n" " add.s32 %r228, %r56, %r657;\n" " shl.b32 %r229, %r228, 2;\n" " add.s32 %r231, %r149, %r229;\n" " st.shared.f32 [%r231], %f255;\n" " add.s32 %r657, %r657, %r30;\n" " setp.lt.s32 %p23, %r657, %r121;\n" " @%p23 bra $L__BB13_18;\n" "\n" "$L__BB13_22:\n" " add.s32 %r656, %r656, 1;\n" " setp.lt.s32 %p26, %r656, %r54;\n" " @%p26 bra $L__BB13_15;\n" "\n" "$L__BB13_36:\n" " setp.lt.s32 %p37, %r121, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p38, %r26, %r54;\n" " or.pred %p39, %p37, %p38;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p39 bra $L__BB13_89;\n" "\n" " and.b32 %r558, %r121, -16;\n" " setp.eq.s32 %p40, %r558, 16;\n" " mov.u32 %r665, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p40 bra $L__BB13_72;\n" "\n" " mov.u32 %r665, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " mov.u32 %r664, %r38;\n" "\n" "$L__BB13_39:\n" " shl.b32 %r277, %r665, 4;\n" " add.s32 %r77, %r277, %r27;\n" " add.s32 %r278, %r31, %r77;\n" " shl.b32 %r279, %r278, 2;\n" " add.s32 %r78, %r149, %r279;\n" " ld.shared.f32 %f17, [%r78];\n" " @%p2 bra $L__BB13_41;\n" "\n" " shl.b32 %r638, %r665, 4;\n" " add.s32 %r637, %r638, %r27;\n" " shl.b32 %r636, %r637, 2;\n" " add.s32 %r635, %r149, %r636;\n" " ld.shared.f32 %f286, [%r635];\n" " fma.rn.f32 %f634, %f17, %f286, %f634;\n" "\n" "$L__BB13_41:\n" " setp.lt.s32 %p42, %r120, 2;\n" " @%p42 bra $L__BB13_43;\n" "\n" " shl.b32 %r643, %r665, 4;\n" " add.s32 %r642, %r643, %r27;\n" " add.s32 %r641, %r642, %r121;\n" " shl.b32 %r640, %r641, 2;\n" " add.s32 %r639, %r149, %r640;\n" " ld.shared.f32 %f287, [%r639];\n" " fma.rn.f32 %f635, %f17, %f287, %f635;\n" "\n" "$L__BB13_43:\n" " setp.lt.s32 %p43, %r120, 3;\n" " @%p43 bra $L__BB13_45;\n" "\n" " shl.b32 %r584, %r665, 4;\n" " add.s32 %r583, %r584, %r27;\n" " add.s32 %r582, %r39, %r583;\n" " shl.b32 %r581, %r582, 2;\n" " add.s32 %r580, %r149, %r581;\n" " ld.shared.f32 %f288, [%r580];\n" " fma.rn.f32 %f636, %f17, %f288, %f636;\n" "\n" "$L__BB13_45:\n" " setp.lt.s32 %p44, %r120, 4;\n" " @%p44 bra $L__BB13_47;\n" "\n" " shl.b32 %r594, %r665, 4;\n" " add.s32 %r593, %r594, %r27;\n" " add.s32 %r592, %r40, %r593;\n" " shl.b32 %r591, %r592, 2;\n" " add.s32 %r590, %r149, %r591;\n" " ld.shared.f32 %f289, [%r590];\n" " fma.rn.f32 %f637, %f17, %f289, %f637;\n" "\n" "$L__BB13_47:\n" " setp.lt.s32 %p45, %r120, 5;\n" " @%p45 bra $L__BB13_49;\n" "\n" " shl.b32 %r604, %r665, 4;\n" " add.s32 %r603, %r604, %r27;\n" " add.s32 %r602, %r41, %r603;\n" " shl.b32 %r601, %r602, 2;\n" " add.s32 %r600, %r149, %r601;\n" " ld.shared.f32 %f290, [%r600];\n" " fma.rn.f32 %f638, %f17, %f290, %f638;\n" "\n" "$L__BB13_49:\n" " setp.lt.s32 %p46, %r120, 6;\n" " @%p46 bra $L__BB13_51;\n" "\n" " shl.b32 %r614, %r665, 4;\n" " add.s32 %r613, %r614, %r27;\n" " add.s32 %r612, %r42, %r613;\n" " shl.b32 %r611, %r612, 2;\n" " add.s32 %r610, %r149, %r611;\n" " ld.shared.f32 %f291, [%r610];\n" " fma.rn.f32 %f639, %f17, %f291, %f639;\n" "\n" "$L__BB13_51:\n" " setp.lt.s32 %p47, %r120, 7;\n" " @%p47 bra $L__BB13_53;\n" "\n" " shl.b32 %r624, %r665, 4;\n" " add.s32 %r623, %r624, %r27;\n" " add.s32 %r622, %r43, %r623;\n" " shl.b32 %r621, %r622, 2;\n" " add.s32 %r620, %r149, %r621;\n" " ld.shared.f32 %f292, [%r620];\n" " fma.rn.f32 %f640, %f17, %f292, %f640;\n" "\n" "$L__BB13_53:\n" " setp.lt.s32 %p48, %r120, 8;\n" " @%p48 bra $L__BB13_55;\n" "\n" " shl.b32 %r634, %r665, 4;\n" " add.s32 %r633, %r634, %r27;\n" " add.s32 %r632, %r44, %r633;\n" " shl.b32 %r631, %r632, 2;\n" " add.s32 %r630, %r149, %r631;\n" " ld.shared.f32 %f293, [%r630];\n" " fma.rn.f32 %f641, %f17, %f293, %f641;\n" "\n" "$L__BB13_55:\n" " shl.b32 %r565, %r665, 4;\n" " add.s32 %r564, %r565, %r27;\n" " add.s32 %r563, %r31, %r564;\n" " shl.b32 %r562, %r563, 2;\n" " add.s32 %r561, %r149, %r562;\n" " ld.shared.f32 %f34, [%r561+64];\n" " @%p2 bra $L__BB13_57;\n" "\n" " shl.b32 %r579, %r665, 4;\n" " add.s32 %r578, %r579, %r27;\n" " shl.b32 %r577, %r578, 2;\n" " add.s32 %r576, %r149, %r577;\n" " ld.shared.f32 %f294, [%r576+64];\n" " fma.rn.f32 %f634, %f34, %f294, %f634;\n" "\n" "$L__BB13_57:\n" " setp.lt.s32 %p188, %r120, 2;\n" " @%p188 bra $L__BB13_59;\n" "\n" " shl.b32 %r575, %r665, 4;\n" " add.s32 %r574, %r575, %r27;\n" " add.s32 %r573, %r574, %r121;\n" " shl.b32 %r572, %r573, 2;\n" " add.s32 %r571, %r149, %r572;\n" " ld.shared.f32 %f295, [%r571+64];\n" " fma.rn.f32 %f635, %f34, %f295, %f635;\n" "\n" "$L__BB13_59:\n" " setp.lt.s32 %p189, %r120, 3;\n" " @%p189 bra $L__BB13_61;\n" "\n" " shl.b32 %r570, %r665, 4;\n" " add.s32 %r569, %r570, %r27;\n" " add.s32 %r568, %r39, %r569;\n" " shl.b32 %r567, %r568, 2;\n" " add.s32 %r566, %r149, %r567;\n" " ld.shared.f32 %f296, [%r566+64];\n" " fma.rn.f32 %f636, %f34, %f296, %f636;\n" "\n" "$L__BB13_61:\n" " setp.lt.s32 %p190, %r120, 4;\n" " @%p190 bra $L__BB13_63;\n" "\n" " shl.b32 %r589, %r665, 4;\n" " add.s32 %r588, %r589, %r27;\n" " add.s32 %r587, %r40, %r588;\n" " shl.b32 %r586, %r587, 2;\n" " add.s32 %r585, %r149, %r586;\n" " ld.shared.f32 %f297, [%r585+64];\n" " fma.rn.f32 %f637, %f34, %f297, %f637;\n" "\n" "$L__BB13_63:\n" " setp.lt.s32 %p191, %r120, 5;\n" " @%p191 bra $L__BB13_65;\n" "\n" " shl.b32 %r599, %r665, 4;\n" " add.s32 %r598, %r599, %r27;\n" " add.s32 %r597, %r41, %r598;\n" " shl.b32 %r596, %r597, 2;\n" " add.s32 %r595, %r149, %r596;\n" " ld.shared.f32 %f298, [%r595+64];\n" " fma.rn.f32 %f638, %f34, %f298, %f638;\n" "\n" "$L__BB13_65:\n" " setp.lt.s32 %p192, %r120, 6;\n" " @%p192 bra $L__BB13_67;\n" "\n" " shl.b32 %r609, %r665, 4;\n" " add.s32 %r608, %r609, %r27;\n" " add.s32 %r607, %r42, %r608;\n" " shl.b32 %r606, %r607, 2;\n" " add.s32 %r605, %r149, %r606;\n" " ld.shared.f32 %f299, [%r605+64];\n" " fma.rn.f32 %f639, %f34, %f299, %f639;\n" "\n" "$L__BB13_67:\n" " setp.lt.s32 %p193, %r120, 7;\n" " @%p193 bra $L__BB13_69;\n" "\n" " shl.b32 %r619, %r665, 4;\n" " add.s32 %r618, %r619, %r27;\n" " add.s32 %r617, %r43, %r618;\n" " shl.b32 %r616, %r617, 2;\n" " add.s32 %r615, %r149, %r616;\n" " ld.shared.f32 %f300, [%r615+64];\n" " fma.rn.f32 %f640, %f34, %f300, %f640;\n" "\n" "$L__BB13_69:\n" " setp.lt.s32 %p194, %r120, 8;\n" " @%p194 bra $L__BB13_71;\n" "\n" " shl.b32 %r629, %r665, 4;\n" " add.s32 %r628, %r629, %r27;\n" " add.s32 %r627, %r44, %r628;\n" " shl.b32 %r626, %r627, 2;\n" " add.s32 %r625, %r149, %r626;\n" " ld.shared.f32 %f301, [%r625+64];\n" " fma.rn.f32 %f641, %f34, %f301, %f641;\n" "\n" "$L__BB13_71:\n" " add.s32 %r665, %r665, 2;\n" " add.s32 %r664, %r664, -2;\n" " setp.ne.s32 %p57, %r664, 0;\n" " @%p57 bra $L__BB13_39;\n" "\n" "$L__BB13_72:\n" " setp.eq.s32 %p58, %r37, 0;\n" " @%p58 bra $L__BB13_89;\n" "\n" " shl.b32 %r303, %r665, 4;\n" " add.s32 %r90, %r303, %r27;\n" " add.s32 %r304, %r31, %r90;\n" " shl.b32 %r305, %r304, 2;\n" " add.s32 %r307, %r149, %r305;\n" " ld.shared.f32 %f59, [%r307];\n" " @%p2 bra $L__BB13_75;\n" "\n" " shl.b32 %r308, %r90, 2;\n" " add.s32 %r310, %r149, %r308;\n" " ld.shared.f32 %f302, [%r310];\n" " fma.rn.f32 %f634, %f59, %f302, %f634;\n" "\n" "$L__BB13_75:\n" " setp.lt.s32 %p60, %r120, 2;\n" " @%p60 bra $L__BB13_77;\n" "\n" " add.s32 %r311, %r90, %r121;\n" " shl.b32 %r312, %r311, 2;\n" " add.s32 %r314, %r149, %r312;\n" " ld.shared.f32 %f303, [%r314];\n" " fma.rn.f32 %f635, %f59, %f303, %f635;\n" "\n" "$L__BB13_77:\n" " setp.lt.s32 %p61, %r120, 3;\n" " @%p61 bra $L__BB13_79;\n" "\n" " add.s32 %r315, %r39, %r90;\n" " shl.b32 %r316, %r315, 2;\n" " add.s32 %r318, %r149, %r316;\n" " ld.shared.f32 %f304, [%r318];\n" " fma.rn.f32 %f636, %f59, %f304, %f636;\n" "\n" "$L__BB13_79:\n" " setp.lt.s32 %p62, %r120, 4;\n" " @%p62 bra $L__BB13_81;\n" "\n" " add.s32 %r319, %r40, %r90;\n" " shl.b32 %r320, %r319, 2;\n" " add.s32 %r322, %r149, %r320;\n" " ld.shared.f32 %f305, [%r322];\n" " fma.rn.f32 %f637, %f59, %f305, %f637;\n" "\n" "$L__BB13_81:\n" " setp.lt.s32 %p63, %r120, 5;\n" " @%p63 bra $L__BB13_83;\n" "\n" " add.s32 %r323, %r41, %r90;\n" " shl.b32 %r324, %r323, 2;\n" " add.s32 %r326, %r149, %r324;\n" " ld.shared.f32 %f306, [%r326];\n" " fma.rn.f32 %f638, %f59, %f306, %f638;\n" "\n" "$L__BB13_83:\n" " setp.lt.s32 %p64, %r120, 6;\n" " @%p64 bra $L__BB13_85;\n" "\n" " add.s32 %r327, %r42, %r90;\n" " shl.b32 %r328, %r327, 2;\n" " add.s32 %r330, %r149, %r328;\n" " ld.shared.f32 %f307, [%r330];\n" " fma.rn.f32 %f639, %f59, %f307, %f639;\n" "\n" "$L__BB13_85:\n" " setp.lt.s32 %p65, %r120, 7;\n" " @%p65 bra $L__BB13_87;\n" "\n" " add.s32 %r331, %r43, %r90;\n" " shl.b32 %r332, %r331, 2;\n" " add.s32 %r334, %r149, %r332;\n" " ld.shared.f32 %f308, [%r334];\n" " fma.rn.f32 %f640, %f59, %f308, %f640;\n" "\n" "$L__BB13_87:\n" " setp.lt.s32 %p66, %r120, 8;\n" " @%p66 bra $L__BB13_89;\n" "\n" " add.s32 %r335, %r44, %r90;\n" " shl.b32 %r336, %r335, 2;\n" " add.s32 %r338, %r149, %r336;\n" " ld.shared.f32 %f309, [%r338];\n" " fma.rn.f32 %f641, %f59, %f309, %f641;\n" "\n" "$L__BB13_89:\n" " setp.eq.s32 %p67, %r27, 0;\n" " setp.lt.s32 %p68, %r26, %r54;\n" " and.pred %p1, %p67, %p68;\n" " @%p2 bra $L__BB13_92;\n" "\n" " mov.b32 %r339, %f634;\n" " mov.u32 %r340, 2;\n" " mov.u32 %r341, 4127;\n" " mov.u32 %r342, 8;\n" " mov.u32 %r343, -1;\n" " shfl.sync.down.b32 %r344|%p70, %r339, %r342, %r341, %r343;\n" " mov.b32 %f310, %r344;\n" " add.f32 %f311, %f634, %f310;\n" " mov.b32 %r345, %f311;\n" " mov.u32 %r346, 4;\n" " shfl.sync.down.b32 %r347|%p71, %r345, %r346, %r341, %r343;\n" " mov.b32 %f312, %r347;\n" " add.f32 %f313, %f311, %f312;\n" " mov.b32 %r348, %f313;\n" " shfl.sync.down.b32 %r349|%p72, %r348, %r340, %r341, %r343;\n" " mov.b32 %f314, %r349;\n" " add.f32 %f315, %f313, %f314;\n" " mov.b32 %r350, %f315;\n" " mov.u32 %r351, 1;\n" " shfl.sync.down.b32 %r352|%p73, %r350, %r351, %r341, %r343;\n" " mov.b32 %f316, %r352;\n" " add.f32 %f83, %f315, %f316;\n" " not.pred %p74, %p1;\n" " @%p74 bra $L__BB13_92;\n" "\n" " mul.f32 %f317, %f83, %f231;\n" " st.shared.f32 [%r45], %f317;\n" "\n" "$L__BB13_92:\n" " setp.lt.s32 %p75, %r120, 2;\n" " @%p75 bra $L__BB13_95;\n" "\n" " mov.b32 %r353, %f635;\n" " mov.u32 %r354, 2;\n" " mov.u32 %r355, 4127;\n" " mov.u32 %r356, 8;\n" " mov.u32 %r357, -1;\n" " shfl.sync.down.b32 %r358|%p76, %r353, %r356, %r355, %r357;\n" " mov.b32 %f318, %r358;\n" " add.f32 %f319, %f635, %f318;\n" " mov.b32 %r359, %f319;\n" " mov.u32 %r360, 4;\n" " shfl.sync.down.b32 %r361|%p77, %r359, %r360, %r355, %r357;\n" " mov.b32 %f320, %r361;\n" " add.f32 %f321, %f319, %f320;\n" " mov.b32 %r362, %f321;\n" " shfl.sync.down.b32 %r363|%p78, %r362, %r354, %r355, %r357;\n" " mov.b32 %f322, %r363;\n" " add.f32 %f323, %f321, %f322;\n" " mov.b32 %r364, %f323;\n" " mov.u32 %r365, 1;\n" " shfl.sync.down.b32 %r366|%p79, %r364, %r365, %r355, %r357;\n" " mov.b32 %f324, %r366;\n" " add.f32 %f84, %f323, %f324;\n" " not.pred %p80, %p1;\n" " @%p80 bra $L__BB13_95;\n" "\n" " mul.f32 %f325, %f84, %f231;\n" " st.shared.f32 [%r45+64], %f325;\n" "\n" "$L__BB13_95:\n" " setp.lt.s32 %p81, %r120, 3;\n" " @%p81 bra $L__BB13_98;\n" "\n" " mov.b32 %r367, %f636;\n" " mov.u32 %r368, 2;\n" " mov.u32 %r369, 4127;\n" " mov.u32 %r370, 8;\n" " mov.u32 %r371, -1;\n" " shfl.sync.down.b32 %r372|%p82, %r367, %r370, %r369, %r371;\n" " mov.b32 %f326, %r372;\n" " add.f32 %f327, %f636, %f326;\n" " mov.b32 %r373, %f327;\n" " mov.u32 %r374, 4;\n" " shfl.sync.down.b32 %r375|%p83, %r373, %r374, %r369, %r371;\n" " mov.b32 %f328, %r375;\n" " add.f32 %f329, %f327, %f328;\n" " mov.b32 %r376, %f329;\n" " shfl.sync.down.b32 %r377|%p84, %r376, %r368, %r369, %r371;\n" " mov.b32 %f330, %r377;\n" " add.f32 %f331, %f329, %f330;\n" " mov.b32 %r378, %f331;\n" " mov.u32 %r379, 1;\n" " shfl.sync.down.b32 %r380|%p85, %r378, %r379, %r369, %r371;\n" " mov.b32 %f332, %r380;\n" " add.f32 %f85, %f331, %f332;\n" " not.pred %p86, %p1;\n" " @%p86 bra $L__BB13_98;\n" "\n" " mul.f32 %f333, %f85, %f231;\n" " st.shared.f32 [%r45+128], %f333;\n" "\n" "$L__BB13_98:\n" " setp.lt.s32 %p87, %r120, 4;\n" " @%p87 bra $L__BB13_101;\n" "\n" " mov.b32 %r381, %f637;\n" " mov.u32 %r382, 2;\n" " mov.u32 %r383, 4127;\n" " mov.u32 %r384, 8;\n" " mov.u32 %r385, -1;\n" " shfl.sync.down.b32 %r386|%p88, %r381, %r384, %r383, %r385;\n" " mov.b32 %f334, %r386;\n" " add.f32 %f335, %f637, %f334;\n" " mov.b32 %r387, %f335;\n" " mov.u32 %r388, 4;\n" " shfl.sync.down.b32 %r389|%p89, %r387, %r388, %r383, %r385;\n" " mov.b32 %f336, %r389;\n" " add.f32 %f337, %f335, %f336;\n" " mov.b32 %r390, %f337;\n" " shfl.sync.down.b32 %r391|%p90, %r390, %r382, %r383, %r385;\n" " mov.b32 %f338, %r391;\n" " add.f32 %f339, %f337, %f338;\n" " mov.b32 %r392, %f339;\n" " mov.u32 %r393, 1;\n" " shfl.sync.down.b32 %r394|%p91, %r392, %r393, %r383, %r385;\n" " mov.b32 %f340, %r394;\n" " add.f32 %f86, %f339, %f340;\n" " not.pred %p92, %p1;\n" " @%p92 bra $L__BB13_101;\n" "\n" " mul.f32 %f341, %f86, %f231;\n" " st.shared.f32 [%r45+192], %f341;\n" "\n" "$L__BB13_101:\n" " setp.lt.s32 %p93, %r120, 5;\n" " @%p93 bra $L__BB13_104;\n" "\n" " mov.b32 %r395, %f638;\n" " mov.u32 %r396, 2;\n" " mov.u32 %r397, 4127;\n" " mov.u32 %r398, 8;\n" " mov.u32 %r399, -1;\n" " shfl.sync.down.b32 %r400|%p94, %r395, %r398, %r397, %r399;\n" " mov.b32 %f342, %r400;\n" " add.f32 %f343, %f638, %f342;\n" " mov.b32 %r401, %f343;\n" " mov.u32 %r402, 4;\n" " shfl.sync.down.b32 %r403|%p95, %r401, %r402, %r397, %r399;\n" " mov.b32 %f344, %r403;\n" " add.f32 %f345, %f343, %f344;\n" " mov.b32 %r404, %f345;\n" " shfl.sync.down.b32 %r405|%p96, %r404, %r396, %r397, %r399;\n" " mov.b32 %f346, %r405;\n" " add.f32 %f347, %f345, %f346;\n" " mov.b32 %r406, %f347;\n" " mov.u32 %r407, 1;\n" " shfl.sync.down.b32 %r408|%p97, %r406, %r407, %r397, %r399;\n" " mov.b32 %f348, %r408;\n" " add.f32 %f87, %f347, %f348;\n" " not.pred %p98, %p1;\n" " @%p98 bra $L__BB13_104;\n" "\n" " mul.f32 %f349, %f87, %f231;\n" " st.shared.f32 [%r45+256], %f349;\n" "\n" "$L__BB13_104:\n" " setp.lt.s32 %p99, %r120, 6;\n" " @%p99 bra $L__BB13_107;\n" "\n" " mov.b32 %r409, %f639;\n" " mov.u32 %r410, 2;\n" " mov.u32 %r411, 4127;\n" " mov.u32 %r412, 8;\n" " mov.u32 %r413, -1;\n" " shfl.sync.down.b32 %r414|%p100, %r409, %r412, %r411, %r413;\n" " mov.b32 %f350, %r414;\n" " add.f32 %f351, %f639, %f350;\n" " mov.b32 %r415, %f351;\n" " mov.u32 %r416, 4;\n" " shfl.sync.down.b32 %r417|%p101, %r415, %r416, %r411, %r413;\n" " mov.b32 %f352, %r417;\n" " add.f32 %f353, %f351, %f352;\n" " mov.b32 %r418, %f353;\n" " shfl.sync.down.b32 %r419|%p102, %r418, %r410, %r411, %r413;\n" " mov.b32 %f354, %r419;\n" " add.f32 %f355, %f353, %f354;\n" " mov.b32 %r420, %f355;\n" " mov.u32 %r421, 1;\n" " shfl.sync.down.b32 %r422|%p103, %r420, %r421, %r411, %r413;\n" " mov.b32 %f356, %r422;\n" " add.f32 %f88, %f355, %f356;\n" " not.pred %p104, %p1;\n" " @%p104 bra $L__BB13_107;\n" "\n" " mul.f32 %f357, %f88, %f231;\n" " st.shared.f32 [%r45+320], %f357;\n" "\n" "$L__BB13_107:\n" " setp.lt.s32 %p105, %r120, 7;\n" " @%p105 bra $L__BB13_110;\n" "\n" " mov.b32 %r423, %f640;\n" " mov.u32 %r424, 2;\n" " mov.u32 %r425, 4127;\n" " mov.u32 %r426, 8;\n" " mov.u32 %r427, -1;\n" " shfl.sync.down.b32 %r428|%p106, %r423, %r426, %r425, %r427;\n" " mov.b32 %f358, %r428;\n" " add.f32 %f359, %f640, %f358;\n" " mov.b32 %r429, %f359;\n" " mov.u32 %r430, 4;\n" " shfl.sync.down.b32 %r431|%p107, %r429, %r430, %r425, %r427;\n" " mov.b32 %f360, %r431;\n" " add.f32 %f361, %f359, %f360;\n" " mov.b32 %r432, %f361;\n" " shfl.sync.down.b32 %r433|%p108, %r432, %r424, %r425, %r427;\n" " mov.b32 %f362, %r433;\n" " add.f32 %f363, %f361, %f362;\n" " mov.b32 %r434, %f363;\n" " mov.u32 %r435, 1;\n" " shfl.sync.down.b32 %r436|%p109, %r434, %r435, %r425, %r427;\n" " mov.b32 %f364, %r436;\n" " add.f32 %f89, %f363, %f364;\n" " not.pred %p110, %p1;\n" " @%p110 bra $L__BB13_110;\n" "\n" " mul.f32 %f365, %f89, %f231;\n" " st.shared.f32 [%r45+384], %f365;\n" "\n" "$L__BB13_110:\n" " setp.lt.s32 %p111, %r120, 8;\n" " @%p111 bra $L__BB13_113;\n" "\n" " mov.b32 %r437, %f641;\n" " mov.u32 %r438, 2;\n" " mov.u32 %r439, 4127;\n" " mov.u32 %r440, 8;\n" " mov.u32 %r441, -1;\n" " shfl.sync.down.b32 %r442|%p112, %r437, %r440, %r439, %r441;\n" " mov.b32 %f366, %r442;\n" " add.f32 %f367, %f641, %f366;\n" " mov.b32 %r443, %f367;\n" " mov.u32 %r444, 4;\n" " shfl.sync.down.b32 %r445|%p113, %r443, %r444, %r439, %r441;\n" " mov.b32 %f368, %r445;\n" " add.f32 %f369, %f367, %f368;\n" " mov.b32 %r446, %f369;\n" " shfl.sync.down.b32 %r447|%p114, %r446, %r438, %r439, %r441;\n" " mov.b32 %f370, %r447;\n" " add.f32 %f371, %f369, %f370;\n" " mov.b32 %r448, %f371;\n" " mov.u32 %r449, 1;\n" " shfl.sync.down.b32 %r450|%p115, %r448, %r449, %r439, %r441;\n" " mov.b32 %f372, %r450;\n" " add.f32 %f90, %f371, %f372;\n" " not.pred %p116, %p1;\n" " @%p116 bra $L__BB13_113;\n" "\n" " mul.f32 %f373, %f90, %f231;\n" " st.shared.f32 [%r45+448], %f373;\n" "\n" "$L__BB13_113:\n" " bar.sync 0;\n" " @%p7 bra $L__BB13_131;\n" "\n" " ld.shared.f32 %f91, [%r23];\n" " mov.f32 %f676, %f91;\n" " @%p18 bra $L__BB13_122;\n" "\n" " and.b32 %r91, %r52, 3;\n" " add.s32 %r452, %r52, -1;\n" " setp.lt.u32 %p119, %r452, 3;\n" " mov.u32 %r668, 0;\n" " mov.f32 %f676, %f91;\n" " @%p119 bra $L__BB13_118;\n" "\n" " sub.s32 %r667, %r52, %r91;\n" " mov.u32 %r668, 0;\n" " mov.f32 %f676, %f91;\n" "\n" "$L__BB13_117:\n" " add.s32 %r454, %r32, %r668;\n" " shl.b32 %r455, %r454, 2;\n" " add.s32 %r457, %r149, %r455;\n" " ld.shared.f32 %f375, [%r457];\n" " max.f32 %f376, %f676, %f375;\n" " ld.shared.f32 %f377, [%r457+4];\n" " max.f32 %f378, %f376, %f377;\n" " ld.shared.f32 %f379, [%r457+8];\n" " max.f32 %f380, %f378, %f379;\n" " ld.shared.f32 %f381, [%r457+12];\n" " max.f32 %f676, %f380, %f381;\n" " add.s32 %r668, %r668, 4;\n" " add.s32 %r667, %r667, -4;\n" " setp.ne.s32 %p120, %r667, 0;\n" " @%p120 bra $L__BB13_117;\n" "\n" "$L__BB13_118:\n" " setp.eq.s32 %p121, %r91, 0;\n" " @%p121 bra $L__BB13_122;\n" "\n" " add.s32 %r458, %r32, %r668;\n" " shl.b32 %r459, %r458, 2;\n" " add.s32 %r98, %r149, %r459;\n" " ld.shared.f32 %f382, [%r98];\n" " max.f32 %f676, %f676, %f382;\n" " setp.eq.s32 %p122, %r91, 1;\n" " @%p122 bra $L__BB13_122;\n" "\n" " ld.shared.f32 %f383, [%r98+4];\n" " max.f32 %f676, %f676, %f383;\n" " setp.eq.s32 %p123, %r91, 2;\n" " @%p123 bra $L__BB13_122;\n" "\n" " ld.shared.f32 %f384, [%r98+8];\n" " max.f32 %f676, %f676, %f384;\n" "\n" "$L__BB13_122:\n" " mov.f32 %f680, 0f00000000;\n" " @%p18 bra $L__BB13_130;\n" "\n" " and.b32 %r99, %r52, 3;\n" " add.s32 %r462, %r52, -1;\n" " setp.lt.u32 %p125, %r462, 3;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r672, 0;\n" " @%p125 bra $L__BB13_126;\n" "\n" " sub.s32 %r670, %r52, %r99;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r672, 0;\n" " mov.u32 %r671, %r47;\n" "\n" "$L__BB13_125:\n" " shl.b32 %r560, %r120, 2;\n" " add.s32 %r559, %r560, 1024;\n" " add.s32 %r464, %r32, %r672;\n" " shl.b32 %r465, %r464, 2;\n" " add.s32 %r467, %r149, %r465;\n" " ld.shared.f32 %f389, [%r467];\n" " sub.f32 %f390, %f389, %f676;\n" " mov.f32 %f391, 0f3F000000;\n" " mov.f32 %f392, 0f3BBB989D;\n" " fma.rn.f32 %f393, %f390, %f392, %f391;\n" " mov.f32 %f394, 0f3FB8AA3B;\n" " mov.f32 %f395, 0f437C0000;\n" " cvt.sat.f32.f32 %f396, %f393;\n" " mov.f32 %f397, 0f4B400001;\n" " fma.rm.f32 %f398, %f396, %f395, %f397;\n" " add.f32 %f399, %f398, 0fCB40007F;\n" " neg.f32 %f400, %f399;\n" " fma.rn.f32 %f401, %f390, %f394, %f400;\n" " mov.f32 %f402, 0f32A57060;\n" " fma.rn.f32 %f403, %f390, %f402, %f401;\n" " mov.b32 %r468, %f398;\n" " shl.b32 %r469, %r468, 23;\n" " mov.b32 %f404, %r469;\n" " ex2.approx.ftz.f32 %f405, %f403;\n" " mul.f32 %f406, %f405, %f404;\n" " st.shared.f32 [%r671+1024], %f406;\n" " add.f32 %f407, %f680, %f406;\n" " ld.shared.f32 %f408, [%r467+4];\n" " sub.f32 %f409, %f408, %f676;\n" " fma.rn.f32 %f410, %f409, %f392, %f391;\n" " cvt.sat.f32.f32 %f411, %f410;\n" " fma.rm.f32 %f412, %f411, %f395, %f397;\n" " add.f32 %f413, %f412, 0fCB40007F;\n" " neg.f32 %f414, %f413;\n" " fma.rn.f32 %f415, %f409, %f394, %f414;\n" " fma.rn.f32 %f416, %f409, %f402, %f415;\n" " mov.b32 %r470, %f412;\n" " shl.b32 %r471, %r470, 23;\n" " mov.b32 %f417, %r471;\n" " ex2.approx.ftz.f32 %f418, %f416;\n" " mul.f32 %f419, %f418, %f417;\n" " add.s32 %r472, %r671, %r560;\n" " st.shared.f32 [%r472+1024], %f419;\n" " add.f32 %f420, %f407, %f419;\n" " ld.shared.f32 %f421, [%r467+8];\n" " sub.f32 %f422, %f421, %f676;\n" " fma.rn.f32 %f423, %f422, %f392, %f391;\n" " cvt.sat.f32.f32 %f424, %f423;\n" " fma.rm.f32 %f425, %f424, %f395, %f397;\n" " add.f32 %f426, %f425, 0fCB40007F;\n" " neg.f32 %f427, %f426;\n" " fma.rn.f32 %f428, %f422, %f394, %f427;\n" " fma.rn.f32 %f429, %f422, %f402, %f428;\n" " mov.b32 %r473, %f425;\n" " shl.b32 %r474, %r473, 23;\n" " mov.b32 %f430, %r474;\n" " ex2.approx.ftz.f32 %f431, %f429;\n" " mul.f32 %f432, %f431, %f430;\n" " add.s32 %r475, %r472, %r560;\n" " add.s32 %r476, %r472, %r559;\n" " st.shared.f32 [%r476], %f432;\n" " add.f32 %f433, %f420, %f432;\n" " ld.shared.f32 %f434, [%r467+12];\n" " sub.f32 %f435, %f434, %f676;\n" " fma.rn.f32 %f436, %f435, %f392, %f391;\n" " cvt.sat.f32.f32 %f437, %f436;\n" " fma.rm.f32 %f438, %f437, %f395, %f397;\n" " add.f32 %f439, %f438, 0fCB40007F;\n" " neg.f32 %f440, %f439;\n" " fma.rn.f32 %f441, %f435, %f394, %f440;\n" " fma.rn.f32 %f442, %f435, %f402, %f441;\n" " mov.b32 %r477, %f438;\n" " shl.b32 %r478, %r477, 23;\n" " mov.b32 %f443, %r478;\n" " ex2.approx.ftz.f32 %f444, %f442;\n" " mul.f32 %f445, %f444, %f443;\n" " add.s32 %r479, %r475, %r560;\n" " add.s32 %r480, %r475, %r559;\n" " add.s32 %r671, %r479, %r560;\n" " st.shared.f32 [%r480], %f445;\n" " add.f32 %f680, %f433, %f445;\n" " add.s32 %r672, %r672, 4;\n" " add.s32 %r670, %r670, -4;\n" " setp.ne.s32 %p126, %r670, 0;\n" " @%p126 bra $L__BB13_125;\n" "\n" "$L__BB13_126:\n" " setp.eq.s32 %p127, %r99, 0;\n" " @%p127 bra $L__BB13_130;\n" "\n" " add.s32 %r481, %r32, %r672;\n" " shl.b32 %r482, %r481, 2;\n" " add.s32 %r108, %r149, %r482;\n" " ld.shared.f32 %f446, [%r108];\n" " sub.f32 %f447, %f446, %f676;\n" " mov.f32 %f448, 0f3F000000;\n" " mov.f32 %f449, 0f3BBB989D;\n" " fma.rn.f32 %f450, %f447, %f449, %f448;\n" " mov.f32 %f451, 0f3FB8AA3B;\n" " mov.f32 %f452, 0f437C0000;\n" " cvt.sat.f32.f32 %f453, %f450;\n" " mov.f32 %f454, 0f4B400001;\n" " fma.rm.f32 %f455, %f453, %f452, %f454;\n" " add.f32 %f456, %f455, 0fCB40007F;\n" " neg.f32 %f457, %f456;\n" " fma.rn.f32 %f458, %f447, %f451, %f457;\n" " mov.f32 %f459, 0f32A57060;\n" " fma.rn.f32 %f460, %f447, %f459, %f458;\n" " mov.b32 %r484, %f455;\n" " shl.b32 %r485, %r484, 23;\n" " mov.b32 %f461, %r485;\n" " ex2.approx.ftz.f32 %f462, %f460;\n" " mul.f32 %f463, %f462, %f461;\n" " cvt.u64.u32 %rd118, %r672;\n" " mul.lo.s64 %rd119, %rd118, %rd13;\n" " add.s64 %rd120, %rd36, %rd119;\n" " cvt.u32.u64 %r486, %rd120;\n" " add.s32 %r487, %r5, %r486;\n" " shl.b32 %r488, %r487, 2;\n" " add.s32 %r489, %r149, %r488;\n" " st.shared.f32 [%r489], %f463;\n" " add.f32 %f680, %f680, %f463;\n" " setp.eq.s32 %p128, %r99, 1;\n" " @%p128 bra $L__BB13_130;\n" "\n" " mov.f32 %f611, 0f32A57060;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f609, 0f4B400001;\n" " mov.f32 %f608, 0f437C0000;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f606, 0f3BBB989D;\n" " add.s32 %r546, %r32, %r672;\n" " shl.b32 %r545, %r546, 2;\n" " add.s32 %r544, %r149, %r545;\n" " ld.shared.f32 %f464, [%r544+4];\n" " sub.f32 %f465, %f464, %f676;\n" " fma.rn.f32 %f468, %f465, %f606, %f607;\n" " cvt.sat.f32.f32 %f471, %f468;\n" " fma.rm.f32 %f473, %f471, %f608, %f609;\n" " add.f32 %f474, %f473, 0fCB40007F;\n" " neg.f32 %f475, %f474;\n" " fma.rn.f32 %f476, %f465, %f610, %f475;\n" " fma.rn.f32 %f478, %f465, %f611, %f476;\n" " mov.b32 %r490, %f473;\n" " shl.b32 %r491, %r490, 23;\n" " mov.b32 %f479, %r491;\n" " ex2.approx.ftz.f32 %f480, %f478;\n" " mul.f32 %f481, %f480, %f479;\n" " add.s32 %r492, %r672, 1;\n" " cvt.u64.u32 %rd121, %r492;\n" " mul.lo.s64 %rd122, %rd121, %rd13;\n" " add.s64 %rd123, %rd36, %rd122;\n" " cvt.u32.u64 %r493, %rd123;\n" " add.s32 %r494, %r5, %r493;\n" " shl.b32 %r495, %r494, 2;\n" " add.s32 %r497, %r149, %r495;\n" " st.shared.f32 [%r497], %f481;\n" " add.f32 %f680, %f680, %f481;\n" " setp.eq.s32 %p129, %r99, 2;\n" " @%p129 bra $L__BB13_130;\n" "\n" " mov.f32 %f617, 0f32A57060;\n" " mov.f32 %f616, 0f3FB8AA3B;\n" " mov.f32 %f615, 0f4B400001;\n" " mov.f32 %f614, 0f437C0000;\n" " mov.f32 %f613, 0f3F000000;\n" " mov.f32 %f612, 0f3BBB989D;\n" " add.s32 %r549, %r32, %r672;\n" " shl.b32 %r548, %r549, 2;\n" " add.s32 %r547, %r149, %r548;\n" " ld.shared.f32 %f482, [%r547+8];\n" " sub.f32 %f483, %f482, %f676;\n" " fma.rn.f32 %f486, %f483, %f612, %f613;\n" " cvt.sat.f32.f32 %f489, %f486;\n" " fma.rm.f32 %f491, %f489, %f614, %f615;\n" " add.f32 %f492, %f491, 0fCB40007F;\n" " neg.f32 %f493, %f492;\n" " fma.rn.f32 %f494, %f483, %f616, %f493;\n" " fma.rn.f32 %f496, %f483, %f617, %f494;\n" " mov.b32 %r498, %f491;\n" " shl.b32 %r499, %r498, 23;\n" " mov.b32 %f497, %r499;\n" " ex2.approx.ftz.f32 %f498, %f496;\n" " mul.f32 %f499, %f498, %f497;\n" " add.s32 %r500, %r672, 2;\n" " cvt.u64.u32 %rd124, %r500;\n" " mul.lo.s64 %rd125, %rd124, %rd13;\n" " add.s64 %rd126, %rd36, %rd125;\n" " cvt.u32.u64 %r501, %rd126;\n" " add.s32 %r502, %r5, %r501;\n" " shl.b32 %r503, %r502, 2;\n" " add.s32 %r505, %r149, %r503;\n" " st.shared.f32 [%r505], %f499;\n" " add.f32 %f680, %f680, %f499;\n" "\n" "$L__BB13_130:\n" " sub.f32 %f500, %f91, %f676;\n" " mov.f32 %f501, 0f3F000000;\n" " mov.f32 %f502, 0f3BBB989D;\n" " fma.rn.f32 %f503, %f500, %f502, %f501;\n" " mov.f32 %f504, 0f3FB8AA3B;\n" " mov.f32 %f505, 0f437C0000;\n" " cvt.sat.f32.f32 %f506, %f503;\n" " mov.f32 %f507, 0f4B400001;\n" " fma.rm.f32 %f508, %f506, %f505, %f507;\n" " add.f32 %f509, %f508, 0fCB40007F;\n" " neg.f32 %f510, %f509;\n" " fma.rn.f32 %f511, %f500, %f504, %f510;\n" " mov.f32 %f512, 0f32A57060;\n" " fma.rn.f32 %f513, %f500, %f512, %f511;\n" " ex2.approx.ftz.f32 %f514, %f513;\n" " mov.b32 %r506, %f508;\n" " shl.b32 %r507, %r506, 23;\n" " mov.b32 %f515, %r507;\n" " mul.f32 %f516, %f514, %f515;\n" " ld.shared.f32 %f517, [%r24];\n" " fma.rn.f32 %f518, %f516, %f517, %f680;\n" " st.shared.f32 [%r24], %f518;\n" " st.shared.f32 [%r23], %f676;\n" " st.shared.f32 [%r34], %f516;\n" "\n" "$L__BB13_131:\n" " bar.sync 0;\n" " @%p18 bra $L__BB13_211;\n" "\n" " @%p19 bra $L__BB13_171;\n" " bra.uni $L__BB13_133;\n" "\n" "$L__BB13_171:\n" " mov.u32 %r674, 0;\n" "\n" "$L__BB13_172:\n" " add.s32 %r113, %r674, %r655;\n" " setp.lt.s32 %p151, %r113, %r126;\n" " @%p151 bra $L__BB13_176;\n" " bra.uni $L__BB13_173;\n" "\n" "$L__BB13_176:\n" " add.s32 %r647, %r674, %r655;\n" " cvt.s64.s32 %rd152, %r647;\n" " mul.lo.s64 %rd153, %rd152, %rd17;\n" " add.s64 %rd154, %rd32, %rd153;\n" " shl.b64 %rd155, %rd154, 1;\n" " add.s64 %rd156, %rd4, %rd155;\n" " ld.global.u16 %rs10, [%rd156];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs10;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB13_177;\n" "\n" "$L__BB13_173:\n" " add.s32 %r644, %r674, %r655;\n" " setp.gt.s32 %p152, %r644, %r29;\n" " @%p152 bra $L__BB13_175;\n" " bra.uni $L__BB13_174;\n" "\n" "$L__BB13_175:\n" " add.s32 %r646, %r674, %r655;\n" " shr.s32 %r529, %r646, 31;\n" " shr.u32 %r530, %r529, 24;\n" " add.s32 %r531, %r646, %r530;\n" " and.b32 %r532, %r531, -256;\n" " sub.s32 %r533, %r646, %r532;\n" " cvt.s64.s32 %rd147, %r533;\n" " mul.lo.s64 %rd148, %rd147, %rd17;\n" " add.s64 %rd149, %rd33, %rd148;\n" " shl.b64 %rd150, %rd149, 1;\n" " add.s64 %rd151, %rd37, %rd150;\n" " ld.global.u16 %rs9, [%rd151];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs9;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB13_177;\n" "\n" "$L__BB13_174:\n" " add.s32 %r645, %r674, %r655;\n" " cvt.s64.s32 %rd140, %r645;\n" " mul.lo.s64 %rd141, %rd140, %rd27;\n" " add.s64 %rd142, %rd141, %rd28;\n" " add.s64 %rd143, %rd142, %rd34;\n" " add.s64 %rd144, %rd5, %rd143;\n" " ld.global.u8 %r522, [%rd144];\n" " ld.global.u8 %r523, [%rd144+1];\n" " prmt.b32 %r524, %r523, %r522, 30212;\n" " cvt.u16.u32 %rs8, %r524;\n" " // begin inline asm\n" " { cvt.f32.f16 %f554, %rs8;}\n" "\n" " // end inline asm\n" " add.s64 %rd145, %rd35, %rd142;\n" " add.s64 %rd146, %rd5, %rd145;\n" " ld.global.u8 %r525, [%rd146];\n" " shr.u32 %r526, %r525, %r33;\n" " and.b32 %r527, %r526, 15;\n" " add.s32 %r528, %r527, -8;\n" " cvt.rn.f32.s32 %f555, %r528;\n" " mul.f32 %f720, %f554, %f555;\n" "\n" "$L__BB13_177:\n" " cvt.u64.u32 %rd157, %r674;\n" " mul.lo.s64 %rd158, %rd157, %rd13;\n" " add.s64 %rd159, %rd158, %rd11;\n" " cvt.u32.u64 %r534, %rd159;\n" " add.s32 %r535, %r5, %r534;\n" " shl.b32 %r536, %r535, 2;\n" " add.s32 %r114, %r149, %r536;\n" " setp.eq.s32 %p153, %r674, 0;\n" " @%p153 bra $L__BB13_194;\n" "\n" " @%p2 bra $L__BB13_180;\n" "\n" " ld.shared.f32 %f558, [%r114];\n" " fma.rn.f32 %f711, %f720, %f558, %f711;\n" "\n" "$L__BB13_180:\n" " @%p75 bra $L__BB13_182;\n" "\n" " ld.shared.f32 %f559, [%r114+4];\n" " fma.rn.f32 %f710, %f720, %f559, %f710;\n" "\n" "$L__BB13_182:\n" " @%p81 bra $L__BB13_184;\n" "\n" " ld.shared.f32 %f560, [%r114+8];\n" " fma.rn.f32 %f709, %f720, %f560, %f709;\n" "\n" "$L__BB13_184:\n" " @%p87 bra $L__BB13_186;\n" "\n" " ld.shared.f32 %f561, [%r114+12];\n" " fma.rn.f32 %f708, %f720, %f561, %f708;\n" "\n" "$L__BB13_186:\n" " @%p93 bra $L__BB13_188;\n" "\n" " ld.shared.f32 %f562, [%r114+16];\n" " fma.rn.f32 %f707, %f720, %f562, %f707;\n" "\n" "$L__BB13_188:\n" " @%p99 bra $L__BB13_190;\n" "\n" " ld.shared.f32 %f563, [%r114+20];\n" " fma.rn.f32 %f706, %f720, %f563, %f706;\n" "\n" "$L__BB13_190:\n" " @%p105 bra $L__BB13_192;\n" "\n" " ld.shared.f32 %f564, [%r114+24];\n" " fma.rn.f32 %f705, %f720, %f564, %f705;\n" "\n" "$L__BB13_192:\n" " @%p111 bra $L__BB13_210;\n" "\n" " ld.shared.f32 %f565, [%r114+28];\n" " fma.rn.f32 %f704, %f720, %f565, %f704;\n" " bra.uni $L__BB13_210;\n" "\n" "$L__BB13_194:\n" " @%p2 bra $L__BB13_196;\n" "\n" " ld.shared.f32 %f566, [%r46];\n" " ld.shared.f32 %f567, [%r114];\n" " mul.f32 %f568, %f720, %f567;\n" " fma.rn.f32 %f711, %f566, %f711, %f568;\n" "\n" "$L__BB13_196:\n" " @%p75 bra $L__BB13_198;\n" "\n" " ld.shared.f32 %f569, [%r46+4];\n" " ld.shared.f32 %f570, [%r114+4];\n" " mul.f32 %f571, %f720, %f570;\n" " fma.rn.f32 %f710, %f569, %f710, %f571;\n" "\n" "$L__BB13_198:\n" " @%p81 bra $L__BB13_200;\n" "\n" " ld.shared.f32 %f572, [%r46+8];\n" " ld.shared.f32 %f573, [%r114+8];\n" " mul.f32 %f574, %f720, %f573;\n" " fma.rn.f32 %f709, %f572, %f709, %f574;\n" "\n" "$L__BB13_200:\n" " @%p87 bra $L__BB13_202;\n" "\n" " ld.shared.f32 %f575, [%r46+12];\n" " ld.shared.f32 %f576, [%r114+12];\n" " mul.f32 %f577, %f720, %f576;\n" " fma.rn.f32 %f708, %f575, %f708, %f577;\n" "\n" "$L__BB13_202:\n" " @%p93 bra $L__BB13_204;\n" "\n" " ld.shared.f32 %f578, [%r46+16];\n" " ld.shared.f32 %f579, [%r114+16];\n" " mul.f32 %f580, %f720, %f579;\n" " fma.rn.f32 %f707, %f578, %f707, %f580;\n" "\n" "$L__BB13_204:\n" " @%p99 bra $L__BB13_206;\n" "\n" " ld.shared.f32 %f581, [%r46+20];\n" " ld.shared.f32 %f582, [%r114+20];\n" " mul.f32 %f583, %f720, %f582;\n" " fma.rn.f32 %f706, %f581, %f706, %f583;\n" "\n" "$L__BB13_206:\n" " @%p105 bra $L__BB13_208;\n" "\n" " ld.shared.f32 %f584, [%r46+24];\n" " ld.shared.f32 %f585, [%r114+24];\n" " mul.f32 %f586, %f720, %f585;\n" " fma.rn.f32 %f705, %f584, %f705, %f586;\n" "\n" "$L__BB13_208:\n" " @%p111 bra $L__BB13_210;\n" "\n" " ld.shared.f32 %f587, [%r46+28];\n" " ld.shared.f32 %f588, [%r114+28];\n" " mul.f32 %f589, %f720, %f588;\n" " fma.rn.f32 %f704, %f587, %f704, %f589;\n" "\n" "$L__BB13_210:\n" " add.s32 %r674, %r674, 1;\n" " setp.lt.s32 %p170, %r674, %r54;\n" " @%p170 bra $L__BB13_172;\n" " bra.uni $L__BB13_211;\n" "\n" "$L__BB13_133:\n" " mov.u32 %r673, 0;\n" "\n" "$L__BB13_134:\n" " add.s32 %r509, %r673, %r655;\n" " setp.lt.s32 %p132, %r509, %r126;\n" " cvt.s64.s32 %rd47, %r509;\n" " @%p132 bra $L__BB13_136;\n" " bra.uni $L__BB13_135;\n" "\n" "$L__BB13_136:\n" " mul.lo.s64 %rd133, %rd47, %rd17;\n" " add.s64 %rd134, %rd32, %rd133;\n" " shl.b64 %rd135, %rd134, 1;\n" " add.s64 %rd136, %rd4, %rd135;\n" " ld.global.u16 %rs7, [%rd136];\n" " // begin inline asm\n" " { cvt.f32.f16 %f689, %rs7;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB13_137;\n" "\n" "$L__BB13_135:\n" " mul.lo.s64 %rd127, %rd47, %rd27;\n" " add.s64 %rd128, %rd127, %rd28;\n" " add.s64 %rd129, %rd128, %rd34;\n" " add.s64 %rd130, %rd5, %rd129;\n" " ld.global.u8 %r510, [%rd130];\n" " ld.global.u8 %r511, [%rd130+1];\n" " prmt.b32 %r512, %r511, %r510, 30212;\n" " cvt.u16.u32 %rs6, %r512;\n" " // begin inline asm\n" " { cvt.f32.f16 %f519, %rs6;}\n" "\n" " // end inline asm\n" " add.s64 %rd131, %rd35, %rd128;\n" " add.s64 %rd132, %rd5, %rd131;\n" " ld.global.u8 %r513, [%rd132];\n" " shr.u32 %r514, %r513, %r33;\n" " and.b32 %r515, %r514, 15;\n" " add.s32 %r516, %r515, -8;\n" " cvt.rn.f32.s32 %f520, %r516;\n" " mul.f32 %f689, %f519, %f520;\n" "\n" "$L__BB13_137:\n" " cvt.u64.u32 %rd137, %r673;\n" " mul.lo.s64 %rd138, %rd137, %rd13;\n" " add.s64 %rd139, %rd138, %rd11;\n" " cvt.u32.u64 %r517, %rd139;\n" " add.s32 %r518, %r5, %r517;\n" " shl.b32 %r519, %r518, 2;\n" " add.s32 %r110, %r149, %r519;\n" " setp.eq.s32 %p133, %r673, 0;\n" " @%p133 bra $L__BB13_154;\n" "\n" " @%p2 bra $L__BB13_140;\n" "\n" " ld.shared.f32 %f522, [%r110];\n" " fma.rn.f32 %f711, %f689, %f522, %f711;\n" "\n" "$L__BB13_140:\n" " @%p75 bra $L__BB13_142;\n" "\n" " ld.shared.f32 %f523, [%r110+4];\n" " fma.rn.f32 %f710, %f689, %f523, %f710;\n" "\n" "$L__BB13_142:\n" " @%p81 bra $L__BB13_144;\n" "\n" " ld.shared.f32 %f524, [%r110+8];\n" " fma.rn.f32 %f709, %f689, %f524, %f709;\n" "\n" "$L__BB13_144:\n" " @%p87 bra $L__BB13_146;\n" "\n" " ld.shared.f32 %f525, [%r110+12];\n" " fma.rn.f32 %f708, %f689, %f525, %f708;\n" "\n" "$L__BB13_146:\n" " @%p93 bra $L__BB13_148;\n" "\n" " ld.shared.f32 %f526, [%r110+16];\n" " fma.rn.f32 %f707, %f689, %f526, %f707;\n" "\n" "$L__BB13_148:\n" " @%p99 bra $L__BB13_150;\n" "\n" " ld.shared.f32 %f527, [%r110+20];\n" " fma.rn.f32 %f706, %f689, %f527, %f706;\n" "\n" "$L__BB13_150:\n" " @%p105 bra $L__BB13_152;\n" "\n" " ld.shared.f32 %f528, [%r110+24];\n" " fma.rn.f32 %f705, %f689, %f528, %f705;\n" "\n" "$L__BB13_152:\n" " @%p111 bra $L__BB13_170;\n" "\n" " ld.shared.f32 %f529, [%r110+28];\n" " fma.rn.f32 %f704, %f689, %f529, %f704;\n" " bra.uni $L__BB13_170;\n" "\n" "$L__BB13_154:\n" " @%p2 bra $L__BB13_156;\n" "\n" " ld.shared.f32 %f530, [%r46];\n" " ld.shared.f32 %f531, [%r110];\n" " mul.f32 %f532, %f689, %f531;\n" " fma.rn.f32 %f711, %f530, %f711, %f532;\n" "\n" "$L__BB13_156:\n" " @%p75 bra $L__BB13_158;\n" "\n" " ld.shared.f32 %f533, [%r46+4];\n" " ld.shared.f32 %f534, [%r110+4];\n" " mul.f32 %f535, %f689, %f534;\n" " fma.rn.f32 %f710, %f533, %f710, %f535;\n" "\n" "$L__BB13_158:\n" " @%p81 bra $L__BB13_160;\n" "\n" " ld.shared.f32 %f536, [%r46+8];\n" " ld.shared.f32 %f537, [%r110+8];\n" " mul.f32 %f538, %f689, %f537;\n" " fma.rn.f32 %f709, %f536, %f709, %f538;\n" "\n" "$L__BB13_160:\n" " @%p87 bra $L__BB13_162;\n" "\n" " ld.shared.f32 %f539, [%r46+12];\n" " ld.shared.f32 %f540, [%r110+12];\n" " mul.f32 %f541, %f689, %f540;\n" " fma.rn.f32 %f708, %f539, %f708, %f541;\n" "\n" "$L__BB13_162:\n" " @%p93 bra $L__BB13_164;\n" "\n" " ld.shared.f32 %f542, [%r46+16];\n" " ld.shared.f32 %f543, [%r110+16];\n" " mul.f32 %f544, %f689, %f543;\n" " fma.rn.f32 %f707, %f542, %f707, %f544;\n" "\n" "$L__BB13_164:\n" " @%p99 bra $L__BB13_166;\n" "\n" " ld.shared.f32 %f545, [%r46+20];\n" " ld.shared.f32 %f546, [%r110+20];\n" " mul.f32 %f547, %f689, %f546;\n" " fma.rn.f32 %f706, %f545, %f706, %f547;\n" "\n" "$L__BB13_166:\n" " @%p105 bra $L__BB13_168;\n" "\n" " ld.shared.f32 %f548, [%r46+24];\n" " ld.shared.f32 %f549, [%r110+24];\n" " mul.f32 %f550, %f689, %f549;\n" " fma.rn.f32 %f705, %f548, %f705, %f550;\n" "\n" "$L__BB13_168:\n" " @%p111 bra $L__BB13_170;\n" "\n" " ld.shared.f32 %f551, [%r46+28];\n" " ld.shared.f32 %f552, [%r110+28];\n" " mul.f32 %f553, %f689, %f552;\n" " fma.rn.f32 %f704, %f551, %f704, %f553;\n" "\n" "$L__BB13_170:\n" " add.s32 %r673, %r673, 1;\n" " setp.lt.s32 %p150, %r673, %r54;\n" " @%p150 bra $L__BB13_134;\n" "\n" "$L__BB13_211:\n" " bar.sync 0;\n" "\n" "$L__BB13_212:\n" " add.s32 %r655, %r655, 16;\n" " setp.gt.s32 %p171, %r4, %r655;\n" " add.s32 %r654, %r654, 1;\n" " @%p171 bra $L__BB13_11;\n" "\n" "$L__BB13_213:\n" " add.s32 %r555, %r121, 16;\n" " mul.wide.s32 %rd235, %r555, 16;\n" " shl.b32 %r554, %r120, 4;\n" " cvt.s64.s32 %rd234, %r554;\n" " add.s64 %rd233, %rd235, %rd234;\n" " add.s64 %rd232, %rd233, %rd234;\n" " cvt.u32.u64 %r553, %rd232;\n" " ld.param.u32 %r552, [fc_attn_flash_decode_q4_param_14];\n" " mov.u32 %r551, %ctaid.x;\n" " add.s64 %rd231, %rd232, %rd13;\n" " cvt.u32.u64 %r550, %rd231;\n" " cvt.s64.s32 %rd48, %r552;\n" " cvt.s64.s32 %rd49, %r551;\n" " add.s32 %r539, %r5, %r553;\n" " shl.b32 %r540, %r539, 2;\n" " add.s32 %r118, %r149, %r540;\n" " add.s32 %r542, %r5, %r550;\n" " shl.b32 %r543, %r542, 2;\n" " add.s32 %r119, %r149, %r543;\n" " @%p2 bra $L__BB13_216;\n" "\n" " setp.ne.s32 %p173, %r1, 0;\n" " mul.lo.s64 %rd160, %rd16, %rd48;\n" " add.s64 %rd50, %rd160, %rd49;\n" " mul.lo.s64 %rd161, %rd50, %rd17;\n" " add.s64 %rd162, %rd161, %rd31;\n" " shl.b64 %rd163, %rd162, 2;\n" " add.s64 %rd164, %rd8, %rd163;\n" " st.global.f32 [%rd164], %f711;\n" " @%p173 bra $L__BB13_216;\n" "\n" " ld.shared.f32 %f590, [%r118];\n" " shl.b64 %rd165, %rd50, 2;\n" " add.s64 %rd166, %rd7, %rd165;\n" " st.global.f32 [%rd166], %f590;\n" " ld.shared.f32 %f591, [%r119];\n" " add.s64 %rd167, %rd6, %rd165;\n" " st.global.f32 [%rd167], %f591;\n" "\n" "$L__BB13_216:\n" " setp.lt.s32 %p174, %r120, 2;\n" " @%p174 bra $L__BB13_219;\n" "\n" " setp.ne.s32 %p175, %r1, 0;\n" " add.s64 %rd168, %rd16, 1;\n" " mul.lo.s64 %rd169, %rd168, %rd48;\n" " add.s64 %rd51, %rd169, %rd49;\n" " mul.lo.s64 %rd170, %rd51, %rd17;\n" " add.s64 %rd171, %rd170, %rd31;\n" " shl.b64 %rd172, %rd171, 2;\n" " add.s64 %rd173, %rd8, %rd172;\n" " st.global.f32 [%rd173], %f710;\n" " @%p175 bra $L__BB13_219;\n" "\n" " ld.shared.f32 %f592, [%r118+4];\n" " shl.b64 %rd174, %rd51, 2;\n" " add.s64 %rd175, %rd7, %rd174;\n" " st.global.f32 [%rd175], %f592;\n" " ld.shared.f32 %f593, [%r119+4];\n" " add.s64 %rd176, %rd6, %rd174;\n" " st.global.f32 [%rd176], %f593;\n" "\n" "$L__BB13_219:\n" " setp.lt.s32 %p176, %r120, 3;\n" " @%p176 bra $L__BB13_222;\n" "\n" " setp.ne.s32 %p177, %r1, 0;\n" " add.s64 %rd177, %rd16, 2;\n" " mul.lo.s64 %rd178, %rd177, %rd48;\n" " add.s64 %rd52, %rd178, %rd49;\n" " mul.lo.s64 %rd179, %rd52, %rd17;\n" " add.s64 %rd180, %rd179, %rd31;\n" " shl.b64 %rd181, %rd180, 2;\n" " add.s64 %rd182, %rd8, %rd181;\n" " st.global.f32 [%rd182], %f709;\n" " @%p177 bra $L__BB13_222;\n" "\n" " ld.shared.f32 %f594, [%r118+8];\n" " shl.b64 %rd183, %rd52, 2;\n" " add.s64 %rd184, %rd7, %rd183;\n" " st.global.f32 [%rd184], %f594;\n" " ld.shared.f32 %f595, [%r119+8];\n" " add.s64 %rd185, %rd6, %rd183;\n" " st.global.f32 [%rd185], %f595;\n" "\n" "$L__BB13_222:\n" " setp.lt.s32 %p178, %r120, 4;\n" " @%p178 bra $L__BB13_225;\n" "\n" " setp.ne.s32 %p179, %r1, 0;\n" " add.s64 %rd186, %rd16, 3;\n" " mul.lo.s64 %rd187, %rd186, %rd48;\n" " add.s64 %rd53, %rd187, %rd49;\n" " mul.lo.s64 %rd188, %rd53, %rd17;\n" " add.s64 %rd189, %rd188, %rd31;\n" " shl.b64 %rd190, %rd189, 2;\n" " add.s64 %rd191, %rd8, %rd190;\n" " st.global.f32 [%rd191], %f708;\n" " @%p179 bra $L__BB13_225;\n" "\n" " ld.shared.f32 %f596, [%r118+12];\n" " shl.b64 %rd192, %rd53, 2;\n" " add.s64 %rd193, %rd7, %rd192;\n" " st.global.f32 [%rd193], %f596;\n" " ld.shared.f32 %f597, [%r119+12];\n" " add.s64 %rd194, %rd6, %rd192;\n" " st.global.f32 [%rd194], %f597;\n" "\n" "$L__BB13_225:\n" " setp.lt.s32 %p180, %r120, 5;\n" " @%p180 bra $L__BB13_228;\n" "\n" " setp.ne.s32 %p181, %r1, 0;\n" " add.s64 %rd195, %rd16, 4;\n" " mul.lo.s64 %rd196, %rd195, %rd48;\n" " add.s64 %rd54, %rd196, %rd49;\n" " mul.lo.s64 %rd197, %rd54, %rd17;\n" " add.s64 %rd198, %rd197, %rd31;\n" " shl.b64 %rd199, %rd198, 2;\n" " add.s64 %rd200, %rd8, %rd199;\n" " st.global.f32 [%rd200], %f707;\n" " @%p181 bra $L__BB13_228;\n" "\n" " ld.shared.f32 %f598, [%r118+16];\n" " shl.b64 %rd201, %rd54, 2;\n" " add.s64 %rd202, %rd7, %rd201;\n" " st.global.f32 [%rd202], %f598;\n" " ld.shared.f32 %f599, [%r119+16];\n" " add.s64 %rd203, %rd6, %rd201;\n" " st.global.f32 [%rd203], %f599;\n" "\n" "$L__BB13_228:\n" " setp.lt.s32 %p182, %r120, 6;\n" " @%p182 bra $L__BB13_231;\n" "\n" " setp.ne.s32 %p183, %r1, 0;\n" " add.s64 %rd204, %rd16, 5;\n" " mul.lo.s64 %rd205, %rd204, %rd48;\n" " add.s64 %rd55, %rd205, %rd49;\n" " mul.lo.s64 %rd206, %rd55, %rd17;\n" " add.s64 %rd207, %rd206, %rd31;\n" " shl.b64 %rd208, %rd207, 2;\n" " add.s64 %rd209, %rd8, %rd208;\n" " st.global.f32 [%rd209], %f706;\n" " @%p183 bra $L__BB13_231;\n" "\n" " ld.shared.f32 %f600, [%r118+20];\n" " shl.b64 %rd210, %rd55, 2;\n" " add.s64 %rd211, %rd7, %rd210;\n" " st.global.f32 [%rd211], %f600;\n" " ld.shared.f32 %f601, [%r119+20];\n" " add.s64 %rd212, %rd6, %rd210;\n" " st.global.f32 [%rd212], %f601;\n" "\n" "$L__BB13_231:\n" " setp.lt.s32 %p184, %r120, 7;\n" " @%p184 bra $L__BB13_234;\n" "\n" " setp.ne.s32 %p185, %r1, 0;\n" " add.s64 %rd213, %rd16, 6;\n" " mul.lo.s64 %rd214, %rd213, %rd48;\n" " add.s64 %rd56, %rd214, %rd49;\n" " mul.lo.s64 %rd215, %rd56, %rd17;\n" " add.s64 %rd216, %rd215, %rd31;\n" " shl.b64 %rd217, %rd216, 2;\n" " add.s64 %rd218, %rd8, %rd217;\n" " st.global.f32 [%rd218], %f705;\n" " @%p185 bra $L__BB13_234;\n" "\n" " ld.shared.f32 %f602, [%r118+24];\n" " shl.b64 %rd219, %rd56, 2;\n" " add.s64 %rd220, %rd7, %rd219;\n" " st.global.f32 [%rd220], %f602;\n" " ld.shared.f32 %f603, [%r119+24];\n" " add.s64 %rd221, %rd6, %rd219;\n" " st.global.f32 [%rd221], %f603;\n" "\n" "$L__BB13_234:\n" " setp.lt.s32 %p186, %r120, 8;\n" " @%p186 bra $L__BB13_237;\n" "\n" " setp.ne.s32 %p187, %r1, 0;\n" " add.s64 %rd222, %rd16, 7;\n" " mul.lo.s64 %rd223, %rd222, %rd48;\n" " add.s64 %rd57, %rd223, %rd49;\n" " mul.lo.s64 %rd224, %rd57, %rd17;\n" " add.s64 %rd225, %rd224, %rd31;\n" " shl.b64 %rd226, %rd225, 2;\n" " add.s64 %rd227, %rd8, %rd226;\n" " st.global.f32 [%rd227], %f704;\n" " @%p187 bra $L__BB13_237;\n" "\n" " ld.shared.f32 %f604, [%r118+28];\n" " shl.b64 %rd228, %rd57, 2;\n" " add.s64 %rd229, %rd7, %rd228;\n" " st.global.f32 [%rd229], %f604;\n" " ld.shared.f32 %f605, [%r119+28];\n" " add.s64 %rd230, %rd6, %rd228;\n" " st.global.f32 [%rd230], %f605;\n" "\n" "$L__BB13_237:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode_q4_dev\n" ".visible .entry fc_attn_flash_decode_q4_dev(\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_0,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_1,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_2,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_3,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_4,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_5,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_6,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_7,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_8,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_9,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_10,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_11,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_12,\n" " .param .u64 fc_attn_flash_decode_q4_dev_param_13,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_14,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_15,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_16,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_17,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_18,\n" " .param .u32 fc_attn_flash_decode_q4_dev_param_19,\n" " .param .f32 fc_attn_flash_decode_q4_dev_param_20\n" ")\n" "{\n" " .reg .pred %p<196>;\n" " .reg .b16 %rs<11>;\n" " .reg .f32 %f<767>;\n" " .reg .b32 %r<689>;\n" " .reg .b64 %rd<240>;\n" "\n" "\n" " ld.param.u64 %rd60, [fc_attn_flash_decode_q4_dev_param_0];\n" " ld.param.u64 %rd61, [fc_attn_flash_decode_q4_dev_param_1];\n" " ld.param.u64 %rd62, [fc_attn_flash_decode_q4_dev_param_2];\n" " ld.param.u64 %rd63, [fc_attn_flash_decode_q4_dev_param_3];\n" " ld.param.u64 %rd64, [fc_attn_flash_decode_q4_dev_param_4];\n" " ld.param.u64 %rd58, [fc_attn_flash_decode_q4_dev_param_5];\n" " ld.param.u64 %rd59, [fc_attn_flash_decode_q4_dev_param_6];\n" " ld.param.u64 %rd65, [fc_attn_flash_decode_q4_dev_param_7];\n" " ld.param.u64 %rd66, [fc_attn_flash_decode_q4_dev_param_8];\n" " ld.param.u64 %rd67, [fc_attn_flash_decode_q4_dev_param_9];\n" " ld.param.u32 %r124, [fc_attn_flash_decode_q4_dev_param_10];\n" " ld.param.u32 %r125, [fc_attn_flash_decode_q4_dev_param_11];\n" " ld.param.u32 %r126, [fc_attn_flash_decode_q4_dev_param_12];\n" " ld.param.u64 %rd68, [fc_attn_flash_decode_q4_dev_param_13];\n" " ld.param.u32 %r131, [fc_attn_flash_decode_q4_dev_param_14];\n" " ld.param.u32 %r132, [fc_attn_flash_decode_q4_dev_param_15];\n" " ld.param.u32 %r127, [fc_attn_flash_decode_q4_dev_param_16];\n" " ld.param.u32 %r128, [fc_attn_flash_decode_q4_dev_param_17];\n" " ld.param.u32 %r130, [fc_attn_flash_decode_q4_dev_param_19];\n" " ld.param.f32 %f231, [fc_attn_flash_decode_q4_dev_param_20];\n" " cvta.to.global.u64 %rd1, %rd60;\n" " cvta.to.global.u64 %rd2, %rd63;\n" " cvta.to.global.u64 %rd3, %rd61;\n" " cvta.to.global.u64 %rd4, %rd64;\n" " cvta.to.global.u64 %rd5, %rd62;\n" " cvta.to.global.u64 %rd6, %rd66;\n" " cvta.to.global.u64 %rd7, %rd65;\n" " cvta.to.global.u64 %rd8, %rd67;\n" " cvta.to.global.u64 %rd69, %rd68;\n" " ld.global.u32 %r1, [%rd69];\n" " add.s32 %r133, %r1, %r131;\n" " div.s32 %r134, %r133, %r131;\n" " max.s32 %r135, %r134, 1;\n" " min.s32 %r136, %r135, %r132;\n" " add.s32 %r137, %r136, %r1;\n" " div.s32 %r138, %r137, %r136;\n" " add.s32 %r2, %r138, 15;\n" " and.b32 %r3, %r2, -16;\n" " add.s32 %r139, %r3, %r1;\n" " div.s32 %r4, %r139, %r3;\n" " mov.u32 %r5, %ctaid.x;\n" " setp.ge.s32 %p2, %r5, %r4;\n" " @%p2 bra $L__BB14_238;\n" "\n" " mov.u32 %r140, %ctaid.y;\n" " mov.u32 %r6, %tid.x;\n" " mul.lo.s32 %r669, %r3, %r5;\n" " add.s32 %r141, %r669, %r3;\n" " add.s32 %r142, %r1, 1;\n" " min.s32 %r8, %r141, %r142;\n" " mul.lo.s32 %r9, %r125, %r124;\n" " add.s32 %r143, %r125, 16;\n" " cvt.s64.s32 %rd9, %r143;\n" " mul.wide.s32 %rd10, %r143, 16;\n" " shl.b32 %r144, %r124, 4;\n" " cvt.s64.s32 %rd70, %r144;\n" " add.s64 %rd11, %rd10, %rd70;\n" " add.s64 %rd12, %rd11, %rd70;\n" " cvt.s64.s32 %rd13, %r124;\n" " add.s64 %rd14, %rd12, %rd13;\n" " cvt.s64.s32 %rd15, %r140;\n" " mul.wide.s32 %rd16, %r124, %r140;\n" " cvt.s64.s32 %rd17, %r125;\n" " setp.lt.s32 %p3, %r124, 1;\n" " @%p3 bra $L__BB14_8;\n" "\n" " cvt.s64.s32 %rd18, %r6;\n" " and.b32 %r667, %r124, 3;\n" " add.s32 %r146, %r124, -1;\n" " setp.lt.u32 %p4, %r146, 3;\n" " mov.u32 %r665, 0;\n" " @%p4 bra $L__BB14_5;\n" "\n" " sub.s32 %r663, %r124, %r667;\n" " mul.lo.s64 %rd71, %rd15, %rd17;\n" " mul.lo.s64 %rd72, %rd71, %rd13;\n" " add.s64 %rd73, %rd72, %rd18;\n" " shl.b64 %rd74, %rd73, 2;\n" " add.s64 %rd238, %rd1, %rd74;\n" " shl.b64 %rd20, %rd17, 2;\n" " shl.b32 %r148, %r6, 2;\n" " mov.u32 %r149, sh;\n" " add.s32 %r664, %r149, %r148;\n" " shl.b32 %r13, %r125, 2;\n" " mov.u32 %r665, 0;\n" "\n" "$L__BB14_4:\n" " ld.global.f32 %f232, [%rd238];\n" " st.shared.f32 [%r664], %f232;\n" " add.s64 %rd75, %rd238, %rd20;\n" " ld.global.f32 %f233, [%rd75];\n" " add.s32 %r150, %r664, %r13;\n" " st.shared.f32 [%r150], %f233;\n" " add.s64 %rd76, %rd75, %rd20;\n" " ld.global.f32 %f234, [%rd76];\n" " add.s32 %r151, %r150, %r13;\n" " st.shared.f32 [%r151], %f234;\n" " add.s64 %rd77, %rd76, %rd20;\n" " add.s64 %rd238, %rd77, %rd20;\n" " ld.global.f32 %f235, [%rd77];\n" " add.s32 %r152, %r151, %r13;\n" " add.s32 %r664, %r152, %r13;\n" " st.shared.f32 [%r152], %f235;\n" " add.s32 %r665, %r665, 4;\n" " add.s32 %r663, %r663, -4;\n" " setp.ne.s32 %p5, %r663, 0;\n" " @%p5 bra $L__BB14_4;\n" "\n" "$L__BB14_5:\n" " setp.eq.s32 %p6, %r667, 0;\n" " @%p6 bra $L__BB14_8;\n" "\n" " mad.lo.s32 %r153, %r665, %r125, %r6;\n" " shl.b32 %r154, %r153, 2;\n" " mov.u32 %r155, sh;\n" " add.s32 %r666, %r155, %r154;\n" " shl.b32 %r22, %r125, 2;\n" " cvt.s64.s32 %rd78, %r665;\n" " add.s64 %rd79, %rd16, %rd78;\n" " mul.lo.s64 %rd80, %rd79, %rd17;\n" " add.s64 %rd81, %rd80, %rd18;\n" " shl.b64 %rd82, %rd81, 2;\n" " add.s64 %rd239, %rd1, %rd82;\n" " shl.b64 %rd24, %rd17, 2;\n" "\n" "$L__BB14_7:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f236, [%rd239];\n" " st.shared.f32 [%r666], %f236;\n" " add.s32 %r666, %r666, %r22;\n" " add.s64 %rd239, %rd239, %rd24;\n" " add.s32 %r667, %r667, -1;\n" " setp.ne.s32 %p7, %r667, 0;\n" " @%p7 bra $L__BB14_7;\n" "\n" "$L__BB14_8:\n" " setp.ge.s32 %p8, %r6, %r124;\n" " add.s32 %r156, %r9, %r6;\n" " cvt.u32.u64 %r157, %rd12;\n" " add.s32 %r158, %r156, %r157;\n" " shl.b32 %r159, %r158, 2;\n" " mov.u32 %r160, sh;\n" " add.s32 %r27, %r160, %r159;\n" " cvt.u32.u64 %r161, %rd14;\n" " add.s32 %r162, %r156, %r161;\n" " shl.b32 %r163, %r162, 2;\n" " add.s32 %r28, %r160, %r163;\n" " @%p8 bra $L__BB14_10;\n" "\n" " mov.u32 %r164, -8388608;\n" " st.shared.u32 [%r27], %r164;\n" " mov.u32 %r165, 0;\n" " st.shared.u32 [%r28], %r165;\n" "\n" "$L__BB14_10:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd83, %r126;\n" " mul.lo.s64 %rd84, %rd83, %rd15;\n" " shr.s32 %r166, %r125, 31;\n" " shr.u32 %r167, %r166, 27;\n" " add.s32 %r168, %r125, %r167;\n" " shr.s32 %r169, %r168, 5;\n" " mul.lo.s32 %r170, %r169, 18;\n" " cvt.s64.s32 %rd27, %r170;\n" " mul.lo.s64 %rd28, %rd84, %rd27;\n" " mul.lo.s64 %rd85, %rd15, %rd17;\n" " shl.b64 %rd29, %rd85, 8;\n" " shl.b64 %rd30, %rd85, 3;\n" " shr.s32 %r171, %r6, 31;\n" " shr.u32 %r172, %r171, 28;\n" " add.s32 %r173, %r6, %r172;\n" " shr.s32 %r30, %r173, 4;\n" " and.b32 %r174, %r173, -16;\n" " sub.s32 %r31, %r6, %r174;\n" " cvt.s64.s32 %rd31, %r6;\n" " setp.le.s32 %p9, %r8, %r669;\n" " mov.f32 %f704, 0f00000000;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " @%p9 bra $L__BB14_214;\n" "\n" " shr.u32 %r177, %r166, 28;\n" " add.s32 %r178, %r125, %r177;\n" " shr.s32 %r179, %r178, 4;\n" " cvt.u32.u64 %r180, %rd9;\n" " sub.s32 %r181, %r1, %r130;\n" " add.s32 %r182, %r181, 1;\n" " mov.u32 %r668, 0;\n" " max.s32 %r32, %r182, 0;\n" " sub.s32 %r33, %r1, %r128;\n" " mov.u32 %r34, %ntid.x;\n" " add.s32 %r183, %r9, %r30;\n" " cvt.u32.u64 %r184, %rd10;\n" " add.s32 %r185, %r183, %r184;\n" " mad.lo.s32 %r35, %r180, %r30, %r9;\n" " shl.b32 %r186, %r6, 4;\n" " add.s32 %r187, %r186, %r184;\n" " add.s32 %r36, %r9, %r187;\n" " add.s64 %rd32, %rd30, %rd31;\n" " add.s32 %r188, %r161, %r124;\n" " add.s32 %r189, %r9, %r188;\n" " add.s64 %rd33, %rd29, %rd31;\n" " shr.u32 %r191, %r171, 27;\n" " add.s32 %r192, %r6, %r191;\n" " shr.s32 %r193, %r192, 5;\n" " mul.lo.s32 %r194, %r193, 18;\n" " cvt.s64.s32 %rd34, %r194;\n" " and.b32 %r195, %r6, 31;\n" " setp.lt.u32 %p10, %r195, 16;\n" " add.s32 %r196, %r195, -16;\n" " selp.b32 %r197, %r195, %r196, %p10;\n" " selp.b32 %r37, 0, 4, %p10;\n" " add.s32 %r198, %r197, 2;\n" " cvt.s64.s32 %rd86, %r198;\n" " add.s64 %rd35, %rd86, %rd34;\n" " add.s32 %r200, %r156, %r188;\n" " shl.b32 %r201, %r200, 2;\n" " add.s32 %r38, %r160, %r201;\n" " cvt.u64.u32 %rd87, %r6;\n" " add.s64 %rd36, %rd11, %rd87;\n" " shl.b32 %r203, %r5, 4;\n" " add.s32 %r204, %r203, 16;\n" " shr.u32 %r205, %r2, 4;\n" " mul.lo.s32 %r206, %r205, %r204;\n" " not.b32 %r207, %r206;\n" " mov.u32 %r208, -2;\n" " sub.s32 %r209, %r208, %r1;\n" " max.s32 %r210, %r209, %r207;\n" " mul.lo.s32 %r211, %r5, %r205;\n" " shl.b32 %r212, %r211, 4;\n" " add.s32 %r39, %r210, %r212;\n" " bfe.u32 %r41, %r178, 4, 1;\n" " sub.s32 %r42, %r179, %r41;\n" " shl.b32 %r43, %r125, 1;\n" " mul.lo.s32 %r44, %r125, 3;\n" " shl.b32 %r45, %r125, 2;\n" " mul.lo.s32 %r46, %r125, 5;\n" " mul.lo.s32 %r47, %r125, 6;\n" " mul.lo.s32 %r48, %r125, 7;\n" " shl.b32 %r213, %r185, 2;\n" " add.s32 %r49, %r160, %r213;\n" " shl.b32 %r214, %r189, 2;\n" " add.s32 %r50, %r160, %r214;\n" " mad.lo.s32 %r215, %r124, %r180, %r6;\n" " add.s32 %r216, %r215, %r184;\n" " shl.b32 %r217, %r216, 2;\n" " add.s32 %r218, %r160, %r217;\n" " add.s32 %r51, %r218, -1024;\n" " cvta.to.global.u64 %rd37, %rd58;\n" " cvta.to.global.u64 %rd38, %rd59;\n" " mov.f32 %f704, 0f00000000;\n" " setp.lt.s32 %p12, %r130, 1;\n" " setp.gt.s32 %p20, %r128, 0;\n" " setp.ge.s32 %p35, %r6, %r125;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" "\n" "$L__BB14_12:\n" " ld.param.u32 %r571, [fc_attn_flash_decode_q4_dev_param_18];\n" " shl.b32 %r219, %r668, 4;\n" " add.s32 %r220, %r39, %r219;\n" " max.s32 %r221, %r220, -17;\n" " not.b32 %r222, %r221;\n" " max.s32 %r56, %r222, 1;\n" " sub.s32 %r57, %r8, %r669;\n" " min.s32 %r58, %r57, 16;\n" " setp.lt.s32 %p11, %r669, %r571;\n" " or.pred %p13, %p12, %p11;\n" " add.s32 %r223, %r58, %r669;\n" " setp.gt.s32 %p14, %r223, %r32;\n" " setp.le.s32 %p15, %r669, %r1;\n" " and.pred %p16, %p15, %p14;\n" " or.pred %p17, %p13, %p16;\n" " not.pred %p18, %p17;\n" " @%p18 bra $L__BB14_213;\n" "\n" " setp.lt.s32 %p19, %r57, 1;\n" " @%p19 bra $L__BB14_37;\n" "\n" " @%p20 bra $L__BB14_24;\n" " bra.uni $L__BB14_15;\n" "\n" "$L__BB14_24:\n" " mov.u32 %r673, 0;\n" "\n" "$L__BB14_25:\n" " cvt.s64.s32 %rd100, %r673;\n" " mul.lo.s64 %rd43, %rd100, %rd9;\n" " add.s32 %r68, %r673, %r669;\n" " setp.lt.s32 %p28, %r68, %r127;\n" " @%p28 bra $L__BB14_33;\n" " bra.uni $L__BB14_26;\n" "\n" "$L__BB14_33:\n" " @%p35 bra $L__BB14_36;\n" "\n" " cvt.u32.u64 %r286, %rd43;\n" " add.s32 %r75, %r9, %r286;\n" " cvt.s64.s32 %rd114, %r68;\n" " mul.lo.s64 %rd115, %rd114, %rd17;\n" " add.s64 %rd46, %rd115, %rd30;\n" " mov.u32 %r676, %r6;\n" "\n" "$L__BB14_35:\n" " cvt.s64.s32 %rd116, %r676;\n" " add.s64 %rd117, %rd46, %rd116;\n" " shl.b64 %rd118, %rd117, 1;\n" " add.s64 %rd119, %rd2, %rd118;\n" " ld.global.u16 %rs5, [%rd119];\n" " // begin inline asm\n" " { cvt.f32.f16 %f261, %rs5;}\n" "\n" " // end inline asm\n" " add.s32 %r287, %r75, %r676;\n" " shl.b32 %r288, %r287, 2;\n" " add.s32 %r290, %r160, %r288;\n" " st.shared.f32 [%r290], %f261;\n" " add.s32 %r676, %r676, %r34;\n" " setp.lt.s32 %p36, %r676, %r125;\n" " @%p36 bra $L__BB14_35;\n" " bra.uni $L__BB14_36;\n" "\n" "$L__BB14_26:\n" " setp.gt.s32 %p29, %r68, %r33;\n" " @%p29 bra $L__BB14_30;\n" " bra.uni $L__BB14_27;\n" "\n" "$L__BB14_30:\n" " @%p35 bra $L__BB14_36;\n" "\n" " cvt.u32.u64 %r276, %rd43;\n" " add.s32 %r72, %r9, %r276;\n" " shr.s32 %r277, %r68, 31;\n" " shr.u32 %r278, %r277, 24;\n" " add.s32 %r279, %r68, %r278;\n" " and.b32 %r280, %r279, -256;\n" " sub.s32 %r281, %r68, %r280;\n" " cvt.s64.s32 %rd108, %r281;\n" " mul.lo.s64 %rd109, %rd108, %rd17;\n" " add.s64 %rd45, %rd109, %rd29;\n" " mov.u32 %r675, %r6;\n" "\n" "$L__BB14_32:\n" " cvt.s64.s32 %rd110, %r675;\n" " add.s64 %rd111, %rd45, %rd110;\n" " shl.b64 %rd112, %rd111, 1;\n" " add.s64 %rd113, %rd37, %rd112;\n" " ld.global.u16 %rs4, [%rd113];\n" " // begin inline asm\n" " { cvt.f32.f16 %f260, %rs4;}\n" "\n" " // end inline asm\n" " add.s32 %r282, %r72, %r675;\n" " shl.b32 %r283, %r282, 2;\n" " add.s32 %r285, %r160, %r283;\n" " st.shared.f32 [%r285], %f260;\n" " add.s32 %r675, %r675, %r34;\n" " setp.lt.s32 %p34, %r675, %r125;\n" " @%p34 bra $L__BB14_32;\n" " bra.uni $L__BB14_36;\n" "\n" "$L__BB14_27:\n" " @%p35 bra $L__BB14_36;\n" "\n" " cvt.u32.u64 %r254, %rd43;\n" " add.s32 %r69, %r9, %r254;\n" " cvt.s64.s32 %rd101, %r68;\n" " mul.lo.s64 %rd102, %rd101, %rd27;\n" " add.s64 %rd44, %rd102, %rd28;\n" " mov.u32 %r674, %r6;\n" "\n" "$L__BB14_29:\n" " shr.s32 %r255, %r674, 31;\n" " shr.u32 %r256, %r255, 27;\n" " add.s32 %r257, %r674, %r256;\n" " shr.s32 %r258, %r257, 5;\n" " mul.lo.s32 %r259, %r258, 18;\n" " cvt.s64.s32 %rd103, %r259;\n" " add.s64 %rd104, %rd44, %rd103;\n" " add.s64 %rd105, %rd3, %rd104;\n" " ld.global.u8 %r260, [%rd105];\n" " ld.global.u8 %r261, [%rd105+1];\n" " prmt.b32 %r262, %r261, %r260, 30212;\n" " cvt.u16.u32 %rs3, %r262;\n" " // begin inline asm\n" " { cvt.f32.f16 %f257, %rs3;}\n" "\n" " // end inline asm\n" " and.b32 %r263, %r674, 31;\n" " setp.lt.u32 %p31, %r263, 16;\n" " add.s32 %r264, %r263, -16;\n" " selp.b32 %r265, %r263, %r264, %p31;\n" " selp.b32 %r266, 0, 4, %p31;\n" " add.s32 %r267, %r265, 2;\n" " cvt.s64.s32 %rd106, %r267;\n" " add.s64 %rd107, %rd105, %rd106;\n" " ld.global.u8 %r268, [%rd107];\n" " shr.u32 %r269, %r268, %r266;\n" " and.b32 %r270, %r269, 15;\n" " add.s32 %r271, %r270, -8;\n" " cvt.rn.f32.s32 %f258, %r271;\n" " mul.f32 %f259, %f257, %f258;\n" " add.s32 %r272, %r69, %r674;\n" " shl.b32 %r273, %r272, 2;\n" " add.s32 %r275, %r160, %r273;\n" " st.shared.f32 [%r275], %f259;\n" " add.s32 %r674, %r674, %r34;\n" " setp.lt.s32 %p32, %r674, %r125;\n" " @%p32 bra $L__BB14_29;\n" "\n" "$L__BB14_36:\n" " add.s32 %r673, %r673, 1;\n" " setp.lt.s32 %p37, %r673, %r58;\n" " @%p37 bra $L__BB14_25;\n" " bra.uni $L__BB14_37;\n" "\n" "$L__BB14_15:\n" " mov.u32 %r670, 0;\n" "\n" "$L__BB14_16:\n" " cvt.s64.s32 %rd88, %r670;\n" " mul.lo.s64 %rd39, %rd88, %rd9;\n" " add.s32 %r225, %r670, %r669;\n" " setp.lt.s32 %p21, %r225, %r127;\n" " cvt.s64.s32 %rd40, %r225;\n" " @%p21 bra $L__BB14_20;\n" " bra.uni $L__BB14_17;\n" "\n" "$L__BB14_20:\n" " @%p35 bra $L__BB14_23;\n" "\n" " cvt.u32.u64 %r248, %rd39;\n" " add.s32 %r63, %r9, %r248;\n" " mul.lo.s64 %rd95, %rd40, %rd17;\n" " add.s64 %rd42, %rd95, %rd30;\n" " mov.u32 %r672, %r6;\n" "\n" "$L__BB14_22:\n" " cvt.s64.s32 %rd96, %r672;\n" " add.s64 %rd97, %rd42, %rd96;\n" " shl.b64 %rd98, %rd97, 1;\n" " add.s64 %rd99, %rd2, %rd98;\n" " ld.global.u16 %rs2, [%rd99];\n" " // begin inline asm\n" " { cvt.f32.f16 %f256, %rs2;}\n" "\n" " // end inline asm\n" " add.s32 %r249, %r63, %r672;\n" " shl.b32 %r250, %r249, 2;\n" " add.s32 %r252, %r160, %r250;\n" " st.shared.f32 [%r252], %f256;\n" " add.s32 %r672, %r672, %r34;\n" " setp.lt.s32 %p26, %r672, %r125;\n" " @%p26 bra $L__BB14_22;\n" " bra.uni $L__BB14_23;\n" "\n" "$L__BB14_17:\n" " @%p35 bra $L__BB14_23;\n" "\n" " cvt.u32.u64 %r226, %rd39;\n" " add.s32 %r60, %r9, %r226;\n" " mul.lo.s64 %rd89, %rd40, %rd27;\n" " add.s64 %rd41, %rd89, %rd28;\n" " mov.u32 %r671, %r6;\n" "\n" "$L__BB14_19:\n" " shr.s32 %r227, %r671, 31;\n" " shr.u32 %r228, %r227, 27;\n" " add.s32 %r229, %r671, %r228;\n" " shr.s32 %r230, %r229, 5;\n" " mul.lo.s32 %r231, %r230, 18;\n" " cvt.s64.s32 %rd90, %r231;\n" " add.s64 %rd91, %rd41, %rd90;\n" " add.s64 %rd92, %rd3, %rd91;\n" " ld.global.u8 %r232, [%rd92];\n" " ld.global.u8 %r233, [%rd92+1];\n" " prmt.b32 %r234, %r233, %r232, 30212;\n" " cvt.u16.u32 %rs1, %r234;\n" " // begin inline asm\n" " { cvt.f32.f16 %f253, %rs1;}\n" "\n" " // end inline asm\n" " and.b32 %r235, %r671, 31;\n" " setp.lt.u32 %p23, %r235, 16;\n" " add.s32 %r236, %r235, -16;\n" " selp.b32 %r237, %r235, %r236, %p23;\n" " selp.b32 %r238, 0, 4, %p23;\n" " add.s32 %r239, %r237, 2;\n" " cvt.s64.s32 %rd93, %r239;\n" " add.s64 %rd94, %rd92, %rd93;\n" " ld.global.u8 %r240, [%rd94];\n" " shr.u32 %r241, %r240, %r238;\n" " and.b32 %r242, %r241, 15;\n" " add.s32 %r243, %r242, -8;\n" " cvt.rn.f32.s32 %f254, %r243;\n" " mul.f32 %f255, %f253, %f254;\n" " add.s32 %r244, %r60, %r671;\n" " shl.b32 %r245, %r244, 2;\n" " add.s32 %r247, %r160, %r245;\n" " st.shared.f32 [%r247], %f255;\n" " add.s32 %r671, %r671, %r34;\n" " setp.lt.s32 %p24, %r671, %r125;\n" " @%p24 bra $L__BB14_19;\n" "\n" "$L__BB14_23:\n" " add.s32 %r670, %r670, 1;\n" " setp.lt.s32 %p27, %r670, %r58;\n" " @%p27 bra $L__BB14_16;\n" "\n" "$L__BB14_37:\n" " setp.lt.s32 %p38, %r125, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p39, %r30, %r58;\n" " or.pred %p40, %p38, %p39;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p40 bra $L__BB14_90;\n" "\n" " and.b32 %r572, %r125, -16;\n" " setp.eq.s32 %p41, %r572, 16;\n" " mov.u32 %r679, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p41 bra $L__BB14_73;\n" "\n" " mov.u32 %r679, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " mov.u32 %r678, %r42;\n" "\n" "$L__BB14_40:\n" " shl.b32 %r293, %r679, 4;\n" " add.s32 %r81, %r293, %r31;\n" " add.s32 %r294, %r35, %r81;\n" " shl.b32 %r295, %r294, 2;\n" " add.s32 %r82, %r160, %r295;\n" " ld.shared.f32 %f17, [%r82];\n" " @%p3 bra $L__BB14_42;\n" "\n" " shl.b32 %r652, %r679, 4;\n" " add.s32 %r651, %r652, %r31;\n" " shl.b32 %r650, %r651, 2;\n" " add.s32 %r649, %r160, %r650;\n" " ld.shared.f32 %f286, [%r649];\n" " fma.rn.f32 %f634, %f17, %f286, %f634;\n" "\n" "$L__BB14_42:\n" " setp.lt.s32 %p43, %r124, 2;\n" " @%p43 bra $L__BB14_44;\n" "\n" " shl.b32 %r657, %r679, 4;\n" " add.s32 %r656, %r657, %r31;\n" " add.s32 %r655, %r656, %r125;\n" " shl.b32 %r654, %r655, 2;\n" " add.s32 %r653, %r160, %r654;\n" " ld.shared.f32 %f287, [%r653];\n" " fma.rn.f32 %f635, %f17, %f287, %f635;\n" "\n" "$L__BB14_44:\n" " setp.lt.s32 %p44, %r124, 3;\n" " @%p44 bra $L__BB14_46;\n" "\n" " shl.b32 %r598, %r679, 4;\n" " add.s32 %r597, %r598, %r31;\n" " add.s32 %r596, %r43, %r597;\n" " shl.b32 %r595, %r596, 2;\n" " add.s32 %r594, %r160, %r595;\n" " ld.shared.f32 %f288, [%r594];\n" " fma.rn.f32 %f636, %f17, %f288, %f636;\n" "\n" "$L__BB14_46:\n" " setp.lt.s32 %p45, %r124, 4;\n" " @%p45 bra $L__BB14_48;\n" "\n" " shl.b32 %r608, %r679, 4;\n" " add.s32 %r607, %r608, %r31;\n" " add.s32 %r606, %r44, %r607;\n" " shl.b32 %r605, %r606, 2;\n" " add.s32 %r604, %r160, %r605;\n" " ld.shared.f32 %f289, [%r604];\n" " fma.rn.f32 %f637, %f17, %f289, %f637;\n" "\n" "$L__BB14_48:\n" " setp.lt.s32 %p46, %r124, 5;\n" " @%p46 bra $L__BB14_50;\n" "\n" " shl.b32 %r618, %r679, 4;\n" " add.s32 %r617, %r618, %r31;\n" " add.s32 %r616, %r45, %r617;\n" " shl.b32 %r615, %r616, 2;\n" " add.s32 %r614, %r160, %r615;\n" " ld.shared.f32 %f290, [%r614];\n" " fma.rn.f32 %f638, %f17, %f290, %f638;\n" "\n" "$L__BB14_50:\n" " setp.lt.s32 %p47, %r124, 6;\n" " @%p47 bra $L__BB14_52;\n" "\n" " shl.b32 %r628, %r679, 4;\n" " add.s32 %r627, %r628, %r31;\n" " add.s32 %r626, %r46, %r627;\n" " shl.b32 %r625, %r626, 2;\n" " add.s32 %r624, %r160, %r625;\n" " ld.shared.f32 %f291, [%r624];\n" " fma.rn.f32 %f639, %f17, %f291, %f639;\n" "\n" "$L__BB14_52:\n" " setp.lt.s32 %p48, %r124, 7;\n" " @%p48 bra $L__BB14_54;\n" "\n" " shl.b32 %r638, %r679, 4;\n" " add.s32 %r637, %r638, %r31;\n" " add.s32 %r636, %r47, %r637;\n" " shl.b32 %r635, %r636, 2;\n" " add.s32 %r634, %r160, %r635;\n" " ld.shared.f32 %f292, [%r634];\n" " fma.rn.f32 %f640, %f17, %f292, %f640;\n" "\n" "$L__BB14_54:\n" " setp.lt.s32 %p49, %r124, 8;\n" " @%p49 bra $L__BB14_56;\n" "\n" " shl.b32 %r648, %r679, 4;\n" " add.s32 %r647, %r648, %r31;\n" " add.s32 %r646, %r48, %r647;\n" " shl.b32 %r645, %r646, 2;\n" " add.s32 %r644, %r160, %r645;\n" " ld.shared.f32 %f293, [%r644];\n" " fma.rn.f32 %f641, %f17, %f293, %f641;\n" "\n" "$L__BB14_56:\n" " shl.b32 %r579, %r679, 4;\n" " add.s32 %r578, %r579, %r31;\n" " add.s32 %r577, %r35, %r578;\n" " shl.b32 %r576, %r577, 2;\n" " add.s32 %r575, %r160, %r576;\n" " ld.shared.f32 %f34, [%r575+64];\n" " @%p3 bra $L__BB14_58;\n" "\n" " shl.b32 %r593, %r679, 4;\n" " add.s32 %r592, %r593, %r31;\n" " shl.b32 %r591, %r592, 2;\n" " add.s32 %r590, %r160, %r591;\n" " ld.shared.f32 %f294, [%r590+64];\n" " fma.rn.f32 %f634, %f34, %f294, %f634;\n" "\n" "$L__BB14_58:\n" " setp.lt.s32 %p189, %r124, 2;\n" " @%p189 bra $L__BB14_60;\n" "\n" " shl.b32 %r589, %r679, 4;\n" " add.s32 %r588, %r589, %r31;\n" " add.s32 %r587, %r588, %r125;\n" " shl.b32 %r586, %r587, 2;\n" " add.s32 %r585, %r160, %r586;\n" " ld.shared.f32 %f295, [%r585+64];\n" " fma.rn.f32 %f635, %f34, %f295, %f635;\n" "\n" "$L__BB14_60:\n" " setp.lt.s32 %p190, %r124, 3;\n" " @%p190 bra $L__BB14_62;\n" "\n" " shl.b32 %r584, %r679, 4;\n" " add.s32 %r583, %r584, %r31;\n" " add.s32 %r582, %r43, %r583;\n" " shl.b32 %r581, %r582, 2;\n" " add.s32 %r580, %r160, %r581;\n" " ld.shared.f32 %f296, [%r580+64];\n" " fma.rn.f32 %f636, %f34, %f296, %f636;\n" "\n" "$L__BB14_62:\n" " setp.lt.s32 %p191, %r124, 4;\n" " @%p191 bra $L__BB14_64;\n" "\n" " shl.b32 %r603, %r679, 4;\n" " add.s32 %r602, %r603, %r31;\n" " add.s32 %r601, %r44, %r602;\n" " shl.b32 %r600, %r601, 2;\n" " add.s32 %r599, %r160, %r600;\n" " ld.shared.f32 %f297, [%r599+64];\n" " fma.rn.f32 %f637, %f34, %f297, %f637;\n" "\n" "$L__BB14_64:\n" " setp.lt.s32 %p192, %r124, 5;\n" " @%p192 bra $L__BB14_66;\n" "\n" " shl.b32 %r613, %r679, 4;\n" " add.s32 %r612, %r613, %r31;\n" " add.s32 %r611, %r45, %r612;\n" " shl.b32 %r610, %r611, 2;\n" " add.s32 %r609, %r160, %r610;\n" " ld.shared.f32 %f298, [%r609+64];\n" " fma.rn.f32 %f638, %f34, %f298, %f638;\n" "\n" "$L__BB14_66:\n" " setp.lt.s32 %p193, %r124, 6;\n" " @%p193 bra $L__BB14_68;\n" "\n" " shl.b32 %r623, %r679, 4;\n" " add.s32 %r622, %r623, %r31;\n" " add.s32 %r621, %r46, %r622;\n" " shl.b32 %r620, %r621, 2;\n" " add.s32 %r619, %r160, %r620;\n" " ld.shared.f32 %f299, [%r619+64];\n" " fma.rn.f32 %f639, %f34, %f299, %f639;\n" "\n" "$L__BB14_68:\n" " setp.lt.s32 %p194, %r124, 7;\n" " @%p194 bra $L__BB14_70;\n" "\n" " shl.b32 %r633, %r679, 4;\n" " add.s32 %r632, %r633, %r31;\n" " add.s32 %r631, %r47, %r632;\n" " shl.b32 %r630, %r631, 2;\n" " add.s32 %r629, %r160, %r630;\n" " ld.shared.f32 %f300, [%r629+64];\n" " fma.rn.f32 %f640, %f34, %f300, %f640;\n" "\n" "$L__BB14_70:\n" " setp.lt.s32 %p195, %r124, 8;\n" " @%p195 bra $L__BB14_72;\n" "\n" " shl.b32 %r643, %r679, 4;\n" " add.s32 %r642, %r643, %r31;\n" " add.s32 %r641, %r48, %r642;\n" " shl.b32 %r640, %r641, 2;\n" " add.s32 %r639, %r160, %r640;\n" " ld.shared.f32 %f301, [%r639+64];\n" " fma.rn.f32 %f641, %f34, %f301, %f641;\n" "\n" "$L__BB14_72:\n" " add.s32 %r679, %r679, 2;\n" " add.s32 %r678, %r678, -2;\n" " setp.ne.s32 %p58, %r678, 0;\n" " @%p58 bra $L__BB14_40;\n" "\n" "$L__BB14_73:\n" " setp.eq.s32 %p59, %r41, 0;\n" " @%p59 bra $L__BB14_90;\n" "\n" " shl.b32 %r319, %r679, 4;\n" " add.s32 %r94, %r319, %r31;\n" " add.s32 %r320, %r35, %r94;\n" " shl.b32 %r321, %r320, 2;\n" " add.s32 %r323, %r160, %r321;\n" " ld.shared.f32 %f59, [%r323];\n" " @%p3 bra $L__BB14_76;\n" "\n" " shl.b32 %r324, %r94, 2;\n" " add.s32 %r326, %r160, %r324;\n" " ld.shared.f32 %f302, [%r326];\n" " fma.rn.f32 %f634, %f59, %f302, %f634;\n" "\n" "$L__BB14_76:\n" " setp.lt.s32 %p61, %r124, 2;\n" " @%p61 bra $L__BB14_78;\n" "\n" " add.s32 %r327, %r94, %r125;\n" " shl.b32 %r328, %r327, 2;\n" " add.s32 %r330, %r160, %r328;\n" " ld.shared.f32 %f303, [%r330];\n" " fma.rn.f32 %f635, %f59, %f303, %f635;\n" "\n" "$L__BB14_78:\n" " setp.lt.s32 %p62, %r124, 3;\n" " @%p62 bra $L__BB14_80;\n" "\n" " add.s32 %r331, %r43, %r94;\n" " shl.b32 %r332, %r331, 2;\n" " add.s32 %r334, %r160, %r332;\n" " ld.shared.f32 %f304, [%r334];\n" " fma.rn.f32 %f636, %f59, %f304, %f636;\n" "\n" "$L__BB14_80:\n" " setp.lt.s32 %p63, %r124, 4;\n" " @%p63 bra $L__BB14_82;\n" "\n" " add.s32 %r335, %r44, %r94;\n" " shl.b32 %r336, %r335, 2;\n" " add.s32 %r338, %r160, %r336;\n" " ld.shared.f32 %f305, [%r338];\n" " fma.rn.f32 %f637, %f59, %f305, %f637;\n" "\n" "$L__BB14_82:\n" " setp.lt.s32 %p64, %r124, 5;\n" " @%p64 bra $L__BB14_84;\n" "\n" " add.s32 %r339, %r45, %r94;\n" " shl.b32 %r340, %r339, 2;\n" " add.s32 %r342, %r160, %r340;\n" " ld.shared.f32 %f306, [%r342];\n" " fma.rn.f32 %f638, %f59, %f306, %f638;\n" "\n" "$L__BB14_84:\n" " setp.lt.s32 %p65, %r124, 6;\n" " @%p65 bra $L__BB14_86;\n" "\n" " add.s32 %r343, %r46, %r94;\n" " shl.b32 %r344, %r343, 2;\n" " add.s32 %r346, %r160, %r344;\n" " ld.shared.f32 %f307, [%r346];\n" " fma.rn.f32 %f639, %f59, %f307, %f639;\n" "\n" "$L__BB14_86:\n" " setp.lt.s32 %p66, %r124, 7;\n" " @%p66 bra $L__BB14_88;\n" "\n" " add.s32 %r347, %r47, %r94;\n" " shl.b32 %r348, %r347, 2;\n" " add.s32 %r350, %r160, %r348;\n" " ld.shared.f32 %f308, [%r350];\n" " fma.rn.f32 %f640, %f59, %f308, %f640;\n" "\n" "$L__BB14_88:\n" " setp.lt.s32 %p67, %r124, 8;\n" " @%p67 bra $L__BB14_90;\n" "\n" " add.s32 %r351, %r48, %r94;\n" " shl.b32 %r352, %r351, 2;\n" " add.s32 %r354, %r160, %r352;\n" " ld.shared.f32 %f309, [%r354];\n" " fma.rn.f32 %f641, %f59, %f309, %f641;\n" "\n" "$L__BB14_90:\n" " setp.eq.s32 %p68, %r31, 0;\n" " setp.lt.s32 %p69, %r30, %r58;\n" " and.pred %p1, %p68, %p69;\n" " @%p3 bra $L__BB14_93;\n" "\n" " mov.b32 %r355, %f634;\n" " mov.u32 %r356, 2;\n" " mov.u32 %r357, 4127;\n" " mov.u32 %r358, 8;\n" " mov.u32 %r359, -1;\n" " shfl.sync.down.b32 %r360|%p71, %r355, %r358, %r357, %r359;\n" " mov.b32 %f310, %r360;\n" " add.f32 %f311, %f634, %f310;\n" " mov.b32 %r361, %f311;\n" " mov.u32 %r362, 4;\n" " shfl.sync.down.b32 %r363|%p72, %r361, %r362, %r357, %r359;\n" " mov.b32 %f312, %r363;\n" " add.f32 %f313, %f311, %f312;\n" " mov.b32 %r364, %f313;\n" " shfl.sync.down.b32 %r365|%p73, %r364, %r356, %r357, %r359;\n" " mov.b32 %f314, %r365;\n" " add.f32 %f315, %f313, %f314;\n" " mov.b32 %r366, %f315;\n" " mov.u32 %r367, 1;\n" " shfl.sync.down.b32 %r368|%p74, %r366, %r367, %r357, %r359;\n" " mov.b32 %f316, %r368;\n" " add.f32 %f83, %f315, %f316;\n" " not.pred %p75, %p1;\n" " @%p75 bra $L__BB14_93;\n" "\n" " mul.f32 %f317, %f83, %f231;\n" " st.shared.f32 [%r49], %f317;\n" "\n" "$L__BB14_93:\n" " setp.lt.s32 %p76, %r124, 2;\n" " @%p76 bra $L__BB14_96;\n" "\n" " mov.b32 %r369, %f635;\n" " mov.u32 %r370, 2;\n" " mov.u32 %r371, 4127;\n" " mov.u32 %r372, 8;\n" " mov.u32 %r373, -1;\n" " shfl.sync.down.b32 %r374|%p77, %r369, %r372, %r371, %r373;\n" " mov.b32 %f318, %r374;\n" " add.f32 %f319, %f635, %f318;\n" " mov.b32 %r375, %f319;\n" " mov.u32 %r376, 4;\n" " shfl.sync.down.b32 %r377|%p78, %r375, %r376, %r371, %r373;\n" " mov.b32 %f320, %r377;\n" " add.f32 %f321, %f319, %f320;\n" " mov.b32 %r378, %f321;\n" " shfl.sync.down.b32 %r379|%p79, %r378, %r370, %r371, %r373;\n" " mov.b32 %f322, %r379;\n" " add.f32 %f323, %f321, %f322;\n" " mov.b32 %r380, %f323;\n" " mov.u32 %r381, 1;\n" " shfl.sync.down.b32 %r382|%p80, %r380, %r381, %r371, %r373;\n" " mov.b32 %f324, %r382;\n" " add.f32 %f84, %f323, %f324;\n" " not.pred %p81, %p1;\n" " @%p81 bra $L__BB14_96;\n" "\n" " mul.f32 %f325, %f84, %f231;\n" " st.shared.f32 [%r49+64], %f325;\n" "\n" "$L__BB14_96:\n" " setp.lt.s32 %p82, %r124, 3;\n" " @%p82 bra $L__BB14_99;\n" "\n" " mov.b32 %r383, %f636;\n" " mov.u32 %r384, 2;\n" " mov.u32 %r385, 4127;\n" " mov.u32 %r386, 8;\n" " mov.u32 %r387, -1;\n" " shfl.sync.down.b32 %r388|%p83, %r383, %r386, %r385, %r387;\n" " mov.b32 %f326, %r388;\n" " add.f32 %f327, %f636, %f326;\n" " mov.b32 %r389, %f327;\n" " mov.u32 %r390, 4;\n" " shfl.sync.down.b32 %r391|%p84, %r389, %r390, %r385, %r387;\n" " mov.b32 %f328, %r391;\n" " add.f32 %f329, %f327, %f328;\n" " mov.b32 %r392, %f329;\n" " shfl.sync.down.b32 %r393|%p85, %r392, %r384, %r385, %r387;\n" " mov.b32 %f330, %r393;\n" " add.f32 %f331, %f329, %f330;\n" " mov.b32 %r394, %f331;\n" " mov.u32 %r395, 1;\n" " shfl.sync.down.b32 %r396|%p86, %r394, %r395, %r385, %r387;\n" " mov.b32 %f332, %r396;\n" " add.f32 %f85, %f331, %f332;\n" " not.pred %p87, %p1;\n" " @%p87 bra $L__BB14_99;\n" "\n" " mul.f32 %f333, %f85, %f231;\n" " st.shared.f32 [%r49+128], %f333;\n" "\n" "$L__BB14_99:\n" " setp.lt.s32 %p88, %r124, 4;\n" " @%p88 bra $L__BB14_102;\n" "\n" " mov.b32 %r397, %f637;\n" " mov.u32 %r398, 2;\n" " mov.u32 %r399, 4127;\n" " mov.u32 %r400, 8;\n" " mov.u32 %r401, -1;\n" " shfl.sync.down.b32 %r402|%p89, %r397, %r400, %r399, %r401;\n" " mov.b32 %f334, %r402;\n" " add.f32 %f335, %f637, %f334;\n" " mov.b32 %r403, %f335;\n" " mov.u32 %r404, 4;\n" " shfl.sync.down.b32 %r405|%p90, %r403, %r404, %r399, %r401;\n" " mov.b32 %f336, %r405;\n" " add.f32 %f337, %f335, %f336;\n" " mov.b32 %r406, %f337;\n" " shfl.sync.down.b32 %r407|%p91, %r406, %r398, %r399, %r401;\n" " mov.b32 %f338, %r407;\n" " add.f32 %f339, %f337, %f338;\n" " mov.b32 %r408, %f339;\n" " mov.u32 %r409, 1;\n" " shfl.sync.down.b32 %r410|%p92, %r408, %r409, %r399, %r401;\n" " mov.b32 %f340, %r410;\n" " add.f32 %f86, %f339, %f340;\n" " not.pred %p93, %p1;\n" " @%p93 bra $L__BB14_102;\n" "\n" " mul.f32 %f341, %f86, %f231;\n" " st.shared.f32 [%r49+192], %f341;\n" "\n" "$L__BB14_102:\n" " setp.lt.s32 %p94, %r124, 5;\n" " @%p94 bra $L__BB14_105;\n" "\n" " mov.b32 %r411, %f638;\n" " mov.u32 %r412, 2;\n" " mov.u32 %r413, 4127;\n" " mov.u32 %r414, 8;\n" " mov.u32 %r415, -1;\n" " shfl.sync.down.b32 %r416|%p95, %r411, %r414, %r413, %r415;\n" " mov.b32 %f342, %r416;\n" " add.f32 %f343, %f638, %f342;\n" " mov.b32 %r417, %f343;\n" " mov.u32 %r418, 4;\n" " shfl.sync.down.b32 %r419|%p96, %r417, %r418, %r413, %r415;\n" " mov.b32 %f344, %r419;\n" " add.f32 %f345, %f343, %f344;\n" " mov.b32 %r420, %f345;\n" " shfl.sync.down.b32 %r421|%p97, %r420, %r412, %r413, %r415;\n" " mov.b32 %f346, %r421;\n" " add.f32 %f347, %f345, %f346;\n" " mov.b32 %r422, %f347;\n" " mov.u32 %r423, 1;\n" " shfl.sync.down.b32 %r424|%p98, %r422, %r423, %r413, %r415;\n" " mov.b32 %f348, %r424;\n" " add.f32 %f87, %f347, %f348;\n" " not.pred %p99, %p1;\n" " @%p99 bra $L__BB14_105;\n" "\n" " mul.f32 %f349, %f87, %f231;\n" " st.shared.f32 [%r49+256], %f349;\n" "\n" "$L__BB14_105:\n" " setp.lt.s32 %p100, %r124, 6;\n" " @%p100 bra $L__BB14_108;\n" "\n" " mov.b32 %r425, %f639;\n" " mov.u32 %r426, 2;\n" " mov.u32 %r427, 4127;\n" " mov.u32 %r428, 8;\n" " mov.u32 %r429, -1;\n" " shfl.sync.down.b32 %r430|%p101, %r425, %r428, %r427, %r429;\n" " mov.b32 %f350, %r430;\n" " add.f32 %f351, %f639, %f350;\n" " mov.b32 %r431, %f351;\n" " mov.u32 %r432, 4;\n" " shfl.sync.down.b32 %r433|%p102, %r431, %r432, %r427, %r429;\n" " mov.b32 %f352, %r433;\n" " add.f32 %f353, %f351, %f352;\n" " mov.b32 %r434, %f353;\n" " shfl.sync.down.b32 %r435|%p103, %r434, %r426, %r427, %r429;\n" " mov.b32 %f354, %r435;\n" " add.f32 %f355, %f353, %f354;\n" " mov.b32 %r436, %f355;\n" " mov.u32 %r437, 1;\n" " shfl.sync.down.b32 %r438|%p104, %r436, %r437, %r427, %r429;\n" " mov.b32 %f356, %r438;\n" " add.f32 %f88, %f355, %f356;\n" " not.pred %p105, %p1;\n" " @%p105 bra $L__BB14_108;\n" "\n" " mul.f32 %f357, %f88, %f231;\n" " st.shared.f32 [%r49+320], %f357;\n" "\n" "$L__BB14_108:\n" " setp.lt.s32 %p106, %r124, 7;\n" " @%p106 bra $L__BB14_111;\n" "\n" " mov.b32 %r439, %f640;\n" " mov.u32 %r440, 2;\n" " mov.u32 %r441, 4127;\n" " mov.u32 %r442, 8;\n" " mov.u32 %r443, -1;\n" " shfl.sync.down.b32 %r444|%p107, %r439, %r442, %r441, %r443;\n" " mov.b32 %f358, %r444;\n" " add.f32 %f359, %f640, %f358;\n" " mov.b32 %r445, %f359;\n" " mov.u32 %r446, 4;\n" " shfl.sync.down.b32 %r447|%p108, %r445, %r446, %r441, %r443;\n" " mov.b32 %f360, %r447;\n" " add.f32 %f361, %f359, %f360;\n" " mov.b32 %r448, %f361;\n" " shfl.sync.down.b32 %r449|%p109, %r448, %r440, %r441, %r443;\n" " mov.b32 %f362, %r449;\n" " add.f32 %f363, %f361, %f362;\n" " mov.b32 %r450, %f363;\n" " mov.u32 %r451, 1;\n" " shfl.sync.down.b32 %r452|%p110, %r450, %r451, %r441, %r443;\n" " mov.b32 %f364, %r452;\n" " add.f32 %f89, %f363, %f364;\n" " not.pred %p111, %p1;\n" " @%p111 bra $L__BB14_111;\n" "\n" " mul.f32 %f365, %f89, %f231;\n" " st.shared.f32 [%r49+384], %f365;\n" "\n" "$L__BB14_111:\n" " setp.lt.s32 %p112, %r124, 8;\n" " @%p112 bra $L__BB14_114;\n" "\n" " mov.b32 %r453, %f641;\n" " mov.u32 %r454, 2;\n" " mov.u32 %r455, 4127;\n" " mov.u32 %r456, 8;\n" " mov.u32 %r457, -1;\n" " shfl.sync.down.b32 %r458|%p113, %r453, %r456, %r455, %r457;\n" " mov.b32 %f366, %r458;\n" " add.f32 %f367, %f641, %f366;\n" " mov.b32 %r459, %f367;\n" " mov.u32 %r460, 4;\n" " shfl.sync.down.b32 %r461|%p114, %r459, %r460, %r455, %r457;\n" " mov.b32 %f368, %r461;\n" " add.f32 %f369, %f367, %f368;\n" " mov.b32 %r462, %f369;\n" " shfl.sync.down.b32 %r463|%p115, %r462, %r454, %r455, %r457;\n" " mov.b32 %f370, %r463;\n" " add.f32 %f371, %f369, %f370;\n" " mov.b32 %r464, %f371;\n" " mov.u32 %r465, 1;\n" " shfl.sync.down.b32 %r466|%p116, %r464, %r465, %r455, %r457;\n" " mov.b32 %f372, %r466;\n" " add.f32 %f90, %f371, %f372;\n" " not.pred %p117, %p1;\n" " @%p117 bra $L__BB14_114;\n" "\n" " mul.f32 %f373, %f90, %f231;\n" " st.shared.f32 [%r49+448], %f373;\n" "\n" "$L__BB14_114:\n" " bar.sync 0;\n" " @%p8 bra $L__BB14_132;\n" "\n" " ld.shared.f32 %f91, [%r27];\n" " mov.f32 %f676, %f91;\n" " @%p19 bra $L__BB14_123;\n" "\n" " and.b32 %r95, %r56, 3;\n" " add.s32 %r468, %r56, -1;\n" " setp.lt.u32 %p120, %r468, 3;\n" " mov.u32 %r682, 0;\n" " mov.f32 %f676, %f91;\n" " @%p120 bra $L__BB14_119;\n" "\n" " sub.s32 %r681, %r56, %r95;\n" " mov.u32 %r682, 0;\n" " mov.f32 %f676, %f91;\n" "\n" "$L__BB14_118:\n" " add.s32 %r470, %r36, %r682;\n" " shl.b32 %r471, %r470, 2;\n" " add.s32 %r473, %r160, %r471;\n" " ld.shared.f32 %f375, [%r473];\n" " max.f32 %f376, %f676, %f375;\n" " ld.shared.f32 %f377, [%r473+4];\n" " max.f32 %f378, %f376, %f377;\n" " ld.shared.f32 %f379, [%r473+8];\n" " max.f32 %f380, %f378, %f379;\n" " ld.shared.f32 %f381, [%r473+12];\n" " max.f32 %f676, %f380, %f381;\n" " add.s32 %r682, %r682, 4;\n" " add.s32 %r681, %r681, -4;\n" " setp.ne.s32 %p121, %r681, 0;\n" " @%p121 bra $L__BB14_118;\n" "\n" "$L__BB14_119:\n" " setp.eq.s32 %p122, %r95, 0;\n" " @%p122 bra $L__BB14_123;\n" "\n" " add.s32 %r474, %r36, %r682;\n" " shl.b32 %r475, %r474, 2;\n" " add.s32 %r102, %r160, %r475;\n" " ld.shared.f32 %f382, [%r102];\n" " max.f32 %f676, %f676, %f382;\n" " setp.eq.s32 %p123, %r95, 1;\n" " @%p123 bra $L__BB14_123;\n" "\n" " ld.shared.f32 %f383, [%r102+4];\n" " max.f32 %f676, %f676, %f383;\n" " setp.eq.s32 %p124, %r95, 2;\n" " @%p124 bra $L__BB14_123;\n" "\n" " ld.shared.f32 %f384, [%r102+8];\n" " max.f32 %f676, %f676, %f384;\n" "\n" "$L__BB14_123:\n" " mov.f32 %f680, 0f00000000;\n" " @%p19 bra $L__BB14_131;\n" "\n" " and.b32 %r103, %r56, 3;\n" " add.s32 %r478, %r56, -1;\n" " setp.lt.u32 %p126, %r478, 3;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r686, 0;\n" " @%p126 bra $L__BB14_127;\n" "\n" " sub.s32 %r684, %r56, %r103;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r686, 0;\n" " mov.u32 %r685, %r51;\n" "\n" "$L__BB14_126:\n" " shl.b32 %r574, %r124, 2;\n" " add.s32 %r573, %r574, 1024;\n" " add.s32 %r480, %r36, %r686;\n" " shl.b32 %r481, %r480, 2;\n" " add.s32 %r483, %r160, %r481;\n" " ld.shared.f32 %f389, [%r483];\n" " sub.f32 %f390, %f389, %f676;\n" " mov.f32 %f391, 0f3F000000;\n" " mov.f32 %f392, 0f3BBB989D;\n" " fma.rn.f32 %f393, %f390, %f392, %f391;\n" " mov.f32 %f394, 0f3FB8AA3B;\n" " mov.f32 %f395, 0f437C0000;\n" " cvt.sat.f32.f32 %f396, %f393;\n" " mov.f32 %f397, 0f4B400001;\n" " fma.rm.f32 %f398, %f396, %f395, %f397;\n" " add.f32 %f399, %f398, 0fCB40007F;\n" " neg.f32 %f400, %f399;\n" " fma.rn.f32 %f401, %f390, %f394, %f400;\n" " mov.f32 %f402, 0f32A57060;\n" " fma.rn.f32 %f403, %f390, %f402, %f401;\n" " mov.b32 %r484, %f398;\n" " shl.b32 %r485, %r484, 23;\n" " mov.b32 %f404, %r485;\n" " ex2.approx.ftz.f32 %f405, %f403;\n" " mul.f32 %f406, %f405, %f404;\n" " st.shared.f32 [%r685+1024], %f406;\n" " add.f32 %f407, %f680, %f406;\n" " ld.shared.f32 %f408, [%r483+4];\n" " sub.f32 %f409, %f408, %f676;\n" " fma.rn.f32 %f410, %f409, %f392, %f391;\n" " cvt.sat.f32.f32 %f411, %f410;\n" " fma.rm.f32 %f412, %f411, %f395, %f397;\n" " add.f32 %f413, %f412, 0fCB40007F;\n" " neg.f32 %f414, %f413;\n" " fma.rn.f32 %f415, %f409, %f394, %f414;\n" " fma.rn.f32 %f416, %f409, %f402, %f415;\n" " mov.b32 %r486, %f412;\n" " shl.b32 %r487, %r486, 23;\n" " mov.b32 %f417, %r487;\n" " ex2.approx.ftz.f32 %f418, %f416;\n" " mul.f32 %f419, %f418, %f417;\n" " add.s32 %r488, %r685, %r574;\n" " st.shared.f32 [%r488+1024], %f419;\n" " add.f32 %f420, %f407, %f419;\n" " ld.shared.f32 %f421, [%r483+8];\n" " sub.f32 %f422, %f421, %f676;\n" " fma.rn.f32 %f423, %f422, %f392, %f391;\n" " cvt.sat.f32.f32 %f424, %f423;\n" " fma.rm.f32 %f425, %f424, %f395, %f397;\n" " add.f32 %f426, %f425, 0fCB40007F;\n" " neg.f32 %f427, %f426;\n" " fma.rn.f32 %f428, %f422, %f394, %f427;\n" " fma.rn.f32 %f429, %f422, %f402, %f428;\n" " mov.b32 %r489, %f425;\n" " shl.b32 %r490, %r489, 23;\n" " mov.b32 %f430, %r490;\n" " ex2.approx.ftz.f32 %f431, %f429;\n" " mul.f32 %f432, %f431, %f430;\n" " add.s32 %r491, %r488, %r574;\n" " add.s32 %r492, %r488, %r573;\n" " st.shared.f32 [%r492], %f432;\n" " add.f32 %f433, %f420, %f432;\n" " ld.shared.f32 %f434, [%r483+12];\n" " sub.f32 %f435, %f434, %f676;\n" " fma.rn.f32 %f436, %f435, %f392, %f391;\n" " cvt.sat.f32.f32 %f437, %f436;\n" " fma.rm.f32 %f438, %f437, %f395, %f397;\n" " add.f32 %f439, %f438, 0fCB40007F;\n" " neg.f32 %f440, %f439;\n" " fma.rn.f32 %f441, %f435, %f394, %f440;\n" " fma.rn.f32 %f442, %f435, %f402, %f441;\n" " mov.b32 %r493, %f438;\n" " shl.b32 %r494, %r493, 23;\n" " mov.b32 %f443, %r494;\n" " ex2.approx.ftz.f32 %f444, %f442;\n" " mul.f32 %f445, %f444, %f443;\n" " add.s32 %r495, %r491, %r574;\n" " add.s32 %r496, %r491, %r573;\n" " add.s32 %r685, %r495, %r574;\n" " st.shared.f32 [%r496], %f445;\n" " add.f32 %f680, %f433, %f445;\n" " add.s32 %r686, %r686, 4;\n" " add.s32 %r684, %r684, -4;\n" " setp.ne.s32 %p127, %r684, 0;\n" " @%p127 bra $L__BB14_126;\n" "\n" "$L__BB14_127:\n" " setp.eq.s32 %p128, %r103, 0;\n" " @%p128 bra $L__BB14_131;\n" "\n" " add.s32 %r497, %r36, %r686;\n" " shl.b32 %r498, %r497, 2;\n" " add.s32 %r112, %r160, %r498;\n" " ld.shared.f32 %f446, [%r112];\n" " sub.f32 %f447, %f446, %f676;\n" " mov.f32 %f448, 0f3F000000;\n" " mov.f32 %f449, 0f3BBB989D;\n" " fma.rn.f32 %f450, %f447, %f449, %f448;\n" " mov.f32 %f451, 0f3FB8AA3B;\n" " mov.f32 %f452, 0f437C0000;\n" " cvt.sat.f32.f32 %f453, %f450;\n" " mov.f32 %f454, 0f4B400001;\n" " fma.rm.f32 %f455, %f453, %f452, %f454;\n" " add.f32 %f456, %f455, 0fCB40007F;\n" " neg.f32 %f457, %f456;\n" " fma.rn.f32 %f458, %f447, %f451, %f457;\n" " mov.f32 %f459, 0f32A57060;\n" " fma.rn.f32 %f460, %f447, %f459, %f458;\n" " mov.b32 %r500, %f455;\n" " shl.b32 %r501, %r500, 23;\n" " mov.b32 %f461, %r501;\n" " ex2.approx.ftz.f32 %f462, %f460;\n" " mul.f32 %f463, %f462, %f461;\n" " cvt.u64.u32 %rd120, %r686;\n" " mul.lo.s64 %rd121, %rd120, %rd13;\n" " add.s64 %rd122, %rd36, %rd121;\n" " cvt.u32.u64 %r502, %rd122;\n" " add.s32 %r503, %r9, %r502;\n" " shl.b32 %r504, %r503, 2;\n" " add.s32 %r505, %r160, %r504;\n" " st.shared.f32 [%r505], %f463;\n" " add.f32 %f680, %f680, %f463;\n" " setp.eq.s32 %p129, %r103, 1;\n" " @%p129 bra $L__BB14_131;\n" "\n" " mov.f32 %f611, 0f32A57060;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f609, 0f4B400001;\n" " mov.f32 %f608, 0f437C0000;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f606, 0f3BBB989D;\n" " add.s32 %r562, %r36, %r686;\n" " shl.b32 %r561, %r562, 2;\n" " add.s32 %r560, %r160, %r561;\n" " ld.shared.f32 %f464, [%r560+4];\n" " sub.f32 %f465, %f464, %f676;\n" " fma.rn.f32 %f468, %f465, %f606, %f607;\n" " cvt.sat.f32.f32 %f471, %f468;\n" " fma.rm.f32 %f473, %f471, %f608, %f609;\n" " add.f32 %f474, %f473, 0fCB40007F;\n" " neg.f32 %f475, %f474;\n" " fma.rn.f32 %f476, %f465, %f610, %f475;\n" " fma.rn.f32 %f478, %f465, %f611, %f476;\n" " mov.b32 %r506, %f473;\n" " shl.b32 %r507, %r506, 23;\n" " mov.b32 %f479, %r507;\n" " ex2.approx.ftz.f32 %f480, %f478;\n" " mul.f32 %f481, %f480, %f479;\n" " add.s32 %r508, %r686, 1;\n" " cvt.u64.u32 %rd123, %r508;\n" " mul.lo.s64 %rd124, %rd123, %rd13;\n" " add.s64 %rd125, %rd36, %rd124;\n" " cvt.u32.u64 %r509, %rd125;\n" " add.s32 %r510, %r9, %r509;\n" " shl.b32 %r511, %r510, 2;\n" " add.s32 %r513, %r160, %r511;\n" " st.shared.f32 [%r513], %f481;\n" " add.f32 %f680, %f680, %f481;\n" " setp.eq.s32 %p130, %r103, 2;\n" " @%p130 bra $L__BB14_131;\n" "\n" " mov.f32 %f617, 0f32A57060;\n" " mov.f32 %f616, 0f3FB8AA3B;\n" " mov.f32 %f615, 0f4B400001;\n" " mov.f32 %f614, 0f437C0000;\n" " mov.f32 %f613, 0f3F000000;\n" " mov.f32 %f612, 0f3BBB989D;\n" " add.s32 %r565, %r36, %r686;\n" " shl.b32 %r564, %r565, 2;\n" " add.s32 %r563, %r160, %r564;\n" " ld.shared.f32 %f482, [%r563+8];\n" " sub.f32 %f483, %f482, %f676;\n" " fma.rn.f32 %f486, %f483, %f612, %f613;\n" " cvt.sat.f32.f32 %f489, %f486;\n" " fma.rm.f32 %f491, %f489, %f614, %f615;\n" " add.f32 %f492, %f491, 0fCB40007F;\n" " neg.f32 %f493, %f492;\n" " fma.rn.f32 %f494, %f483, %f616, %f493;\n" " fma.rn.f32 %f496, %f483, %f617, %f494;\n" " mov.b32 %r514, %f491;\n" " shl.b32 %r515, %r514, 23;\n" " mov.b32 %f497, %r515;\n" " ex2.approx.ftz.f32 %f498, %f496;\n" " mul.f32 %f499, %f498, %f497;\n" " add.s32 %r516, %r686, 2;\n" " cvt.u64.u32 %rd126, %r516;\n" " mul.lo.s64 %rd127, %rd126, %rd13;\n" " add.s64 %rd128, %rd36, %rd127;\n" " cvt.u32.u64 %r517, %rd128;\n" " add.s32 %r518, %r9, %r517;\n" " shl.b32 %r519, %r518, 2;\n" " add.s32 %r521, %r160, %r519;\n" " st.shared.f32 [%r521], %f499;\n" " add.f32 %f680, %f680, %f499;\n" "\n" "$L__BB14_131:\n" " sub.f32 %f500, %f91, %f676;\n" " mov.f32 %f501, 0f3F000000;\n" " mov.f32 %f502, 0f3BBB989D;\n" " fma.rn.f32 %f503, %f500, %f502, %f501;\n" " mov.f32 %f504, 0f3FB8AA3B;\n" " mov.f32 %f505, 0f437C0000;\n" " cvt.sat.f32.f32 %f506, %f503;\n" " mov.f32 %f507, 0f4B400001;\n" " fma.rm.f32 %f508, %f506, %f505, %f507;\n" " add.f32 %f509, %f508, 0fCB40007F;\n" " neg.f32 %f510, %f509;\n" " fma.rn.f32 %f511, %f500, %f504, %f510;\n" " mov.f32 %f512, 0f32A57060;\n" " fma.rn.f32 %f513, %f500, %f512, %f511;\n" " ex2.approx.ftz.f32 %f514, %f513;\n" " mov.b32 %r522, %f508;\n" " shl.b32 %r523, %r522, 23;\n" " mov.b32 %f515, %r523;\n" " mul.f32 %f516, %f514, %f515;\n" " ld.shared.f32 %f517, [%r28];\n" " fma.rn.f32 %f518, %f516, %f517, %f680;\n" " st.shared.f32 [%r28], %f518;\n" " st.shared.f32 [%r27], %f676;\n" " st.shared.f32 [%r38], %f516;\n" "\n" "$L__BB14_132:\n" " bar.sync 0;\n" " @%p19 bra $L__BB14_212;\n" "\n" " @%p20 bra $L__BB14_172;\n" " bra.uni $L__BB14_134;\n" "\n" "$L__BB14_172:\n" " mov.u32 %r688, 0;\n" "\n" "$L__BB14_173:\n" " add.s32 %r117, %r688, %r669;\n" " setp.lt.s32 %p152, %r117, %r127;\n" " @%p152 bra $L__BB14_177;\n" " bra.uni $L__BB14_174;\n" "\n" "$L__BB14_177:\n" " add.s32 %r661, %r688, %r669;\n" " cvt.s64.s32 %rd154, %r661;\n" " mul.lo.s64 %rd155, %rd154, %rd17;\n" " add.s64 %rd156, %rd32, %rd155;\n" " shl.b64 %rd157, %rd156, 1;\n" " add.s64 %rd158, %rd4, %rd157;\n" " ld.global.u16 %rs10, [%rd158];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs10;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB14_178;\n" "\n" "$L__BB14_174:\n" " add.s32 %r658, %r688, %r669;\n" " setp.gt.s32 %p153, %r658, %r33;\n" " @%p153 bra $L__BB14_176;\n" " bra.uni $L__BB14_175;\n" "\n" "$L__BB14_176:\n" " add.s32 %r660, %r688, %r669;\n" " shr.s32 %r545, %r660, 31;\n" " shr.u32 %r546, %r545, 24;\n" " add.s32 %r547, %r660, %r546;\n" " and.b32 %r548, %r547, -256;\n" " sub.s32 %r549, %r660, %r548;\n" " cvt.s64.s32 %rd149, %r549;\n" " mul.lo.s64 %rd150, %rd149, %rd17;\n" " add.s64 %rd151, %rd33, %rd150;\n" " shl.b64 %rd152, %rd151, 1;\n" " add.s64 %rd153, %rd38, %rd152;\n" " ld.global.u16 %rs9, [%rd153];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs9;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB14_178;\n" "\n" "$L__BB14_175:\n" " add.s32 %r659, %r688, %r669;\n" " cvt.s64.s32 %rd142, %r659;\n" " mul.lo.s64 %rd143, %rd142, %rd27;\n" " add.s64 %rd144, %rd143, %rd28;\n" " add.s64 %rd145, %rd144, %rd34;\n" " add.s64 %rd146, %rd5, %rd145;\n" " ld.global.u8 %r538, [%rd146];\n" " ld.global.u8 %r539, [%rd146+1];\n" " prmt.b32 %r540, %r539, %r538, 30212;\n" " cvt.u16.u32 %rs8, %r540;\n" " // begin inline asm\n" " { cvt.f32.f16 %f554, %rs8;}\n" "\n" " // end inline asm\n" " add.s64 %rd147, %rd35, %rd144;\n" " add.s64 %rd148, %rd5, %rd147;\n" " ld.global.u8 %r541, [%rd148];\n" " shr.u32 %r542, %r541, %r37;\n" " and.b32 %r543, %r542, 15;\n" " add.s32 %r544, %r543, -8;\n" " cvt.rn.f32.s32 %f555, %r544;\n" " mul.f32 %f720, %f554, %f555;\n" "\n" "$L__BB14_178:\n" " cvt.u64.u32 %rd159, %r688;\n" " mul.lo.s64 %rd160, %rd159, %rd13;\n" " add.s64 %rd161, %rd160, %rd11;\n" " cvt.u32.u64 %r550, %rd161;\n" " add.s32 %r551, %r9, %r550;\n" " shl.b32 %r552, %r551, 2;\n" " add.s32 %r118, %r160, %r552;\n" " setp.eq.s32 %p154, %r688, 0;\n" " @%p154 bra $L__BB14_195;\n" "\n" " @%p3 bra $L__BB14_181;\n" "\n" " ld.shared.f32 %f558, [%r118];\n" " fma.rn.f32 %f711, %f720, %f558, %f711;\n" "\n" "$L__BB14_181:\n" " @%p76 bra $L__BB14_183;\n" "\n" " ld.shared.f32 %f559, [%r118+4];\n" " fma.rn.f32 %f710, %f720, %f559, %f710;\n" "\n" "$L__BB14_183:\n" " @%p82 bra $L__BB14_185;\n" "\n" " ld.shared.f32 %f560, [%r118+8];\n" " fma.rn.f32 %f709, %f720, %f560, %f709;\n" "\n" "$L__BB14_185:\n" " @%p88 bra $L__BB14_187;\n" "\n" " ld.shared.f32 %f561, [%r118+12];\n" " fma.rn.f32 %f708, %f720, %f561, %f708;\n" "\n" "$L__BB14_187:\n" " @%p94 bra $L__BB14_189;\n" "\n" " ld.shared.f32 %f562, [%r118+16];\n" " fma.rn.f32 %f707, %f720, %f562, %f707;\n" "\n" "$L__BB14_189:\n" " @%p100 bra $L__BB14_191;\n" "\n" " ld.shared.f32 %f563, [%r118+20];\n" " fma.rn.f32 %f706, %f720, %f563, %f706;\n" "\n" "$L__BB14_191:\n" " @%p106 bra $L__BB14_193;\n" "\n" " ld.shared.f32 %f564, [%r118+24];\n" " fma.rn.f32 %f705, %f720, %f564, %f705;\n" "\n" "$L__BB14_193:\n" " @%p112 bra $L__BB14_211;\n" "\n" " ld.shared.f32 %f565, [%r118+28];\n" " fma.rn.f32 %f704, %f720, %f565, %f704;\n" " bra.uni $L__BB14_211;\n" "\n" "$L__BB14_195:\n" " @%p3 bra $L__BB14_197;\n" "\n" " ld.shared.f32 %f566, [%r50];\n" " ld.shared.f32 %f567, [%r118];\n" " mul.f32 %f568, %f720, %f567;\n" " fma.rn.f32 %f711, %f566, %f711, %f568;\n" "\n" "$L__BB14_197:\n" " @%p76 bra $L__BB14_199;\n" "\n" " ld.shared.f32 %f569, [%r50+4];\n" " ld.shared.f32 %f570, [%r118+4];\n" " mul.f32 %f571, %f720, %f570;\n" " fma.rn.f32 %f710, %f569, %f710, %f571;\n" "\n" "$L__BB14_199:\n" " @%p82 bra $L__BB14_201;\n" "\n" " ld.shared.f32 %f572, [%r50+8];\n" " ld.shared.f32 %f573, [%r118+8];\n" " mul.f32 %f574, %f720, %f573;\n" " fma.rn.f32 %f709, %f572, %f709, %f574;\n" "\n" "$L__BB14_201:\n" " @%p88 bra $L__BB14_203;\n" "\n" " ld.shared.f32 %f575, [%r50+12];\n" " ld.shared.f32 %f576, [%r118+12];\n" " mul.f32 %f577, %f720, %f576;\n" " fma.rn.f32 %f708, %f575, %f708, %f577;\n" "\n" "$L__BB14_203:\n" " @%p94 bra $L__BB14_205;\n" "\n" " ld.shared.f32 %f578, [%r50+16];\n" " ld.shared.f32 %f579, [%r118+16];\n" " mul.f32 %f580, %f720, %f579;\n" " fma.rn.f32 %f707, %f578, %f707, %f580;\n" "\n" "$L__BB14_205:\n" " @%p100 bra $L__BB14_207;\n" "\n" " ld.shared.f32 %f581, [%r50+20];\n" " ld.shared.f32 %f582, [%r118+20];\n" " mul.f32 %f583, %f720, %f582;\n" " fma.rn.f32 %f706, %f581, %f706, %f583;\n" "\n" "$L__BB14_207:\n" " @%p106 bra $L__BB14_209;\n" "\n" " ld.shared.f32 %f584, [%r50+24];\n" " ld.shared.f32 %f585, [%r118+24];\n" " mul.f32 %f586, %f720, %f585;\n" " fma.rn.f32 %f705, %f584, %f705, %f586;\n" "\n" "$L__BB14_209:\n" " @%p112 bra $L__BB14_211;\n" "\n" " ld.shared.f32 %f587, [%r50+28];\n" " ld.shared.f32 %f588, [%r118+28];\n" " mul.f32 %f589, %f720, %f588;\n" " fma.rn.f32 %f704, %f587, %f704, %f589;\n" "\n" "$L__BB14_211:\n" " add.s32 %r688, %r688, 1;\n" " setp.lt.s32 %p171, %r688, %r58;\n" " @%p171 bra $L__BB14_173;\n" " bra.uni $L__BB14_212;\n" "\n" "$L__BB14_134:\n" " mov.u32 %r687, 0;\n" "\n" "$L__BB14_135:\n" " add.s32 %r525, %r687, %r669;\n" " setp.lt.s32 %p133, %r525, %r127;\n" " cvt.s64.s32 %rd47, %r525;\n" " @%p133 bra $L__BB14_137;\n" " bra.uni $L__BB14_136;\n" "\n" "$L__BB14_137:\n" " mul.lo.s64 %rd135, %rd47, %rd17;\n" " add.s64 %rd136, %rd32, %rd135;\n" " shl.b64 %rd137, %rd136, 1;\n" " add.s64 %rd138, %rd4, %rd137;\n" " ld.global.u16 %rs7, [%rd138];\n" " // begin inline asm\n" " { cvt.f32.f16 %f689, %rs7;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB14_138;\n" "\n" "$L__BB14_136:\n" " mul.lo.s64 %rd129, %rd47, %rd27;\n" " add.s64 %rd130, %rd129, %rd28;\n" " add.s64 %rd131, %rd130, %rd34;\n" " add.s64 %rd132, %rd5, %rd131;\n" " ld.global.u8 %r526, [%rd132];\n" " ld.global.u8 %r527, [%rd132+1];\n" " prmt.b32 %r528, %r527, %r526, 30212;\n" " cvt.u16.u32 %rs6, %r528;\n" " // begin inline asm\n" " { cvt.f32.f16 %f519, %rs6;}\n" "\n" " // end inline asm\n" " add.s64 %rd133, %rd35, %rd130;\n" " add.s64 %rd134, %rd5, %rd133;\n" " ld.global.u8 %r529, [%rd134];\n" " shr.u32 %r530, %r529, %r37;\n" " and.b32 %r531, %r530, 15;\n" " add.s32 %r532, %r531, -8;\n" " cvt.rn.f32.s32 %f520, %r532;\n" " mul.f32 %f689, %f519, %f520;\n" "\n" "$L__BB14_138:\n" " cvt.u64.u32 %rd139, %r687;\n" " mul.lo.s64 %rd140, %rd139, %rd13;\n" " add.s64 %rd141, %rd140, %rd11;\n" " cvt.u32.u64 %r533, %rd141;\n" " add.s32 %r534, %r9, %r533;\n" " shl.b32 %r535, %r534, 2;\n" " add.s32 %r114, %r160, %r535;\n" " setp.eq.s32 %p134, %r687, 0;\n" " @%p134 bra $L__BB14_155;\n" "\n" " @%p3 bra $L__BB14_141;\n" "\n" " ld.shared.f32 %f522, [%r114];\n" " fma.rn.f32 %f711, %f689, %f522, %f711;\n" "\n" "$L__BB14_141:\n" " @%p76 bra $L__BB14_143;\n" "\n" " ld.shared.f32 %f523, [%r114+4];\n" " fma.rn.f32 %f710, %f689, %f523, %f710;\n" "\n" "$L__BB14_143:\n" " @%p82 bra $L__BB14_145;\n" "\n" " ld.shared.f32 %f524, [%r114+8];\n" " fma.rn.f32 %f709, %f689, %f524, %f709;\n" "\n" "$L__BB14_145:\n" " @%p88 bra $L__BB14_147;\n" "\n" " ld.shared.f32 %f525, [%r114+12];\n" " fma.rn.f32 %f708, %f689, %f525, %f708;\n" "\n" "$L__BB14_147:\n" " @%p94 bra $L__BB14_149;\n" "\n" " ld.shared.f32 %f526, [%r114+16];\n" " fma.rn.f32 %f707, %f689, %f526, %f707;\n" "\n" "$L__BB14_149:\n" " @%p100 bra $L__BB14_151;\n" "\n" " ld.shared.f32 %f527, [%r114+20];\n" " fma.rn.f32 %f706, %f689, %f527, %f706;\n" "\n" "$L__BB14_151:\n" " @%p106 bra $L__BB14_153;\n" "\n" " ld.shared.f32 %f528, [%r114+24];\n" " fma.rn.f32 %f705, %f689, %f528, %f705;\n" "\n" "$L__BB14_153:\n" " @%p112 bra $L__BB14_171;\n" "\n" " ld.shared.f32 %f529, [%r114+28];\n" " fma.rn.f32 %f704, %f689, %f529, %f704;\n" " bra.uni $L__BB14_171;\n" "\n" "$L__BB14_155:\n" " @%p3 bra $L__BB14_157;\n" "\n" " ld.shared.f32 %f530, [%r50];\n" " ld.shared.f32 %f531, [%r114];\n" " mul.f32 %f532, %f689, %f531;\n" " fma.rn.f32 %f711, %f530, %f711, %f532;\n" "\n" "$L__BB14_157:\n" " @%p76 bra $L__BB14_159;\n" "\n" " ld.shared.f32 %f533, [%r50+4];\n" " ld.shared.f32 %f534, [%r114+4];\n" " mul.f32 %f535, %f689, %f534;\n" " fma.rn.f32 %f710, %f533, %f710, %f535;\n" "\n" "$L__BB14_159:\n" " @%p82 bra $L__BB14_161;\n" "\n" " ld.shared.f32 %f536, [%r50+8];\n" " ld.shared.f32 %f537, [%r114+8];\n" " mul.f32 %f538, %f689, %f537;\n" " fma.rn.f32 %f709, %f536, %f709, %f538;\n" "\n" "$L__BB14_161:\n" " @%p88 bra $L__BB14_163;\n" "\n" " ld.shared.f32 %f539, [%r50+12];\n" " ld.shared.f32 %f540, [%r114+12];\n" " mul.f32 %f541, %f689, %f540;\n" " fma.rn.f32 %f708, %f539, %f708, %f541;\n" "\n" "$L__BB14_163:\n" " @%p94 bra $L__BB14_165;\n" "\n" " ld.shared.f32 %f542, [%r50+16];\n" " ld.shared.f32 %f543, [%r114+16];\n" " mul.f32 %f544, %f689, %f543;\n" " fma.rn.f32 %f707, %f542, %f707, %f544;\n" "\n" "$L__BB14_165:\n" " @%p100 bra $L__BB14_167;\n" "\n" " ld.shared.f32 %f545, [%r50+20];\n" " ld.shared.f32 %f546, [%r114+20];\n" " mul.f32 %f547, %f689, %f546;\n" " fma.rn.f32 %f706, %f545, %f706, %f547;\n" "\n" "$L__BB14_167:\n" " @%p106 bra $L__BB14_169;\n" "\n" " ld.shared.f32 %f548, [%r50+24];\n" " ld.shared.f32 %f549, [%r114+24];\n" " mul.f32 %f550, %f689, %f549;\n" " fma.rn.f32 %f705, %f548, %f705, %f550;\n" "\n" "$L__BB14_169:\n" " @%p112 bra $L__BB14_171;\n" "\n" " ld.shared.f32 %f551, [%r50+28];\n" " ld.shared.f32 %f552, [%r114+28];\n" " mul.f32 %f553, %f689, %f552;\n" " fma.rn.f32 %f704, %f551, %f704, %f553;\n" "\n" "$L__BB14_171:\n" " add.s32 %r687, %r687, 1;\n" " setp.lt.s32 %p151, %r687, %r58;\n" " @%p151 bra $L__BB14_135;\n" "\n" "$L__BB14_212:\n" " bar.sync 0;\n" "\n" "$L__BB14_213:\n" " add.s32 %r669, %r669, 16;\n" " setp.gt.s32 %p172, %r8, %r669;\n" " add.s32 %r668, %r668, 1;\n" " @%p172 bra $L__BB14_12;\n" "\n" "$L__BB14_214:\n" " shl.b32 %r570, %r124, 4;\n" " cvt.s64.s32 %rd237, %r570;\n" " add.s32 %r569, %r125, 16;\n" " mul.wide.s32 %rd236, %r569, 16;\n" " add.s64 %rd235, %rd236, %rd237;\n" " add.s64 %rd234, %rd235, %rd237;\n" " cvt.u32.u64 %r568, %rd234;\n" " add.s64 %rd233, %rd234, %rd13;\n" " cvt.u32.u64 %r567, %rd233;\n" " mov.u32 %r566, %ctaid.x;\n" " cvt.s64.s32 %rd48, %r4;\n" " cvt.s64.s32 %rd49, %r566;\n" " add.s32 %r555, %r9, %r568;\n" " shl.b32 %r556, %r555, 2;\n" " add.s32 %r122, %r160, %r556;\n" " add.s32 %r558, %r9, %r567;\n" " shl.b32 %r559, %r558, 2;\n" " add.s32 %r123, %r160, %r559;\n" " @%p3 bra $L__BB14_217;\n" "\n" " setp.ne.s32 %p174, %r6, 0;\n" " mul.lo.s64 %rd162, %rd16, %rd48;\n" " add.s64 %rd50, %rd162, %rd49;\n" " mul.lo.s64 %rd163, %rd50, %rd17;\n" " add.s64 %rd164, %rd163, %rd31;\n" " shl.b64 %rd165, %rd164, 2;\n" " add.s64 %rd166, %rd8, %rd165;\n" " st.global.f32 [%rd166], %f711;\n" " @%p174 bra $L__BB14_217;\n" "\n" " ld.shared.f32 %f590, [%r122];\n" " shl.b64 %rd167, %rd50, 2;\n" " add.s64 %rd168, %rd7, %rd167;\n" " st.global.f32 [%rd168], %f590;\n" " ld.shared.f32 %f591, [%r123];\n" " add.s64 %rd169, %rd6, %rd167;\n" " st.global.f32 [%rd169], %f591;\n" "\n" "$L__BB14_217:\n" " setp.lt.s32 %p175, %r124, 2;\n" " @%p175 bra $L__BB14_220;\n" "\n" " setp.ne.s32 %p176, %r6, 0;\n" " add.s64 %rd170, %rd16, 1;\n" " mul.lo.s64 %rd171, %rd170, %rd48;\n" " add.s64 %rd51, %rd171, %rd49;\n" " mul.lo.s64 %rd172, %rd51, %rd17;\n" " add.s64 %rd173, %rd172, %rd31;\n" " shl.b64 %rd174, %rd173, 2;\n" " add.s64 %rd175, %rd8, %rd174;\n" " st.global.f32 [%rd175], %f710;\n" " @%p176 bra $L__BB14_220;\n" "\n" " ld.shared.f32 %f592, [%r122+4];\n" " shl.b64 %rd176, %rd51, 2;\n" " add.s64 %rd177, %rd7, %rd176;\n" " st.global.f32 [%rd177], %f592;\n" " ld.shared.f32 %f593, [%r123+4];\n" " add.s64 %rd178, %rd6, %rd176;\n" " st.global.f32 [%rd178], %f593;\n" "\n" "$L__BB14_220:\n" " setp.lt.s32 %p177, %r124, 3;\n" " @%p177 bra $L__BB14_223;\n" "\n" " setp.ne.s32 %p178, %r6, 0;\n" " add.s64 %rd179, %rd16, 2;\n" " mul.lo.s64 %rd180, %rd179, %rd48;\n" " add.s64 %rd52, %rd180, %rd49;\n" " mul.lo.s64 %rd181, %rd52, %rd17;\n" " add.s64 %rd182, %rd181, %rd31;\n" " shl.b64 %rd183, %rd182, 2;\n" " add.s64 %rd184, %rd8, %rd183;\n" " st.global.f32 [%rd184], %f709;\n" " @%p178 bra $L__BB14_223;\n" "\n" " ld.shared.f32 %f594, [%r122+8];\n" " shl.b64 %rd185, %rd52, 2;\n" " add.s64 %rd186, %rd7, %rd185;\n" " st.global.f32 [%rd186], %f594;\n" " ld.shared.f32 %f595, [%r123+8];\n" " add.s64 %rd187, %rd6, %rd185;\n" " st.global.f32 [%rd187], %f595;\n" "\n" "$L__BB14_223:\n" " setp.lt.s32 %p179, %r124, 4;\n" " @%p179 bra $L__BB14_226;\n" "\n" " setp.ne.s32 %p180, %r6, 0;\n" " add.s64 %rd188, %rd16, 3;\n" " mul.lo.s64 %rd189, %rd188, %rd48;\n" " add.s64 %rd53, %rd189, %rd49;\n" " mul.lo.s64 %rd190, %rd53, %rd17;\n" " add.s64 %rd191, %rd190, %rd31;\n" " shl.b64 %rd192, %rd191, 2;\n" " add.s64 %rd193, %rd8, %rd192;\n" " st.global.f32 [%rd193], %f708;\n" " @%p180 bra $L__BB14_226;\n" "\n" " ld.shared.f32 %f596, [%r122+12];\n" " shl.b64 %rd194, %rd53, 2;\n" " add.s64 %rd195, %rd7, %rd194;\n" " st.global.f32 [%rd195], %f596;\n" " ld.shared.f32 %f597, [%r123+12];\n" " add.s64 %rd196, %rd6, %rd194;\n" " st.global.f32 [%rd196], %f597;\n" "\n" "$L__BB14_226:\n" " setp.lt.s32 %p181, %r124, 5;\n" " @%p181 bra $L__BB14_229;\n" "\n" " setp.ne.s32 %p182, %r6, 0;\n" " add.s64 %rd197, %rd16, 4;\n" " mul.lo.s64 %rd198, %rd197, %rd48;\n" " add.s64 %rd54, %rd198, %rd49;\n" " mul.lo.s64 %rd199, %rd54, %rd17;\n" " add.s64 %rd200, %rd199, %rd31;\n" " shl.b64 %rd201, %rd200, 2;\n" " add.s64 %rd202, %rd8, %rd201;\n" " st.global.f32 [%rd202], %f707;\n" " @%p182 bra $L__BB14_229;\n" "\n" " ld.shared.f32 %f598, [%r122+16];\n" " shl.b64 %rd203, %rd54, 2;\n" " add.s64 %rd204, %rd7, %rd203;\n" " st.global.f32 [%rd204], %f598;\n" " ld.shared.f32 %f599, [%r123+16];\n" " add.s64 %rd205, %rd6, %rd203;\n" " st.global.f32 [%rd205], %f599;\n" "\n" "$L__BB14_229:\n" " setp.lt.s32 %p183, %r124, 6;\n" " @%p183 bra $L__BB14_232;\n" "\n" " setp.ne.s32 %p184, %r6, 0;\n" " add.s64 %rd206, %rd16, 5;\n" " mul.lo.s64 %rd207, %rd206, %rd48;\n" " add.s64 %rd55, %rd207, %rd49;\n" " mul.lo.s64 %rd208, %rd55, %rd17;\n" " add.s64 %rd209, %rd208, %rd31;\n" " shl.b64 %rd210, %rd209, 2;\n" " add.s64 %rd211, %rd8, %rd210;\n" " st.global.f32 [%rd211], %f706;\n" " @%p184 bra $L__BB14_232;\n" "\n" " ld.shared.f32 %f600, [%r122+20];\n" " shl.b64 %rd212, %rd55, 2;\n" " add.s64 %rd213, %rd7, %rd212;\n" " st.global.f32 [%rd213], %f600;\n" " ld.shared.f32 %f601, [%r123+20];\n" " add.s64 %rd214, %rd6, %rd212;\n" " st.global.f32 [%rd214], %f601;\n" "\n" "$L__BB14_232:\n" " setp.lt.s32 %p185, %r124, 7;\n" " @%p185 bra $L__BB14_235;\n" "\n" " setp.ne.s32 %p186, %r6, 0;\n" " add.s64 %rd215, %rd16, 6;\n" " mul.lo.s64 %rd216, %rd215, %rd48;\n" " add.s64 %rd56, %rd216, %rd49;\n" " mul.lo.s64 %rd217, %rd56, %rd17;\n" " add.s64 %rd218, %rd217, %rd31;\n" " shl.b64 %rd219, %rd218, 2;\n" " add.s64 %rd220, %rd8, %rd219;\n" " st.global.f32 [%rd220], %f705;\n" " @%p186 bra $L__BB14_235;\n" "\n" " ld.shared.f32 %f602, [%r122+24];\n" " shl.b64 %rd221, %rd56, 2;\n" " add.s64 %rd222, %rd7, %rd221;\n" " st.global.f32 [%rd222], %f602;\n" " ld.shared.f32 %f603, [%r123+24];\n" " add.s64 %rd223, %rd6, %rd221;\n" " st.global.f32 [%rd223], %f603;\n" "\n" "$L__BB14_235:\n" " setp.lt.s32 %p187, %r124, 8;\n" " @%p187 bra $L__BB14_238;\n" "\n" " setp.ne.s32 %p188, %r6, 0;\n" " add.s64 %rd224, %rd16, 7;\n" " mul.lo.s64 %rd225, %rd224, %rd48;\n" " add.s64 %rd57, %rd225, %rd49;\n" " mul.lo.s64 %rd226, %rd57, %rd17;\n" " add.s64 %rd227, %rd226, %rd31;\n" " shl.b64 %rd228, %rd227, 2;\n" " add.s64 %rd229, %rd8, %rd228;\n" " st.global.f32 [%rd229], %f704;\n" " @%p188 bra $L__BB14_238;\n" "\n" " ld.shared.f32 %f604, [%r122+28];\n" " shl.b64 %rd230, %rd57, 2;\n" " add.s64 %rd231, %rd7, %rd230;\n" " st.global.f32 [%rd231], %f604;\n" " ld.shared.f32 %f605, [%r123+28];\n" " add.s64 %rd232, %rd6, %rd230;\n" " st.global.f32 [%rd232], %f605;\n" "\n" "$L__BB14_238:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_kv_append_dev\n" ".visible .entry fc_attn_kv_append_dev(\n" " .param .u64 fc_attn_kv_append_dev_param_0,\n" " .param .u64 fc_attn_kv_append_dev_param_1,\n" " .param .u64 fc_attn_kv_append_dev_param_2,\n" " .param .u64 fc_attn_kv_append_dev_param_3,\n" " .param .u32 fc_attn_kv_append_dev_param_4,\n" " .param .u32 fc_attn_kv_append_dev_param_5,\n" " .param .u32 fc_attn_kv_append_dev_param_6,\n" " .param .u64 fc_attn_kv_append_dev_param_7\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<3>;\n" " .reg .b32 %r<10>;\n" " .reg .b64 %rd<29>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_attn_kv_append_dev_param_0];\n" " ld.param.u64 %rd2, [fc_attn_kv_append_dev_param_1];\n" " ld.param.u64 %rd3, [fc_attn_kv_append_dev_param_2];\n" " ld.param.u64 %rd4, [fc_attn_kv_append_dev_param_3];\n" " ld.param.u32 %r2, [fc_attn_kv_append_dev_param_4];\n" " ld.param.u32 %r3, [fc_attn_kv_append_dev_param_5];\n" " ld.param.u32 %r4, [fc_attn_kv_append_dev_param_6];\n" " ld.param.u64 %rd5, [fc_attn_kv_append_dev_param_7];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r3;\n" " @%p1 bra $L__BB15_2;\n" "\n" " mov.u32 %r5, %ctaid.x;\n" " mov.u32 %r6, %ctaid.y;\n" " cvta.to.global.u64 %rd6, %rd5;\n" " ld.global.u32 %r7, [%rd6];\n" " add.s32 %r8, %r7, %r5;\n" " rem.s32 %r9, %r8, %r4;\n" " mul.wide.s32 %rd7, %r2, %r5;\n" " cvt.s64.s32 %rd8, %r6;\n" " add.s64 %rd9, %rd7, %rd8;\n" " cvt.s64.s32 %rd10, %r3;\n" " mul.lo.s64 %rd11, %rd9, %rd10;\n" " cvt.s64.s32 %rd12, %r1;\n" " add.s64 %rd13, %rd11, %rd12;\n" " mul.wide.s32 %rd14, %r4, %r6;\n" " cvt.s64.s32 %rd15, %r9;\n" " add.s64 %rd16, %rd14, %rd15;\n" " mul.lo.s64 %rd17, %rd16, %rd10;\n" " add.s64 %rd18, %rd17, %rd12;\n" " cvta.to.global.u64 %rd19, %rd1;\n" " shl.b64 %rd20, %rd13, 2;\n" " add.s64 %rd21, %rd19, %rd20;\n" " ld.global.f32 %f1, [%rd21];\n" " cvta.to.global.u64 %rd22, %rd3;\n" " shl.b64 %rd23, %rd18, 2;\n" " add.s64 %rd24, %rd22, %rd23;\n" " st.global.f32 [%rd24], %f1;\n" " cvta.to.global.u64 %rd25, %rd2;\n" " add.s64 %rd26, %rd25, %rd20;\n" " ld.global.f32 %f2, [%rd26];\n" " cvta.to.global.u64 %rd27, %rd4;\n" " add.s64 %rd28, %rd27, %rd23;\n" " st.global.f32 [%rd28], %f2;\n" "\n" "$L__BB15_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_kv_append_slots\n" ".visible .entry fc_attn_kv_append_slots(\n" " .param .u64 fc_attn_kv_append_slots_param_0,\n" " .param .u64 fc_attn_kv_append_slots_param_1,\n" " .param .u64 fc_attn_kv_append_slots_param_2,\n" " .param .u64 fc_attn_kv_append_slots_param_3,\n" " .param .u32 fc_attn_kv_append_slots_param_4,\n" " .param .u32 fc_attn_kv_append_slots_param_5,\n" " .param .u32 fc_attn_kv_append_slots_param_6,\n" " .param .u64 fc_attn_kv_append_slots_param_7\n" ")\n" "{\n" " .reg .pred %p<2>;\n" " .reg .f32 %f<3>;\n" " .reg .b32 %r<9>;\n" " .reg .b64 %rd<32>;\n" "\n" "\n" " ld.param.u64 %rd1, [fc_attn_kv_append_slots_param_0];\n" " ld.param.u64 %rd2, [fc_attn_kv_append_slots_param_1];\n" " ld.param.u64 %rd3, [fc_attn_kv_append_slots_param_2];\n" " ld.param.u64 %rd4, [fc_attn_kv_append_slots_param_3];\n" " ld.param.u32 %r2, [fc_attn_kv_append_slots_param_4];\n" " ld.param.u32 %r3, [fc_attn_kv_append_slots_param_5];\n" " ld.param.u32 %r4, [fc_attn_kv_append_slots_param_6];\n" " ld.param.u64 %rd5, [fc_attn_kv_append_slots_param_7];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r3;\n" " @%p1 bra $L__BB16_2;\n" "\n" " mov.u32 %r5, %ctaid.y;\n" " mov.u32 %r6, %ctaid.x;\n" " cvta.to.global.u64 %rd6, %rd5;\n" " mul.wide.s32 %rd7, %r6, 4;\n" " add.s64 %rd8, %rd6, %rd7;\n" " ld.global.u32 %r7, [%rd8];\n" " rem.s32 %r8, %r7, %r4;\n" " mul.wide.s32 %rd9, %r2, %r6;\n" " cvt.s64.s32 %rd10, %r5;\n" " add.s64 %rd11, %rd9, %rd10;\n" " cvt.s64.s32 %rd12, %r3;\n" " mul.lo.s64 %rd13, %rd11, %rd12;\n" " cvt.s64.s32 %rd14, %r1;\n" " add.s64 %rd15, %rd13, %rd14;\n" " cvt.s64.s32 %rd16, %r4;\n" " mul.lo.s64 %rd17, %rd11, %rd16;\n" " cvt.s64.s32 %rd18, %r8;\n" " add.s64 %rd19, %rd17, %rd18;\n" " mul.lo.s64 %rd20, %rd19, %rd12;\n" " add.s64 %rd21, %rd20, %rd14;\n" " cvta.to.global.u64 %rd22, %rd1;\n" " shl.b64 %rd23, %rd15, 2;\n" " add.s64 %rd24, %rd22, %rd23;\n" " ld.global.f32 %f1, [%rd24];\n" " cvta.to.global.u64 %rd25, %rd3;\n" " shl.b64 %rd26, %rd21, 2;\n" " add.s64 %rd27, %rd25, %rd26;\n" " st.global.f32 [%rd27], %f1;\n" " cvta.to.global.u64 %rd28, %rd2;\n" " add.s64 %rd29, %rd28, %rd23;\n" " ld.global.f32 %f2, [%rd29];\n" " cvta.to.global.u64 %rd30, %rd4;\n" " add.s64 %rd31, %rd30, %rd26;\n" " st.global.f32 [%rd31], %f2;\n" "\n" "$L__BB16_2:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_kv_append_slots_q4\n" ".visible .entry fc_attn_kv_append_slots_q4(\n" " .param .u64 fc_attn_kv_append_slots_q4_param_0,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_1,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_2,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_3,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_4,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_5,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_6,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_7,\n" " .param .u32 fc_attn_kv_append_slots_q4_param_8,\n" " .param .u32 fc_attn_kv_append_slots_q4_param_9,\n" " .param .u32 fc_attn_kv_append_slots_q4_param_10,\n" " .param .u64 fc_attn_kv_append_slots_q4_param_11,\n" " .param .u32 fc_attn_kv_append_slots_q4_param_12,\n" " .param .u32 fc_attn_kv_append_slots_q4_param_13\n" ")\n" "{\n" " .reg .pred %p<10>;\n" " .reg .b16 %rs<7>;\n" " .reg .f32 %f<223>;\n" " .reg .b32 %r<64>;\n" " .reg .b64 %rd<60>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ26fc_attn_kv_append_slots_q4E3s_k[4096];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ26fc_attn_kv_append_slots_q4E3s_v[4096];\n" "\n" " ld.param.u64 %rd8, [fc_attn_kv_append_slots_q4_param_0];\n" " ld.param.u64 %rd9, [fc_attn_kv_append_slots_q4_param_1];\n" " ld.param.u64 %rd10, [fc_attn_kv_append_slots_q4_param_2];\n" " ld.param.u64 %rd11, [fc_attn_kv_append_slots_q4_param_3];\n" " ld.param.u64 %rd12, [fc_attn_kv_append_slots_q4_param_4];\n" " ld.param.u64 %rd13, [fc_attn_kv_append_slots_q4_param_5];\n" " ld.param.u64 %rd14, [fc_attn_kv_append_slots_q4_param_6];\n" " ld.param.u64 %rd15, [fc_attn_kv_append_slots_q4_param_7];\n" " ld.param.u32 %r6, [fc_attn_kv_append_slots_q4_param_8];\n" " ld.param.u32 %r7, [fc_attn_kv_append_slots_q4_param_9];\n" " ld.param.u32 %r8, [fc_attn_kv_append_slots_q4_param_10];\n" " ld.param.u64 %rd16, [fc_attn_kv_append_slots_q4_param_11];\n" " ld.param.u32 %r9, [fc_attn_kv_append_slots_q4_param_12];\n" " ld.param.u32 %r10, [fc_attn_kv_append_slots_q4_param_13];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r7;\n" " @%p1 bra $L__BB17_17;\n" "\n" " mov.u32 %r11, %ctaid.y;\n" " mov.u32 %r12, %ctaid.x;\n" " cvta.to.global.u64 %rd17, %rd16;\n" " mul.wide.s32 %rd18, %r12, 4;\n" " add.s64 %rd19, %rd17, %rd18;\n" " shr.s32 %r13, %r7, 31;\n" " shr.u32 %r14, %r13, 27;\n" " add.s32 %r15, %r7, %r14;\n" " shr.s32 %r16, %r15, 5;\n" " mul.lo.s32 %r17, %r16, 18;\n" " mul.wide.s32 %rd20, %r6, %r12;\n" " cvt.s64.s32 %rd21, %r11;\n" " add.s64 %rd1, %rd20, %rd21;\n" " cvt.s64.s32 %rd2, %r7;\n" " mul.lo.s64 %rd22, %rd1, %rd2;\n" " cvt.s64.s32 %rd3, %r1;\n" " add.s64 %rd23, %rd22, %rd3;\n" " cvt.s64.s32 %rd24, %r8;\n" " mul.lo.s64 %rd25, %rd1, %rd24;\n" " ld.global.s32 %rd4, [%rd19];\n" " add.s64 %rd26, %rd25, %rd4;\n" " cvt.s64.s32 %rd27, %r17;\n" " mul.lo.s64 %rd28, %rd26, %rd27;\n" " cvta.to.global.u64 %rd29, %rd8;\n" " shl.b64 %rd30, %rd23, 2;\n" " add.s64 %rd31, %rd29, %rd30;\n" " ld.global.f32 %f1, [%rd31];\n" " cvta.to.global.u64 %rd32, %rd9;\n" " add.s64 %rd33, %rd32, %rd30;\n" " ld.global.f32 %f2, [%rd33];\n" " shl.b32 %r18, %r1, 2;\n" " mov.u32 %r19, _ZZ26fc_attn_kv_append_slots_q4E3s_k;\n" " add.s32 %r20, %r19, %r18;\n" " st.shared.f32 [%r20], %f1;\n" " mov.u32 %r21, _ZZ26fc_attn_kv_append_slots_q4E3s_v;\n" " add.s32 %r22, %r21, %r18;\n" " st.shared.f32 [%r22], %f2;\n" " bar.sync 0;\n" " shr.s32 %r23, %r1, 31;\n" " shr.u32 %r24, %r23, 27;\n" " add.s32 %r25, %r1, %r24;\n" " shr.s32 %r26, %r25, 5;\n" " and.b32 %r2, %r25, -32;\n" " mul.lo.s32 %r27, %r26, 18;\n" " cvt.s64.s32 %rd34, %r27;\n" " add.s64 %rd5, %rd28, %rd34;\n" " shl.b32 %r28, %r26, 7;\n" " add.s32 %r3, %r19, %r28;\n" " ld.shared.f32 %f10, [%r3];\n" " abs.f32 %f11, %f10;\n" " mov.f32 %f221, 0f00000000;\n" " max.f32 %f12, %f221, %f11;\n" " ld.shared.f32 %f13, [%r3+4];\n" " abs.f32 %f14, %f13;\n" " max.f32 %f15, %f12, %f14;\n" " ld.shared.f32 %f16, [%r3+8];\n" " abs.f32 %f17, %f16;\n" " max.f32 %f18, %f15, %f17;\n" " ld.shared.f32 %f19, [%r3+12];\n" " abs.f32 %f20, %f19;\n" " max.f32 %f21, %f18, %f20;\n" " ld.shared.f32 %f22, [%r3+16];\n" " abs.f32 %f23, %f22;\n" " max.f32 %f24, %f21, %f23;\n" " ld.shared.f32 %f25, [%r3+20];\n" " abs.f32 %f26, %f25;\n" " max.f32 %f27, %f24, %f26;\n" " ld.shared.f32 %f28, [%r3+24];\n" " abs.f32 %f29, %f28;\n" " max.f32 %f30, %f27, %f29;\n" " ld.shared.f32 %f31, [%r3+28];\n" " abs.f32 %f32, %f31;\n" " max.f32 %f33, %f30, %f32;\n" " ld.shared.f32 %f34, [%r3+32];\n" " abs.f32 %f35, %f34;\n" " max.f32 %f36, %f33, %f35;\n" " ld.shared.f32 %f37, [%r3+36];\n" " abs.f32 %f38, %f37;\n" " max.f32 %f39, %f36, %f38;\n" " ld.shared.f32 %f40, [%r3+40];\n" " abs.f32 %f41, %f40;\n" " max.f32 %f42, %f39, %f41;\n" " ld.shared.f32 %f43, [%r3+44];\n" " abs.f32 %f44, %f43;\n" " max.f32 %f45, %f42, %f44;\n" " ld.shared.f32 %f46, [%r3+48];\n" " abs.f32 %f47, %f46;\n" " max.f32 %f48, %f45, %f47;\n" " ld.shared.f32 %f49, [%r3+52];\n" " abs.f32 %f50, %f49;\n" " max.f32 %f51, %f48, %f50;\n" " ld.shared.f32 %f52, [%r3+56];\n" " abs.f32 %f53, %f52;\n" " max.f32 %f54, %f51, %f53;\n" " ld.shared.f32 %f55, [%r3+60];\n" " abs.f32 %f56, %f55;\n" " max.f32 %f57, %f54, %f56;\n" " ld.shared.f32 %f58, [%r3+64];\n" " abs.f32 %f59, %f58;\n" " max.f32 %f60, %f57, %f59;\n" " ld.shared.f32 %f61, [%r3+68];\n" " abs.f32 %f62, %f61;\n" " max.f32 %f63, %f60, %f62;\n" " ld.shared.f32 %f64, [%r3+72];\n" " abs.f32 %f65, %f64;\n" " max.f32 %f66, %f63, %f65;\n" " ld.shared.f32 %f67, [%r3+76];\n" " abs.f32 %f68, %f67;\n" " max.f32 %f69, %f66, %f68;\n" " ld.shared.f32 %f70, [%r3+80];\n" " abs.f32 %f71, %f70;\n" " max.f32 %f72, %f69, %f71;\n" " ld.shared.f32 %f73, [%r3+84];\n" " abs.f32 %f74, %f73;\n" " max.f32 %f75, %f72, %f74;\n" " ld.shared.f32 %f76, [%r3+88];\n" " abs.f32 %f77, %f76;\n" " max.f32 %f78, %f75, %f77;\n" " ld.shared.f32 %f79, [%r3+92];\n" " abs.f32 %f80, %f79;\n" " max.f32 %f81, %f78, %f80;\n" " ld.shared.f32 %f82, [%r3+96];\n" " abs.f32 %f83, %f82;\n" " max.f32 %f84, %f81, %f83;\n" " ld.shared.f32 %f85, [%r3+100];\n" " abs.f32 %f86, %f85;\n" " max.f32 %f87, %f84, %f86;\n" " ld.shared.f32 %f88, [%r3+104];\n" " abs.f32 %f89, %f88;\n" " max.f32 %f90, %f87, %f89;\n" " ld.shared.f32 %f91, [%r3+108];\n" " abs.f32 %f92, %f91;\n" " max.f32 %f93, %f90, %f92;\n" " ld.shared.f32 %f94, [%r3+112];\n" " abs.f32 %f95, %f94;\n" " max.f32 %f96, %f93, %f95;\n" " ld.shared.f32 %f97, [%r3+116];\n" " abs.f32 %f98, %f97;\n" " max.f32 %f99, %f96, %f98;\n" " ld.shared.f32 %f100, [%r3+120];\n" " abs.f32 %f101, %f100;\n" " max.f32 %f102, %f99, %f101;\n" " ld.shared.f32 %f103, [%r3+124];\n" " abs.f32 %f104, %f103;\n" " max.f32 %f105, %f102, %f104;\n" " and.b32 %r4, %r1, 31;\n" " mul.f32 %f3, %f105, 0fBE000000;\n" " setp.eq.f32 %p2, %f3, 0f00000000;\n" " @%p2 bra $L__BB17_3;\n" "\n" " rcp.rn.f32 %f221, %f3;\n" "\n" "$L__BB17_3:\n" " cvta.to.global.u64 %rd35, %rd10;\n" " add.s64 %rd6, %rd35, %rd5;\n" " setp.ne.s32 %p3, %r4, 0;\n" " @%p3 bra $L__BB17_5;\n" "\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs1, %f3;}\n" "\n" " // end inline asm\n" " st.global.u16 [%rd6], %rs1;\n" "\n" "$L__BB17_5:\n" " setp.gt.u32 %p4, %r4, 15;\n" " @%p4 bra $L__BB17_7;\n" "\n" " shl.b32 %r29, %r4, 2;\n" " add.s32 %r30, %r3, %r29;\n" " ld.shared.f32 %f107, [%r30];\n" " mul.f32 %f108, %f221, %f107;\n" " cvt.rni.f32.f32 %f109, %f108;\n" " cvt.rzi.s32.f32 %r31, %f109;\n" " add.s32 %r32, %r31, 8;\n" " ld.shared.f32 %f110, [%r30+64];\n" " mul.f32 %f111, %f221, %f110;\n" " cvt.rni.f32.f32 %f112, %f111;\n" " cvt.rzi.s32.f32 %r33, %f112;\n" " add.s32 %r34, %r33, 8;\n" " min.s32 %r35, %r32, 15;\n" " max.s32 %r36, %r35, 0;\n" " min.s32 %r37, %r34, 15;\n" " max.s32 %r38, %r37, 0;\n" " shl.b32 %r39, %r38, 4;\n" " or.b32 %r40, %r39, %r36;\n" " add.s32 %r41, %r4, 2;\n" " cvt.u64.u32 %rd36, %r41;\n" " add.s64 %rd37, %rd6, %rd36;\n" " st.global.u8 [%rd37], %r40;\n" "\n" "$L__BB17_7:\n" " shl.b32 %r42, %r2, 2;\n" " add.s32 %r5, %r21, %r42;\n" " ld.shared.f32 %f114, [%r5];\n" " abs.f32 %f115, %f114;\n" " mov.f32 %f222, 0f00000000;\n" " max.f32 %f116, %f222, %f115;\n" " ld.shared.f32 %f117, [%r5+4];\n" " abs.f32 %f118, %f117;\n" " max.f32 %f119, %f116, %f118;\n" " ld.shared.f32 %f120, [%r5+8];\n" " abs.f32 %f121, %f120;\n" " max.f32 %f122, %f119, %f121;\n" " ld.shared.f32 %f123, [%r5+12];\n" " abs.f32 %f124, %f123;\n" " max.f32 %f125, %f122, %f124;\n" " ld.shared.f32 %f126, [%r5+16];\n" " abs.f32 %f127, %f126;\n" " max.f32 %f128, %f125, %f127;\n" " ld.shared.f32 %f129, [%r5+20];\n" " abs.f32 %f130, %f129;\n" " max.f32 %f131, %f128, %f130;\n" " ld.shared.f32 %f132, [%r5+24];\n" " abs.f32 %f133, %f132;\n" " max.f32 %f134, %f131, %f133;\n" " ld.shared.f32 %f135, [%r5+28];\n" " abs.f32 %f136, %f135;\n" " max.f32 %f137, %f134, %f136;\n" " ld.shared.f32 %f138, [%r5+32];\n" " abs.f32 %f139, %f138;\n" " max.f32 %f140, %f137, %f139;\n" " ld.shared.f32 %f141, [%r5+36];\n" " abs.f32 %f142, %f141;\n" " max.f32 %f143, %f140, %f142;\n" " ld.shared.f32 %f144, [%r5+40];\n" " abs.f32 %f145, %f144;\n" " max.f32 %f146, %f143, %f145;\n" " ld.shared.f32 %f147, [%r5+44];\n" " abs.f32 %f148, %f147;\n" " max.f32 %f149, %f146, %f148;\n" " ld.shared.f32 %f150, [%r5+48];\n" " abs.f32 %f151, %f150;\n" " max.f32 %f152, %f149, %f151;\n" " ld.shared.f32 %f153, [%r5+52];\n" " abs.f32 %f154, %f153;\n" " max.f32 %f155, %f152, %f154;\n" " ld.shared.f32 %f156, [%r5+56];\n" " abs.f32 %f157, %f156;\n" " max.f32 %f158, %f155, %f157;\n" " ld.shared.f32 %f159, [%r5+60];\n" " abs.f32 %f160, %f159;\n" " max.f32 %f161, %f158, %f160;\n" " ld.shared.f32 %f162, [%r5+64];\n" " abs.f32 %f163, %f162;\n" " max.f32 %f164, %f161, %f163;\n" " ld.shared.f32 %f165, [%r5+68];\n" " abs.f32 %f166, %f165;\n" " max.f32 %f167, %f164, %f166;\n" " ld.shared.f32 %f168, [%r5+72];\n" " abs.f32 %f169, %f168;\n" " max.f32 %f170, %f167, %f169;\n" " ld.shared.f32 %f171, [%r5+76];\n" " abs.f32 %f172, %f171;\n" " max.f32 %f173, %f170, %f172;\n" " ld.shared.f32 %f174, [%r5+80];\n" " abs.f32 %f175, %f174;\n" " max.f32 %f176, %f173, %f175;\n" " ld.shared.f32 %f177, [%r5+84];\n" " abs.f32 %f178, %f177;\n" " max.f32 %f179, %f176, %f178;\n" " ld.shared.f32 %f180, [%r5+88];\n" " abs.f32 %f181, %f180;\n" " max.f32 %f182, %f179, %f181;\n" " ld.shared.f32 %f183, [%r5+92];\n" " abs.f32 %f184, %f183;\n" " max.f32 %f185, %f182, %f184;\n" " ld.shared.f32 %f186, [%r5+96];\n" " abs.f32 %f187, %f186;\n" " max.f32 %f188, %f185, %f187;\n" " ld.shared.f32 %f189, [%r5+100];\n" " abs.f32 %f190, %f189;\n" " max.f32 %f191, %f188, %f190;\n" " ld.shared.f32 %f192, [%r5+104];\n" " abs.f32 %f193, %f192;\n" " max.f32 %f194, %f191, %f193;\n" " ld.shared.f32 %f195, [%r5+108];\n" " abs.f32 %f196, %f195;\n" " max.f32 %f197, %f194, %f196;\n" " ld.shared.f32 %f198, [%r5+112];\n" " abs.f32 %f199, %f198;\n" " max.f32 %f200, %f197, %f199;\n" " ld.shared.f32 %f201, [%r5+116];\n" " abs.f32 %f202, %f201;\n" " max.f32 %f203, %f200, %f202;\n" " ld.shared.f32 %f204, [%r5+120];\n" " abs.f32 %f205, %f204;\n" " max.f32 %f206, %f203, %f205;\n" " ld.shared.f32 %f207, [%r5+124];\n" " abs.f32 %f208, %f207;\n" " max.f32 %f209, %f206, %f208;\n" " mul.f32 %f6, %f209, 0fBE000000;\n" " setp.eq.f32 %p5, %f6, 0f00000000;\n" " @%p5 bra $L__BB17_9;\n" "\n" " rcp.rn.f32 %f222, %f6;\n" "\n" "$L__BB17_9:\n" " cvta.to.global.u64 %rd38, %rd11;\n" " add.s64 %rd7, %rd38, %rd5;\n" " @%p3 bra $L__BB17_11;\n" "\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs2, %f6;}\n" "\n" " // end inline asm\n" " st.global.u16 [%rd7], %rs2;\n" "\n" "$L__BB17_11:\n" " @%p4 bra $L__BB17_13;\n" "\n" " shl.b32 %r44, %r4, 2;\n" " add.s32 %r45, %r5, %r44;\n" " ld.shared.f32 %f211, [%r45];\n" " mul.f32 %f212, %f222, %f211;\n" " cvt.rni.f32.f32 %f213, %f212;\n" " cvt.rzi.s32.f32 %r46, %f213;\n" " add.s32 %r47, %r46, 8;\n" " ld.shared.f32 %f214, [%r45+64];\n" " mul.f32 %f215, %f222, %f214;\n" " cvt.rni.f32.f32 %f216, %f215;\n" " cvt.rzi.s32.f32 %r48, %f216;\n" " add.s32 %r49, %r48, 8;\n" " min.s32 %r50, %r47, 15;\n" " max.s32 %r51, %r50, 0;\n" " min.s32 %r52, %r49, 15;\n" " max.s32 %r53, %r52, 0;\n" " shl.b32 %r54, %r53, 4;\n" " or.b32 %r55, %r54, %r51;\n" " add.s32 %r56, %r4, 2;\n" " cvt.u64.u32 %rd39, %r56;\n" " add.s64 %rd40, %rd7, %rd39;\n" " st.global.u8 [%rd40], %r55;\n" "\n" "$L__BB17_13:\n" " setp.lt.s32 %p8, %r10, 1;\n" " @%p8 bra $L__BB17_15;\n" "\n" " cvt.u32.u64 %r57, %rd4;\n" " shr.s32 %r58, %r57, 31;\n" " shr.u32 %r59, %r58, 24;\n" " add.s32 %r60, %r57, %r59;\n" " and.b32 %r61, %r60, -256;\n" " sub.s32 %r62, %r57, %r61;\n" " cvt.s64.s32 %rd41, %r62;\n" " shl.b64 %rd42, %rd1, 8;\n" " add.s64 %rd43, %rd42, %rd41;\n" " mul.lo.s64 %rd44, %rd43, %rd2;\n" " add.s64 %rd45, %rd44, %rd3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs3, %f1;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd46, %rd14;\n" " shl.b64 %rd47, %rd45, 1;\n" " add.s64 %rd48, %rd46, %rd47;\n" " st.global.u16 [%rd48], %rs3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs4, %f2;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd49, %rd15;\n" " add.s64 %rd50, %rd49, %rd47;\n" " st.global.u16 [%rd50], %rs4;\n" "\n" "$L__BB17_15:\n" " cvt.u32.u64 %r63, %rd4;\n" " setp.ge.s32 %p9, %r63, %r9;\n" " @%p9 bra $L__BB17_17;\n" "\n" " shl.b64 %rd51, %rd1, 3;\n" " add.s64 %rd52, %rd51, %rd4;\n" " mul.lo.s64 %rd53, %rd52, %rd2;\n" " add.s64 %rd54, %rd53, %rd3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs5, %f1;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd55, %rd12;\n" " shl.b64 %rd56, %rd54, 1;\n" " add.s64 %rd57, %rd55, %rd56;\n" " st.global.u16 [%rd57], %rs5;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs6, %f2;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd58, %rd13;\n" " add.s64 %rd59, %rd58, %rd56;\n" " st.global.u16 [%rd59], %rs6;\n" "\n" "$L__BB17_17:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode_slots\n" ".visible .entry fc_attn_flash_decode_slots(\n" " .param .u64 fc_attn_flash_decode_slots_param_0,\n" " .param .u64 fc_attn_flash_decode_slots_param_1,\n" " .param .u64 fc_attn_flash_decode_slots_param_2,\n" " .param .u64 fc_attn_flash_decode_slots_param_3,\n" " .param .u64 fc_attn_flash_decode_slots_param_4,\n" " .param .u64 fc_attn_flash_decode_slots_param_5,\n" " .param .u32 fc_attn_flash_decode_slots_param_6,\n" " .param .u32 fc_attn_flash_decode_slots_param_7,\n" " .param .u32 fc_attn_flash_decode_slots_param_8,\n" " .param .u64 fc_attn_flash_decode_slots_param_9,\n" " .param .u32 fc_attn_flash_decode_slots_param_10,\n" " .param .u32 fc_attn_flash_decode_slots_param_11,\n" " .param .u32 fc_attn_flash_decode_slots_param_12,\n" " .param .u32 fc_attn_flash_decode_slots_param_13,\n" " .param .f32 fc_attn_flash_decode_slots_param_14\n" ")\n" "{\n" " .reg .pred %p<182>;\n" " .reg .f32 %f<637>;\n" " .reg .b32 %r<605>;\n" " .reg .b64 %rd<238>;\n" "\n" "\n" " ld.param.u64 %rd62, [fc_attn_flash_decode_slots_param_0];\n" " ld.param.u64 %rd60, [fc_attn_flash_decode_slots_param_1];\n" " ld.param.u64 %rd63, [fc_attn_flash_decode_slots_param_3];\n" " ld.param.u64 %rd64, [fc_attn_flash_decode_slots_param_4];\n" " ld.param.u64 %rd65, [fc_attn_flash_decode_slots_param_5];\n" " ld.param.u32 %r113, [fc_attn_flash_decode_slots_param_6];\n" " ld.param.u32 %r114, [fc_attn_flash_decode_slots_param_7];\n" " ld.param.u32 %r115, [fc_attn_flash_decode_slots_param_8];\n" " ld.param.u64 %rd66, [fc_attn_flash_decode_slots_param_9];\n" " ld.param.u32 %r118, [fc_attn_flash_decode_slots_param_10];\n" " ld.param.u32 %r119, [fc_attn_flash_decode_slots_param_11];\n" " ld.param.u32 %r117, [fc_attn_flash_decode_slots_param_13];\n" " ld.param.f32 %f179, [fc_attn_flash_decode_slots_param_14];\n" " cvta.to.global.u64 %rd1, %rd62;\n" " cvta.to.global.u64 %rd2, %rd64;\n" " cvta.to.global.u64 %rd3, %rd63;\n" " cvta.to.global.u64 %rd4, %rd65;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %nctaid.x;\n" " mov.u32 %r120, %ctaid.z;\n" " cvta.to.global.u64 %rd67, %rd66;\n" " mul.wide.s32 %rd68, %r120, 4;\n" " add.s64 %rd69, %rd67, %rd68;\n" " ld.global.u32 %r3, [%rd69];\n" " add.s32 %r121, %r3, %r118;\n" " div.s32 %r122, %r121, %r118;\n" " max.s32 %r123, %r122, 1;\n" " min.s32 %r124, %r123, %r119;\n" " add.s32 %r125, %r124, %r3;\n" " div.s32 %r126, %r125, %r124;\n" " add.s32 %r4, %r126, 15;\n" " and.b32 %r5, %r4, -16;\n" " add.s32 %r127, %r5, %r3;\n" " div.s32 %r128, %r127, %r5;\n" " mov.u32 %r129, %nctaid.y;\n" " mul.wide.s32 %rd70, %r120, %r129;\n" " mov.u32 %r130, %ctaid.y;\n" " cvt.s64.s32 %rd71, %r130;\n" " add.s64 %rd5, %rd70, %rd71;\n" " cvt.s64.s32 %rd6, %r113;\n" " mul.lo.s64 %rd7, %rd5, %rd6;\n" " mov.u32 %r6, %ctaid.x;\n" " setp.lt.s32 %p2, %r6, %r128;\n" " @%p2 bra $L__BB18_18;\n" " bra.uni $L__BB18_1;\n" "\n" "$L__BB18_18:\n" " setp.lt.s32 %p13, %r113, 1;\n" " mul.lo.s32 %r591, %r5, %r6;\n" " add.s32 %r152, %r591, %r5;\n" " add.s32 %r153, %r3, 1;\n" " min.s32 %r17, %r152, %r153;\n" " mul.lo.s32 %r18, %r114, %r113;\n" " add.s32 %r19, %r114, 16;\n" " mul.wide.s32 %rd30, %r19, 16;\n" " shl.b32 %r154, %r113, 4;\n" " cvt.s64.s32 %rd116, %r154;\n" " add.s64 %rd31, %rd30, %rd116;\n" " add.s64 %rd32, %rd31, %rd116;\n" " add.s64 %rd33, %rd32, %rd6;\n" " cvt.s64.s32 %rd34, %r114;\n" " @%p13 bra $L__BB18_25;\n" "\n" " cvt.s64.s32 %rd35, %r1;\n" " and.b32 %r589, %r113, 3;\n" " add.s32 %r156, %r113, -1;\n" " setp.lt.u32 %p14, %r156, 3;\n" " mov.u32 %r587, 0;\n" " @%p14 bra $L__BB18_22;\n" "\n" " sub.s32 %r585, %r113, %r589;\n" " mul.lo.s64 %rd117, %rd5, %rd34;\n" " mul.lo.s64 %rd118, %rd117, %rd6;\n" " add.s64 %rd119, %rd118, %rd35;\n" " shl.b64 %rd120, %rd119, 2;\n" " add.s64 %rd236, %rd1, %rd120;\n" " shl.b64 %rd37, %rd34, 2;\n" " shl.b32 %r158, %r1, 2;\n" " mov.u32 %r159, sh;\n" " add.s32 %r586, %r159, %r158;\n" " shl.b32 %r23, %r114, 2;\n" " mov.u32 %r587, 0;\n" "\n" "$L__BB18_21:\n" " ld.global.f32 %f180, [%rd236];\n" " st.shared.f32 [%r586], %f180;\n" " add.s64 %rd121, %rd236, %rd37;\n" " ld.global.f32 %f181, [%rd121];\n" " add.s32 %r160, %r586, %r23;\n" " st.shared.f32 [%r160], %f181;\n" " add.s64 %rd122, %rd121, %rd37;\n" " ld.global.f32 %f182, [%rd122];\n" " add.s32 %r161, %r160, %r23;\n" " st.shared.f32 [%r161], %f182;\n" " add.s64 %rd123, %rd122, %rd37;\n" " add.s64 %rd236, %rd123, %rd37;\n" " ld.global.f32 %f183, [%rd123];\n" " add.s32 %r162, %r161, %r23;\n" " add.s32 %r586, %r162, %r23;\n" " st.shared.f32 [%r162], %f183;\n" " add.s32 %r587, %r587, 4;\n" " add.s32 %r585, %r585, -4;\n" " setp.ne.s32 %p15, %r585, 0;\n" " @%p15 bra $L__BB18_21;\n" "\n" "$L__BB18_22:\n" " setp.eq.s32 %p16, %r589, 0;\n" " @%p16 bra $L__BB18_25;\n" "\n" " mad.lo.s32 %r163, %r587, %r114, %r1;\n" " shl.b32 %r164, %r163, 2;\n" " mov.u32 %r165, sh;\n" " add.s32 %r588, %r165, %r164;\n" " shl.b32 %r32, %r114, 2;\n" " cvt.s64.s32 %rd124, %r587;\n" " add.s64 %rd125, %rd7, %rd124;\n" " mul.lo.s64 %rd126, %rd125, %rd34;\n" " add.s64 %rd127, %rd126, %rd35;\n" " shl.b64 %rd128, %rd127, 2;\n" " add.s64 %rd237, %rd1, %rd128;\n" " shl.b64 %rd41, %rd34, 2;\n" "\n" "$L__BB18_24:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f184, [%rd237];\n" " st.shared.f32 [%r588], %f184;\n" " add.s32 %r588, %r588, %r32;\n" " add.s64 %rd237, %rd237, %rd41;\n" " add.s32 %r589, %r589, -1;\n" " setp.ne.s32 %p17, %r589, 0;\n" " @%p17 bra $L__BB18_24;\n" "\n" "$L__BB18_25:\n" " setp.ge.s32 %p18, %r1, %r113;\n" " add.s32 %r166, %r18, %r1;\n" " cvt.u32.u64 %r167, %rd32;\n" " add.s32 %r168, %r166, %r167;\n" " shl.b32 %r169, %r168, 2;\n" " mov.u32 %r170, sh;\n" " add.s32 %r37, %r170, %r169;\n" " cvt.u32.u64 %r171, %rd33;\n" " add.s32 %r172, %r166, %r171;\n" " shl.b32 %r173, %r172, 2;\n" " add.s32 %r38, %r170, %r173;\n" " @%p18 bra $L__BB18_27;\n" "\n" " mov.u32 %r174, -8388608;\n" " st.shared.u32 [%r37], %r174;\n" " mov.u32 %r175, 0;\n" " st.shared.u32 [%r38], %r175;\n" "\n" "$L__BB18_27:\n" " bar.sync 0;\n" " mul.lo.s64 %rd129, %rd5, %rd34;\n" " cvt.s64.s32 %rd130, %r115;\n" " mul.lo.s64 %rd44, %rd129, %rd130;\n" " shr.s32 %r176, %r1, 31;\n" " shr.u32 %r177, %r176, 28;\n" " add.s32 %r178, %r1, %r177;\n" " shr.s32 %r40, %r178, 4;\n" " and.b32 %r179, %r178, -16;\n" " sub.s32 %r41, %r1, %r179;\n" " cvt.s64.s32 %rd45, %r1;\n" " setp.le.s32 %p19, %r17, %r591;\n" " mov.f32 %f605, 0f00000000;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" " @%p19 bra $L__BB18_169;\n" "\n" " shr.s32 %r181, %r114, 31;\n" " shr.u32 %r182, %r181, 28;\n" " add.s32 %r183, %r114, %r182;\n" " shr.s32 %r184, %r183, 4;\n" " sub.s32 %r185, %r3, %r117;\n" " add.s32 %r186, %r185, 1;\n" " mov.u32 %r590, 0;\n" " max.s32 %r42, %r186, 0;\n" " add.s32 %r187, %r18, %r40;\n" " cvt.u32.u64 %r188, %rd30;\n" " add.s32 %r189, %r187, %r188;\n" " mad.lo.s32 %r44, %r19, %r40, %r18;\n" " shl.b32 %r190, %r1, 4;\n" " add.s32 %r191, %r190, %r188;\n" " add.s32 %r45, %r18, %r191;\n" " add.s64 %rd46, %rd44, %rd45;\n" " add.s32 %r192, %r171, %r113;\n" " add.s32 %r193, %r18, %r192;\n" " add.s32 %r195, %r166, %r192;\n" " shl.b32 %r196, %r195, 2;\n" " add.s32 %r46, %r170, %r196;\n" " cvt.u64.u32 %rd131, %r1;\n" " add.s64 %rd47, %rd31, %rd131;\n" " shl.b32 %r198, %r6, 4;\n" " add.s32 %r199, %r198, 16;\n" " shr.u32 %r200, %r4, 4;\n" " mul.lo.s32 %r201, %r200, %r199;\n" " not.b32 %r202, %r201;\n" " mov.u32 %r203, -2;\n" " sub.s32 %r204, %r203, %r3;\n" " max.s32 %r205, %r204, %r202;\n" " mul.lo.s32 %r206, %r6, %r200;\n" " shl.b32 %r207, %r206, 4;\n" " add.s32 %r47, %r205, %r207;\n" " bfe.u32 %r49, %r183, 4, 1;\n" " sub.s32 %r50, %r184, %r49;\n" " shl.b32 %r51, %r114, 1;\n" " mul.lo.s32 %r52, %r114, 3;\n" " shl.b32 %r53, %r114, 2;\n" " mul.lo.s32 %r54, %r114, 5;\n" " mul.lo.s32 %r55, %r114, 6;\n" " mul.lo.s32 %r56, %r114, 7;\n" " shl.b32 %r208, %r189, 2;\n" " add.s32 %r57, %r170, %r208;\n" " shl.b32 %r209, %r193, 2;\n" " add.s32 %r58, %r170, %r209;\n" " mad.lo.s32 %r211, %r113, %r19, %r1;\n" " add.s32 %r212, %r211, %r188;\n" " shl.b32 %r213, %r212, 2;\n" " add.s32 %r214, %r170, %r213;\n" " add.s32 %r59, %r214, -1024;\n" " cvta.to.global.u64 %rd48, %rd60;\n" " mov.f32 %f605, 0f00000000;\n" " setp.lt.s32 %p21, %r117, 1;\n" " setp.ge.s32 %p29, %r1, %r114;\n" " setp.lt.s32 %p32, %r114, 16;\n" " setp.eq.s32 %p62, %r41, 0;\n" " mov.f32 %f606, %f605;\n" " mov.f32 %f607, %f605;\n" " mov.f32 %f608, %f605;\n" " mov.f32 %f609, %f605;\n" " mov.f32 %f610, %f605;\n" " mov.f32 %f611, %f605;\n" " mov.f32 %f612, %f605;\n" "\n" "$L__BB18_29:\n" " ld.param.u32 %r483, [fc_attn_flash_decode_slots_param_12];\n" " shl.b32 %r215, %r590, 4;\n" " add.s32 %r216, %r47, %r215;\n" " max.s32 %r217, %r216, -17;\n" " not.b32 %r218, %r217;\n" " max.s32 %r64, %r218, 1;\n" " sub.s32 %r65, %r17, %r591;\n" " min.s32 %r66, %r65, 16;\n" " setp.lt.s32 %p20, %r591, %r483;\n" " or.pred %p22, %p21, %p20;\n" " add.s32 %r219, %r66, %r591;\n" " setp.gt.s32 %p23, %r219, %r42;\n" " setp.le.s32 %p24, %r591, %r3;\n" " and.pred %p25, %p24, %p23;\n" " or.pred %p26, %p22, %p25;\n" " not.pred %p27, %p26;\n" " @%p27 bra $L__BB18_168;\n" "\n" " setp.lt.s32 %p28, %r65, 1;\n" " @%p28 bra $L__BB18_36;\n" "\n" " mov.u32 %r592, 0;\n" "\n" "$L__BB18_32:\n" " @%p29 bra $L__BB18_35;\n" "\n" " add.s32 %r493, %r114, 16;\n" " add.s32 %r221, %r592, %r591;\n" " cvt.s64.s32 %rd132, %r221;\n" " mul.lo.s64 %rd133, %rd132, %rd34;\n" " add.s64 %rd49, %rd133, %rd44;\n" " mad.lo.s32 %r68, %r592, %r493, %r18;\n" " mov.u32 %r593, %r1;\n" "\n" "$L__BB18_34:\n" " mov.u32 %r494, %ntid.x;\n" " cvt.s64.s32 %rd134, %r593;\n" " add.s64 %rd135, %rd49, %rd134;\n" " shl.b64 %rd136, %rd135, 2;\n" " add.s64 %rd137, %rd48, %rd136;\n" " ld.global.f32 %f201, [%rd137];\n" " add.s32 %r222, %r68, %r593;\n" " shl.b32 %r223, %r222, 2;\n" " add.s32 %r225, %r170, %r223;\n" " st.shared.f32 [%r225], %f201;\n" " add.s32 %r593, %r593, %r494;\n" " setp.lt.s32 %p30, %r593, %r114;\n" " @%p30 bra $L__BB18_34;\n" "\n" "$L__BB18_35:\n" " sub.s32 %r492, %r17, %r591;\n" " min.s32 %r491, %r492, 16;\n" " add.s32 %r592, %r592, 1;\n" " setp.lt.s32 %p31, %r592, %r491;\n" " @%p31 bra $L__BB18_32;\n" "\n" "$L__BB18_36:\n" " sub.s32 %r485, %r17, %r591;\n" " min.s32 %r484, %r485, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p33, %r40, %r484;\n" " or.pred %p34, %p32, %p33;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p34 bra $L__BB18_89;\n" "\n" " and.b32 %r486, %r114, -16;\n" " setp.eq.s32 %p35, %r486, 16;\n" " mov.u32 %r596, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " @%p35 bra $L__BB18_72;\n" "\n" " mov.u32 %r596, 0;\n" " mov.f32 %f543, 0f00000000;\n" " mov.f32 %f542, %f543;\n" " mov.f32 %f541, %f543;\n" " mov.f32 %f540, %f543;\n" " mov.f32 %f539, %f543;\n" " mov.f32 %f538, %f543;\n" " mov.f32 %f537, %f543;\n" " mov.f32 %f536, %f543;\n" " mov.u32 %r595, %r50;\n" "\n" "$L__BB18_39:\n" " shl.b32 %r228, %r596, 4;\n" " add.s32 %r74, %r228, %r41;\n" " add.s32 %r229, %r44, %r74;\n" " shl.b32 %r230, %r229, 2;\n" " add.s32 %r75, %r170, %r230;\n" " ld.shared.f32 %f17, [%r75];\n" " @%p13 bra $L__BB18_41;\n" "\n" " shl.b32 %r574, %r596, 4;\n" " add.s32 %r573, %r574, %r41;\n" " shl.b32 %r572, %r573, 2;\n" " add.s32 %r571, %r170, %r572;\n" " ld.shared.f32 %f226, [%r571];\n" " fma.rn.f32 %f536, %f17, %f226, %f536;\n" "\n" "$L__BB18_41:\n" " setp.lt.s32 %p37, %r113, 2;\n" " @%p37 bra $L__BB18_43;\n" "\n" " shl.b32 %r579, %r596, 4;\n" " add.s32 %r578, %r579, %r41;\n" " add.s32 %r577, %r578, %r114;\n" " shl.b32 %r576, %r577, 2;\n" " add.s32 %r575, %r170, %r576;\n" " ld.shared.f32 %f227, [%r575];\n" " fma.rn.f32 %f537, %f17, %f227, %f537;\n" "\n" "$L__BB18_43:\n" " setp.lt.s32 %p38, %r113, 3;\n" " @%p38 bra $L__BB18_45;\n" "\n" " shl.b32 %r520, %r596, 4;\n" " add.s32 %r519, %r520, %r41;\n" " add.s32 %r518, %r51, %r519;\n" " shl.b32 %r517, %r518, 2;\n" " add.s32 %r516, %r170, %r517;\n" " ld.shared.f32 %f228, [%r516];\n" " fma.rn.f32 %f538, %f17, %f228, %f538;\n" "\n" "$L__BB18_45:\n" " setp.lt.s32 %p39, %r113, 4;\n" " @%p39 bra $L__BB18_47;\n" "\n" " shl.b32 %r530, %r596, 4;\n" " add.s32 %r529, %r530, %r41;\n" " add.s32 %r528, %r52, %r529;\n" " shl.b32 %r527, %r528, 2;\n" " add.s32 %r526, %r170, %r527;\n" " ld.shared.f32 %f229, [%r526];\n" " fma.rn.f32 %f539, %f17, %f229, %f539;\n" "\n" "$L__BB18_47:\n" " setp.lt.s32 %p40, %r113, 5;\n" " @%p40 bra $L__BB18_49;\n" "\n" " shl.b32 %r540, %r596, 4;\n" " add.s32 %r539, %r540, %r41;\n" " add.s32 %r538, %r53, %r539;\n" " shl.b32 %r537, %r538, 2;\n" " add.s32 %r536, %r170, %r537;\n" " ld.shared.f32 %f230, [%r536];\n" " fma.rn.f32 %f540, %f17, %f230, %f540;\n" "\n" "$L__BB18_49:\n" " setp.lt.s32 %p41, %r113, 6;\n" " @%p41 bra $L__BB18_51;\n" "\n" " shl.b32 %r550, %r596, 4;\n" " add.s32 %r549, %r550, %r41;\n" " add.s32 %r548, %r54, %r549;\n" " shl.b32 %r547, %r548, 2;\n" " add.s32 %r546, %r170, %r547;\n" " ld.shared.f32 %f231, [%r546];\n" " fma.rn.f32 %f541, %f17, %f231, %f541;\n" "\n" "$L__BB18_51:\n" " setp.lt.s32 %p42, %r113, 7;\n" " @%p42 bra $L__BB18_53;\n" "\n" " shl.b32 %r560, %r596, 4;\n" " add.s32 %r559, %r560, %r41;\n" " add.s32 %r558, %r55, %r559;\n" " shl.b32 %r557, %r558, 2;\n" " add.s32 %r556, %r170, %r557;\n" " ld.shared.f32 %f232, [%r556];\n" " fma.rn.f32 %f542, %f17, %f232, %f542;\n" "\n" "$L__BB18_53:\n" " setp.lt.s32 %p43, %r113, 8;\n" " @%p43 bra $L__BB18_55;\n" "\n" " shl.b32 %r570, %r596, 4;\n" " add.s32 %r569, %r570, %r41;\n" " add.s32 %r568, %r56, %r569;\n" " shl.b32 %r567, %r568, 2;\n" " add.s32 %r566, %r170, %r567;\n" " ld.shared.f32 %f233, [%r566];\n" " fma.rn.f32 %f543, %f17, %f233, %f543;\n" "\n" "$L__BB18_55:\n" " shl.b32 %r501, %r596, 4;\n" " add.s32 %r500, %r501, %r41;\n" " add.s32 %r499, %r44, %r500;\n" " shl.b32 %r498, %r499, 2;\n" " add.s32 %r497, %r170, %r498;\n" " ld.shared.f32 %f34, [%r497+64];\n" " @%p13 bra $L__BB18_57;\n" "\n" " shl.b32 %r515, %r596, 4;\n" " add.s32 %r514, %r515, %r41;\n" " shl.b32 %r513, %r514, 2;\n" " add.s32 %r512, %r170, %r513;\n" " ld.shared.f32 %f234, [%r512+64];\n" " fma.rn.f32 %f536, %f34, %f234, %f536;\n" "\n" "$L__BB18_57:\n" " setp.lt.s32 %p175, %r113, 2;\n" " @%p175 bra $L__BB18_59;\n" "\n" " shl.b32 %r511, %r596, 4;\n" " add.s32 %r510, %r511, %r41;\n" " add.s32 %r509, %r510, %r114;\n" " shl.b32 %r508, %r509, 2;\n" " add.s32 %r507, %r170, %r508;\n" " ld.shared.f32 %f235, [%r507+64];\n" " fma.rn.f32 %f537, %f34, %f235, %f537;\n" "\n" "$L__BB18_59:\n" " setp.lt.s32 %p176, %r113, 3;\n" " @%p176 bra $L__BB18_61;\n" "\n" " shl.b32 %r506, %r596, 4;\n" " add.s32 %r505, %r506, %r41;\n" " add.s32 %r504, %r51, %r505;\n" " shl.b32 %r503, %r504, 2;\n" " add.s32 %r502, %r170, %r503;\n" " ld.shared.f32 %f236, [%r502+64];\n" " fma.rn.f32 %f538, %f34, %f236, %f538;\n" "\n" "$L__BB18_61:\n" " setp.lt.s32 %p177, %r113, 4;\n" " @%p177 bra $L__BB18_63;\n" "\n" " shl.b32 %r525, %r596, 4;\n" " add.s32 %r524, %r525, %r41;\n" " add.s32 %r523, %r52, %r524;\n" " shl.b32 %r522, %r523, 2;\n" " add.s32 %r521, %r170, %r522;\n" " ld.shared.f32 %f237, [%r521+64];\n" " fma.rn.f32 %f539, %f34, %f237, %f539;\n" "\n" "$L__BB18_63:\n" " setp.lt.s32 %p178, %r113, 5;\n" " @%p178 bra $L__BB18_65;\n" "\n" " shl.b32 %r535, %r596, 4;\n" " add.s32 %r534, %r535, %r41;\n" " add.s32 %r533, %r53, %r534;\n" " shl.b32 %r532, %r533, 2;\n" " add.s32 %r531, %r170, %r532;\n" " ld.shared.f32 %f238, [%r531+64];\n" " fma.rn.f32 %f540, %f34, %f238, %f540;\n" "\n" "$L__BB18_65:\n" " setp.lt.s32 %p179, %r113, 6;\n" " @%p179 bra $L__BB18_67;\n" "\n" " shl.b32 %r545, %r596, 4;\n" " add.s32 %r544, %r545, %r41;\n" " add.s32 %r543, %r54, %r544;\n" " shl.b32 %r542, %r543, 2;\n" " add.s32 %r541, %r170, %r542;\n" " ld.shared.f32 %f239, [%r541+64];\n" " fma.rn.f32 %f541, %f34, %f239, %f541;\n" "\n" "$L__BB18_67:\n" " setp.lt.s32 %p180, %r113, 7;\n" " @%p180 bra $L__BB18_69;\n" "\n" " shl.b32 %r555, %r596, 4;\n" " add.s32 %r554, %r555, %r41;\n" " add.s32 %r553, %r55, %r554;\n" " shl.b32 %r552, %r553, 2;\n" " add.s32 %r551, %r170, %r552;\n" " ld.shared.f32 %f240, [%r551+64];\n" " fma.rn.f32 %f542, %f34, %f240, %f542;\n" "\n" "$L__BB18_69:\n" " setp.lt.s32 %p181, %r113, 8;\n" " @%p181 bra $L__BB18_71;\n" "\n" " shl.b32 %r565, %r596, 4;\n" " add.s32 %r564, %r565, %r41;\n" " add.s32 %r563, %r56, %r564;\n" " shl.b32 %r562, %r563, 2;\n" " add.s32 %r561, %r170, %r562;\n" " ld.shared.f32 %f241, [%r561+64];\n" " fma.rn.f32 %f543, %f34, %f241, %f543;\n" "\n" "$L__BB18_71:\n" " add.s32 %r596, %r596, 2;\n" " add.s32 %r595, %r595, -2;\n" " setp.ne.s32 %p52, %r595, 0;\n" " @%p52 bra $L__BB18_39;\n" "\n" "$L__BB18_72:\n" " setp.eq.s32 %p53, %r49, 0;\n" " @%p53 bra $L__BB18_89;\n" "\n" " shl.b32 %r254, %r596, 4;\n" " add.s32 %r87, %r254, %r41;\n" " add.s32 %r255, %r44, %r87;\n" " shl.b32 %r256, %r255, 2;\n" " add.s32 %r258, %r170, %r256;\n" " ld.shared.f32 %f59, [%r258];\n" " @%p13 bra $L__BB18_75;\n" "\n" " shl.b32 %r259, %r87, 2;\n" " add.s32 %r261, %r170, %r259;\n" " ld.shared.f32 %f242, [%r261];\n" " fma.rn.f32 %f536, %f59, %f242, %f536;\n" "\n" "$L__BB18_75:\n" " setp.lt.s32 %p55, %r113, 2;\n" " @%p55 bra $L__BB18_77;\n" "\n" " add.s32 %r262, %r87, %r114;\n" " shl.b32 %r263, %r262, 2;\n" " add.s32 %r265, %r170, %r263;\n" " ld.shared.f32 %f243, [%r265];\n" " fma.rn.f32 %f537, %f59, %f243, %f537;\n" "\n" "$L__BB18_77:\n" " setp.lt.s32 %p56, %r113, 3;\n" " @%p56 bra $L__BB18_79;\n" "\n" " add.s32 %r266, %r51, %r87;\n" " shl.b32 %r267, %r266, 2;\n" " add.s32 %r269, %r170, %r267;\n" " ld.shared.f32 %f244, [%r269];\n" " fma.rn.f32 %f538, %f59, %f244, %f538;\n" "\n" "$L__BB18_79:\n" " setp.lt.s32 %p57, %r113, 4;\n" " @%p57 bra $L__BB18_81;\n" "\n" " add.s32 %r270, %r52, %r87;\n" " shl.b32 %r271, %r270, 2;\n" " add.s32 %r273, %r170, %r271;\n" " ld.shared.f32 %f245, [%r273];\n" " fma.rn.f32 %f539, %f59, %f245, %f539;\n" "\n" "$L__BB18_81:\n" " setp.lt.s32 %p58, %r113, 5;\n" " @%p58 bra $L__BB18_83;\n" "\n" " add.s32 %r274, %r53, %r87;\n" " shl.b32 %r275, %r274, 2;\n" " add.s32 %r277, %r170, %r275;\n" " ld.shared.f32 %f246, [%r277];\n" " fma.rn.f32 %f540, %f59, %f246, %f540;\n" "\n" "$L__BB18_83:\n" " setp.lt.s32 %p59, %r113, 6;\n" " @%p59 bra $L__BB18_85;\n" "\n" " add.s32 %r278, %r54, %r87;\n" " shl.b32 %r279, %r278, 2;\n" " add.s32 %r281, %r170, %r279;\n" " ld.shared.f32 %f247, [%r281];\n" " fma.rn.f32 %f541, %f59, %f247, %f541;\n" "\n" "$L__BB18_85:\n" " setp.lt.s32 %p60, %r113, 7;\n" " @%p60 bra $L__BB18_87;\n" "\n" " add.s32 %r282, %r55, %r87;\n" " shl.b32 %r283, %r282, 2;\n" " add.s32 %r285, %r170, %r283;\n" " ld.shared.f32 %f248, [%r285];\n" " fma.rn.f32 %f542, %f59, %f248, %f542;\n" "\n" "$L__BB18_87:\n" " setp.lt.s32 %p61, %r113, 8;\n" " @%p61 bra $L__BB18_89;\n" "\n" " add.s32 %r286, %r56, %r87;\n" " shl.b32 %r287, %r286, 2;\n" " add.s32 %r289, %r170, %r287;\n" " ld.shared.f32 %f249, [%r289];\n" " fma.rn.f32 %f543, %f59, %f249, %f543;\n" "\n" "$L__BB18_89:\n" " sub.s32 %r488, %r17, %r591;\n" " min.s32 %r487, %r488, 16;\n" " setp.lt.s32 %p63, %r40, %r487;\n" " and.pred %p1, %p62, %p63;\n" " @%p13 bra $L__BB18_92;\n" "\n" " mov.b32 %r290, %f536;\n" " mov.u32 %r291, 2;\n" " mov.u32 %r292, 4127;\n" " mov.u32 %r293, 8;\n" " mov.u32 %r294, -1;\n" " shfl.sync.down.b32 %r295|%p65, %r290, %r293, %r292, %r294;\n" " mov.b32 %f250, %r295;\n" " add.f32 %f251, %f536, %f250;\n" " mov.b32 %r296, %f251;\n" " mov.u32 %r297, 4;\n" " shfl.sync.down.b32 %r298|%p66, %r296, %r297, %r292, %r294;\n" " mov.b32 %f252, %r298;\n" " add.f32 %f253, %f251, %f252;\n" " mov.b32 %r299, %f253;\n" " shfl.sync.down.b32 %r300|%p67, %r299, %r291, %r292, %r294;\n" " mov.b32 %f254, %r300;\n" " add.f32 %f255, %f253, %f254;\n" " mov.b32 %r301, %f255;\n" " mov.u32 %r302, 1;\n" " shfl.sync.down.b32 %r303|%p68, %r301, %r302, %r292, %r294;\n" " mov.b32 %f256, %r303;\n" " add.f32 %f83, %f255, %f256;\n" " not.pred %p69, %p1;\n" " @%p69 bra $L__BB18_92;\n" "\n" " mul.f32 %f257, %f83, %f179;\n" " st.shared.f32 [%r57], %f257;\n" "\n" "$L__BB18_92:\n" " setp.lt.s32 %p70, %r113, 2;\n" " @%p70 bra $L__BB18_95;\n" "\n" " mov.b32 %r304, %f537;\n" " mov.u32 %r305, 2;\n" " mov.u32 %r306, 4127;\n" " mov.u32 %r307, 8;\n" " mov.u32 %r308, -1;\n" " shfl.sync.down.b32 %r309|%p71, %r304, %r307, %r306, %r308;\n" " mov.b32 %f258, %r309;\n" " add.f32 %f259, %f537, %f258;\n" " mov.b32 %r310, %f259;\n" " mov.u32 %r311, 4;\n" " shfl.sync.down.b32 %r312|%p72, %r310, %r311, %r306, %r308;\n" " mov.b32 %f260, %r312;\n" " add.f32 %f261, %f259, %f260;\n" " mov.b32 %r313, %f261;\n" " shfl.sync.down.b32 %r314|%p73, %r313, %r305, %r306, %r308;\n" " mov.b32 %f262, %r314;\n" " add.f32 %f263, %f261, %f262;\n" " mov.b32 %r315, %f263;\n" " mov.u32 %r316, 1;\n" " shfl.sync.down.b32 %r317|%p74, %r315, %r316, %r306, %r308;\n" " mov.b32 %f264, %r317;\n" " add.f32 %f84, %f263, %f264;\n" " not.pred %p75, %p1;\n" " @%p75 bra $L__BB18_95;\n" "\n" " mul.f32 %f265, %f84, %f179;\n" " st.shared.f32 [%r57+64], %f265;\n" "\n" "$L__BB18_95:\n" " setp.lt.s32 %p76, %r113, 3;\n" " @%p76 bra $L__BB18_98;\n" "\n" " mov.b32 %r318, %f538;\n" " mov.u32 %r319, 2;\n" " mov.u32 %r320, 4127;\n" " mov.u32 %r321, 8;\n" " mov.u32 %r322, -1;\n" " shfl.sync.down.b32 %r323|%p77, %r318, %r321, %r320, %r322;\n" " mov.b32 %f266, %r323;\n" " add.f32 %f267, %f538, %f266;\n" " mov.b32 %r324, %f267;\n" " mov.u32 %r325, 4;\n" " shfl.sync.down.b32 %r326|%p78, %r324, %r325, %r320, %r322;\n" " mov.b32 %f268, %r326;\n" " add.f32 %f269, %f267, %f268;\n" " mov.b32 %r327, %f269;\n" " shfl.sync.down.b32 %r328|%p79, %r327, %r319, %r320, %r322;\n" " mov.b32 %f270, %r328;\n" " add.f32 %f271, %f269, %f270;\n" " mov.b32 %r329, %f271;\n" " mov.u32 %r330, 1;\n" " shfl.sync.down.b32 %r331|%p80, %r329, %r330, %r320, %r322;\n" " mov.b32 %f272, %r331;\n" " add.f32 %f85, %f271, %f272;\n" " not.pred %p81, %p1;\n" " @%p81 bra $L__BB18_98;\n" "\n" " mul.f32 %f273, %f85, %f179;\n" " st.shared.f32 [%r57+128], %f273;\n" "\n" "$L__BB18_98:\n" " setp.lt.s32 %p82, %r113, 4;\n" " @%p82 bra $L__BB18_101;\n" "\n" " mov.b32 %r332, %f539;\n" " mov.u32 %r333, 2;\n" " mov.u32 %r334, 4127;\n" " mov.u32 %r335, 8;\n" " mov.u32 %r336, -1;\n" " shfl.sync.down.b32 %r337|%p83, %r332, %r335, %r334, %r336;\n" " mov.b32 %f274, %r337;\n" " add.f32 %f275, %f539, %f274;\n" " mov.b32 %r338, %f275;\n" " mov.u32 %r339, 4;\n" " shfl.sync.down.b32 %r340|%p84, %r338, %r339, %r334, %r336;\n" " mov.b32 %f276, %r340;\n" " add.f32 %f277, %f275, %f276;\n" " mov.b32 %r341, %f277;\n" " shfl.sync.down.b32 %r342|%p85, %r341, %r333, %r334, %r336;\n" " mov.b32 %f278, %r342;\n" " add.f32 %f279, %f277, %f278;\n" " mov.b32 %r343, %f279;\n" " mov.u32 %r344, 1;\n" " shfl.sync.down.b32 %r345|%p86, %r343, %r344, %r334, %r336;\n" " mov.b32 %f280, %r345;\n" " add.f32 %f86, %f279, %f280;\n" " not.pred %p87, %p1;\n" " @%p87 bra $L__BB18_101;\n" "\n" " mul.f32 %f281, %f86, %f179;\n" " st.shared.f32 [%r57+192], %f281;\n" "\n" "$L__BB18_101:\n" " setp.lt.s32 %p88, %r113, 5;\n" " @%p88 bra $L__BB18_104;\n" "\n" " mov.b32 %r346, %f540;\n" " mov.u32 %r347, 2;\n" " mov.u32 %r348, 4127;\n" " mov.u32 %r349, 8;\n" " mov.u32 %r350, -1;\n" " shfl.sync.down.b32 %r351|%p89, %r346, %r349, %r348, %r350;\n" " mov.b32 %f282, %r351;\n" " add.f32 %f283, %f540, %f282;\n" " mov.b32 %r352, %f283;\n" " mov.u32 %r353, 4;\n" " shfl.sync.down.b32 %r354|%p90, %r352, %r353, %r348, %r350;\n" " mov.b32 %f284, %r354;\n" " add.f32 %f285, %f283, %f284;\n" " mov.b32 %r355, %f285;\n" " shfl.sync.down.b32 %r356|%p91, %r355, %r347, %r348, %r350;\n" " mov.b32 %f286, %r356;\n" " add.f32 %f287, %f285, %f286;\n" " mov.b32 %r357, %f287;\n" " mov.u32 %r358, 1;\n" " shfl.sync.down.b32 %r359|%p92, %r357, %r358, %r348, %r350;\n" " mov.b32 %f288, %r359;\n" " add.f32 %f87, %f287, %f288;\n" " not.pred %p93, %p1;\n" " @%p93 bra $L__BB18_104;\n" "\n" " mul.f32 %f289, %f87, %f179;\n" " st.shared.f32 [%r57+256], %f289;\n" "\n" "$L__BB18_104:\n" " setp.lt.s32 %p94, %r113, 6;\n" " @%p94 bra $L__BB18_107;\n" "\n" " mov.b32 %r360, %f541;\n" " mov.u32 %r361, 2;\n" " mov.u32 %r362, 4127;\n" " mov.u32 %r363, 8;\n" " mov.u32 %r364, -1;\n" " shfl.sync.down.b32 %r365|%p95, %r360, %r363, %r362, %r364;\n" " mov.b32 %f290, %r365;\n" " add.f32 %f291, %f541, %f290;\n" " mov.b32 %r366, %f291;\n" " mov.u32 %r367, 4;\n" " shfl.sync.down.b32 %r368|%p96, %r366, %r367, %r362, %r364;\n" " mov.b32 %f292, %r368;\n" " add.f32 %f293, %f291, %f292;\n" " mov.b32 %r369, %f293;\n" " shfl.sync.down.b32 %r370|%p97, %r369, %r361, %r362, %r364;\n" " mov.b32 %f294, %r370;\n" " add.f32 %f295, %f293, %f294;\n" " mov.b32 %r371, %f295;\n" " mov.u32 %r372, 1;\n" " shfl.sync.down.b32 %r373|%p98, %r371, %r372, %r362, %r364;\n" " mov.b32 %f296, %r373;\n" " add.f32 %f88, %f295, %f296;\n" " not.pred %p99, %p1;\n" " @%p99 bra $L__BB18_107;\n" "\n" " mul.f32 %f297, %f88, %f179;\n" " st.shared.f32 [%r57+320], %f297;\n" "\n" "$L__BB18_107:\n" " setp.lt.s32 %p100, %r113, 7;\n" " @%p100 bra $L__BB18_110;\n" "\n" " mov.b32 %r374, %f542;\n" " mov.u32 %r375, 2;\n" " mov.u32 %r376, 4127;\n" " mov.u32 %r377, 8;\n" " mov.u32 %r378, -1;\n" " shfl.sync.down.b32 %r379|%p101, %r374, %r377, %r376, %r378;\n" " mov.b32 %f298, %r379;\n" " add.f32 %f299, %f542, %f298;\n" " mov.b32 %r380, %f299;\n" " mov.u32 %r381, 4;\n" " shfl.sync.down.b32 %r382|%p102, %r380, %r381, %r376, %r378;\n" " mov.b32 %f300, %r382;\n" " add.f32 %f301, %f299, %f300;\n" " mov.b32 %r383, %f301;\n" " shfl.sync.down.b32 %r384|%p103, %r383, %r375, %r376, %r378;\n" " mov.b32 %f302, %r384;\n" " add.f32 %f303, %f301, %f302;\n" " mov.b32 %r385, %f303;\n" " mov.u32 %r386, 1;\n" " shfl.sync.down.b32 %r387|%p104, %r385, %r386, %r376, %r378;\n" " mov.b32 %f304, %r387;\n" " add.f32 %f89, %f303, %f304;\n" " not.pred %p105, %p1;\n" " @%p105 bra $L__BB18_110;\n" "\n" " mul.f32 %f305, %f89, %f179;\n" " st.shared.f32 [%r57+384], %f305;\n" "\n" "$L__BB18_110:\n" " setp.lt.s32 %p106, %r113, 8;\n" " @%p106 bra $L__BB18_113;\n" "\n" " mov.b32 %r388, %f543;\n" " mov.u32 %r389, 2;\n" " mov.u32 %r390, 4127;\n" " mov.u32 %r391, 8;\n" " mov.u32 %r392, -1;\n" " shfl.sync.down.b32 %r393|%p107, %r388, %r391, %r390, %r392;\n" " mov.b32 %f306, %r393;\n" " add.f32 %f307, %f543, %f306;\n" " mov.b32 %r394, %f307;\n" " mov.u32 %r395, 4;\n" " shfl.sync.down.b32 %r396|%p108, %r394, %r395, %r390, %r392;\n" " mov.b32 %f308, %r396;\n" " add.f32 %f309, %f307, %f308;\n" " mov.b32 %r397, %f309;\n" " shfl.sync.down.b32 %r398|%p109, %r397, %r389, %r390, %r392;\n" " mov.b32 %f310, %r398;\n" " add.f32 %f311, %f309, %f310;\n" " mov.b32 %r399, %f311;\n" " mov.u32 %r400, 1;\n" " shfl.sync.down.b32 %r401|%p110, %r399, %r400, %r390, %r392;\n" " mov.b32 %f312, %r401;\n" " add.f32 %f90, %f311, %f312;\n" " not.pred %p111, %p1;\n" " @%p111 bra $L__BB18_113;\n" "\n" " mul.f32 %f313, %f90, %f179;\n" " st.shared.f32 [%r57+448], %f313;\n" "\n" "$L__BB18_113:\n" " bar.sync 0;\n" " @%p18 bra $L__BB18_131;\n" "\n" " ld.shared.f32 %f91, [%r37];\n" " mov.f32 %f578, %f91;\n" " @%p28 bra $L__BB18_122;\n" "\n" " and.b32 %r88, %r64, 3;\n" " add.s32 %r403, %r64, -1;\n" " setp.lt.u32 %p114, %r403, 3;\n" " mov.u32 %r599, 0;\n" " mov.f32 %f578, %f91;\n" " @%p114 bra $L__BB18_118;\n" "\n" " sub.s32 %r598, %r64, %r88;\n" " mov.u32 %r599, 0;\n" " mov.f32 %f578, %f91;\n" "\n" "$L__BB18_117:\n" " add.s32 %r405, %r45, %r599;\n" " shl.b32 %r406, %r405, 2;\n" " add.s32 %r408, %r170, %r406;\n" " ld.shared.f32 %f315, [%r408];\n" " max.f32 %f316, %f578, %f315;\n" " ld.shared.f32 %f317, [%r408+4];\n" " max.f32 %f318, %f316, %f317;\n" " ld.shared.f32 %f319, [%r408+8];\n" " max.f32 %f320, %f318, %f319;\n" " ld.shared.f32 %f321, [%r408+12];\n" " max.f32 %f578, %f320, %f321;\n" " add.s32 %r599, %r599, 4;\n" " add.s32 %r598, %r598, -4;\n" " setp.ne.s32 %p115, %r598, 0;\n" " @%p115 bra $L__BB18_117;\n" "\n" "$L__BB18_118:\n" " setp.eq.s32 %p116, %r88, 0;\n" " @%p116 bra $L__BB18_122;\n" "\n" " add.s32 %r409, %r45, %r599;\n" " shl.b32 %r410, %r409, 2;\n" " add.s32 %r95, %r170, %r410;\n" " ld.shared.f32 %f322, [%r95];\n" " max.f32 %f578, %f578, %f322;\n" " setp.eq.s32 %p117, %r88, 1;\n" " @%p117 bra $L__BB18_122;\n" "\n" " ld.shared.f32 %f323, [%r95+4];\n" " max.f32 %f578, %f578, %f323;\n" " setp.eq.s32 %p118, %r88, 2;\n" " @%p118 bra $L__BB18_122;\n" "\n" " ld.shared.f32 %f324, [%r95+8];\n" " max.f32 %f578, %f578, %f324;\n" "\n" "$L__BB18_122:\n" " mov.f32 %f582, 0f00000000;\n" " @%p28 bra $L__BB18_130;\n" "\n" " and.b32 %r96, %r64, 3;\n" " add.s32 %r413, %r64, -1;\n" " setp.lt.u32 %p120, %r413, 3;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r603, 0;\n" " @%p120 bra $L__BB18_126;\n" "\n" " sub.s32 %r601, %r64, %r96;\n" " mov.f32 %f582, 0f00000000;\n" " mov.u32 %r603, 0;\n" " mov.u32 %r602, %r59;\n" "\n" "$L__BB18_125:\n" " shl.b32 %r490, %r113, 2;\n" " add.s32 %r489, %r490, 1024;\n" " add.s32 %r415, %r45, %r603;\n" " shl.b32 %r416, %r415, 2;\n" " add.s32 %r418, %r170, %r416;\n" " ld.shared.f32 %f329, [%r418];\n" " sub.f32 %f330, %f329, %f578;\n" " mov.f32 %f331, 0f3F000000;\n" " mov.f32 %f332, 0f3BBB989D;\n" " fma.rn.f32 %f333, %f330, %f332, %f331;\n" " mov.f32 %f334, 0f3FB8AA3B;\n" " mov.f32 %f335, 0f437C0000;\n" " cvt.sat.f32.f32 %f336, %f333;\n" " mov.f32 %f337, 0f4B400001;\n" " fma.rm.f32 %f338, %f336, %f335, %f337;\n" " add.f32 %f339, %f338, 0fCB40007F;\n" " neg.f32 %f340, %f339;\n" " fma.rn.f32 %f341, %f330, %f334, %f340;\n" " mov.f32 %f342, 0f32A57060;\n" " fma.rn.f32 %f343, %f330, %f342, %f341;\n" " mov.b32 %r419, %f338;\n" " shl.b32 %r420, %r419, 23;\n" " mov.b32 %f344, %r420;\n" " ex2.approx.ftz.f32 %f345, %f343;\n" " mul.f32 %f346, %f345, %f344;\n" " st.shared.f32 [%r602+1024], %f346;\n" " add.f32 %f347, %f582, %f346;\n" " ld.shared.f32 %f348, [%r418+4];\n" " sub.f32 %f349, %f348, %f578;\n" " fma.rn.f32 %f350, %f349, %f332, %f331;\n" " cvt.sat.f32.f32 %f351, %f350;\n" " fma.rm.f32 %f352, %f351, %f335, %f337;\n" " add.f32 %f353, %f352, 0fCB40007F;\n" " neg.f32 %f354, %f353;\n" " fma.rn.f32 %f355, %f349, %f334, %f354;\n" " fma.rn.f32 %f356, %f349, %f342, %f355;\n" " mov.b32 %r421, %f352;\n" " shl.b32 %r422, %r421, 23;\n" " mov.b32 %f357, %r422;\n" " ex2.approx.ftz.f32 %f358, %f356;\n" " mul.f32 %f359, %f358, %f357;\n" " add.s32 %r423, %r602, %r490;\n" " st.shared.f32 [%r423+1024], %f359;\n" " add.f32 %f360, %f347, %f359;\n" " ld.shared.f32 %f361, [%r418+8];\n" " sub.f32 %f362, %f361, %f578;\n" " fma.rn.f32 %f363, %f362, %f332, %f331;\n" " cvt.sat.f32.f32 %f364, %f363;\n" " fma.rm.f32 %f365, %f364, %f335, %f337;\n" " add.f32 %f366, %f365, 0fCB40007F;\n" " neg.f32 %f367, %f366;\n" " fma.rn.f32 %f368, %f362, %f334, %f367;\n" " fma.rn.f32 %f369, %f362, %f342, %f368;\n" " mov.b32 %r424, %f365;\n" " shl.b32 %r425, %r424, 23;\n" " mov.b32 %f370, %r425;\n" " ex2.approx.ftz.f32 %f371, %f369;\n" " mul.f32 %f372, %f371, %f370;\n" " add.s32 %r426, %r423, %r490;\n" " add.s32 %r427, %r423, %r489;\n" " st.shared.f32 [%r427], %f372;\n" " add.f32 %f373, %f360, %f372;\n" " ld.shared.f32 %f374, [%r418+12];\n" " sub.f32 %f375, %f374, %f578;\n" " fma.rn.f32 %f376, %f375, %f332, %f331;\n" " cvt.sat.f32.f32 %f377, %f376;\n" " fma.rm.f32 %f378, %f377, %f335, %f337;\n" " add.f32 %f379, %f378, 0fCB40007F;\n" " neg.f32 %f380, %f379;\n" " fma.rn.f32 %f381, %f375, %f334, %f380;\n" " fma.rn.f32 %f382, %f375, %f342, %f381;\n" " mov.b32 %r428, %f378;\n" " shl.b32 %r429, %r428, 23;\n" " mov.b32 %f383, %r429;\n" " ex2.approx.ftz.f32 %f384, %f382;\n" " mul.f32 %f385, %f384, %f383;\n" " add.s32 %r430, %r426, %r490;\n" " add.s32 %r431, %r426, %r489;\n" " add.s32 %r602, %r430, %r490;\n" " st.shared.f32 [%r431], %f385;\n" " add.f32 %f582, %f373, %f385;\n" " add.s32 %r603, %r603, 4;\n" " add.s32 %r601, %r601, -4;\n" " setp.ne.s32 %p121, %r601, 0;\n" " @%p121 bra $L__BB18_125;\n" "\n" "$L__BB18_126:\n" " setp.eq.s32 %p122, %r96, 0;\n" " @%p122 bra $L__BB18_130;\n" "\n" " add.s32 %r432, %r45, %r603;\n" " shl.b32 %r433, %r432, 2;\n" " add.s32 %r105, %r170, %r433;\n" " ld.shared.f32 %f386, [%r105];\n" " sub.f32 %f387, %f386, %f578;\n" " mov.f32 %f388, 0f3F000000;\n" " mov.f32 %f389, 0f3BBB989D;\n" " fma.rn.f32 %f390, %f387, %f389, %f388;\n" " mov.f32 %f391, 0f3FB8AA3B;\n" " mov.f32 %f392, 0f437C0000;\n" " cvt.sat.f32.f32 %f393, %f390;\n" " mov.f32 %f394, 0f4B400001;\n" " fma.rm.f32 %f395, %f393, %f392, %f394;\n" " add.f32 %f396, %f395, 0fCB40007F;\n" " neg.f32 %f397, %f396;\n" " fma.rn.f32 %f398, %f387, %f391, %f397;\n" " mov.f32 %f399, 0f32A57060;\n" " fma.rn.f32 %f400, %f387, %f399, %f398;\n" " mov.b32 %r435, %f395;\n" " shl.b32 %r436, %r435, 23;\n" " mov.b32 %f401, %r436;\n" " ex2.approx.ftz.f32 %f402, %f400;\n" " mul.f32 %f403, %f402, %f401;\n" " cvt.u64.u32 %rd138, %r603;\n" " mul.lo.s64 %rd139, %rd138, %rd6;\n" " add.s64 %rd140, %rd47, %rd139;\n" " cvt.u32.u64 %r437, %rd140;\n" " add.s32 %r438, %r18, %r437;\n" " shl.b32 %r439, %r438, 2;\n" " add.s32 %r440, %r170, %r439;\n" " st.shared.f32 [%r440], %f403;\n" " add.f32 %f582, %f582, %f403;\n" " setp.eq.s32 %p123, %r96, 1;\n" " @%p123 bra $L__BB18_130;\n" "\n" " mov.f32 %f513, 0f32A57060;\n" " mov.f32 %f512, 0f3FB8AA3B;\n" " mov.f32 %f511, 0f4B400001;\n" " mov.f32 %f510, 0f437C0000;\n" " mov.f32 %f509, 0f3F000000;\n" " mov.f32 %f508, 0f3BBB989D;\n" " add.s32 %r473, %r45, %r603;\n" " shl.b32 %r472, %r473, 2;\n" " add.s32 %r471, %r170, %r472;\n" " ld.shared.f32 %f404, [%r471+4];\n" " sub.f32 %f405, %f404, %f578;\n" " fma.rn.f32 %f408, %f405, %f508, %f509;\n" " cvt.sat.f32.f32 %f411, %f408;\n" " fma.rm.f32 %f413, %f411, %f510, %f511;\n" " add.f32 %f414, %f413, 0fCB40007F;\n" " neg.f32 %f415, %f414;\n" " fma.rn.f32 %f416, %f405, %f512, %f415;\n" " fma.rn.f32 %f418, %f405, %f513, %f416;\n" " mov.b32 %r441, %f413;\n" " shl.b32 %r442, %r441, 23;\n" " mov.b32 %f419, %r442;\n" " ex2.approx.ftz.f32 %f420, %f418;\n" " mul.f32 %f421, %f420, %f419;\n" " add.s32 %r443, %r603, 1;\n" " cvt.u64.u32 %rd141, %r443;\n" " mul.lo.s64 %rd142, %rd141, %rd6;\n" " add.s64 %rd143, %rd47, %rd142;\n" " cvt.u32.u64 %r444, %rd143;\n" " add.s32 %r445, %r18, %r444;\n" " shl.b32 %r446, %r445, 2;\n" " add.s32 %r448, %r170, %r446;\n" " st.shared.f32 [%r448], %f421;\n" " add.f32 %f582, %f582, %f421;\n" " setp.eq.s32 %p124, %r96, 2;\n" " @%p124 bra $L__BB18_130;\n" "\n" " mov.f32 %f519, 0f32A57060;\n" " mov.f32 %f518, 0f3FB8AA3B;\n" " mov.f32 %f517, 0f4B400001;\n" " mov.f32 %f516, 0f437C0000;\n" " mov.f32 %f515, 0f3F000000;\n" " mov.f32 %f514, 0f3BBB989D;\n" " add.s32 %r476, %r45, %r603;\n" " shl.b32 %r475, %r476, 2;\n" " add.s32 %r474, %r170, %r475;\n" " ld.shared.f32 %f422, [%r474+8];\n" " sub.f32 %f423, %f422, %f578;\n" " fma.rn.f32 %f426, %f423, %f514, %f515;\n" " cvt.sat.f32.f32 %f429, %f426;\n" " fma.rm.f32 %f431, %f429, %f516, %f517;\n" " add.f32 %f432, %f431, 0fCB40007F;\n" " neg.f32 %f433, %f432;\n" " fma.rn.f32 %f434, %f423, %f518, %f433;\n" " fma.rn.f32 %f436, %f423, %f519, %f434;\n" " mov.b32 %r449, %f431;\n" " shl.b32 %r450, %r449, 23;\n" " mov.b32 %f437, %r450;\n" " ex2.approx.ftz.f32 %f438, %f436;\n" " mul.f32 %f439, %f438, %f437;\n" " add.s32 %r451, %r603, 2;\n" " cvt.u64.u32 %rd144, %r451;\n" " mul.lo.s64 %rd145, %rd144, %rd6;\n" " add.s64 %rd146, %rd47, %rd145;\n" " cvt.u32.u64 %r452, %rd146;\n" " add.s32 %r453, %r18, %r452;\n" " shl.b32 %r454, %r453, 2;\n" " add.s32 %r456, %r170, %r454;\n" " st.shared.f32 [%r456], %f439;\n" " add.f32 %f582, %f582, %f439;\n" "\n" "$L__BB18_130:\n" " sub.f32 %f440, %f91, %f578;\n" " mov.f32 %f441, 0f3F000000;\n" " mov.f32 %f442, 0f3BBB989D;\n" " fma.rn.f32 %f443, %f440, %f442, %f441;\n" " mov.f32 %f444, 0f3FB8AA3B;\n" " mov.f32 %f445, 0f437C0000;\n" " cvt.sat.f32.f32 %f446, %f443;\n" " mov.f32 %f447, 0f4B400001;\n" " fma.rm.f32 %f448, %f446, %f445, %f447;\n" " add.f32 %f449, %f448, 0fCB40007F;\n" " neg.f32 %f450, %f449;\n" " fma.rn.f32 %f451, %f440, %f444, %f450;\n" " mov.f32 %f452, 0f32A57060;\n" " fma.rn.f32 %f453, %f440, %f452, %f451;\n" " ex2.approx.ftz.f32 %f454, %f453;\n" " mov.b32 %r457, %f448;\n" " shl.b32 %r458, %r457, 23;\n" " mov.b32 %f455, %r458;\n" " mul.f32 %f456, %f454, %f455;\n" " ld.shared.f32 %f457, [%r38];\n" " fma.rn.f32 %f458, %f456, %f457, %f582;\n" " st.shared.f32 [%r38], %f458;\n" " st.shared.f32 [%r37], %f578;\n" " st.shared.f32 [%r46], %f456;\n" "\n" "$L__BB18_131:\n" " bar.sync 0;\n" " @%p28 bra $L__BB18_167;\n" "\n" " mov.u32 %r604, 0;\n" "\n" "$L__BB18_133:\n" " ld.param.u64 %rd231, [fc_attn_flash_decode_slots_param_2];\n" " add.s32 %r460, %r604, %r591;\n" " cvt.s64.s32 %rd148, %r460;\n" " mul.lo.s64 %rd149, %rd148, %rd34;\n" " add.s64 %rd150, %rd46, %rd149;\n" " shl.b64 %rd151, %rd150, 2;\n" " add.s64 %rd147, %rd231, %rd151;\n" " // begin inline asm\n" " ld.global.nc.f32 %f459, [%rd147];\n" " // end inline asm\n" " cvt.u64.u32 %rd152, %r604;\n" " mul.lo.s64 %rd153, %rd152, %rd6;\n" " add.s64 %rd154, %rd153, %rd31;\n" " cvt.u32.u64 %r461, %rd154;\n" " add.s32 %r462, %r18, %r461;\n" " shl.b32 %r463, %r462, 2;\n" " add.s32 %r107, %r170, %r463;\n" " setp.eq.s32 %p126, %r604, 0;\n" " @%p126 bra $L__BB18_150;\n" "\n" " @%p13 bra $L__BB18_136;\n" "\n" " ld.shared.f32 %f460, [%r107];\n" " fma.rn.f32 %f612, %f459, %f460, %f612;\n" "\n" "$L__BB18_136:\n" " setp.lt.s32 %p161, %r113, 2;\n" " @%p161 bra $L__BB18_138;\n" "\n" " ld.shared.f32 %f461, [%r107+4];\n" " fma.rn.f32 %f611, %f459, %f461, %f611;\n" "\n" "$L__BB18_138:\n" " setp.lt.s32 %p162, %r113, 3;\n" " @%p162 bra $L__BB18_140;\n" "\n" " ld.shared.f32 %f462, [%r107+8];\n" " fma.rn.f32 %f610, %f459, %f462, %f610;\n" "\n" "$L__BB18_140:\n" " setp.lt.s32 %p163, %r113, 4;\n" " @%p163 bra $L__BB18_142;\n" "\n" " ld.shared.f32 %f463, [%r107+12];\n" " fma.rn.f32 %f609, %f459, %f463, %f609;\n" "\n" "$L__BB18_142:\n" " setp.lt.s32 %p164, %r113, 5;\n" " @%p164 bra $L__BB18_144;\n" "\n" " ld.shared.f32 %f464, [%r107+16];\n" " fma.rn.f32 %f608, %f459, %f464, %f608;\n" "\n" "$L__BB18_144:\n" " setp.lt.s32 %p165, %r113, 6;\n" " @%p165 bra $L__BB18_146;\n" "\n" " ld.shared.f32 %f465, [%r107+20];\n" " fma.rn.f32 %f607, %f459, %f465, %f607;\n" "\n" "$L__BB18_146:\n" " setp.lt.s32 %p166, %r113, 7;\n" " @%p166 bra $L__BB18_148;\n" "\n" " ld.shared.f32 %f466, [%r107+24];\n" " fma.rn.f32 %f606, %f459, %f466, %f606;\n" "\n" "$L__BB18_148:\n" " setp.lt.s32 %p167, %r113, 8;\n" " @%p167 bra $L__BB18_166;\n" "\n" " ld.shared.f32 %f467, [%r107+28];\n" " fma.rn.f32 %f605, %f459, %f467, %f605;\n" " bra.uni $L__BB18_166;\n" "\n" "$L__BB18_150:\n" " @%p13 bra $L__BB18_152;\n" "\n" " ld.shared.f32 %f468, [%r58];\n" " ld.shared.f32 %f469, [%r107];\n" " mul.f32 %f470, %f459, %f469;\n" " fma.rn.f32 %f612, %f468, %f612, %f470;\n" "\n" "$L__BB18_152:\n" " setp.lt.s32 %p168, %r113, 2;\n" " @%p168 bra $L__BB18_154;\n" "\n" " ld.shared.f32 %f471, [%r58+4];\n" " ld.shared.f32 %f472, [%r107+4];\n" " mul.f32 %f473, %f459, %f472;\n" " fma.rn.f32 %f611, %f471, %f611, %f473;\n" "\n" "$L__BB18_154:\n" " setp.lt.s32 %p169, %r113, 3;\n" " @%p169 bra $L__BB18_156;\n" "\n" " ld.shared.f32 %f474, [%r58+8];\n" " ld.shared.f32 %f475, [%r107+8];\n" " mul.f32 %f476, %f459, %f475;\n" " fma.rn.f32 %f610, %f474, %f610, %f476;\n" "\n" "$L__BB18_156:\n" " setp.lt.s32 %p170, %r113, 4;\n" " @%p170 bra $L__BB18_158;\n" "\n" " ld.shared.f32 %f477, [%r58+12];\n" " ld.shared.f32 %f478, [%r107+12];\n" " mul.f32 %f479, %f459, %f478;\n" " fma.rn.f32 %f609, %f477, %f609, %f479;\n" "\n" "$L__BB18_158:\n" " setp.lt.s32 %p171, %r113, 5;\n" " @%p171 bra $L__BB18_160;\n" "\n" " ld.shared.f32 %f480, [%r58+16];\n" " ld.shared.f32 %f481, [%r107+16];\n" " mul.f32 %f482, %f459, %f481;\n" " fma.rn.f32 %f608, %f480, %f608, %f482;\n" "\n" "$L__BB18_160:\n" " setp.lt.s32 %p172, %r113, 6;\n" " @%p172 bra $L__BB18_162;\n" "\n" " ld.shared.f32 %f483, [%r58+20];\n" " ld.shared.f32 %f484, [%r107+20];\n" " mul.f32 %f485, %f459, %f484;\n" " fma.rn.f32 %f607, %f483, %f607, %f485;\n" "\n" "$L__BB18_162:\n" " setp.lt.s32 %p173, %r113, 7;\n" " @%p173 bra $L__BB18_164;\n" "\n" " ld.shared.f32 %f486, [%r58+24];\n" " ld.shared.f32 %f487, [%r107+24];\n" " mul.f32 %f488, %f459, %f487;\n" " fma.rn.f32 %f606, %f486, %f606, %f488;\n" "\n" "$L__BB18_164:\n" " setp.lt.s32 %p174, %r113, 8;\n" " @%p174 bra $L__BB18_166;\n" "\n" " ld.shared.f32 %f489, [%r58+28];\n" " ld.shared.f32 %f490, [%r107+28];\n" " mul.f32 %f491, %f459, %f490;\n" " fma.rn.f32 %f605, %f489, %f605, %f491;\n" "\n" "$L__BB18_166:\n" " sub.s32 %r496, %r17, %r591;\n" " min.s32 %r495, %r496, 16;\n" " add.s32 %r604, %r604, 1;\n" " setp.lt.s32 %p143, %r604, %r495;\n" " @%p143 bra $L__BB18_133;\n" "\n" "$L__BB18_167:\n" " bar.sync 0;\n" "\n" "$L__BB18_168:\n" " add.s32 %r591, %r591, 16;\n" " setp.gt.s32 %p144, %r17, %r591;\n" " add.s32 %r590, %r590, 1;\n" " @%p144 bra $L__BB18_29;\n" "\n" "$L__BB18_169:\n" " shl.b32 %r482, %r113, 4;\n" " cvt.s64.s32 %rd230, %r482;\n" " add.s32 %r481, %r114, 16;\n" " mul.wide.s32 %rd229, %r481, 16;\n" " add.s64 %rd228, %rd229, %rd230;\n" " add.s64 %rd227, %rd228, %rd230;\n" " cvt.u32.u64 %r480, %rd227;\n" " add.s64 %rd226, %rd227, %rd6;\n" " cvt.u32.u64 %r479, %rd226;\n" " mov.u32 %r478, %ctaid.x;\n" " mov.u32 %r477, %nctaid.x;\n" " cvt.s64.s32 %rd50, %r477;\n" " cvt.s64.s32 %rd51, %r478;\n" " add.s32 %r466, %r18, %r480;\n" " shl.b32 %r467, %r466, 2;\n" " add.s32 %r111, %r170, %r467;\n" " add.s32 %r469, %r18, %r479;\n" " shl.b32 %r470, %r469, 2;\n" " add.s32 %r112, %r170, %r470;\n" " @%p13 bra $L__BB18_172;\n" "\n" " setp.ne.s32 %p146, %r1, 0;\n" " mul.lo.s64 %rd155, %rd7, %rd50;\n" " add.s64 %rd52, %rd155, %rd51;\n" " mul.lo.s64 %rd156, %rd52, %rd34;\n" " add.s64 %rd157, %rd156, %rd45;\n" " shl.b64 %rd158, %rd157, 2;\n" " add.s64 %rd159, %rd4, %rd158;\n" " st.global.f32 [%rd159], %f612;\n" " @%p146 bra $L__BB18_172;\n" "\n" " ld.shared.f32 %f492, [%r111];\n" " shl.b64 %rd160, %rd52, 2;\n" " add.s64 %rd161, %rd3, %rd160;\n" " st.global.f32 [%rd161], %f492;\n" " ld.shared.f32 %f493, [%r112];\n" " add.s64 %rd162, %rd2, %rd160;\n" " st.global.f32 [%rd162], %f493;\n" "\n" "$L__BB18_172:\n" " setp.lt.s32 %p147, %r113, 2;\n" " @%p147 bra $L__BB18_175;\n" "\n" " setp.ne.s32 %p148, %r1, 0;\n" " add.s64 %rd163, %rd7, 1;\n" " mul.lo.s64 %rd164, %rd163, %rd50;\n" " add.s64 %rd53, %rd164, %rd51;\n" " mul.lo.s64 %rd165, %rd53, %rd34;\n" " add.s64 %rd166, %rd165, %rd45;\n" " shl.b64 %rd167, %rd166, 2;\n" " add.s64 %rd168, %rd4, %rd167;\n" " st.global.f32 [%rd168], %f611;\n" " @%p148 bra $L__BB18_175;\n" "\n" " ld.shared.f32 %f494, [%r111+4];\n" " shl.b64 %rd169, %rd53, 2;\n" " add.s64 %rd170, %rd3, %rd169;\n" " st.global.f32 [%rd170], %f494;\n" " ld.shared.f32 %f495, [%r112+4];\n" " add.s64 %rd171, %rd2, %rd169;\n" " st.global.f32 [%rd171], %f495;\n" "\n" "$L__BB18_175:\n" " setp.lt.s32 %p149, %r113, 3;\n" " @%p149 bra $L__BB18_178;\n" "\n" " setp.ne.s32 %p150, %r1, 0;\n" " add.s64 %rd172, %rd7, 2;\n" " mul.lo.s64 %rd173, %rd172, %rd50;\n" " add.s64 %rd54, %rd173, %rd51;\n" " mul.lo.s64 %rd174, %rd54, %rd34;\n" " add.s64 %rd175, %rd174, %rd45;\n" " shl.b64 %rd176, %rd175, 2;\n" " add.s64 %rd177, %rd4, %rd176;\n" " st.global.f32 [%rd177], %f610;\n" " @%p150 bra $L__BB18_178;\n" "\n" " ld.shared.f32 %f496, [%r111+8];\n" " shl.b64 %rd178, %rd54, 2;\n" " add.s64 %rd179, %rd3, %rd178;\n" " st.global.f32 [%rd179], %f496;\n" " ld.shared.f32 %f497, [%r112+8];\n" " add.s64 %rd180, %rd2, %rd178;\n" " st.global.f32 [%rd180], %f497;\n" "\n" "$L__BB18_178:\n" " setp.lt.s32 %p151, %r113, 4;\n" " @%p151 bra $L__BB18_181;\n" "\n" " setp.ne.s32 %p152, %r1, 0;\n" " add.s64 %rd181, %rd7, 3;\n" " mul.lo.s64 %rd182, %rd181, %rd50;\n" " add.s64 %rd55, %rd182, %rd51;\n" " mul.lo.s64 %rd183, %rd55, %rd34;\n" " add.s64 %rd184, %rd183, %rd45;\n" " shl.b64 %rd185, %rd184, 2;\n" " add.s64 %rd186, %rd4, %rd185;\n" " st.global.f32 [%rd186], %f609;\n" " @%p152 bra $L__BB18_181;\n" "\n" " ld.shared.f32 %f498, [%r111+12];\n" " shl.b64 %rd187, %rd55, 2;\n" " add.s64 %rd188, %rd3, %rd187;\n" " st.global.f32 [%rd188], %f498;\n" " ld.shared.f32 %f499, [%r112+12];\n" " add.s64 %rd189, %rd2, %rd187;\n" " st.global.f32 [%rd189], %f499;\n" "\n" "$L__BB18_181:\n" " setp.lt.s32 %p153, %r113, 5;\n" " @%p153 bra $L__BB18_184;\n" "\n" " setp.ne.s32 %p154, %r1, 0;\n" " add.s64 %rd190, %rd7, 4;\n" " mul.lo.s64 %rd191, %rd190, %rd50;\n" " add.s64 %rd56, %rd191, %rd51;\n" " mul.lo.s64 %rd192, %rd56, %rd34;\n" " add.s64 %rd193, %rd192, %rd45;\n" " shl.b64 %rd194, %rd193, 2;\n" " add.s64 %rd195, %rd4, %rd194;\n" " st.global.f32 [%rd195], %f608;\n" " @%p154 bra $L__BB18_184;\n" "\n" " ld.shared.f32 %f500, [%r111+16];\n" " shl.b64 %rd196, %rd56, 2;\n" " add.s64 %rd197, %rd3, %rd196;\n" " st.global.f32 [%rd197], %f500;\n" " ld.shared.f32 %f501, [%r112+16];\n" " add.s64 %rd198, %rd2, %rd196;\n" " st.global.f32 [%rd198], %f501;\n" "\n" "$L__BB18_184:\n" " setp.lt.s32 %p155, %r113, 6;\n" " @%p155 bra $L__BB18_187;\n" "\n" " setp.ne.s32 %p156, %r1, 0;\n" " add.s64 %rd199, %rd7, 5;\n" " mul.lo.s64 %rd200, %rd199, %rd50;\n" " add.s64 %rd57, %rd200, %rd51;\n" " mul.lo.s64 %rd201, %rd57, %rd34;\n" " add.s64 %rd202, %rd201, %rd45;\n" " shl.b64 %rd203, %rd202, 2;\n" " add.s64 %rd204, %rd4, %rd203;\n" " st.global.f32 [%rd204], %f607;\n" " @%p156 bra $L__BB18_187;\n" "\n" " ld.shared.f32 %f502, [%r111+20];\n" " shl.b64 %rd205, %rd57, 2;\n" " add.s64 %rd206, %rd3, %rd205;\n" " st.global.f32 [%rd206], %f502;\n" " ld.shared.f32 %f503, [%r112+20];\n" " add.s64 %rd207, %rd2, %rd205;\n" " st.global.f32 [%rd207], %f503;\n" "\n" "$L__BB18_187:\n" " setp.lt.s32 %p157, %r113, 7;\n" " @%p157 bra $L__BB18_190;\n" "\n" " setp.ne.s32 %p158, %r1, 0;\n" " add.s64 %rd208, %rd7, 6;\n" " mul.lo.s64 %rd209, %rd208, %rd50;\n" " add.s64 %rd58, %rd209, %rd51;\n" " mul.lo.s64 %rd210, %rd58, %rd34;\n" " add.s64 %rd211, %rd210, %rd45;\n" " shl.b64 %rd212, %rd211, 2;\n" " add.s64 %rd213, %rd4, %rd212;\n" " st.global.f32 [%rd213], %f606;\n" " @%p158 bra $L__BB18_190;\n" "\n" " ld.shared.f32 %f504, [%r111+24];\n" " shl.b64 %rd214, %rd58, 2;\n" " add.s64 %rd215, %rd3, %rd214;\n" " st.global.f32 [%rd215], %f504;\n" " ld.shared.f32 %f505, [%r112+24];\n" " add.s64 %rd216, %rd2, %rd214;\n" " st.global.f32 [%rd216], %f505;\n" "\n" "$L__BB18_190:\n" " setp.lt.s32 %p159, %r113, 8;\n" " @%p159 bra $L__BB18_193;\n" "\n" " setp.ne.s32 %p160, %r1, 0;\n" " add.s64 %rd217, %rd7, 7;\n" " mul.lo.s64 %rd218, %rd217, %rd50;\n" " add.s64 %rd59, %rd218, %rd51;\n" " mul.lo.s64 %rd219, %rd59, %rd34;\n" " add.s64 %rd220, %rd219, %rd45;\n" " shl.b64 %rd221, %rd220, 2;\n" " add.s64 %rd222, %rd4, %rd221;\n" " st.global.f32 [%rd222], %f605;\n" " @%p160 bra $L__BB18_193;\n" "\n" " ld.shared.f32 %f506, [%r111+28];\n" " shl.b64 %rd223, %rd59, 2;\n" " add.s64 %rd224, %rd3, %rd223;\n" " st.global.f32 [%rd224], %f506;\n" " ld.shared.f32 %f507, [%r112+28];\n" " add.s64 %rd225, %rd2, %rd223;\n" " st.global.f32 [%rd225], %f507;\n" " bra.uni $L__BB18_193;\n" "\n" "$L__BB18_1:\n" " setp.lt.s32 %p3, %r113, 1;\n" " @%p3 bra $L__BB18_193;\n" "\n" " cvt.s64.s32 %rd8, %r2;\n" " cvt.s64.s32 %rd9, %r6;\n" " cvt.s64.s32 %rd10, %r114;\n" " cvt.s64.s32 %rd11, %r1;\n" " and.b32 %r583, %r113, 3;\n" " add.s32 %r132, %r113, -1;\n" " setp.lt.u32 %p4, %r132, 3;\n" " mov.u32 %r582, 0;\n" " @%p4 bra $L__BB18_13;\n" "\n" " sub.s32 %r581, %r113, %r583;\n" " mul.lo.s64 %rd72, %rd5, %rd8;\n" " mul.lo.s64 %rd73, %rd72, %rd6;\n" " add.s64 %rd74, %rd73, %rd9;\n" " mul.lo.s64 %rd75, %rd74, %rd10;\n" " add.s64 %rd76, %rd75, %rd11;\n" " shl.b64 %rd77, %rd76, 2;\n" " add.s64 %rd232, %rd4, %rd77;\n" " mul.lo.s64 %rd78, %rd8, %rd10;\n" " shl.b64 %rd13, %rd78, 2;\n" " mov.u32 %r133, 0;\n" " setp.ne.s32 %p5, %r1, 0;\n" " mov.u32 %r582, %r133;\n" "\n" "$L__BB18_4:\n" " st.global.u32 [%rd232], %r133;\n" " @%p5 bra $L__BB18_6;\n" "\n" " cvt.s64.s32 %rd79, %r582;\n" " add.s64 %rd80, %rd7, %rd79;\n" " mul.lo.s64 %rd81, %rd80, %rd8;\n" " add.s64 %rd82, %rd81, %rd9;\n" " shl.b64 %rd83, %rd82, 2;\n" " add.s64 %rd84, %rd3, %rd83;\n" " mov.u32 %r135, -8388608;\n" " st.global.u32 [%rd84], %r135;\n" " add.s64 %rd85, %rd2, %rd83;\n" " mov.u32 %r136, 0;\n" " st.global.u32 [%rd85], %r136;\n" "\n" "$L__BB18_6:\n" " mov.u32 %r137, 0;\n" " add.s64 %rd15, %rd232, %rd13;\n" " st.global.u32 [%rd15], %r137;\n" " @%p5 bra $L__BB18_8;\n" "\n" " add.s32 %r138, %r582, 1;\n" " cvt.s64.s32 %rd86, %r138;\n" " add.s64 %rd87, %rd7, %rd86;\n" " mul.lo.s64 %rd88, %rd87, %rd8;\n" " add.s64 %rd89, %rd88, %rd9;\n" " shl.b64 %rd90, %rd89, 2;\n" " add.s64 %rd91, %rd3, %rd90;\n" " mov.u32 %r139, -8388608;\n" " st.global.u32 [%rd91], %r139;\n" " add.s64 %rd92, %rd2, %rd90;\n" " st.global.u32 [%rd92], %r137;\n" "\n" "$L__BB18_8:\n" " add.s64 %rd16, %rd15, %rd13;\n" " st.global.u32 [%rd16], %r137;\n" " @%p5 bra $L__BB18_10;\n" "\n" " add.s32 %r142, %r582, 2;\n" " cvt.s64.s32 %rd93, %r142;\n" " add.s64 %rd94, %rd7, %rd93;\n" " mul.lo.s64 %rd95, %rd94, %rd8;\n" " add.s64 %rd96, %rd95, %rd9;\n" " shl.b64 %rd97, %rd96, 2;\n" " add.s64 %rd98, %rd3, %rd97;\n" " mov.u32 %r143, -8388608;\n" " st.global.u32 [%rd98], %r143;\n" " add.s64 %rd99, %rd2, %rd97;\n" " st.global.u32 [%rd99], %r137;\n" "\n" "$L__BB18_10:\n" " add.s64 %rd17, %rd16, %rd13;\n" " st.global.u32 [%rd17], %r137;\n" " @%p5 bra $L__BB18_12;\n" "\n" " add.s32 %r146, %r582, 3;\n" " cvt.s64.s32 %rd100, %r146;\n" " add.s64 %rd101, %rd7, %rd100;\n" " mul.lo.s64 %rd102, %rd101, %rd8;\n" " add.s64 %rd103, %rd102, %rd9;\n" " shl.b64 %rd104, %rd103, 2;\n" " add.s64 %rd105, %rd3, %rd104;\n" " mov.u32 %r147, -8388608;\n" " st.global.u32 [%rd105], %r147;\n" " add.s64 %rd106, %rd2, %rd104;\n" " st.global.u32 [%rd106], %r137;\n" "\n" "$L__BB18_12:\n" " add.s32 %r582, %r582, 4;\n" " add.s32 %r581, %r581, -4;\n" " setp.ne.s32 %p9, %r581, 0;\n" " add.s64 %rd232, %rd17, %rd13;\n" " @%p9 bra $L__BB18_4;\n" "\n" "$L__BB18_13:\n" " setp.eq.s32 %p10, %r583, 0;\n" " @%p10 bra $L__BB18_193;\n" "\n" " cvt.s64.s32 %rd107, %r582;\n" " add.s64 %rd108, %rd7, %rd107;\n" " mul.lo.s64 %rd109, %rd108, %rd8;\n" " add.s64 %rd110, %rd109, %rd9;\n" " shl.b64 %rd111, %rd110, 2;\n" " add.s64 %rd235, %rd2, %rd111;\n" " shl.b64 %rd20, %rd8, 2;\n" " add.s64 %rd234, %rd3, %rd111;\n" " mul.lo.s64 %rd112, %rd110, %rd10;\n" " add.s64 %rd113, %rd112, %rd11;\n" " shl.b64 %rd114, %rd113, 2;\n" " add.s64 %rd233, %rd4, %rd114;\n" " mul.lo.s64 %rd115, %rd8, %rd10;\n" " shl.b64 %rd23, %rd115, 2;\n" " mov.u32 %r149, 0;\n" " setp.ne.s32 %p11, %r1, 0;\n" " mov.u32 %r150, -8388608;\n" "\n" "$L__BB18_15:\n" " .pragma \"nounroll\";\n" " st.global.u32 [%rd233], %r149;\n" " @%p11 bra $L__BB18_17;\n" "\n" " st.global.u32 [%rd234], %r150;\n" " st.global.u32 [%rd235], %r149;\n" "\n" "$L__BB18_17:\n" " add.s64 %rd235, %rd235, %rd20;\n" " add.s64 %rd234, %rd234, %rd20;\n" " add.s64 %rd233, %rd233, %rd23;\n" " add.s32 %r583, %r583, -1;\n" " setp.eq.s32 %p12, %r583, 0;\n" " @%p12 bra $L__BB18_193;\n" " bra.uni $L__BB18_15;\n" "\n" "$L__BB18_193:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_flash_decode_slots_q4\n" ".visible .entry fc_attn_flash_decode_slots_q4(\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_0,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_1,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_2,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_3,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_4,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_5,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_6,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_7,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_8,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_9,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_10,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_11,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_12,\n" " .param .u64 fc_attn_flash_decode_slots_q4_param_13,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_14,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_15,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_16,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_17,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_18,\n" " .param .u32 fc_attn_flash_decode_slots_q4_param_19,\n" " .param .f32 fc_attn_flash_decode_slots_q4_param_20\n" ")\n" "{\n" " .reg .pred %p<206>;\n" " .reg .b16 %rs<11>;\n" " .reg .f32 %f<767>;\n" " .reg .b32 %r<727>;\n" " .reg .b64 %rd<314>;\n" "\n" "\n" " ld.param.u64 %rd82, [fc_attn_flash_decode_slots_q4_param_0];\n" " ld.param.u64 %rd83, [fc_attn_flash_decode_slots_q4_param_1];\n" " ld.param.u64 %rd84, [fc_attn_flash_decode_slots_q4_param_2];\n" " ld.param.u64 %rd85, [fc_attn_flash_decode_slots_q4_param_3];\n" " ld.param.u64 %rd86, [fc_attn_flash_decode_slots_q4_param_4];\n" " ld.param.u64 %rd80, [fc_attn_flash_decode_slots_q4_param_5];\n" " ld.param.u64 %rd81, [fc_attn_flash_decode_slots_q4_param_6];\n" " ld.param.u64 %rd87, [fc_attn_flash_decode_slots_q4_param_7];\n" " ld.param.u64 %rd88, [fc_attn_flash_decode_slots_q4_param_8];\n" " ld.param.u64 %rd89, [fc_attn_flash_decode_slots_q4_param_9];\n" " ld.param.u32 %r133, [fc_attn_flash_decode_slots_q4_param_10];\n" " ld.param.u32 %r134, [fc_attn_flash_decode_slots_q4_param_11];\n" " ld.param.u32 %r135, [fc_attn_flash_decode_slots_q4_param_12];\n" " ld.param.u64 %rd90, [fc_attn_flash_decode_slots_q4_param_13];\n" " ld.param.u32 %r140, [fc_attn_flash_decode_slots_q4_param_14];\n" " ld.param.u32 %r141, [fc_attn_flash_decode_slots_q4_param_15];\n" " ld.param.u32 %r136, [fc_attn_flash_decode_slots_q4_param_16];\n" " ld.param.u32 %r137, [fc_attn_flash_decode_slots_q4_param_17];\n" " ld.param.u32 %r139, [fc_attn_flash_decode_slots_q4_param_19];\n" " ld.param.f32 %f231, [fc_attn_flash_decode_slots_q4_param_20];\n" " cvta.to.global.u64 %rd1, %rd82;\n" " cvta.to.global.u64 %rd2, %rd85;\n" " cvta.to.global.u64 %rd3, %rd83;\n" " cvta.to.global.u64 %rd4, %rd86;\n" " cvta.to.global.u64 %rd5, %rd84;\n" " cvta.to.global.u64 %rd6, %rd88;\n" " cvta.to.global.u64 %rd7, %rd87;\n" " cvta.to.global.u64 %rd8, %rd89;\n" " mov.u32 %r1, %tid.x;\n" " mov.u32 %r2, %nctaid.x;\n" " mov.u32 %r142, %ctaid.z;\n" " cvta.to.global.u64 %rd91, %rd90;\n" " mul.wide.s32 %rd92, %r142, 4;\n" " add.s64 %rd93, %rd91, %rd92;\n" " ld.global.u32 %r3, [%rd93];\n" " add.s32 %r143, %r3, %r140;\n" " div.s32 %r144, %r143, %r140;\n" " max.s32 %r145, %r144, 1;\n" " min.s32 %r146, %r145, %r141;\n" " add.s32 %r147, %r146, %r3;\n" " div.s32 %r148, %r147, %r146;\n" " add.s32 %r4, %r148, 15;\n" " and.b32 %r5, %r4, -16;\n" " add.s32 %r149, %r5, %r3;\n" " div.s32 %r150, %r149, %r5;\n" " mov.u32 %r151, %nctaid.y;\n" " mul.wide.s32 %rd94, %r142, %r151;\n" " mov.u32 %r152, %ctaid.y;\n" " cvt.s64.s32 %rd95, %r152;\n" " add.s64 %rd9, %rd94, %rd95;\n" " cvt.s64.s32 %rd10, %r133;\n" " mul.lo.s64 %rd11, %rd9, %rd10;\n" " mov.u32 %r6, %ctaid.x;\n" " setp.lt.s32 %p2, %r6, %r150;\n" " @%p2 bra $L__BB19_18;\n" " bra.uni $L__BB19_1;\n" "\n" "$L__BB19_18:\n" " setp.lt.s32 %p13, %r133, 1;\n" " mul.lo.s32 %r707, %r5, %r6;\n" " add.s32 %r174, %r707, %r5;\n" " add.s32 %r175, %r3, 1;\n" " min.s32 %r17, %r174, %r175;\n" " mul.lo.s32 %r18, %r134, %r133;\n" " add.s32 %r176, %r134, 16;\n" " cvt.s64.s32 %rd34, %r176;\n" " mul.wide.s32 %rd35, %r176, 16;\n" " shl.b32 %r177, %r133, 4;\n" " cvt.s64.s32 %rd140, %r177;\n" " add.s64 %rd36, %rd35, %rd140;\n" " add.s64 %rd37, %rd36, %rd140;\n" " add.s64 %rd38, %rd37, %rd10;\n" " cvt.s64.s32 %rd39, %r134;\n" " @%p13 bra $L__BB19_25;\n" "\n" " cvt.s64.s32 %rd40, %r1;\n" " and.b32 %r705, %r133, 3;\n" " add.s32 %r179, %r133, -1;\n" " setp.lt.u32 %p14, %r179, 3;\n" " mov.u32 %r703, 0;\n" " @%p14 bra $L__BB19_22;\n" "\n" " sub.s32 %r701, %r133, %r705;\n" " mul.lo.s64 %rd141, %rd9, %rd39;\n" " mul.lo.s64 %rd142, %rd141, %rd10;\n" " add.s64 %rd143, %rd142, %rd40;\n" " shl.b64 %rd144, %rd143, 2;\n" " add.s64 %rd312, %rd1, %rd144;\n" " shl.b64 %rd42, %rd39, 2;\n" " shl.b32 %r181, %r1, 2;\n" " mov.u32 %r182, sh;\n" " add.s32 %r702, %r182, %r181;\n" " shl.b32 %r22, %r134, 2;\n" " mov.u32 %r703, 0;\n" "\n" "$L__BB19_21:\n" " ld.global.f32 %f232, [%rd312];\n" " st.shared.f32 [%r702], %f232;\n" " add.s64 %rd145, %rd312, %rd42;\n" " ld.global.f32 %f233, [%rd145];\n" " add.s32 %r183, %r702, %r22;\n" " st.shared.f32 [%r183], %f233;\n" " add.s64 %rd146, %rd145, %rd42;\n" " ld.global.f32 %f234, [%rd146];\n" " add.s32 %r184, %r183, %r22;\n" " st.shared.f32 [%r184], %f234;\n" " add.s64 %rd147, %rd146, %rd42;\n" " add.s64 %rd312, %rd147, %rd42;\n" " ld.global.f32 %f235, [%rd147];\n" " add.s32 %r185, %r184, %r22;\n" " add.s32 %r702, %r185, %r22;\n" " st.shared.f32 [%r185], %f235;\n" " add.s32 %r703, %r703, 4;\n" " add.s32 %r701, %r701, -4;\n" " setp.ne.s32 %p15, %r701, 0;\n" " @%p15 bra $L__BB19_21;\n" "\n" "$L__BB19_22:\n" " setp.eq.s32 %p16, %r705, 0;\n" " @%p16 bra $L__BB19_25;\n" "\n" " mad.lo.s32 %r186, %r703, %r134, %r1;\n" " shl.b32 %r187, %r186, 2;\n" " mov.u32 %r188, sh;\n" " add.s32 %r704, %r188, %r187;\n" " shl.b32 %r31, %r134, 2;\n" " cvt.s64.s32 %rd148, %r703;\n" " add.s64 %rd149, %rd11, %rd148;\n" " mul.lo.s64 %rd150, %rd149, %rd39;\n" " add.s64 %rd151, %rd150, %rd40;\n" " shl.b64 %rd152, %rd151, 2;\n" " add.s64 %rd313, %rd1, %rd152;\n" " shl.b64 %rd46, %rd39, 2;\n" "\n" "$L__BB19_24:\n" " .pragma \"nounroll\";\n" " ld.global.f32 %f236, [%rd313];\n" " st.shared.f32 [%r704], %f236;\n" " add.s32 %r704, %r704, %r31;\n" " add.s64 %rd313, %rd313, %rd46;\n" " add.s32 %r705, %r705, -1;\n" " setp.ne.s32 %p17, %r705, 0;\n" " @%p17 bra $L__BB19_24;\n" "\n" "$L__BB19_25:\n" " setp.ge.s32 %p18, %r1, %r133;\n" " add.s32 %r189, %r18, %r1;\n" " cvt.u32.u64 %r190, %rd37;\n" " add.s32 %r191, %r189, %r190;\n" " shl.b32 %r192, %r191, 2;\n" " mov.u32 %r193, sh;\n" " add.s32 %r36, %r193, %r192;\n" " cvt.u32.u64 %r194, %rd38;\n" " add.s32 %r195, %r189, %r194;\n" " shl.b32 %r196, %r195, 2;\n" " add.s32 %r37, %r193, %r196;\n" " @%p18 bra $L__BB19_27;\n" "\n" " mov.u32 %r197, -8388608;\n" " st.shared.u32 [%r36], %r197;\n" " mov.u32 %r198, 0;\n" " st.shared.u32 [%r37], %r198;\n" "\n" "$L__BB19_27:\n" " bar.sync 0;\n" " cvt.s64.s32 %rd153, %r135;\n" " mul.lo.s64 %rd154, %rd9, %rd153;\n" " shr.s32 %r199, %r134, 31;\n" " shr.u32 %r200, %r199, 27;\n" " add.s32 %r201, %r134, %r200;\n" " shr.s32 %r202, %r201, 5;\n" " mul.lo.s32 %r203, %r202, 18;\n" " cvt.s64.s32 %rd49, %r203;\n" " mul.lo.s64 %rd50, %rd154, %rd49;\n" " mul.lo.s64 %rd155, %rd9, %rd39;\n" " shl.b64 %rd51, %rd155, 8;\n" " shl.b64 %rd52, %rd155, 3;\n" " shr.s32 %r204, %r1, 31;\n" " shr.u32 %r205, %r204, 28;\n" " add.s32 %r206, %r1, %r205;\n" " shr.s32 %r39, %r206, 4;\n" " and.b32 %r207, %r206, -16;\n" " sub.s32 %r40, %r1, %r207;\n" " cvt.s64.s32 %rd53, %r1;\n" " setp.le.s32 %p19, %r17, %r707;\n" " mov.f32 %f704, 0f00000000;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" " @%p19 bra $L__BB19_231;\n" "\n" " shr.u32 %r210, %r199, 28;\n" " add.s32 %r211, %r134, %r210;\n" " shr.s32 %r212, %r211, 4;\n" " cvt.u32.u64 %r213, %rd34;\n" " sub.s32 %r214, %r3, %r139;\n" " add.s32 %r215, %r214, 1;\n" " mov.u32 %r706, 0;\n" " max.s32 %r41, %r215, 0;\n" " sub.s32 %r42, %r3, %r137;\n" " mov.u32 %r43, %ntid.x;\n" " add.s32 %r216, %r18, %r39;\n" " cvt.u32.u64 %r217, %rd35;\n" " add.s32 %r218, %r216, %r217;\n" " mad.lo.s32 %r44, %r213, %r39, %r18;\n" " shl.b32 %r219, %r1, 4;\n" " add.s32 %r220, %r219, %r217;\n" " add.s32 %r45, %r18, %r220;\n" " add.s64 %rd54, %rd52, %rd53;\n" " add.s32 %r221, %r194, %r133;\n" " add.s32 %r222, %r18, %r221;\n" " add.s64 %rd55, %rd51, %rd53;\n" " shr.u32 %r224, %r204, 27;\n" " add.s32 %r225, %r1, %r224;\n" " shr.s32 %r226, %r225, 5;\n" " mul.lo.s32 %r227, %r226, 18;\n" " cvt.s64.s32 %rd56, %r227;\n" " and.b32 %r228, %r1, 31;\n" " setp.lt.u32 %p20, %r228, 16;\n" " add.s32 %r229, %r228, -16;\n" " selp.b32 %r230, %r228, %r229, %p20;\n" " selp.b32 %r46, 0, 4, %p20;\n" " add.s32 %r231, %r230, 2;\n" " cvt.s64.s32 %rd156, %r231;\n" " add.s64 %rd57, %rd156, %rd56;\n" " add.s32 %r233, %r189, %r221;\n" " shl.b32 %r234, %r233, 2;\n" " add.s32 %r47, %r193, %r234;\n" " cvt.u64.u32 %rd157, %r1;\n" " add.s64 %rd58, %rd36, %rd157;\n" " shl.b32 %r236, %r6, 4;\n" " add.s32 %r237, %r236, 16;\n" " shr.u32 %r238, %r4, 4;\n" " mul.lo.s32 %r239, %r238, %r237;\n" " not.b32 %r240, %r239;\n" " mov.u32 %r241, -2;\n" " sub.s32 %r242, %r241, %r3;\n" " max.s32 %r243, %r242, %r240;\n" " mul.lo.s32 %r244, %r6, %r238;\n" " shl.b32 %r245, %r244, 4;\n" " add.s32 %r48, %r243, %r245;\n" " bfe.u32 %r50, %r211, 4, 1;\n" " sub.s32 %r51, %r212, %r50;\n" " shl.b32 %r52, %r134, 1;\n" " mul.lo.s32 %r53, %r134, 3;\n" " shl.b32 %r54, %r134, 2;\n" " mul.lo.s32 %r55, %r134, 5;\n" " mul.lo.s32 %r56, %r134, 6;\n" " mul.lo.s32 %r57, %r134, 7;\n" " shl.b32 %r246, %r218, 2;\n" " add.s32 %r58, %r193, %r246;\n" " shl.b32 %r247, %r222, 2;\n" " add.s32 %r59, %r193, %r247;\n" " mad.lo.s32 %r248, %r133, %r213, %r1;\n" " add.s32 %r249, %r248, %r217;\n" " shl.b32 %r250, %r249, 2;\n" " add.s32 %r251, %r193, %r250;\n" " add.s32 %r60, %r251, -1024;\n" " cvta.to.global.u64 %rd59, %rd80;\n" " cvta.to.global.u64 %rd60, %rd81;\n" " mov.f32 %f704, 0f00000000;\n" " setp.lt.s32 %p22, %r139, 1;\n" " setp.gt.s32 %p30, %r137, 0;\n" " setp.ge.s32 %p45, %r1, %r134;\n" " mov.f32 %f705, %f704;\n" " mov.f32 %f706, %f704;\n" " mov.f32 %f707, %f704;\n" " mov.f32 %f708, %f704;\n" " mov.f32 %f709, %f704;\n" " mov.f32 %f710, %f704;\n" " mov.f32 %f711, %f704;\n" "\n" "$L__BB19_29:\n" " ld.param.u32 %r605, [fc_attn_flash_decode_slots_q4_param_18];\n" " shl.b32 %r252, %r706, 4;\n" " add.s32 %r253, %r48, %r252;\n" " max.s32 %r254, %r253, -17;\n" " not.b32 %r255, %r254;\n" " max.s32 %r65, %r255, 1;\n" " sub.s32 %r66, %r17, %r707;\n" " min.s32 %r67, %r66, 16;\n" " setp.lt.s32 %p21, %r707, %r605;\n" " or.pred %p23, %p22, %p21;\n" " add.s32 %r256, %r67, %r707;\n" " setp.gt.s32 %p24, %r256, %r41;\n" " setp.le.s32 %p25, %r707, %r3;\n" " and.pred %p26, %p25, %p24;\n" " or.pred %p27, %p23, %p26;\n" " not.pred %p28, %p27;\n" " @%p28 bra $L__BB19_230;\n" "\n" " setp.lt.s32 %p29, %r66, 1;\n" " @%p29 bra $L__BB19_54;\n" "\n" " @%p30 bra $L__BB19_41;\n" " bra.uni $L__BB19_32;\n" "\n" "$L__BB19_41:\n" " mov.u32 %r711, 0;\n" "\n" "$L__BB19_42:\n" " cvt.s64.s32 %rd170, %r711;\n" " mul.lo.s64 %rd65, %rd170, %rd34;\n" " add.s32 %r77, %r711, %r707;\n" " setp.lt.s32 %p38, %r77, %r136;\n" " @%p38 bra $L__BB19_50;\n" " bra.uni $L__BB19_43;\n" "\n" "$L__BB19_50:\n" " @%p45 bra $L__BB19_53;\n" "\n" " cvt.u32.u64 %r319, %rd65;\n" " add.s32 %r84, %r18, %r319;\n" " cvt.s64.s32 %rd184, %r77;\n" " mul.lo.s64 %rd185, %rd184, %rd39;\n" " add.s64 %rd68, %rd185, %rd52;\n" " mov.u32 %r714, %r1;\n" "\n" "$L__BB19_52:\n" " cvt.s64.s32 %rd186, %r714;\n" " add.s64 %rd187, %rd68, %rd186;\n" " shl.b64 %rd188, %rd187, 1;\n" " add.s64 %rd189, %rd2, %rd188;\n" " ld.global.u16 %rs5, [%rd189];\n" " // begin inline asm\n" " { cvt.f32.f16 %f261, %rs5;}\n" "\n" " // end inline asm\n" " add.s32 %r320, %r84, %r714;\n" " shl.b32 %r321, %r320, 2;\n" " add.s32 %r323, %r193, %r321;\n" " st.shared.f32 [%r323], %f261;\n" " add.s32 %r714, %r714, %r43;\n" " setp.lt.s32 %p46, %r714, %r134;\n" " @%p46 bra $L__BB19_52;\n" " bra.uni $L__BB19_53;\n" "\n" "$L__BB19_43:\n" " setp.gt.s32 %p39, %r77, %r42;\n" " @%p39 bra $L__BB19_47;\n" " bra.uni $L__BB19_44;\n" "\n" "$L__BB19_47:\n" " @%p45 bra $L__BB19_53;\n" "\n" " cvt.u32.u64 %r309, %rd65;\n" " add.s32 %r81, %r18, %r309;\n" " shr.s32 %r310, %r77, 31;\n" " shr.u32 %r311, %r310, 24;\n" " add.s32 %r312, %r77, %r311;\n" " and.b32 %r313, %r312, -256;\n" " sub.s32 %r314, %r77, %r313;\n" " cvt.s64.s32 %rd178, %r314;\n" " mul.lo.s64 %rd179, %rd178, %rd39;\n" " add.s64 %rd67, %rd179, %rd51;\n" " mov.u32 %r713, %r1;\n" "\n" "$L__BB19_49:\n" " cvt.s64.s32 %rd180, %r713;\n" " add.s64 %rd181, %rd67, %rd180;\n" " shl.b64 %rd182, %rd181, 1;\n" " add.s64 %rd183, %rd59, %rd182;\n" " ld.global.u16 %rs4, [%rd183];\n" " // begin inline asm\n" " { cvt.f32.f16 %f260, %rs4;}\n" "\n" " // end inline asm\n" " add.s32 %r315, %r81, %r713;\n" " shl.b32 %r316, %r315, 2;\n" " add.s32 %r318, %r193, %r316;\n" " st.shared.f32 [%r318], %f260;\n" " add.s32 %r713, %r713, %r43;\n" " setp.lt.s32 %p44, %r713, %r134;\n" " @%p44 bra $L__BB19_49;\n" " bra.uni $L__BB19_53;\n" "\n" "$L__BB19_44:\n" " @%p45 bra $L__BB19_53;\n" "\n" " cvt.u32.u64 %r287, %rd65;\n" " add.s32 %r78, %r18, %r287;\n" " cvt.s64.s32 %rd171, %r77;\n" " mul.lo.s64 %rd172, %rd171, %rd49;\n" " add.s64 %rd66, %rd172, %rd50;\n" " mov.u32 %r712, %r1;\n" "\n" "$L__BB19_46:\n" " shr.s32 %r288, %r712, 31;\n" " shr.u32 %r289, %r288, 27;\n" " add.s32 %r290, %r712, %r289;\n" " shr.s32 %r291, %r290, 5;\n" " mul.lo.s32 %r292, %r291, 18;\n" " cvt.s64.s32 %rd173, %r292;\n" " add.s64 %rd174, %rd66, %rd173;\n" " add.s64 %rd175, %rd3, %rd174;\n" " ld.global.u8 %r293, [%rd175];\n" " ld.global.u8 %r294, [%rd175+1];\n" " prmt.b32 %r295, %r294, %r293, 30212;\n" " cvt.u16.u32 %rs3, %r295;\n" " // begin inline asm\n" " { cvt.f32.f16 %f257, %rs3;}\n" "\n" " // end inline asm\n" " and.b32 %r296, %r712, 31;\n" " setp.lt.u32 %p41, %r296, 16;\n" " add.s32 %r297, %r296, -16;\n" " selp.b32 %r298, %r296, %r297, %p41;\n" " selp.b32 %r299, 0, 4, %p41;\n" " add.s32 %r300, %r298, 2;\n" " cvt.s64.s32 %rd176, %r300;\n" " add.s64 %rd177, %rd175, %rd176;\n" " ld.global.u8 %r301, [%rd177];\n" " shr.u32 %r302, %r301, %r299;\n" " and.b32 %r303, %r302, 15;\n" " add.s32 %r304, %r303, -8;\n" " cvt.rn.f32.s32 %f258, %r304;\n" " mul.f32 %f259, %f257, %f258;\n" " add.s32 %r305, %r78, %r712;\n" " shl.b32 %r306, %r305, 2;\n" " add.s32 %r308, %r193, %r306;\n" " st.shared.f32 [%r308], %f259;\n" " add.s32 %r712, %r712, %r43;\n" " setp.lt.s32 %p42, %r712, %r134;\n" " @%p42 bra $L__BB19_46;\n" "\n" "$L__BB19_53:\n" " add.s32 %r711, %r711, 1;\n" " setp.lt.s32 %p47, %r711, %r67;\n" " @%p47 bra $L__BB19_42;\n" " bra.uni $L__BB19_54;\n" "\n" "$L__BB19_32:\n" " mov.u32 %r708, 0;\n" "\n" "$L__BB19_33:\n" " cvt.s64.s32 %rd158, %r708;\n" " mul.lo.s64 %rd61, %rd158, %rd34;\n" " add.s32 %r258, %r708, %r707;\n" " setp.lt.s32 %p31, %r258, %r136;\n" " cvt.s64.s32 %rd62, %r258;\n" " @%p31 bra $L__BB19_37;\n" " bra.uni $L__BB19_34;\n" "\n" "$L__BB19_37:\n" " @%p45 bra $L__BB19_40;\n" "\n" " cvt.u32.u64 %r281, %rd61;\n" " add.s32 %r72, %r18, %r281;\n" " mul.lo.s64 %rd165, %rd62, %rd39;\n" " add.s64 %rd64, %rd165, %rd52;\n" " mov.u32 %r710, %r1;\n" "\n" "$L__BB19_39:\n" " cvt.s64.s32 %rd166, %r710;\n" " add.s64 %rd167, %rd64, %rd166;\n" " shl.b64 %rd168, %rd167, 1;\n" " add.s64 %rd169, %rd2, %rd168;\n" " ld.global.u16 %rs2, [%rd169];\n" " // begin inline asm\n" " { cvt.f32.f16 %f256, %rs2;}\n" "\n" " // end inline asm\n" " add.s32 %r282, %r72, %r710;\n" " shl.b32 %r283, %r282, 2;\n" " add.s32 %r285, %r193, %r283;\n" " st.shared.f32 [%r285], %f256;\n" " add.s32 %r710, %r710, %r43;\n" " setp.lt.s32 %p36, %r710, %r134;\n" " @%p36 bra $L__BB19_39;\n" " bra.uni $L__BB19_40;\n" "\n" "$L__BB19_34:\n" " @%p45 bra $L__BB19_40;\n" "\n" " cvt.u32.u64 %r259, %rd61;\n" " add.s32 %r69, %r18, %r259;\n" " mul.lo.s64 %rd159, %rd62, %rd49;\n" " add.s64 %rd63, %rd159, %rd50;\n" " mov.u32 %r709, %r1;\n" "\n" "$L__BB19_36:\n" " shr.s32 %r260, %r709, 31;\n" " shr.u32 %r261, %r260, 27;\n" " add.s32 %r262, %r709, %r261;\n" " shr.s32 %r263, %r262, 5;\n" " mul.lo.s32 %r264, %r263, 18;\n" " cvt.s64.s32 %rd160, %r264;\n" " add.s64 %rd161, %rd63, %rd160;\n" " add.s64 %rd162, %rd3, %rd161;\n" " ld.global.u8 %r265, [%rd162];\n" " ld.global.u8 %r266, [%rd162+1];\n" " prmt.b32 %r267, %r266, %r265, 30212;\n" " cvt.u16.u32 %rs1, %r267;\n" " // begin inline asm\n" " { cvt.f32.f16 %f253, %rs1;}\n" "\n" " // end inline asm\n" " and.b32 %r268, %r709, 31;\n" " setp.lt.u32 %p33, %r268, 16;\n" " add.s32 %r269, %r268, -16;\n" " selp.b32 %r270, %r268, %r269, %p33;\n" " selp.b32 %r271, 0, 4, %p33;\n" " add.s32 %r272, %r270, 2;\n" " cvt.s64.s32 %rd163, %r272;\n" " add.s64 %rd164, %rd162, %rd163;\n" " ld.global.u8 %r273, [%rd164];\n" " shr.u32 %r274, %r273, %r271;\n" " and.b32 %r275, %r274, 15;\n" " add.s32 %r276, %r275, -8;\n" " cvt.rn.f32.s32 %f254, %r276;\n" " mul.f32 %f255, %f253, %f254;\n" " add.s32 %r277, %r69, %r709;\n" " shl.b32 %r278, %r277, 2;\n" " add.s32 %r280, %r193, %r278;\n" " st.shared.f32 [%r280], %f255;\n" " add.s32 %r709, %r709, %r43;\n" " setp.lt.s32 %p34, %r709, %r134;\n" " @%p34 bra $L__BB19_36;\n" "\n" "$L__BB19_40:\n" " add.s32 %r708, %r708, 1;\n" " setp.lt.s32 %p37, %r708, %r67;\n" " @%p37 bra $L__BB19_33;\n" "\n" "$L__BB19_54:\n" " setp.lt.s32 %p48, %r134, 16;\n" " bar.sync 0;\n" " setp.ge.s32 %p49, %r39, %r67;\n" " or.pred %p50, %p48, %p49;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p50 bra $L__BB19_107;\n" "\n" " and.b32 %r606, %r134, -16;\n" " setp.eq.s32 %p51, %r606, 16;\n" " mov.u32 %r717, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " @%p51 bra $L__BB19_90;\n" "\n" " mov.u32 %r717, 0;\n" " mov.f32 %f641, 0f00000000;\n" " mov.f32 %f640, %f641;\n" " mov.f32 %f639, %f641;\n" " mov.f32 %f638, %f641;\n" " mov.f32 %f637, %f641;\n" " mov.f32 %f636, %f641;\n" " mov.f32 %f635, %f641;\n" " mov.f32 %f634, %f641;\n" " mov.u32 %r716, %r51;\n" "\n" "$L__BB19_57:\n" " shl.b32 %r326, %r717, 4;\n" " add.s32 %r90, %r326, %r40;\n" " add.s32 %r327, %r44, %r90;\n" " shl.b32 %r328, %r327, 2;\n" " add.s32 %r91, %r193, %r328;\n" " ld.shared.f32 %f17, [%r91];\n" " @%p13 bra $L__BB19_59;\n" "\n" " shl.b32 %r686, %r717, 4;\n" " add.s32 %r685, %r686, %r40;\n" " shl.b32 %r684, %r685, 2;\n" " add.s32 %r683, %r193, %r684;\n" " ld.shared.f32 %f286, [%r683];\n" " fma.rn.f32 %f634, %f17, %f286, %f634;\n" "\n" "$L__BB19_59:\n" " setp.lt.s32 %p53, %r133, 2;\n" " @%p53 bra $L__BB19_61;\n" "\n" " shl.b32 %r691, %r717, 4;\n" " add.s32 %r690, %r691, %r40;\n" " add.s32 %r689, %r690, %r134;\n" " shl.b32 %r688, %r689, 2;\n" " add.s32 %r687, %r193, %r688;\n" " ld.shared.f32 %f287, [%r687];\n" " fma.rn.f32 %f635, %f17, %f287, %f635;\n" "\n" "$L__BB19_61:\n" " setp.lt.s32 %p54, %r133, 3;\n" " @%p54 bra $L__BB19_63;\n" "\n" " shl.b32 %r632, %r717, 4;\n" " add.s32 %r631, %r632, %r40;\n" " add.s32 %r630, %r52, %r631;\n" " shl.b32 %r629, %r630, 2;\n" " add.s32 %r628, %r193, %r629;\n" " ld.shared.f32 %f288, [%r628];\n" " fma.rn.f32 %f636, %f17, %f288, %f636;\n" "\n" "$L__BB19_63:\n" " setp.lt.s32 %p55, %r133, 4;\n" " @%p55 bra $L__BB19_65;\n" "\n" " shl.b32 %r642, %r717, 4;\n" " add.s32 %r641, %r642, %r40;\n" " add.s32 %r640, %r53, %r641;\n" " shl.b32 %r639, %r640, 2;\n" " add.s32 %r638, %r193, %r639;\n" " ld.shared.f32 %f289, [%r638];\n" " fma.rn.f32 %f637, %f17, %f289, %f637;\n" "\n" "$L__BB19_65:\n" " setp.lt.s32 %p56, %r133, 5;\n" " @%p56 bra $L__BB19_67;\n" "\n" " shl.b32 %r652, %r717, 4;\n" " add.s32 %r651, %r652, %r40;\n" " add.s32 %r650, %r54, %r651;\n" " shl.b32 %r649, %r650, 2;\n" " add.s32 %r648, %r193, %r649;\n" " ld.shared.f32 %f290, [%r648];\n" " fma.rn.f32 %f638, %f17, %f290, %f638;\n" "\n" "$L__BB19_67:\n" " setp.lt.s32 %p57, %r133, 6;\n" " @%p57 bra $L__BB19_69;\n" "\n" " shl.b32 %r662, %r717, 4;\n" " add.s32 %r661, %r662, %r40;\n" " add.s32 %r660, %r55, %r661;\n" " shl.b32 %r659, %r660, 2;\n" " add.s32 %r658, %r193, %r659;\n" " ld.shared.f32 %f291, [%r658];\n" " fma.rn.f32 %f639, %f17, %f291, %f639;\n" "\n" "$L__BB19_69:\n" " setp.lt.s32 %p58, %r133, 7;\n" " @%p58 bra $L__BB19_71;\n" "\n" " shl.b32 %r672, %r717, 4;\n" " add.s32 %r671, %r672, %r40;\n" " add.s32 %r670, %r56, %r671;\n" " shl.b32 %r669, %r670, 2;\n" " add.s32 %r668, %r193, %r669;\n" " ld.shared.f32 %f292, [%r668];\n" " fma.rn.f32 %f640, %f17, %f292, %f640;\n" "\n" "$L__BB19_71:\n" " setp.lt.s32 %p59, %r133, 8;\n" " @%p59 bra $L__BB19_73;\n" "\n" " shl.b32 %r682, %r717, 4;\n" " add.s32 %r681, %r682, %r40;\n" " add.s32 %r680, %r57, %r681;\n" " shl.b32 %r679, %r680, 2;\n" " add.s32 %r678, %r193, %r679;\n" " ld.shared.f32 %f293, [%r678];\n" " fma.rn.f32 %f641, %f17, %f293, %f641;\n" "\n" "$L__BB19_73:\n" " shl.b32 %r613, %r717, 4;\n" " add.s32 %r612, %r613, %r40;\n" " add.s32 %r611, %r44, %r612;\n" " shl.b32 %r610, %r611, 2;\n" " add.s32 %r609, %r193, %r610;\n" " ld.shared.f32 %f34, [%r609+64];\n" " @%p13 bra $L__BB19_75;\n" "\n" " shl.b32 %r627, %r717, 4;\n" " add.s32 %r626, %r627, %r40;\n" " shl.b32 %r625, %r626, 2;\n" " add.s32 %r624, %r193, %r625;\n" " ld.shared.f32 %f294, [%r624+64];\n" " fma.rn.f32 %f634, %f34, %f294, %f634;\n" "\n" "$L__BB19_75:\n" " setp.lt.s32 %p199, %r133, 2;\n" " @%p199 bra $L__BB19_77;\n" "\n" " shl.b32 %r623, %r717, 4;\n" " add.s32 %r622, %r623, %r40;\n" " add.s32 %r621, %r622, %r134;\n" " shl.b32 %r620, %r621, 2;\n" " add.s32 %r619, %r193, %r620;\n" " ld.shared.f32 %f295, [%r619+64];\n" " fma.rn.f32 %f635, %f34, %f295, %f635;\n" "\n" "$L__BB19_77:\n" " setp.lt.s32 %p200, %r133, 3;\n" " @%p200 bra $L__BB19_79;\n" "\n" " shl.b32 %r618, %r717, 4;\n" " add.s32 %r617, %r618, %r40;\n" " add.s32 %r616, %r52, %r617;\n" " shl.b32 %r615, %r616, 2;\n" " add.s32 %r614, %r193, %r615;\n" " ld.shared.f32 %f296, [%r614+64];\n" " fma.rn.f32 %f636, %f34, %f296, %f636;\n" "\n" "$L__BB19_79:\n" " setp.lt.s32 %p201, %r133, 4;\n" " @%p201 bra $L__BB19_81;\n" "\n" " shl.b32 %r637, %r717, 4;\n" " add.s32 %r636, %r637, %r40;\n" " add.s32 %r635, %r53, %r636;\n" " shl.b32 %r634, %r635, 2;\n" " add.s32 %r633, %r193, %r634;\n" " ld.shared.f32 %f297, [%r633+64];\n" " fma.rn.f32 %f637, %f34, %f297, %f637;\n" "\n" "$L__BB19_81:\n" " setp.lt.s32 %p202, %r133, 5;\n" " @%p202 bra $L__BB19_83;\n" "\n" " shl.b32 %r647, %r717, 4;\n" " add.s32 %r646, %r647, %r40;\n" " add.s32 %r645, %r54, %r646;\n" " shl.b32 %r644, %r645, 2;\n" " add.s32 %r643, %r193, %r644;\n" " ld.shared.f32 %f298, [%r643+64];\n" " fma.rn.f32 %f638, %f34, %f298, %f638;\n" "\n" "$L__BB19_83:\n" " setp.lt.s32 %p203, %r133, 6;\n" " @%p203 bra $L__BB19_85;\n" "\n" " shl.b32 %r657, %r717, 4;\n" " add.s32 %r656, %r657, %r40;\n" " add.s32 %r655, %r55, %r656;\n" " shl.b32 %r654, %r655, 2;\n" " add.s32 %r653, %r193, %r654;\n" " ld.shared.f32 %f299, [%r653+64];\n" " fma.rn.f32 %f639, %f34, %f299, %f639;\n" "\n" "$L__BB19_85:\n" " setp.lt.s32 %p204, %r133, 7;\n" " @%p204 bra $L__BB19_87;\n" "\n" " shl.b32 %r667, %r717, 4;\n" " add.s32 %r666, %r667, %r40;\n" " add.s32 %r665, %r56, %r666;\n" " shl.b32 %r664, %r665, 2;\n" " add.s32 %r663, %r193, %r664;\n" " ld.shared.f32 %f300, [%r663+64];\n" " fma.rn.f32 %f640, %f34, %f300, %f640;\n" "\n" "$L__BB19_87:\n" " setp.lt.s32 %p205, %r133, 8;\n" " @%p205 bra $L__BB19_89;\n" "\n" " shl.b32 %r677, %r717, 4;\n" " add.s32 %r676, %r677, %r40;\n" " add.s32 %r675, %r57, %r676;\n" " shl.b32 %r674, %r675, 2;\n" " add.s32 %r673, %r193, %r674;\n" " ld.shared.f32 %f301, [%r673+64];\n" " fma.rn.f32 %f641, %f34, %f301, %f641;\n" "\n" "$L__BB19_89:\n" " add.s32 %r717, %r717, 2;\n" " add.s32 %r716, %r716, -2;\n" " setp.ne.s32 %p68, %r716, 0;\n" " @%p68 bra $L__BB19_57;\n" "\n" "$L__BB19_90:\n" " setp.eq.s32 %p69, %r50, 0;\n" " @%p69 bra $L__BB19_107;\n" "\n" " shl.b32 %r352, %r717, 4;\n" " add.s32 %r103, %r352, %r40;\n" " add.s32 %r353, %r44, %r103;\n" " shl.b32 %r354, %r353, 2;\n" " add.s32 %r356, %r193, %r354;\n" " ld.shared.f32 %f59, [%r356];\n" " @%p13 bra $L__BB19_93;\n" "\n" " shl.b32 %r357, %r103, 2;\n" " add.s32 %r359, %r193, %r357;\n" " ld.shared.f32 %f302, [%r359];\n" " fma.rn.f32 %f634, %f59, %f302, %f634;\n" "\n" "$L__BB19_93:\n" " setp.lt.s32 %p71, %r133, 2;\n" " @%p71 bra $L__BB19_95;\n" "\n" " add.s32 %r360, %r103, %r134;\n" " shl.b32 %r361, %r360, 2;\n" " add.s32 %r363, %r193, %r361;\n" " ld.shared.f32 %f303, [%r363];\n" " fma.rn.f32 %f635, %f59, %f303, %f635;\n" "\n" "$L__BB19_95:\n" " setp.lt.s32 %p72, %r133, 3;\n" " @%p72 bra $L__BB19_97;\n" "\n" " add.s32 %r364, %r52, %r103;\n" " shl.b32 %r365, %r364, 2;\n" " add.s32 %r367, %r193, %r365;\n" " ld.shared.f32 %f304, [%r367];\n" " fma.rn.f32 %f636, %f59, %f304, %f636;\n" "\n" "$L__BB19_97:\n" " setp.lt.s32 %p73, %r133, 4;\n" " @%p73 bra $L__BB19_99;\n" "\n" " add.s32 %r368, %r53, %r103;\n" " shl.b32 %r369, %r368, 2;\n" " add.s32 %r371, %r193, %r369;\n" " ld.shared.f32 %f305, [%r371];\n" " fma.rn.f32 %f637, %f59, %f305, %f637;\n" "\n" "$L__BB19_99:\n" " setp.lt.s32 %p74, %r133, 5;\n" " @%p74 bra $L__BB19_101;\n" "\n" " add.s32 %r372, %r54, %r103;\n" " shl.b32 %r373, %r372, 2;\n" " add.s32 %r375, %r193, %r373;\n" " ld.shared.f32 %f306, [%r375];\n" " fma.rn.f32 %f638, %f59, %f306, %f638;\n" "\n" "$L__BB19_101:\n" " setp.lt.s32 %p75, %r133, 6;\n" " @%p75 bra $L__BB19_103;\n" "\n" " add.s32 %r376, %r55, %r103;\n" " shl.b32 %r377, %r376, 2;\n" " add.s32 %r379, %r193, %r377;\n" " ld.shared.f32 %f307, [%r379];\n" " fma.rn.f32 %f639, %f59, %f307, %f639;\n" "\n" "$L__BB19_103:\n" " setp.lt.s32 %p76, %r133, 7;\n" " @%p76 bra $L__BB19_105;\n" "\n" " add.s32 %r380, %r56, %r103;\n" " shl.b32 %r381, %r380, 2;\n" " add.s32 %r383, %r193, %r381;\n" " ld.shared.f32 %f308, [%r383];\n" " fma.rn.f32 %f640, %f59, %f308, %f640;\n" "\n" "$L__BB19_105:\n" " setp.lt.s32 %p77, %r133, 8;\n" " @%p77 bra $L__BB19_107;\n" "\n" " add.s32 %r384, %r57, %r103;\n" " shl.b32 %r385, %r384, 2;\n" " add.s32 %r387, %r193, %r385;\n" " ld.shared.f32 %f309, [%r387];\n" " fma.rn.f32 %f641, %f59, %f309, %f641;\n" "\n" "$L__BB19_107:\n" " setp.eq.s32 %p78, %r40, 0;\n" " setp.lt.s32 %p79, %r39, %r67;\n" " and.pred %p1, %p78, %p79;\n" " @%p13 bra $L__BB19_110;\n" "\n" " mov.b32 %r388, %f634;\n" " mov.u32 %r389, 2;\n" " mov.u32 %r390, 4127;\n" " mov.u32 %r391, 8;\n" " mov.u32 %r392, -1;\n" " shfl.sync.down.b32 %r393|%p81, %r388, %r391, %r390, %r392;\n" " mov.b32 %f310, %r393;\n" " add.f32 %f311, %f634, %f310;\n" " mov.b32 %r394, %f311;\n" " mov.u32 %r395, 4;\n" " shfl.sync.down.b32 %r396|%p82, %r394, %r395, %r390, %r392;\n" " mov.b32 %f312, %r396;\n" " add.f32 %f313, %f311, %f312;\n" " mov.b32 %r397, %f313;\n" " shfl.sync.down.b32 %r398|%p83, %r397, %r389, %r390, %r392;\n" " mov.b32 %f314, %r398;\n" " add.f32 %f315, %f313, %f314;\n" " mov.b32 %r399, %f315;\n" " mov.u32 %r400, 1;\n" " shfl.sync.down.b32 %r401|%p84, %r399, %r400, %r390, %r392;\n" " mov.b32 %f316, %r401;\n" " add.f32 %f83, %f315, %f316;\n" " not.pred %p85, %p1;\n" " @%p85 bra $L__BB19_110;\n" "\n" " mul.f32 %f317, %f83, %f231;\n" " st.shared.f32 [%r58], %f317;\n" "\n" "$L__BB19_110:\n" " setp.lt.s32 %p86, %r133, 2;\n" " @%p86 bra $L__BB19_113;\n" "\n" " mov.b32 %r402, %f635;\n" " mov.u32 %r403, 2;\n" " mov.u32 %r404, 4127;\n" " mov.u32 %r405, 8;\n" " mov.u32 %r406, -1;\n" " shfl.sync.down.b32 %r407|%p87, %r402, %r405, %r404, %r406;\n" " mov.b32 %f318, %r407;\n" " add.f32 %f319, %f635, %f318;\n" " mov.b32 %r408, %f319;\n" " mov.u32 %r409, 4;\n" " shfl.sync.down.b32 %r410|%p88, %r408, %r409, %r404, %r406;\n" " mov.b32 %f320, %r410;\n" " add.f32 %f321, %f319, %f320;\n" " mov.b32 %r411, %f321;\n" " shfl.sync.down.b32 %r412|%p89, %r411, %r403, %r404, %r406;\n" " mov.b32 %f322, %r412;\n" " add.f32 %f323, %f321, %f322;\n" " mov.b32 %r413, %f323;\n" " mov.u32 %r414, 1;\n" " shfl.sync.down.b32 %r415|%p90, %r413, %r414, %r404, %r406;\n" " mov.b32 %f324, %r415;\n" " add.f32 %f84, %f323, %f324;\n" " not.pred %p91, %p1;\n" " @%p91 bra $L__BB19_113;\n" "\n" " mul.f32 %f325, %f84, %f231;\n" " st.shared.f32 [%r58+64], %f325;\n" "\n" "$L__BB19_113:\n" " setp.lt.s32 %p92, %r133, 3;\n" " @%p92 bra $L__BB19_116;\n" "\n" " mov.b32 %r416, %f636;\n" " mov.u32 %r417, 2;\n" " mov.u32 %r418, 4127;\n" " mov.u32 %r419, 8;\n" " mov.u32 %r420, -1;\n" " shfl.sync.down.b32 %r421|%p93, %r416, %r419, %r418, %r420;\n" " mov.b32 %f326, %r421;\n" " add.f32 %f327, %f636, %f326;\n" " mov.b32 %r422, %f327;\n" " mov.u32 %r423, 4;\n" " shfl.sync.down.b32 %r424|%p94, %r422, %r423, %r418, %r420;\n" " mov.b32 %f328, %r424;\n" " add.f32 %f329, %f327, %f328;\n" " mov.b32 %r425, %f329;\n" " shfl.sync.down.b32 %r426|%p95, %r425, %r417, %r418, %r420;\n" " mov.b32 %f330, %r426;\n" " add.f32 %f331, %f329, %f330;\n" " mov.b32 %r427, %f331;\n" " mov.u32 %r428, 1;\n" " shfl.sync.down.b32 %r429|%p96, %r427, %r428, %r418, %r420;\n" " mov.b32 %f332, %r429;\n" " add.f32 %f85, %f331, %f332;\n" " not.pred %p97, %p1;\n" " @%p97 bra $L__BB19_116;\n" "\n" " mul.f32 %f333, %f85, %f231;\n" " st.shared.f32 [%r58+128], %f333;\n" "\n" "$L__BB19_116:\n" " setp.lt.s32 %p98, %r133, 4;\n" " @%p98 bra $L__BB19_119;\n" "\n" " mov.b32 %r430, %f637;\n" " mov.u32 %r431, 2;\n" " mov.u32 %r432, 4127;\n" " mov.u32 %r433, 8;\n" " mov.u32 %r434, -1;\n" " shfl.sync.down.b32 %r435|%p99, %r430, %r433, %r432, %r434;\n" " mov.b32 %f334, %r435;\n" " add.f32 %f335, %f637, %f334;\n" " mov.b32 %r436, %f335;\n" " mov.u32 %r437, 4;\n" " shfl.sync.down.b32 %r438|%p100, %r436, %r437, %r432, %r434;\n" " mov.b32 %f336, %r438;\n" " add.f32 %f337, %f335, %f336;\n" " mov.b32 %r439, %f337;\n" " shfl.sync.down.b32 %r440|%p101, %r439, %r431, %r432, %r434;\n" " mov.b32 %f338, %r440;\n" " add.f32 %f339, %f337, %f338;\n" " mov.b32 %r441, %f339;\n" " mov.u32 %r442, 1;\n" " shfl.sync.down.b32 %r443|%p102, %r441, %r442, %r432, %r434;\n" " mov.b32 %f340, %r443;\n" " add.f32 %f86, %f339, %f340;\n" " not.pred %p103, %p1;\n" " @%p103 bra $L__BB19_119;\n" "\n" " mul.f32 %f341, %f86, %f231;\n" " st.shared.f32 [%r58+192], %f341;\n" "\n" "$L__BB19_119:\n" " setp.lt.s32 %p104, %r133, 5;\n" " @%p104 bra $L__BB19_122;\n" "\n" " mov.b32 %r444, %f638;\n" " mov.u32 %r445, 2;\n" " mov.u32 %r446, 4127;\n" " mov.u32 %r447, 8;\n" " mov.u32 %r448, -1;\n" " shfl.sync.down.b32 %r449|%p105, %r444, %r447, %r446, %r448;\n" " mov.b32 %f342, %r449;\n" " add.f32 %f343, %f638, %f342;\n" " mov.b32 %r450, %f343;\n" " mov.u32 %r451, 4;\n" " shfl.sync.down.b32 %r452|%p106, %r450, %r451, %r446, %r448;\n" " mov.b32 %f344, %r452;\n" " add.f32 %f345, %f343, %f344;\n" " mov.b32 %r453, %f345;\n" " shfl.sync.down.b32 %r454|%p107, %r453, %r445, %r446, %r448;\n" " mov.b32 %f346, %r454;\n" " add.f32 %f347, %f345, %f346;\n" " mov.b32 %r455, %f347;\n" " mov.u32 %r456, 1;\n" " shfl.sync.down.b32 %r457|%p108, %r455, %r456, %r446, %r448;\n" " mov.b32 %f348, %r457;\n" " add.f32 %f87, %f347, %f348;\n" " not.pred %p109, %p1;\n" " @%p109 bra $L__BB19_122;\n" "\n" " mul.f32 %f349, %f87, %f231;\n" " st.shared.f32 [%r58+256], %f349;\n" "\n" "$L__BB19_122:\n" " setp.lt.s32 %p110, %r133, 6;\n" " @%p110 bra $L__BB19_125;\n" "\n" " mov.b32 %r458, %f639;\n" " mov.u32 %r459, 2;\n" " mov.u32 %r460, 4127;\n" " mov.u32 %r461, 8;\n" " mov.u32 %r462, -1;\n" " shfl.sync.down.b32 %r463|%p111, %r458, %r461, %r460, %r462;\n" " mov.b32 %f350, %r463;\n" " add.f32 %f351, %f639, %f350;\n" " mov.b32 %r464, %f351;\n" " mov.u32 %r465, 4;\n" " shfl.sync.down.b32 %r466|%p112, %r464, %r465, %r460, %r462;\n" " mov.b32 %f352, %r466;\n" " add.f32 %f353, %f351, %f352;\n" " mov.b32 %r467, %f353;\n" " shfl.sync.down.b32 %r468|%p113, %r467, %r459, %r460, %r462;\n" " mov.b32 %f354, %r468;\n" " add.f32 %f355, %f353, %f354;\n" " mov.b32 %r469, %f355;\n" " mov.u32 %r470, 1;\n" " shfl.sync.down.b32 %r471|%p114, %r469, %r470, %r460, %r462;\n" " mov.b32 %f356, %r471;\n" " add.f32 %f88, %f355, %f356;\n" " not.pred %p115, %p1;\n" " @%p115 bra $L__BB19_125;\n" "\n" " mul.f32 %f357, %f88, %f231;\n" " st.shared.f32 [%r58+320], %f357;\n" "\n" "$L__BB19_125:\n" " setp.lt.s32 %p116, %r133, 7;\n" " @%p116 bra $L__BB19_128;\n" "\n" " mov.b32 %r472, %f640;\n" " mov.u32 %r473, 2;\n" " mov.u32 %r474, 4127;\n" " mov.u32 %r475, 8;\n" " mov.u32 %r476, -1;\n" " shfl.sync.down.b32 %r477|%p117, %r472, %r475, %r474, %r476;\n" " mov.b32 %f358, %r477;\n" " add.f32 %f359, %f640, %f358;\n" " mov.b32 %r478, %f359;\n" " mov.u32 %r479, 4;\n" " shfl.sync.down.b32 %r480|%p118, %r478, %r479, %r474, %r476;\n" " mov.b32 %f360, %r480;\n" " add.f32 %f361, %f359, %f360;\n" " mov.b32 %r481, %f361;\n" " shfl.sync.down.b32 %r482|%p119, %r481, %r473, %r474, %r476;\n" " mov.b32 %f362, %r482;\n" " add.f32 %f363, %f361, %f362;\n" " mov.b32 %r483, %f363;\n" " mov.u32 %r484, 1;\n" " shfl.sync.down.b32 %r485|%p120, %r483, %r484, %r474, %r476;\n" " mov.b32 %f364, %r485;\n" " add.f32 %f89, %f363, %f364;\n" " not.pred %p121, %p1;\n" " @%p121 bra $L__BB19_128;\n" "\n" " mul.f32 %f365, %f89, %f231;\n" " st.shared.f32 [%r58+384], %f365;\n" "\n" "$L__BB19_128:\n" " setp.lt.s32 %p122, %r133, 8;\n" " @%p122 bra $L__BB19_131;\n" "\n" " mov.b32 %r486, %f641;\n" " mov.u32 %r487, 2;\n" " mov.u32 %r488, 4127;\n" " mov.u32 %r489, 8;\n" " mov.u32 %r490, -1;\n" " shfl.sync.down.b32 %r491|%p123, %r486, %r489, %r488, %r490;\n" " mov.b32 %f366, %r491;\n" " add.f32 %f367, %f641, %f366;\n" " mov.b32 %r492, %f367;\n" " mov.u32 %r493, 4;\n" " shfl.sync.down.b32 %r494|%p124, %r492, %r493, %r488, %r490;\n" " mov.b32 %f368, %r494;\n" " add.f32 %f369, %f367, %f368;\n" " mov.b32 %r495, %f369;\n" " shfl.sync.down.b32 %r496|%p125, %r495, %r487, %r488, %r490;\n" " mov.b32 %f370, %r496;\n" " add.f32 %f371, %f369, %f370;\n" " mov.b32 %r497, %f371;\n" " mov.u32 %r498, 1;\n" " shfl.sync.down.b32 %r499|%p126, %r497, %r498, %r488, %r490;\n" " mov.b32 %f372, %r499;\n" " add.f32 %f90, %f371, %f372;\n" " not.pred %p127, %p1;\n" " @%p127 bra $L__BB19_131;\n" "\n" " mul.f32 %f373, %f90, %f231;\n" " st.shared.f32 [%r58+448], %f373;\n" "\n" "$L__BB19_131:\n" " bar.sync 0;\n" " @%p18 bra $L__BB19_149;\n" "\n" " ld.shared.f32 %f91, [%r36];\n" " mov.f32 %f676, %f91;\n" " @%p29 bra $L__BB19_140;\n" "\n" " and.b32 %r104, %r65, 3;\n" " add.s32 %r501, %r65, -1;\n" " setp.lt.u32 %p130, %r501, 3;\n" " mov.u32 %r720, 0;\n" " mov.f32 %f676, %f91;\n" " @%p130 bra $L__BB19_136;\n" "\n" " sub.s32 %r719, %r65, %r104;\n" " mov.u32 %r720, 0;\n" " mov.f32 %f676, %f91;\n" "\n" "$L__BB19_135:\n" " add.s32 %r503, %r45, %r720;\n" " shl.b32 %r504, %r503, 2;\n" " add.s32 %r506, %r193, %r504;\n" " ld.shared.f32 %f375, [%r506];\n" " max.f32 %f376, %f676, %f375;\n" " ld.shared.f32 %f377, [%r506+4];\n" " max.f32 %f378, %f376, %f377;\n" " ld.shared.f32 %f379, [%r506+8];\n" " max.f32 %f380, %f378, %f379;\n" " ld.shared.f32 %f381, [%r506+12];\n" " max.f32 %f676, %f380, %f381;\n" " add.s32 %r720, %r720, 4;\n" " add.s32 %r719, %r719, -4;\n" " setp.ne.s32 %p131, %r719, 0;\n" " @%p131 bra $L__BB19_135;\n" "\n" "$L__BB19_136:\n" " setp.eq.s32 %p132, %r104, 0;\n" " @%p132 bra $L__BB19_140;\n" "\n" " add.s32 %r507, %r45, %r720;\n" " shl.b32 %r508, %r507, 2;\n" " add.s32 %r111, %r193, %r508;\n" " ld.shared.f32 %f382, [%r111];\n" " max.f32 %f676, %f676, %f382;\n" " setp.eq.s32 %p133, %r104, 1;\n" " @%p133 bra $L__BB19_140;\n" "\n" " ld.shared.f32 %f383, [%r111+4];\n" " max.f32 %f676, %f676, %f383;\n" " setp.eq.s32 %p134, %r104, 2;\n" " @%p134 bra $L__BB19_140;\n" "\n" " ld.shared.f32 %f384, [%r111+8];\n" " max.f32 %f676, %f676, %f384;\n" "\n" "$L__BB19_140:\n" " mov.f32 %f680, 0f00000000;\n" " @%p29 bra $L__BB19_148;\n" "\n" " and.b32 %r112, %r65, 3;\n" " add.s32 %r511, %r65, -1;\n" " setp.lt.u32 %p136, %r511, 3;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r724, 0;\n" " @%p136 bra $L__BB19_144;\n" "\n" " sub.s32 %r722, %r65, %r112;\n" " mov.f32 %f680, 0f00000000;\n" " mov.u32 %r724, 0;\n" " mov.u32 %r723, %r60;\n" "\n" "$L__BB19_143:\n" " shl.b32 %r608, %r133, 2;\n" " add.s32 %r607, %r608, 1024;\n" " add.s32 %r513, %r45, %r724;\n" " shl.b32 %r514, %r513, 2;\n" " add.s32 %r516, %r193, %r514;\n" " ld.shared.f32 %f389, [%r516];\n" " sub.f32 %f390, %f389, %f676;\n" " mov.f32 %f391, 0f3F000000;\n" " mov.f32 %f392, 0f3BBB989D;\n" " fma.rn.f32 %f393, %f390, %f392, %f391;\n" " mov.f32 %f394, 0f3FB8AA3B;\n" " mov.f32 %f395, 0f437C0000;\n" " cvt.sat.f32.f32 %f396, %f393;\n" " mov.f32 %f397, 0f4B400001;\n" " fma.rm.f32 %f398, %f396, %f395, %f397;\n" " add.f32 %f399, %f398, 0fCB40007F;\n" " neg.f32 %f400, %f399;\n" " fma.rn.f32 %f401, %f390, %f394, %f400;\n" " mov.f32 %f402, 0f32A57060;\n" " fma.rn.f32 %f403, %f390, %f402, %f401;\n" " mov.b32 %r517, %f398;\n" " shl.b32 %r518, %r517, 23;\n" " mov.b32 %f404, %r518;\n" " ex2.approx.ftz.f32 %f405, %f403;\n" " mul.f32 %f406, %f405, %f404;\n" " st.shared.f32 [%r723+1024], %f406;\n" " add.f32 %f407, %f680, %f406;\n" " ld.shared.f32 %f408, [%r516+4];\n" " sub.f32 %f409, %f408, %f676;\n" " fma.rn.f32 %f410, %f409, %f392, %f391;\n" " cvt.sat.f32.f32 %f411, %f410;\n" " fma.rm.f32 %f412, %f411, %f395, %f397;\n" " add.f32 %f413, %f412, 0fCB40007F;\n" " neg.f32 %f414, %f413;\n" " fma.rn.f32 %f415, %f409, %f394, %f414;\n" " fma.rn.f32 %f416, %f409, %f402, %f415;\n" " mov.b32 %r519, %f412;\n" " shl.b32 %r520, %r519, 23;\n" " mov.b32 %f417, %r520;\n" " ex2.approx.ftz.f32 %f418, %f416;\n" " mul.f32 %f419, %f418, %f417;\n" " add.s32 %r521, %r723, %r608;\n" " st.shared.f32 [%r521+1024], %f419;\n" " add.f32 %f420, %f407, %f419;\n" " ld.shared.f32 %f421, [%r516+8];\n" " sub.f32 %f422, %f421, %f676;\n" " fma.rn.f32 %f423, %f422, %f392, %f391;\n" " cvt.sat.f32.f32 %f424, %f423;\n" " fma.rm.f32 %f425, %f424, %f395, %f397;\n" " add.f32 %f426, %f425, 0fCB40007F;\n" " neg.f32 %f427, %f426;\n" " fma.rn.f32 %f428, %f422, %f394, %f427;\n" " fma.rn.f32 %f429, %f422, %f402, %f428;\n" " mov.b32 %r522, %f425;\n" " shl.b32 %r523, %r522, 23;\n" " mov.b32 %f430, %r523;\n" " ex2.approx.ftz.f32 %f431, %f429;\n" " mul.f32 %f432, %f431, %f430;\n" " add.s32 %r524, %r521, %r608;\n" " add.s32 %r525, %r521, %r607;\n" " st.shared.f32 [%r525], %f432;\n" " add.f32 %f433, %f420, %f432;\n" " ld.shared.f32 %f434, [%r516+12];\n" " sub.f32 %f435, %f434, %f676;\n" " fma.rn.f32 %f436, %f435, %f392, %f391;\n" " cvt.sat.f32.f32 %f437, %f436;\n" " fma.rm.f32 %f438, %f437, %f395, %f397;\n" " add.f32 %f439, %f438, 0fCB40007F;\n" " neg.f32 %f440, %f439;\n" " fma.rn.f32 %f441, %f435, %f394, %f440;\n" " fma.rn.f32 %f442, %f435, %f402, %f441;\n" " mov.b32 %r526, %f438;\n" " shl.b32 %r527, %r526, 23;\n" " mov.b32 %f443, %r527;\n" " ex2.approx.ftz.f32 %f444, %f442;\n" " mul.f32 %f445, %f444, %f443;\n" " add.s32 %r528, %r524, %r608;\n" " add.s32 %r529, %r524, %r607;\n" " add.s32 %r723, %r528, %r608;\n" " st.shared.f32 [%r529], %f445;\n" " add.f32 %f680, %f433, %f445;\n" " add.s32 %r724, %r724, 4;\n" " add.s32 %r722, %r722, -4;\n" " setp.ne.s32 %p137, %r722, 0;\n" " @%p137 bra $L__BB19_143;\n" "\n" "$L__BB19_144:\n" " setp.eq.s32 %p138, %r112, 0;\n" " @%p138 bra $L__BB19_148;\n" "\n" " add.s32 %r530, %r45, %r724;\n" " shl.b32 %r531, %r530, 2;\n" " add.s32 %r121, %r193, %r531;\n" " ld.shared.f32 %f446, [%r121];\n" " sub.f32 %f447, %f446, %f676;\n" " mov.f32 %f448, 0f3F000000;\n" " mov.f32 %f449, 0f3BBB989D;\n" " fma.rn.f32 %f450, %f447, %f449, %f448;\n" " mov.f32 %f451, 0f3FB8AA3B;\n" " mov.f32 %f452, 0f437C0000;\n" " cvt.sat.f32.f32 %f453, %f450;\n" " mov.f32 %f454, 0f4B400001;\n" " fma.rm.f32 %f455, %f453, %f452, %f454;\n" " add.f32 %f456, %f455, 0fCB40007F;\n" " neg.f32 %f457, %f456;\n" " fma.rn.f32 %f458, %f447, %f451, %f457;\n" " mov.f32 %f459, 0f32A57060;\n" " fma.rn.f32 %f460, %f447, %f459, %f458;\n" " mov.b32 %r533, %f455;\n" " shl.b32 %r534, %r533, 23;\n" " mov.b32 %f461, %r534;\n" " ex2.approx.ftz.f32 %f462, %f460;\n" " mul.f32 %f463, %f462, %f461;\n" " cvt.u64.u32 %rd190, %r724;\n" " mul.lo.s64 %rd191, %rd190, %rd10;\n" " add.s64 %rd192, %rd58, %rd191;\n" " cvt.u32.u64 %r535, %rd192;\n" " add.s32 %r536, %r18, %r535;\n" " shl.b32 %r537, %r536, 2;\n" " add.s32 %r538, %r193, %r537;\n" " st.shared.f32 [%r538], %f463;\n" " add.f32 %f680, %f680, %f463;\n" " setp.eq.s32 %p139, %r112, 1;\n" " @%p139 bra $L__BB19_148;\n" "\n" " mov.f32 %f611, 0f32A57060;\n" " mov.f32 %f610, 0f3FB8AA3B;\n" " mov.f32 %f609, 0f4B400001;\n" " mov.f32 %f608, 0f437C0000;\n" " mov.f32 %f607, 0f3F000000;\n" " mov.f32 %f606, 0f3BBB989D;\n" " add.s32 %r595, %r45, %r724;\n" " shl.b32 %r594, %r595, 2;\n" " add.s32 %r593, %r193, %r594;\n" " ld.shared.f32 %f464, [%r593+4];\n" " sub.f32 %f465, %f464, %f676;\n" " fma.rn.f32 %f468, %f465, %f606, %f607;\n" " cvt.sat.f32.f32 %f471, %f468;\n" " fma.rm.f32 %f473, %f471, %f608, %f609;\n" " add.f32 %f474, %f473, 0fCB40007F;\n" " neg.f32 %f475, %f474;\n" " fma.rn.f32 %f476, %f465, %f610, %f475;\n" " fma.rn.f32 %f478, %f465, %f611, %f476;\n" " mov.b32 %r539, %f473;\n" " shl.b32 %r540, %r539, 23;\n" " mov.b32 %f479, %r540;\n" " ex2.approx.ftz.f32 %f480, %f478;\n" " mul.f32 %f481, %f480, %f479;\n" " add.s32 %r541, %r724, 1;\n" " cvt.u64.u32 %rd193, %r541;\n" " mul.lo.s64 %rd194, %rd193, %rd10;\n" " add.s64 %rd195, %rd58, %rd194;\n" " cvt.u32.u64 %r542, %rd195;\n" " add.s32 %r543, %r18, %r542;\n" " shl.b32 %r544, %r543, 2;\n" " add.s32 %r546, %r193, %r544;\n" " st.shared.f32 [%r546], %f481;\n" " add.f32 %f680, %f680, %f481;\n" " setp.eq.s32 %p140, %r112, 2;\n" " @%p140 bra $L__BB19_148;\n" "\n" " mov.f32 %f617, 0f32A57060;\n" " mov.f32 %f616, 0f3FB8AA3B;\n" " mov.f32 %f615, 0f4B400001;\n" " mov.f32 %f614, 0f437C0000;\n" " mov.f32 %f613, 0f3F000000;\n" " mov.f32 %f612, 0f3BBB989D;\n" " add.s32 %r598, %r45, %r724;\n" " shl.b32 %r597, %r598, 2;\n" " add.s32 %r596, %r193, %r597;\n" " ld.shared.f32 %f482, [%r596+8];\n" " sub.f32 %f483, %f482, %f676;\n" " fma.rn.f32 %f486, %f483, %f612, %f613;\n" " cvt.sat.f32.f32 %f489, %f486;\n" " fma.rm.f32 %f491, %f489, %f614, %f615;\n" " add.f32 %f492, %f491, 0fCB40007F;\n" " neg.f32 %f493, %f492;\n" " fma.rn.f32 %f494, %f483, %f616, %f493;\n" " fma.rn.f32 %f496, %f483, %f617, %f494;\n" " mov.b32 %r547, %f491;\n" " shl.b32 %r548, %r547, 23;\n" " mov.b32 %f497, %r548;\n" " ex2.approx.ftz.f32 %f498, %f496;\n" " mul.f32 %f499, %f498, %f497;\n" " add.s32 %r549, %r724, 2;\n" " cvt.u64.u32 %rd196, %r549;\n" " mul.lo.s64 %rd197, %rd196, %rd10;\n" " add.s64 %rd198, %rd58, %rd197;\n" " cvt.u32.u64 %r550, %rd198;\n" " add.s32 %r551, %r18, %r550;\n" " shl.b32 %r552, %r551, 2;\n" " add.s32 %r554, %r193, %r552;\n" " st.shared.f32 [%r554], %f499;\n" " add.f32 %f680, %f680, %f499;\n" "\n" "$L__BB19_148:\n" " sub.f32 %f500, %f91, %f676;\n" " mov.f32 %f501, 0f3F000000;\n" " mov.f32 %f502, 0f3BBB989D;\n" " fma.rn.f32 %f503, %f500, %f502, %f501;\n" " mov.f32 %f504, 0f3FB8AA3B;\n" " mov.f32 %f505, 0f437C0000;\n" " cvt.sat.f32.f32 %f506, %f503;\n" " mov.f32 %f507, 0f4B400001;\n" " fma.rm.f32 %f508, %f506, %f505, %f507;\n" " add.f32 %f509, %f508, 0fCB40007F;\n" " neg.f32 %f510, %f509;\n" " fma.rn.f32 %f511, %f500, %f504, %f510;\n" " mov.f32 %f512, 0f32A57060;\n" " fma.rn.f32 %f513, %f500, %f512, %f511;\n" " ex2.approx.ftz.f32 %f514, %f513;\n" " mov.b32 %r555, %f508;\n" " shl.b32 %r556, %r555, 23;\n" " mov.b32 %f515, %r556;\n" " mul.f32 %f516, %f514, %f515;\n" " ld.shared.f32 %f517, [%r37];\n" " fma.rn.f32 %f518, %f516, %f517, %f680;\n" " st.shared.f32 [%r37], %f518;\n" " st.shared.f32 [%r36], %f676;\n" " st.shared.f32 [%r47], %f516;\n" "\n" "$L__BB19_149:\n" " bar.sync 0;\n" " @%p29 bra $L__BB19_229;\n" "\n" " @%p30 bra $L__BB19_189;\n" " bra.uni $L__BB19_151;\n" "\n" "$L__BB19_189:\n" " mov.u32 %r726, 0;\n" "\n" "$L__BB19_190:\n" " add.s32 %r126, %r726, %r707;\n" " setp.lt.s32 %p162, %r126, %r136;\n" " @%p162 bra $L__BB19_194;\n" " bra.uni $L__BB19_191;\n" "\n" "$L__BB19_194:\n" " add.s32 %r695, %r726, %r707;\n" " cvt.s64.s32 %rd224, %r695;\n" " mul.lo.s64 %rd225, %rd224, %rd39;\n" " add.s64 %rd226, %rd54, %rd225;\n" " shl.b64 %rd227, %rd226, 1;\n" " add.s64 %rd228, %rd4, %rd227;\n" " ld.global.u16 %rs10, [%rd228];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs10;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB19_195;\n" "\n" "$L__BB19_191:\n" " add.s32 %r692, %r726, %r707;\n" " setp.gt.s32 %p163, %r692, %r42;\n" " @%p163 bra $L__BB19_193;\n" " bra.uni $L__BB19_192;\n" "\n" "$L__BB19_193:\n" " add.s32 %r694, %r726, %r707;\n" " shr.s32 %r578, %r694, 31;\n" " shr.u32 %r579, %r578, 24;\n" " add.s32 %r580, %r694, %r579;\n" " and.b32 %r581, %r580, -256;\n" " sub.s32 %r582, %r694, %r581;\n" " cvt.s64.s32 %rd219, %r582;\n" " mul.lo.s64 %rd220, %rd219, %rd39;\n" " add.s64 %rd221, %rd55, %rd220;\n" " shl.b64 %rd222, %rd221, 1;\n" " add.s64 %rd223, %rd60, %rd222;\n" " ld.global.u16 %rs9, [%rd223];\n" " // begin inline asm\n" " { cvt.f32.f16 %f720, %rs9;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB19_195;\n" "\n" "$L__BB19_192:\n" " add.s32 %r693, %r726, %r707;\n" " cvt.s64.s32 %rd212, %r693;\n" " mul.lo.s64 %rd213, %rd212, %rd49;\n" " add.s64 %rd214, %rd213, %rd50;\n" " add.s64 %rd215, %rd214, %rd56;\n" " add.s64 %rd216, %rd5, %rd215;\n" " ld.global.u8 %r571, [%rd216];\n" " ld.global.u8 %r572, [%rd216+1];\n" " prmt.b32 %r573, %r572, %r571, 30212;\n" " cvt.u16.u32 %rs8, %r573;\n" " // begin inline asm\n" " { cvt.f32.f16 %f554, %rs8;}\n" "\n" " // end inline asm\n" " add.s64 %rd217, %rd57, %rd214;\n" " add.s64 %rd218, %rd5, %rd217;\n" " ld.global.u8 %r574, [%rd218];\n" " shr.u32 %r575, %r574, %r46;\n" " and.b32 %r576, %r575, 15;\n" " add.s32 %r577, %r576, -8;\n" " cvt.rn.f32.s32 %f555, %r577;\n" " mul.f32 %f720, %f554, %f555;\n" "\n" "$L__BB19_195:\n" " cvt.u64.u32 %rd229, %r726;\n" " mul.lo.s64 %rd230, %rd229, %rd10;\n" " add.s64 %rd231, %rd230, %rd36;\n" " cvt.u32.u64 %r583, %rd231;\n" " add.s32 %r584, %r18, %r583;\n" " shl.b32 %r585, %r584, 2;\n" " add.s32 %r127, %r193, %r585;\n" " setp.eq.s32 %p164, %r726, 0;\n" " @%p164 bra $L__BB19_212;\n" "\n" " @%p13 bra $L__BB19_198;\n" "\n" " ld.shared.f32 %f558, [%r127];\n" " fma.rn.f32 %f711, %f720, %f558, %f711;\n" "\n" "$L__BB19_198:\n" " @%p86 bra $L__BB19_200;\n" "\n" " ld.shared.f32 %f559, [%r127+4];\n" " fma.rn.f32 %f710, %f720, %f559, %f710;\n" "\n" "$L__BB19_200:\n" " @%p92 bra $L__BB19_202;\n" "\n" " ld.shared.f32 %f560, [%r127+8];\n" " fma.rn.f32 %f709, %f720, %f560, %f709;\n" "\n" "$L__BB19_202:\n" " @%p98 bra $L__BB19_204;\n" "\n" " ld.shared.f32 %f561, [%r127+12];\n" " fma.rn.f32 %f708, %f720, %f561, %f708;\n" "\n" "$L__BB19_204:\n" " @%p104 bra $L__BB19_206;\n" "\n" " ld.shared.f32 %f562, [%r127+16];\n" " fma.rn.f32 %f707, %f720, %f562, %f707;\n" "\n" "$L__BB19_206:\n" " @%p110 bra $L__BB19_208;\n" "\n" " ld.shared.f32 %f563, [%r127+20];\n" " fma.rn.f32 %f706, %f720, %f563, %f706;\n" "\n" "$L__BB19_208:\n" " @%p116 bra $L__BB19_210;\n" "\n" " ld.shared.f32 %f564, [%r127+24];\n" " fma.rn.f32 %f705, %f720, %f564, %f705;\n" "\n" "$L__BB19_210:\n" " @%p122 bra $L__BB19_228;\n" "\n" " ld.shared.f32 %f565, [%r127+28];\n" " fma.rn.f32 %f704, %f720, %f565, %f704;\n" " bra.uni $L__BB19_228;\n" "\n" "$L__BB19_212:\n" " @%p13 bra $L__BB19_214;\n" "\n" " ld.shared.f32 %f566, [%r59];\n" " ld.shared.f32 %f567, [%r127];\n" " mul.f32 %f568, %f720, %f567;\n" " fma.rn.f32 %f711, %f566, %f711, %f568;\n" "\n" "$L__BB19_214:\n" " @%p86 bra $L__BB19_216;\n" "\n" " ld.shared.f32 %f569, [%r59+4];\n" " ld.shared.f32 %f570, [%r127+4];\n" " mul.f32 %f571, %f720, %f570;\n" " fma.rn.f32 %f710, %f569, %f710, %f571;\n" "\n" "$L__BB19_216:\n" " @%p92 bra $L__BB19_218;\n" "\n" " ld.shared.f32 %f572, [%r59+8];\n" " ld.shared.f32 %f573, [%r127+8];\n" " mul.f32 %f574, %f720, %f573;\n" " fma.rn.f32 %f709, %f572, %f709, %f574;\n" "\n" "$L__BB19_218:\n" " @%p98 bra $L__BB19_220;\n" "\n" " ld.shared.f32 %f575, [%r59+12];\n" " ld.shared.f32 %f576, [%r127+12];\n" " mul.f32 %f577, %f720, %f576;\n" " fma.rn.f32 %f708, %f575, %f708, %f577;\n" "\n" "$L__BB19_220:\n" " @%p104 bra $L__BB19_222;\n" "\n" " ld.shared.f32 %f578, [%r59+16];\n" " ld.shared.f32 %f579, [%r127+16];\n" " mul.f32 %f580, %f720, %f579;\n" " fma.rn.f32 %f707, %f578, %f707, %f580;\n" "\n" "$L__BB19_222:\n" " @%p110 bra $L__BB19_224;\n" "\n" " ld.shared.f32 %f581, [%r59+20];\n" " ld.shared.f32 %f582, [%r127+20];\n" " mul.f32 %f583, %f720, %f582;\n" " fma.rn.f32 %f706, %f581, %f706, %f583;\n" "\n" "$L__BB19_224:\n" " @%p116 bra $L__BB19_226;\n" "\n" " ld.shared.f32 %f584, [%r59+24];\n" " ld.shared.f32 %f585, [%r127+24];\n" " mul.f32 %f586, %f720, %f585;\n" " fma.rn.f32 %f705, %f584, %f705, %f586;\n" "\n" "$L__BB19_226:\n" " @%p122 bra $L__BB19_228;\n" "\n" " ld.shared.f32 %f587, [%r59+28];\n" " ld.shared.f32 %f588, [%r127+28];\n" " mul.f32 %f589, %f720, %f588;\n" " fma.rn.f32 %f704, %f587, %f704, %f589;\n" "\n" "$L__BB19_228:\n" " add.s32 %r726, %r726, 1;\n" " setp.lt.s32 %p181, %r726, %r67;\n" " @%p181 bra $L__BB19_190;\n" " bra.uni $L__BB19_229;\n" "\n" "$L__BB19_151:\n" " mov.u32 %r725, 0;\n" "\n" "$L__BB19_152:\n" " add.s32 %r558, %r725, %r707;\n" " setp.lt.s32 %p143, %r558, %r136;\n" " cvt.s64.s32 %rd69, %r558;\n" " @%p143 bra $L__BB19_154;\n" " bra.uni $L__BB19_153;\n" "\n" "$L__BB19_154:\n" " mul.lo.s64 %rd205, %rd69, %rd39;\n" " add.s64 %rd206, %rd54, %rd205;\n" " shl.b64 %rd207, %rd206, 1;\n" " add.s64 %rd208, %rd4, %rd207;\n" " ld.global.u16 %rs7, [%rd208];\n" " // begin inline asm\n" " { cvt.f32.f16 %f689, %rs7;}\n" "\n" " // end inline asm\n" " bra.uni $L__BB19_155;\n" "\n" "$L__BB19_153:\n" " mul.lo.s64 %rd199, %rd69, %rd49;\n" " add.s64 %rd200, %rd199, %rd50;\n" " add.s64 %rd201, %rd200, %rd56;\n" " add.s64 %rd202, %rd5, %rd201;\n" " ld.global.u8 %r559, [%rd202];\n" " ld.global.u8 %r560, [%rd202+1];\n" " prmt.b32 %r561, %r560, %r559, 30212;\n" " cvt.u16.u32 %rs6, %r561;\n" " // begin inline asm\n" " { cvt.f32.f16 %f519, %rs6;}\n" "\n" " // end inline asm\n" " add.s64 %rd203, %rd57, %rd200;\n" " add.s64 %rd204, %rd5, %rd203;\n" " ld.global.u8 %r562, [%rd204];\n" " shr.u32 %r563, %r562, %r46;\n" " and.b32 %r564, %r563, 15;\n" " add.s32 %r565, %r564, -8;\n" " cvt.rn.f32.s32 %f520, %r565;\n" " mul.f32 %f689, %f519, %f520;\n" "\n" "$L__BB19_155:\n" " cvt.u64.u32 %rd209, %r725;\n" " mul.lo.s64 %rd210, %rd209, %rd10;\n" " add.s64 %rd211, %rd210, %rd36;\n" " cvt.u32.u64 %r566, %rd211;\n" " add.s32 %r567, %r18, %r566;\n" " shl.b32 %r568, %r567, 2;\n" " add.s32 %r123, %r193, %r568;\n" " setp.eq.s32 %p144, %r725, 0;\n" " @%p144 bra $L__BB19_172;\n" "\n" " @%p13 bra $L__BB19_158;\n" "\n" " ld.shared.f32 %f522, [%r123];\n" " fma.rn.f32 %f711, %f689, %f522, %f711;\n" "\n" "$L__BB19_158:\n" " @%p86 bra $L__BB19_160;\n" "\n" " ld.shared.f32 %f523, [%r123+4];\n" " fma.rn.f32 %f710, %f689, %f523, %f710;\n" "\n" "$L__BB19_160:\n" " @%p92 bra $L__BB19_162;\n" "\n" " ld.shared.f32 %f524, [%r123+8];\n" " fma.rn.f32 %f709, %f689, %f524, %f709;\n" "\n" "$L__BB19_162:\n" " @%p98 bra $L__BB19_164;\n" "\n" " ld.shared.f32 %f525, [%r123+12];\n" " fma.rn.f32 %f708, %f689, %f525, %f708;\n" "\n" "$L__BB19_164:\n" " @%p104 bra $L__BB19_166;\n" "\n" " ld.shared.f32 %f526, [%r123+16];\n" " fma.rn.f32 %f707, %f689, %f526, %f707;\n" "\n" "$L__BB19_166:\n" " @%p110 bra $L__BB19_168;\n" "\n" " ld.shared.f32 %f527, [%r123+20];\n" " fma.rn.f32 %f706, %f689, %f527, %f706;\n" "\n" "$L__BB19_168:\n" " @%p116 bra $L__BB19_170;\n" "\n" " ld.shared.f32 %f528, [%r123+24];\n" " fma.rn.f32 %f705, %f689, %f528, %f705;\n" "\n" "$L__BB19_170:\n" " @%p122 bra $L__BB19_188;\n" "\n" " ld.shared.f32 %f529, [%r123+28];\n" " fma.rn.f32 %f704, %f689, %f529, %f704;\n" " bra.uni $L__BB19_188;\n" "\n" "$L__BB19_172:\n" " @%p13 bra $L__BB19_174;\n" "\n" " ld.shared.f32 %f530, [%r59];\n" " ld.shared.f32 %f531, [%r123];\n" " mul.f32 %f532, %f689, %f531;\n" " fma.rn.f32 %f711, %f530, %f711, %f532;\n" "\n" "$L__BB19_174:\n" " @%p86 bra $L__BB19_176;\n" "\n" " ld.shared.f32 %f533, [%r59+4];\n" " ld.shared.f32 %f534, [%r123+4];\n" " mul.f32 %f535, %f689, %f534;\n" " fma.rn.f32 %f710, %f533, %f710, %f535;\n" "\n" "$L__BB19_176:\n" " @%p92 bra $L__BB19_178;\n" "\n" " ld.shared.f32 %f536, [%r59+8];\n" " ld.shared.f32 %f537, [%r123+8];\n" " mul.f32 %f538, %f689, %f537;\n" " fma.rn.f32 %f709, %f536, %f709, %f538;\n" "\n" "$L__BB19_178:\n" " @%p98 bra $L__BB19_180;\n" "\n" " ld.shared.f32 %f539, [%r59+12];\n" " ld.shared.f32 %f540, [%r123+12];\n" " mul.f32 %f541, %f689, %f540;\n" " fma.rn.f32 %f708, %f539, %f708, %f541;\n" "\n" "$L__BB19_180:\n" " @%p104 bra $L__BB19_182;\n" "\n" " ld.shared.f32 %f542, [%r59+16];\n" " ld.shared.f32 %f543, [%r123+16];\n" " mul.f32 %f544, %f689, %f543;\n" " fma.rn.f32 %f707, %f542, %f707, %f544;\n" "\n" "$L__BB19_182:\n" " @%p110 bra $L__BB19_184;\n" "\n" " ld.shared.f32 %f545, [%r59+20];\n" " ld.shared.f32 %f546, [%r123+20];\n" " mul.f32 %f547, %f689, %f546;\n" " fma.rn.f32 %f706, %f545, %f706, %f547;\n" "\n" "$L__BB19_184:\n" " @%p116 bra $L__BB19_186;\n" "\n" " ld.shared.f32 %f548, [%r59+24];\n" " ld.shared.f32 %f549, [%r123+24];\n" " mul.f32 %f550, %f689, %f549;\n" " fma.rn.f32 %f705, %f548, %f705, %f550;\n" "\n" "$L__BB19_186:\n" " @%p122 bra $L__BB19_188;\n" "\n" " ld.shared.f32 %f551, [%r59+28];\n" " ld.shared.f32 %f552, [%r123+28];\n" " mul.f32 %f553, %f689, %f552;\n" " fma.rn.f32 %f704, %f551, %f704, %f553;\n" "\n" "$L__BB19_188:\n" " add.s32 %r725, %r725, 1;\n" " setp.lt.s32 %p161, %r725, %r67;\n" " @%p161 bra $L__BB19_152;\n" "\n" "$L__BB19_229:\n" " bar.sync 0;\n" "\n" "$L__BB19_230:\n" " add.s32 %r707, %r707, 16;\n" " setp.gt.s32 %p182, %r17, %r707;\n" " add.s32 %r706, %r706, 1;\n" " @%p182 bra $L__BB19_29;\n" "\n" "$L__BB19_231:\n" " shl.b32 %r604, %r133, 4;\n" " cvt.s64.s32 %rd307, %r604;\n" " add.s32 %r603, %r134, 16;\n" " mul.wide.s32 %rd306, %r603, 16;\n" " add.s64 %rd305, %rd306, %rd307;\n" " add.s64 %rd304, %rd305, %rd307;\n" " cvt.u32.u64 %r602, %rd304;\n" " add.s64 %rd303, %rd304, %rd10;\n" " cvt.u32.u64 %r601, %rd303;\n" " mov.u32 %r600, %ctaid.x;\n" " mov.u32 %r599, %nctaid.x;\n" " cvt.s64.s32 %rd70, %r599;\n" " cvt.s64.s32 %rd71, %r600;\n" " add.s32 %r588, %r18, %r602;\n" " shl.b32 %r589, %r588, 2;\n" " add.s32 %r131, %r193, %r589;\n" " add.s32 %r591, %r18, %r601;\n" " shl.b32 %r592, %r591, 2;\n" " add.s32 %r132, %r193, %r592;\n" " @%p13 bra $L__BB19_234;\n" "\n" " setp.ne.s32 %p184, %r1, 0;\n" " mul.lo.s64 %rd232, %rd11, %rd70;\n" " add.s64 %rd72, %rd232, %rd71;\n" " mul.lo.s64 %rd233, %rd72, %rd39;\n" " add.s64 %rd234, %rd233, %rd53;\n" " shl.b64 %rd235, %rd234, 2;\n" " add.s64 %rd236, %rd8, %rd235;\n" " st.global.f32 [%rd236], %f711;\n" " @%p184 bra $L__BB19_234;\n" "\n" " ld.shared.f32 %f590, [%r131];\n" " shl.b64 %rd237, %rd72, 2;\n" " add.s64 %rd238, %rd7, %rd237;\n" " st.global.f32 [%rd238], %f590;\n" " ld.shared.f32 %f591, [%r132];\n" " add.s64 %rd239, %rd6, %rd237;\n" " st.global.f32 [%rd239], %f591;\n" "\n" "$L__BB19_234:\n" " setp.lt.s32 %p185, %r133, 2;\n" " @%p185 bra $L__BB19_237;\n" "\n" " setp.ne.s32 %p186, %r1, 0;\n" " add.s64 %rd240, %rd11, 1;\n" " mul.lo.s64 %rd241, %rd240, %rd70;\n" " add.s64 %rd73, %rd241, %rd71;\n" " mul.lo.s64 %rd242, %rd73, %rd39;\n" " add.s64 %rd243, %rd242, %rd53;\n" " shl.b64 %rd244, %rd243, 2;\n" " add.s64 %rd245, %rd8, %rd244;\n" " st.global.f32 [%rd245], %f710;\n" " @%p186 bra $L__BB19_237;\n" "\n" " ld.shared.f32 %f592, [%r131+4];\n" " shl.b64 %rd246, %rd73, 2;\n" " add.s64 %rd247, %rd7, %rd246;\n" " st.global.f32 [%rd247], %f592;\n" " ld.shared.f32 %f593, [%r132+4];\n" " add.s64 %rd248, %rd6, %rd246;\n" " st.global.f32 [%rd248], %f593;\n" "\n" "$L__BB19_237:\n" " setp.lt.s32 %p187, %r133, 3;\n" " @%p187 bra $L__BB19_240;\n" "\n" " setp.ne.s32 %p188, %r1, 0;\n" " add.s64 %rd249, %rd11, 2;\n" " mul.lo.s64 %rd250, %rd249, %rd70;\n" " add.s64 %rd74, %rd250, %rd71;\n" " mul.lo.s64 %rd251, %rd74, %rd39;\n" " add.s64 %rd252, %rd251, %rd53;\n" " shl.b64 %rd253, %rd252, 2;\n" " add.s64 %rd254, %rd8, %rd253;\n" " st.global.f32 [%rd254], %f709;\n" " @%p188 bra $L__BB19_240;\n" "\n" " ld.shared.f32 %f594, [%r131+8];\n" " shl.b64 %rd255, %rd74, 2;\n" " add.s64 %rd256, %rd7, %rd255;\n" " st.global.f32 [%rd256], %f594;\n" " ld.shared.f32 %f595, [%r132+8];\n" " add.s64 %rd257, %rd6, %rd255;\n" " st.global.f32 [%rd257], %f595;\n" "\n" "$L__BB19_240:\n" " setp.lt.s32 %p189, %r133, 4;\n" " @%p189 bra $L__BB19_243;\n" "\n" " setp.ne.s32 %p190, %r1, 0;\n" " add.s64 %rd258, %rd11, 3;\n" " mul.lo.s64 %rd259, %rd258, %rd70;\n" " add.s64 %rd75, %rd259, %rd71;\n" " mul.lo.s64 %rd260, %rd75, %rd39;\n" " add.s64 %rd261, %rd260, %rd53;\n" " shl.b64 %rd262, %rd261, 2;\n" " add.s64 %rd263, %rd8, %rd262;\n" " st.global.f32 [%rd263], %f708;\n" " @%p190 bra $L__BB19_243;\n" "\n" " ld.shared.f32 %f596, [%r131+12];\n" " shl.b64 %rd264, %rd75, 2;\n" " add.s64 %rd265, %rd7, %rd264;\n" " st.global.f32 [%rd265], %f596;\n" " ld.shared.f32 %f597, [%r132+12];\n" " add.s64 %rd266, %rd6, %rd264;\n" " st.global.f32 [%rd266], %f597;\n" "\n" "$L__BB19_243:\n" " setp.lt.s32 %p191, %r133, 5;\n" " @%p191 bra $L__BB19_246;\n" "\n" " setp.ne.s32 %p192, %r1, 0;\n" " add.s64 %rd267, %rd11, 4;\n" " mul.lo.s64 %rd268, %rd267, %rd70;\n" " add.s64 %rd76, %rd268, %rd71;\n" " mul.lo.s64 %rd269, %rd76, %rd39;\n" " add.s64 %rd270, %rd269, %rd53;\n" " shl.b64 %rd271, %rd270, 2;\n" " add.s64 %rd272, %rd8, %rd271;\n" " st.global.f32 [%rd272], %f707;\n" " @%p192 bra $L__BB19_246;\n" "\n" " ld.shared.f32 %f598, [%r131+16];\n" " shl.b64 %rd273, %rd76, 2;\n" " add.s64 %rd274, %rd7, %rd273;\n" " st.global.f32 [%rd274], %f598;\n" " ld.shared.f32 %f599, [%r132+16];\n" " add.s64 %rd275, %rd6, %rd273;\n" " st.global.f32 [%rd275], %f599;\n" "\n" "$L__BB19_246:\n" " setp.lt.s32 %p193, %r133, 6;\n" " @%p193 bra $L__BB19_249;\n" "\n" " setp.ne.s32 %p194, %r1, 0;\n" " add.s64 %rd276, %rd11, 5;\n" " mul.lo.s64 %rd277, %rd276, %rd70;\n" " add.s64 %rd77, %rd277, %rd71;\n" " mul.lo.s64 %rd278, %rd77, %rd39;\n" " add.s64 %rd279, %rd278, %rd53;\n" " shl.b64 %rd280, %rd279, 2;\n" " add.s64 %rd281, %rd8, %rd280;\n" " st.global.f32 [%rd281], %f706;\n" " @%p194 bra $L__BB19_249;\n" "\n" " ld.shared.f32 %f600, [%r131+20];\n" " shl.b64 %rd282, %rd77, 2;\n" " add.s64 %rd283, %rd7, %rd282;\n" " st.global.f32 [%rd283], %f600;\n" " ld.shared.f32 %f601, [%r132+20];\n" " add.s64 %rd284, %rd6, %rd282;\n" " st.global.f32 [%rd284], %f601;\n" "\n" "$L__BB19_249:\n" " setp.lt.s32 %p195, %r133, 7;\n" " @%p195 bra $L__BB19_252;\n" "\n" " setp.ne.s32 %p196, %r1, 0;\n" " add.s64 %rd285, %rd11, 6;\n" " mul.lo.s64 %rd286, %rd285, %rd70;\n" " add.s64 %rd78, %rd286, %rd71;\n" " mul.lo.s64 %rd287, %rd78, %rd39;\n" " add.s64 %rd288, %rd287, %rd53;\n" " shl.b64 %rd289, %rd288, 2;\n" " add.s64 %rd290, %rd8, %rd289;\n" " st.global.f32 [%rd290], %f705;\n" " @%p196 bra $L__BB19_252;\n" "\n" " ld.shared.f32 %f602, [%r131+24];\n" " shl.b64 %rd291, %rd78, 2;\n" " add.s64 %rd292, %rd7, %rd291;\n" " st.global.f32 [%rd292], %f602;\n" " ld.shared.f32 %f603, [%r132+24];\n" " add.s64 %rd293, %rd6, %rd291;\n" " st.global.f32 [%rd293], %f603;\n" "\n" "$L__BB19_252:\n" " setp.lt.s32 %p197, %r133, 8;\n" " @%p197 bra $L__BB19_255;\n" "\n" " setp.ne.s32 %p198, %r1, 0;\n" " add.s64 %rd294, %rd11, 7;\n" " mul.lo.s64 %rd295, %rd294, %rd70;\n" " add.s64 %rd79, %rd295, %rd71;\n" " mul.lo.s64 %rd296, %rd79, %rd39;\n" " add.s64 %rd297, %rd296, %rd53;\n" " shl.b64 %rd298, %rd297, 2;\n" " add.s64 %rd299, %rd8, %rd298;\n" " st.global.f32 [%rd299], %f704;\n" " @%p198 bra $L__BB19_255;\n" "\n" " ld.shared.f32 %f604, [%r131+28];\n" " shl.b64 %rd300, %rd79, 2;\n" " add.s64 %rd301, %rd7, %rd300;\n" " st.global.f32 [%rd301], %f604;\n" " ld.shared.f32 %f605, [%r132+28];\n" " add.s64 %rd302, %rd6, %rd300;\n" " st.global.f32 [%rd302], %f605;\n" " bra.uni $L__BB19_255;\n" "\n" "$L__BB19_1:\n" " setp.lt.s32 %p3, %r133, 1;\n" " @%p3 bra $L__BB19_255;\n" "\n" " cvt.s64.s32 %rd12, %r2;\n" " cvt.s64.s32 %rd13, %r6;\n" " cvt.s64.s32 %rd14, %r134;\n" " cvt.s64.s32 %rd15, %r1;\n" " and.b32 %r699, %r133, 3;\n" " add.s32 %r154, %r133, -1;\n" " setp.lt.u32 %p4, %r154, 3;\n" " mov.u32 %r698, 0;\n" " @%p4 bra $L__BB19_13;\n" "\n" " sub.s32 %r697, %r133, %r699;\n" " mul.lo.s64 %rd96, %rd9, %rd12;\n" " mul.lo.s64 %rd97, %rd96, %rd10;\n" " add.s64 %rd98, %rd97, %rd13;\n" " mul.lo.s64 %rd99, %rd98, %rd14;\n" " add.s64 %rd100, %rd99, %rd15;\n" " shl.b64 %rd101, %rd100, 2;\n" " add.s64 %rd308, %rd8, %rd101;\n" " mul.lo.s64 %rd102, %rd12, %rd14;\n" " shl.b64 %rd17, %rd102, 2;\n" " mov.u32 %r155, 0;\n" " setp.ne.s32 %p5, %r1, 0;\n" " mov.u32 %r698, %r155;\n" "\n" "$L__BB19_4:\n" " st.global.u32 [%rd308], %r155;\n" " @%p5 bra $L__BB19_6;\n" "\n" " cvt.s64.s32 %rd103, %r698;\n" " add.s64 %rd104, %rd11, %rd103;\n" " mul.lo.s64 %rd105, %rd104, %rd12;\n" " add.s64 %rd106, %rd105, %rd13;\n" " shl.b64 %rd107, %rd106, 2;\n" " add.s64 %rd108, %rd7, %rd107;\n" " mov.u32 %r157, -8388608;\n" " st.global.u32 [%rd108], %r157;\n" " add.s64 %rd109, %rd6, %rd107;\n" " mov.u32 %r158, 0;\n" " st.global.u32 [%rd109], %r158;\n" "\n" "$L__BB19_6:\n" " mov.u32 %r159, 0;\n" " add.s64 %rd19, %rd308, %rd17;\n" " st.global.u32 [%rd19], %r159;\n" " @%p5 bra $L__BB19_8;\n" "\n" " add.s32 %r160, %r698, 1;\n" " cvt.s64.s32 %rd110, %r160;\n" " add.s64 %rd111, %rd11, %rd110;\n" " mul.lo.s64 %rd112, %rd111, %rd12;\n" " add.s64 %rd113, %rd112, %rd13;\n" " shl.b64 %rd114, %rd113, 2;\n" " add.s64 %rd115, %rd7, %rd114;\n" " mov.u32 %r161, -8388608;\n" " st.global.u32 [%rd115], %r161;\n" " add.s64 %rd116, %rd6, %rd114;\n" " st.global.u32 [%rd116], %r159;\n" "\n" "$L__BB19_8:\n" " add.s64 %rd20, %rd19, %rd17;\n" " st.global.u32 [%rd20], %r159;\n" " @%p5 bra $L__BB19_10;\n" "\n" " add.s32 %r164, %r698, 2;\n" " cvt.s64.s32 %rd117, %r164;\n" " add.s64 %rd118, %rd11, %rd117;\n" " mul.lo.s64 %rd119, %rd118, %rd12;\n" " add.s64 %rd120, %rd119, %rd13;\n" " shl.b64 %rd121, %rd120, 2;\n" " add.s64 %rd122, %rd7, %rd121;\n" " mov.u32 %r165, -8388608;\n" " st.global.u32 [%rd122], %r165;\n" " add.s64 %rd123, %rd6, %rd121;\n" " st.global.u32 [%rd123], %r159;\n" "\n" "$L__BB19_10:\n" " add.s64 %rd21, %rd20, %rd17;\n" " st.global.u32 [%rd21], %r159;\n" " @%p5 bra $L__BB19_12;\n" "\n" " add.s32 %r168, %r698, 3;\n" " cvt.s64.s32 %rd124, %r168;\n" " add.s64 %rd125, %rd11, %rd124;\n" " mul.lo.s64 %rd126, %rd125, %rd12;\n" " add.s64 %rd127, %rd126, %rd13;\n" " shl.b64 %rd128, %rd127, 2;\n" " add.s64 %rd129, %rd7, %rd128;\n" " mov.u32 %r169, -8388608;\n" " st.global.u32 [%rd129], %r169;\n" " add.s64 %rd130, %rd6, %rd128;\n" " st.global.u32 [%rd130], %r159;\n" "\n" "$L__BB19_12:\n" " add.s32 %r698, %r698, 4;\n" " add.s32 %r697, %r697, -4;\n" " setp.ne.s32 %p9, %r697, 0;\n" " add.s64 %rd308, %rd21, %rd17;\n" " @%p9 bra $L__BB19_4;\n" "\n" "$L__BB19_13:\n" " setp.eq.s32 %p10, %r699, 0;\n" " @%p10 bra $L__BB19_255;\n" "\n" " cvt.s64.s32 %rd131, %r698;\n" " add.s64 %rd132, %rd11, %rd131;\n" " mul.lo.s64 %rd133, %rd132, %rd12;\n" " add.s64 %rd134, %rd133, %rd13;\n" " shl.b64 %rd135, %rd134, 2;\n" " add.s64 %rd311, %rd6, %rd135;\n" " shl.b64 %rd24, %rd12, 2;\n" " add.s64 %rd310, %rd7, %rd135;\n" " mul.lo.s64 %rd136, %rd134, %rd14;\n" " add.s64 %rd137, %rd136, %rd15;\n" " shl.b64 %rd138, %rd137, 2;\n" " add.s64 %rd309, %rd8, %rd138;\n" " mul.lo.s64 %rd139, %rd12, %rd14;\n" " shl.b64 %rd27, %rd139, 2;\n" " mov.u32 %r171, 0;\n" " setp.ne.s32 %p11, %r1, 0;\n" " mov.u32 %r172, -8388608;\n" "\n" "$L__BB19_15:\n" " .pragma \"nounroll\";\n" " st.global.u32 [%rd309], %r171;\n" " @%p11 bra $L__BB19_17;\n" "\n" " st.global.u32 [%rd310], %r172;\n" " st.global.u32 [%rd311], %r171;\n" "\n" "$L__BB19_17:\n" " add.s64 %rd311, %rd311, %rd24;\n" " add.s64 %rd310, %rd310, %rd24;\n" " add.s64 %rd309, %rd309, %rd27;\n" " add.s32 %r699, %r699, -1;\n" " setp.eq.s32 %p12, %r699, 0;\n" " @%p12 bra $L__BB19_255;\n" " bra.uni $L__BB19_15;\n" "\n" "$L__BB19_255:\n" " ret;\n" "\n" "}\n" " // .globl fc_attn_kv_append_q4_dev\n" ".visible .entry fc_attn_kv_append_q4_dev(\n" " .param .u64 fc_attn_kv_append_q4_dev_param_0,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_1,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_2,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_3,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_4,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_5,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_6,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_7,\n" " .param .u32 fc_attn_kv_append_q4_dev_param_8,\n" " .param .u32 fc_attn_kv_append_q4_dev_param_9,\n" " .param .u32 fc_attn_kv_append_q4_dev_param_10,\n" " .param .u64 fc_attn_kv_append_q4_dev_param_11,\n" " .param .u32 fc_attn_kv_append_q4_dev_param_12,\n" " .param .u32 fc_attn_kv_append_q4_dev_param_13\n" ")\n" "{\n" " .reg .pred %p<10>;\n" " .reg .b16 %rs<7>;\n" " .reg .f32 %f<223>;\n" " .reg .b32 %r<66>;\n" " .reg .b64 %rd<57>;\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ24fc_attn_kv_append_q4_devE3s_k[4096];\n" " // demoted variable\n" " .shared .align 4 .b8 _ZZ24fc_attn_kv_append_q4_devE3s_v[4096];\n" "\n" " ld.param.u64 %rd8, [fc_attn_kv_append_q4_dev_param_0];\n" " ld.param.u64 %rd9, [fc_attn_kv_append_q4_dev_param_1];\n" " ld.param.u64 %rd10, [fc_attn_kv_append_q4_dev_param_2];\n" " ld.param.u64 %rd11, [fc_attn_kv_append_q4_dev_param_3];\n" " ld.param.u64 %rd12, [fc_attn_kv_append_q4_dev_param_4];\n" " ld.param.u64 %rd13, [fc_attn_kv_append_q4_dev_param_5];\n" " ld.param.u64 %rd14, [fc_attn_kv_append_q4_dev_param_6];\n" " ld.param.u64 %rd15, [fc_attn_kv_append_q4_dev_param_7];\n" " ld.param.u32 %r6, [fc_attn_kv_append_q4_dev_param_8];\n" " ld.param.u32 %r7, [fc_attn_kv_append_q4_dev_param_9];\n" " ld.param.u32 %r8, [fc_attn_kv_append_q4_dev_param_10];\n" " ld.param.u64 %rd16, [fc_attn_kv_append_q4_dev_param_11];\n" " ld.param.u32 %r9, [fc_attn_kv_append_q4_dev_param_12];\n" " ld.param.u32 %r10, [fc_attn_kv_append_q4_dev_param_13];\n" " mov.u32 %r1, %tid.x;\n" " setp.ge.s32 %p1, %r1, %r7;\n" " @%p1 bra $L__BB20_17;\n" "\n" " mov.u32 %r11, %ctaid.x;\n" " mov.u32 %r12, %ctaid.y;\n" " cvta.to.global.u64 %rd17, %rd16;\n" " ld.global.u32 %r13, [%rd17];\n" " add.s32 %r14, %r13, %r11;\n" " shr.s32 %r15, %r7, 31;\n" " shr.u32 %r16, %r15, 27;\n" " add.s32 %r17, %r7, %r16;\n" " shr.s32 %r18, %r17, 5;\n" " mul.lo.s32 %r19, %r18, 18;\n" " mul.wide.s32 %rd18, %r6, %r11;\n" " cvt.s64.s32 %rd1, %r12;\n" " add.s64 %rd19, %rd18, %rd1;\n" " cvt.s64.s32 %rd2, %r7;\n" " mul.lo.s64 %rd20, %rd19, %rd2;\n" " cvt.s64.s32 %rd3, %r1;\n" " add.s64 %rd21, %rd20, %rd3;\n" " mul.wide.s32 %rd22, %r8, %r12;\n" " cvt.s64.s32 %rd4, %r14;\n" " add.s64 %rd23, %rd22, %rd4;\n" " cvt.s64.s32 %rd24, %r19;\n" " mul.lo.s64 %rd25, %rd23, %rd24;\n" " cvta.to.global.u64 %rd26, %rd8;\n" " shl.b64 %rd27, %rd21, 2;\n" " add.s64 %rd28, %rd26, %rd27;\n" " ld.global.f32 %f1, [%rd28];\n" " cvta.to.global.u64 %rd29, %rd9;\n" " add.s64 %rd30, %rd29, %rd27;\n" " ld.global.f32 %f2, [%rd30];\n" " shl.b32 %r20, %r1, 2;\n" " mov.u32 %r21, _ZZ24fc_attn_kv_append_q4_devE3s_k;\n" " add.s32 %r22, %r21, %r20;\n" " st.shared.f32 [%r22], %f1;\n" " mov.u32 %r23, _ZZ24fc_attn_kv_append_q4_devE3s_v;\n" " add.s32 %r24, %r23, %r20;\n" " st.shared.f32 [%r24], %f2;\n" " bar.sync 0;\n" " shr.s32 %r25, %r1, 31;\n" " shr.u32 %r26, %r25, 27;\n" " add.s32 %r27, %r1, %r26;\n" " shr.s32 %r28, %r27, 5;\n" " and.b32 %r2, %r27, -32;\n" " mul.lo.s32 %r29, %r28, 18;\n" " cvt.s64.s32 %rd31, %r29;\n" " add.s64 %rd5, %rd25, %rd31;\n" " shl.b32 %r30, %r28, 7;\n" " add.s32 %r3, %r21, %r30;\n" " ld.shared.f32 %f10, [%r3];\n" " abs.f32 %f11, %f10;\n" " mov.f32 %f221, 0f00000000;\n" " max.f32 %f12, %f221, %f11;\n" " ld.shared.f32 %f13, [%r3+4];\n" " abs.f32 %f14, %f13;\n" " max.f32 %f15, %f12, %f14;\n" " ld.shared.f32 %f16, [%r3+8];\n" " abs.f32 %f17, %f16;\n" " max.f32 %f18, %f15, %f17;\n" " ld.shared.f32 %f19, [%r3+12];\n" " abs.f32 %f20, %f19;\n" " max.f32 %f21, %f18, %f20;\n" " ld.shared.f32 %f22, [%r3+16];\n" " abs.f32 %f23, %f22;\n" " max.f32 %f24, %f21, %f23;\n" " ld.shared.f32 %f25, [%r3+20];\n" " abs.f32 %f26, %f25;\n" " max.f32 %f27, %f24, %f26;\n" " ld.shared.f32 %f28, [%r3+24];\n" " abs.f32 %f29, %f28;\n" " max.f32 %f30, %f27, %f29;\n" " ld.shared.f32 %f31, [%r3+28];\n" " abs.f32 %f32, %f31;\n" " max.f32 %f33, %f30, %f32;\n" " ld.shared.f32 %f34, [%r3+32];\n" " abs.f32 %f35, %f34;\n" " max.f32 %f36, %f33, %f35;\n" " ld.shared.f32 %f37, [%r3+36];\n" " abs.f32 %f38, %f37;\n" " max.f32 %f39, %f36, %f38;\n" " ld.shared.f32 %f40, [%r3+40];\n" " abs.f32 %f41, %f40;\n" " max.f32 %f42, %f39, %f41;\n" " ld.shared.f32 %f43, [%r3+44];\n" " abs.f32 %f44, %f43;\n" " max.f32 %f45, %f42, %f44;\n" " ld.shared.f32 %f46, [%r3+48];\n" " abs.f32 %f47, %f46;\n" " max.f32 %f48, %f45, %f47;\n" " ld.shared.f32 %f49, [%r3+52];\n" " abs.f32 %f50, %f49;\n" " max.f32 %f51, %f48, %f50;\n" " ld.shared.f32 %f52, [%r3+56];\n" " abs.f32 %f53, %f52;\n" " max.f32 %f54, %f51, %f53;\n" " ld.shared.f32 %f55, [%r3+60];\n" " abs.f32 %f56, %f55;\n" " max.f32 %f57, %f54, %f56;\n" " ld.shared.f32 %f58, [%r3+64];\n" " abs.f32 %f59, %f58;\n" " max.f32 %f60, %f57, %f59;\n" " ld.shared.f32 %f61, [%r3+68];\n" " abs.f32 %f62, %f61;\n" " max.f32 %f63, %f60, %f62;\n" " ld.shared.f32 %f64, [%r3+72];\n" " abs.f32 %f65, %f64;\n" " max.f32 %f66, %f63, %f65;\n" " ld.shared.f32 %f67, [%r3+76];\n" " abs.f32 %f68, %f67;\n" " max.f32 %f69, %f66, %f68;\n" " ld.shared.f32 %f70, [%r3+80];\n" " abs.f32 %f71, %f70;\n" " max.f32 %f72, %f69, %f71;\n" " ld.shared.f32 %f73, [%r3+84];\n" " abs.f32 %f74, %f73;\n" " max.f32 %f75, %f72, %f74;\n" " ld.shared.f32 %f76, [%r3+88];\n" " abs.f32 %f77, %f76;\n" " max.f32 %f78, %f75, %f77;\n" " ld.shared.f32 %f79, [%r3+92];\n" " abs.f32 %f80, %f79;\n" " max.f32 %f81, %f78, %f80;\n" " ld.shared.f32 %f82, [%r3+96];\n" " abs.f32 %f83, %f82;\n" " max.f32 %f84, %f81, %f83;\n" " ld.shared.f32 %f85, [%r3+100];\n" " abs.f32 %f86, %f85;\n" " max.f32 %f87, %f84, %f86;\n" " ld.shared.f32 %f88, [%r3+104];\n" " abs.f32 %f89, %f88;\n" " max.f32 %f90, %f87, %f89;\n" " ld.shared.f32 %f91, [%r3+108];\n" " abs.f32 %f92, %f91;\n" " max.f32 %f93, %f90, %f92;\n" " ld.shared.f32 %f94, [%r3+112];\n" " abs.f32 %f95, %f94;\n" " max.f32 %f96, %f93, %f95;\n" " ld.shared.f32 %f97, [%r3+116];\n" " abs.f32 %f98, %f97;\n" " max.f32 %f99, %f96, %f98;\n" " ld.shared.f32 %f100, [%r3+120];\n" " abs.f32 %f101, %f100;\n" " max.f32 %f102, %f99, %f101;\n" " ld.shared.f32 %f103, [%r3+124];\n" " abs.f32 %f104, %f103;\n" " max.f32 %f105, %f102, %f104;\n" " and.b32 %r4, %r1, 31;\n" " mul.f32 %f3, %f105, 0fBE000000;\n" " setp.eq.f32 %p2, %f3, 0f00000000;\n" " @%p2 bra $L__BB20_3;\n" "\n" " rcp.rn.f32 %f221, %f3;\n" "\n" "$L__BB20_3:\n" " cvta.to.global.u64 %rd32, %rd10;\n" " add.s64 %rd6, %rd32, %rd5;\n" " setp.ne.s32 %p3, %r4, 0;\n" " @%p3 bra $L__BB20_5;\n" "\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs1, %f3;}\n" "\n" " // end inline asm\n" " st.global.u16 [%rd6], %rs1;\n" "\n" "$L__BB20_5:\n" " setp.gt.u32 %p4, %r4, 15;\n" " @%p4 bra $L__BB20_7;\n" "\n" " shl.b32 %r31, %r4, 2;\n" " add.s32 %r32, %r3, %r31;\n" " ld.shared.f32 %f107, [%r32];\n" " mul.f32 %f108, %f221, %f107;\n" " cvt.rni.f32.f32 %f109, %f108;\n" " cvt.rzi.s32.f32 %r33, %f109;\n" " add.s32 %r34, %r33, 8;\n" " ld.shared.f32 %f110, [%r32+64];\n" " mul.f32 %f111, %f221, %f110;\n" " cvt.rni.f32.f32 %f112, %f111;\n" " cvt.rzi.s32.f32 %r35, %f112;\n" " add.s32 %r36, %r35, 8;\n" " min.s32 %r37, %r34, 15;\n" " max.s32 %r38, %r37, 0;\n" " min.s32 %r39, %r36, 15;\n" " max.s32 %r40, %r39, 0;\n" " shl.b32 %r41, %r40, 4;\n" " or.b32 %r42, %r41, %r38;\n" " add.s32 %r43, %r4, 2;\n" " cvt.u64.u32 %rd33, %r43;\n" " add.s64 %rd34, %rd6, %rd33;\n" " st.global.u8 [%rd34], %r42;\n" "\n" "$L__BB20_7:\n" " shl.b32 %r44, %r2, 2;\n" " add.s32 %r5, %r23, %r44;\n" " ld.shared.f32 %f114, [%r5];\n" " abs.f32 %f115, %f114;\n" " mov.f32 %f222, 0f00000000;\n" " max.f32 %f116, %f222, %f115;\n" " ld.shared.f32 %f117, [%r5+4];\n" " abs.f32 %f118, %f117;\n" " max.f32 %f119, %f116, %f118;\n" " ld.shared.f32 %f120, [%r5+8];\n" " abs.f32 %f121, %f120;\n" " max.f32 %f122, %f119, %f121;\n" " ld.shared.f32 %f123, [%r5+12];\n" " abs.f32 %f124, %f123;\n" " max.f32 %f125, %f122, %f124;\n" " ld.shared.f32 %f126, [%r5+16];\n" " abs.f32 %f127, %f126;\n" " max.f32 %f128, %f125, %f127;\n" " ld.shared.f32 %f129, [%r5+20];\n" " abs.f32 %f130, %f129;\n" " max.f32 %f131, %f128, %f130;\n" " ld.shared.f32 %f132, [%r5+24];\n" " abs.f32 %f133, %f132;\n" " max.f32 %f134, %f131, %f133;\n" " ld.shared.f32 %f135, [%r5+28];\n" " abs.f32 %f136, %f135;\n" " max.f32 %f137, %f134, %f136;\n" " ld.shared.f32 %f138, [%r5+32];\n" " abs.f32 %f139, %f138;\n" " max.f32 %f140, %f137, %f139;\n" " ld.shared.f32 %f141, [%r5+36];\n" " abs.f32 %f142, %f141;\n" " max.f32 %f143, %f140, %f142;\n" " ld.shared.f32 %f144, [%r5+40];\n" " abs.f32 %f145, %f144;\n" " max.f32 %f146, %f143, %f145;\n" " ld.shared.f32 %f147, [%r5+44];\n" " abs.f32 %f148, %f147;\n" " max.f32 %f149, %f146, %f148;\n" " ld.shared.f32 %f150, [%r5+48];\n" " abs.f32 %f151, %f150;\n" " max.f32 %f152, %f149, %f151;\n" " ld.shared.f32 %f153, [%r5+52];\n" " abs.f32 %f154, %f153;\n" " max.f32 %f155, %f152, %f154;\n" " ld.shared.f32 %f156, [%r5+56];\n" " abs.f32 %f157, %f156;\n" " max.f32 %f158, %f155, %f157;\n" " ld.shared.f32 %f159, [%r5+60];\n" " abs.f32 %f160, %f159;\n" " max.f32 %f161, %f158, %f160;\n" " ld.shared.f32 %f162, [%r5+64];\n" " abs.f32 %f163, %f162;\n" " max.f32 %f164, %f161, %f163;\n" " ld.shared.f32 %f165, [%r5+68];\n" " abs.f32 %f166, %f165;\n" " max.f32 %f167, %f164, %f166;\n" " ld.shared.f32 %f168, [%r5+72];\n" " abs.f32 %f169, %f168;\n" " max.f32 %f170, %f167, %f169;\n" " ld.shared.f32 %f171, [%r5+76];\n" " abs.f32 %f172, %f171;\n" " max.f32 %f173, %f170, %f172;\n" " ld.shared.f32 %f174, [%r5+80];\n" " abs.f32 %f175, %f174;\n" " max.f32 %f176, %f173, %f175;\n" " ld.shared.f32 %f177, [%r5+84];\n" " abs.f32 %f178, %f177;\n" " max.f32 %f179, %f176, %f178;\n" " ld.shared.f32 %f180, [%r5+88];\n" " abs.f32 %f181, %f180;\n" " max.f32 %f182, %f179, %f181;\n" " ld.shared.f32 %f183, [%r5+92];\n" " abs.f32 %f184, %f183;\n" " max.f32 %f185, %f182, %f184;\n" " ld.shared.f32 %f186, [%r5+96];\n" " abs.f32 %f187, %f186;\n" " max.f32 %f188, %f185, %f187;\n" " ld.shared.f32 %f189, [%r5+100];\n" " abs.f32 %f190, %f189;\n" " max.f32 %f191, %f188, %f190;\n" " ld.shared.f32 %f192, [%r5+104];\n" " abs.f32 %f193, %f192;\n" " max.f32 %f194, %f191, %f193;\n" " ld.shared.f32 %f195, [%r5+108];\n" " abs.f32 %f196, %f195;\n" " max.f32 %f197, %f194, %f196;\n" " ld.shared.f32 %f198, [%r5+112];\n" " abs.f32 %f199, %f198;\n" " max.f32 %f200, %f197, %f199;\n" " ld.shared.f32 %f201, [%r5+116];\n" " abs.f32 %f202, %f201;\n" " max.f32 %f203, %f200, %f202;\n" " ld.shared.f32 %f204, [%r5+120];\n" " abs.f32 %f205, %f204;\n" " max.f32 %f206, %f203, %f205;\n" " ld.shared.f32 %f207, [%r5+124];\n" " abs.f32 %f208, %f207;\n" " max.f32 %f209, %f206, %f208;\n" " mul.f32 %f6, %f209, 0fBE000000;\n" " setp.eq.f32 %p5, %f6, 0f00000000;\n" " @%p5 bra $L__BB20_9;\n" "\n" " rcp.rn.f32 %f222, %f6;\n" "\n" "$L__BB20_9:\n" " cvta.to.global.u64 %rd35, %rd11;\n" " add.s64 %rd7, %rd35, %rd5;\n" " @%p3 bra $L__BB20_11;\n" "\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs2, %f6;}\n" "\n" " // end inline asm\n" " st.global.u16 [%rd7], %rs2;\n" "\n" "$L__BB20_11:\n" " @%p4 bra $L__BB20_13;\n" "\n" " shl.b32 %r46, %r4, 2;\n" " add.s32 %r47, %r5, %r46;\n" " ld.shared.f32 %f211, [%r47];\n" " mul.f32 %f212, %f222, %f211;\n" " cvt.rni.f32.f32 %f213, %f212;\n" " cvt.rzi.s32.f32 %r48, %f213;\n" " add.s32 %r49, %r48, 8;\n" " ld.shared.f32 %f214, [%r47+64];\n" " mul.f32 %f215, %f222, %f214;\n" " cvt.rni.f32.f32 %f216, %f215;\n" " cvt.rzi.s32.f32 %r50, %f216;\n" " add.s32 %r51, %r50, 8;\n" " min.s32 %r52, %r49, 15;\n" " max.s32 %r53, %r52, 0;\n" " min.s32 %r54, %r51, 15;\n" " max.s32 %r55, %r54, 0;\n" " shl.b32 %r56, %r55, 4;\n" " or.b32 %r57, %r56, %r53;\n" " add.s32 %r58, %r4, 2;\n" " cvt.u64.u32 %rd36, %r58;\n" " add.s64 %rd37, %rd7, %rd36;\n" " st.global.u8 [%rd37], %r57;\n" "\n" "$L__BB20_13:\n" " setp.lt.s32 %p8, %r10, 1;\n" " @%p8 bra $L__BB20_15;\n" "\n" " cvt.u32.u64 %r59, %rd4;\n" " shr.s32 %r60, %r59, 31;\n" " shr.u32 %r61, %r60, 24;\n" " add.s32 %r62, %r59, %r61;\n" " and.b32 %r63, %r62, -256;\n" " sub.s32 %r64, %r59, %r63;\n" " cvt.s64.s32 %rd38, %r64;\n" " shl.b64 %rd39, %rd1, 8;\n" " add.s64 %rd40, %rd39, %rd38;\n" " mul.lo.s64 %rd41, %rd40, %rd2;\n" " add.s64 %rd42, %rd41, %rd3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs3, %f1;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd43, %rd14;\n" " shl.b64 %rd44, %rd42, 1;\n" " add.s64 %rd45, %rd43, %rd44;\n" " st.global.u16 [%rd45], %rs3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs4, %f2;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd46, %rd15;\n" " add.s64 %rd47, %rd46, %rd44;\n" " st.global.u16 [%rd47], %rs4;\n" "\n" "$L__BB20_15:\n" " cvt.u32.u64 %r65, %rd4;\n" " setp.ge.s32 %p9, %r65, %r9;\n" " @%p9 bra $L__BB20_17;\n" "\n" " shl.b64 %rd48, %rd1, 3;\n" " add.s64 %rd49, %rd48, %rd4;\n" " mul.lo.s64 %rd50, %rd49, %rd2;\n" " add.s64 %rd51, %rd50, %rd3;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs5, %f1;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd52, %rd12;\n" " shl.b64 %rd53, %rd51, 1;\n" " add.s64 %rd54, %rd52, %rd53;\n" " st.global.u16 [%rd54], %rs5;\n" " // begin inline asm\n" " { cvt.rn.f16.f32 %rs6, %f2;}\n" "\n" " // end inline asm\n" " cvta.to.global.u64 %rd55, %rd13;\n" " add.s64 %rd56, %rd55, %rd53;\n" " st.global.u16 [%rd56], %rs6;\n" "\n" "$L__BB20_17:\n" " ret;\n" "\n" "}\n" "\n" ; #endif /* FC_CUDA_PTX_H */