diff --git a/qemu/aarch64.h b/qemu/aarch64.h index 7311584a..3d92629e 100644 --- a/qemu/aarch64.h +++ b/qemu/aarch64.h @@ -3516,6 +3516,9 @@ #define helper_sve_fadds_d helper_sve_fadds_d_aarch64 #define helper_sve_fadds_h helper_sve_fadds_h_aarch64 #define helper_sve_fadds_s helper_sve_fadds_s_aarch64 +#define helper_sve_faddv_d helper_sve_faddv_d_aarch64 +#define helper_sve_faddv_h helper_sve_faddv_h_aarch64 +#define helper_sve_faddv_s helper_sve_faddv_s_aarch64 #define helper_sve_facge_d helper_sve_facge_d_aarch64 #define helper_sve_facge_h helper_sve_facge_h_aarch64 #define helper_sve_facge_s helper_sve_facge_s_aarch64 @@ -3549,9 +3552,15 @@ #define helper_sve_fmaxs_d helper_sve_fmaxs_d_aarch64 #define helper_sve_fmaxs_h helper_sve_fmaxs_h_aarch64 #define helper_sve_fmaxs_s helper_sve_fmaxs_s_aarch64 +#define helper_sve_fmaxv_d helper_sve_fmaxv_d_aarch64 +#define helper_sve_fmaxv_h helper_sve_fmaxv_h_aarch64 +#define helper_sve_fmaxv_s helper_sve_fmaxv_s_aarch64 #define helper_sve_fmaxnms_d helper_sve_fmaxnms_d_aarch64 #define helper_sve_fmaxnms_h helper_sve_fmaxnms_h_aarch64 #define helper_sve_fmaxnms_s helper_sve_fmaxnms_s_aarch64 +#define helper_sve_fmaxnmv_d helper_sve_fmaxnmv_d_aarch64 +#define helper_sve_fmaxnmv_h helper_sve_fmaxnmv_h_aarch64 +#define helper_sve_fmaxnmv_s helper_sve_fmaxnmv_s_aarch64 #define helper_sve_fmaxnum_d helper_sve_fmaxnum_d_aarch64 #define helper_sve_fmaxnum_h helper_sve_fmaxnum_h_aarch64 #define helper_sve_fmaxnum_s helper_sve_fmaxnum_s_aarch64 @@ -3561,9 +3570,15 @@ #define helper_sve_fmins_d helper_sve_fmins_d_aarch64 #define helper_sve_fmins_h helper_sve_fmins_h_aarch64 #define helper_sve_fmins_s helper_sve_fmins_s_aarch64 +#define helper_sve_fminv_d helper_sve_fminv_d_aarch64 +#define helper_sve_fminv_h helper_sve_fminv_h_aarch64 +#define helper_sve_fminv_s helper_sve_fminv_s_aarch64 #define helper_sve_fminnms_d helper_sve_fminnms_d_aarch64 #define helper_sve_fminnms_h helper_sve_fminnms_h_aarch64 #define helper_sve_fminnms_s helper_sve_fminnms_s_aarch64 +#define helper_sve_fminnmv_d helper_sve_fminnmv_d_aarch64 +#define helper_sve_fminnmv_h helper_sve_fminnmv_h_aarch64 +#define helper_sve_fminnmv_s helper_sve_fminnmv_s_aarch64 #define helper_sve_fminnum_d helper_sve_fminnum_d_aarch64 #define helper_sve_fminnum_h helper_sve_fminnum_h_aarch64 #define helper_sve_fminnum_s helper_sve_fminnum_s_aarch64 diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h index c775b421..1eaf1b1b 100644 --- a/qemu/aarch64eb.h +++ b/qemu/aarch64eb.h @@ -3516,6 +3516,9 @@ #define helper_sve_fadds_d helper_sve_fadds_d_aarch64eb #define helper_sve_fadds_h helper_sve_fadds_h_aarch64eb #define helper_sve_fadds_s helper_sve_fadds_s_aarch64eb +#define helper_sve_faddv_d helper_sve_faddv_d_aarch64eb +#define helper_sve_faddv_h helper_sve_faddv_h_aarch64eb +#define helper_sve_faddv_s helper_sve_faddv_s_aarch64eb #define helper_sve_facge_d helper_sve_facge_d_aarch64eb #define helper_sve_facge_h helper_sve_facge_h_aarch64eb #define helper_sve_facge_s helper_sve_facge_s_aarch64eb @@ -3549,9 +3552,15 @@ #define helper_sve_fmaxs_d helper_sve_fmaxs_d_aarch64eb #define helper_sve_fmaxs_h helper_sve_fmaxs_h_aarch64eb #define helper_sve_fmaxs_s helper_sve_fmaxs_s_aarch64eb +#define helper_sve_fmaxv_d helper_sve_fmaxv_d_aarch64eb +#define helper_sve_fmaxv_h helper_sve_fmaxv_h_aarch64eb +#define helper_sve_fmaxv_s helper_sve_fmaxv_s_aarch64eb #define helper_sve_fmaxnms_d helper_sve_fmaxnms_d_aarch64eb #define helper_sve_fmaxnms_h helper_sve_fmaxnms_h_aarch64eb #define helper_sve_fmaxnms_s helper_sve_fmaxnms_s_aarch64eb +#define helper_sve_fmaxnmv_d helper_sve_fmaxnmv_d_aarch64eb +#define helper_sve_fmaxnmv_h helper_sve_fmaxnmv_h_aarch64eb +#define helper_sve_fmaxnmv_s helper_sve_fmaxnmv_s_aarch64eb #define helper_sve_fmaxnum_d helper_sve_fmaxnum_d_aarch64eb #define helper_sve_fmaxnum_h helper_sve_fmaxnum_h_aarch64eb #define helper_sve_fmaxnum_s helper_sve_fmaxnum_s_aarch64eb @@ -3561,9 +3570,15 @@ #define helper_sve_fmins_d helper_sve_fmins_d_aarch64eb #define helper_sve_fmins_h helper_sve_fmins_h_aarch64eb #define helper_sve_fmins_s helper_sve_fmins_s_aarch64eb +#define helper_sve_fminv_d helper_sve_fminv_d_aarch64eb +#define helper_sve_fminv_h helper_sve_fminv_h_aarch64eb +#define helper_sve_fminv_s helper_sve_fminv_s_aarch64eb #define helper_sve_fminnms_d helper_sve_fminnms_d_aarch64eb #define helper_sve_fminnms_h helper_sve_fminnms_h_aarch64eb #define helper_sve_fminnms_s helper_sve_fminnms_s_aarch64eb +#define helper_sve_fminnmv_d helper_sve_fminnmv_d_aarch64eb +#define helper_sve_fminnmv_h helper_sve_fminnmv_h_aarch64eb +#define helper_sve_fminnmv_s helper_sve_fminnmv_s_aarch64eb #define helper_sve_fminnum_d helper_sve_fminnum_d_aarch64eb #define helper_sve_fminnum_h helper_sve_fminnum_h_aarch64eb #define helper_sve_fminnum_s helper_sve_fminnum_s_aarch64eb diff --git a/qemu/header_gen.py b/qemu/header_gen.py index 12331a38..767bb396 100644 --- a/qemu/header_gen.py +++ b/qemu/header_gen.py @@ -3537,6 +3537,9 @@ aarch64_symbols = ( 'helper_sve_fadds_d', 'helper_sve_fadds_h', 'helper_sve_fadds_s', + 'helper_sve_faddv_d', + 'helper_sve_faddv_h', + 'helper_sve_faddv_s', 'helper_sve_facge_d', 'helper_sve_facge_h', 'helper_sve_facge_s', @@ -3570,9 +3573,15 @@ aarch64_symbols = ( 'helper_sve_fmaxs_d', 'helper_sve_fmaxs_h', 'helper_sve_fmaxs_s', + 'helper_sve_fmaxv_d', + 'helper_sve_fmaxv_h', + 'helper_sve_fmaxv_s', 'helper_sve_fmaxnms_d', 'helper_sve_fmaxnms_h', 'helper_sve_fmaxnms_s', + 'helper_sve_fmaxnmv_d', + 'helper_sve_fmaxnmv_h', + 'helper_sve_fmaxnmv_s', 'helper_sve_fmaxnum_d', 'helper_sve_fmaxnum_h', 'helper_sve_fmaxnum_s', @@ -3582,9 +3591,15 @@ aarch64_symbols = ( 'helper_sve_fmins_d', 'helper_sve_fmins_h', 'helper_sve_fmins_s', + 'helper_sve_fminv_d', + 'helper_sve_fminv_h', + 'helper_sve_fminv_s', 'helper_sve_fminnms_d', 'helper_sve_fminnms_h', 'helper_sve_fminnms_s', + 'helper_sve_fminnmv_d', + 'helper_sve_fminnmv_h', + 'helper_sve_fminnmv_s', 'helper_sve_fminnum_d', 'helper_sve_fminnum_h', 'helper_sve_fminnum_s', diff --git a/qemu/target/arm/helper-sve.h b/qemu/target/arm/helper-sve.h index 087819ec..ff69d143 100644 --- a/qemu/target/arm/helper-sve.h +++ b/qemu/target/arm/helper-sve.h @@ -725,6 +725,41 @@ DEF_HELPER_FLAGS_5(gvec_rsqrts_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(gvec_rsqrts_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_faddv_h, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_faddv_s, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_faddv_d, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve_fmaxnmv_h, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fmaxnmv_s, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fmaxnmv_d, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve_fminnmv_h, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fminnmv_s, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fminnmv_d, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve_fmaxv_h, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fmaxv_s, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fmaxv_d, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_4(sve_fminv_h, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fminv_s, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(sve_fminv_d, TCG_CALL_NO_RWG, + i64, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_fadda_h, TCG_CALL_NO_RWG, i64, i64, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_fadda_s, TCG_CALL_NO_RWG, diff --git a/qemu/target/arm/sve.decode b/qemu/target/arm/sve.decode index 7a17e50f..5b515821 100644 --- a/qemu/target/arm/sve.decode +++ b/qemu/target/arm/sve.decode @@ -745,6 +745,14 @@ FMUL_zzx 01100100 0.1 .. rm:3 001000 rn:5 rd:5 \ FMUL_zzx 01100100 101 index:2 rm:3 001000 rn:5 rd:5 esz=2 FMUL_zzx 01100100 111 index:1 rm:4 001000 rn:5 rd:5 esz=3 +### SVE FP Fast Reduction Group + +FADDV 01100101 .. 000 000 001 ... ..... ..... @rd_pg_rn +FMAXNMV 01100101 .. 000 100 001 ... ..... ..... @rd_pg_rn +FMINNMV 01100101 .. 000 101 001 ... ..... ..... @rd_pg_rn +FMAXV 01100101 .. 000 110 001 ... ..... ..... @rd_pg_rn +FMINV 01100101 .. 000 111 001 ... ..... ..... @rd_pg_rn + ### SVE FP Accumulating Reduction Group # SVE floating-point serial reduction (predicated) diff --git a/qemu/target/arm/sve_helper.c b/qemu/target/arm/sve_helper.c index 6302def4..8fc493e0 100644 --- a/qemu/target/arm/sve_helper.c +++ b/qemu/target/arm/sve_helper.c @@ -3592,6 +3592,67 @@ void HELPER(sve_st4dd_r)(CPUARMState *env, void *vg, } } +/* Recursive reduction on a function; + * C.f. the ARM ARM function ReducePredicated. + * + * While it would be possible to write this without the DATA temporary, + * it is much simpler to process the predicate register this way. + * The recursion is bounded to depth 7 (128 fp16 elements), so there's + * little to gain with a more complex non-recursive form. + */ +#define DO_REDUCE(NAME, TYPE, H, FUNC, IDENT) \ +static TYPE NAME##_reduce(TYPE *data, float_status *status, uintptr_t n) \ +{ \ + if (n == 1) { \ + return *data; \ + } else { \ + uintptr_t half = n / 2; \ + TYPE lo = NAME##_reduce(data, status, half); \ + TYPE hi = NAME##_reduce(data + half, status, half); \ + return TYPE##_##FUNC(lo, hi, status); \ + } \ +} \ +uint64_t HELPER(NAME)(void *vn, void *vg, void *vs, uint32_t desc) \ +{ \ + uintptr_t i, oprsz = simd_oprsz(desc), maxsz = simd_maxsz(desc); \ + TYPE data[sizeof(ARMVectorReg) / sizeof(TYPE)]; \ + for (i = 0; i < oprsz; ) { \ + uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ + do { \ + TYPE nn = *(TYPE *)(vn + H(i)); \ + *(TYPE *)((void *)data + i) = (pg & 1 ? nn : IDENT); \ + i += sizeof(TYPE), pg >>= sizeof(TYPE); \ + } while (i & 15); \ + } \ + for (; i < maxsz; i += sizeof(TYPE)) { \ + *(TYPE *)((void *)data + i) = IDENT; \ + } \ + return NAME##_reduce(data, vs, maxsz / sizeof(TYPE)); \ +} + +DO_REDUCE(sve_faddv_h, float16, H1_2, add, float16_zero) +DO_REDUCE(sve_faddv_s, float32, H1_4, add, float32_zero) +DO_REDUCE(sve_faddv_d, float64, , add, float64_zero) + +/* Identity is floatN_default_nan, without the function call. */ +DO_REDUCE(sve_fminnmv_h, float16, H1_2, minnum, 0x7E00) +DO_REDUCE(sve_fminnmv_s, float32, H1_4, minnum, 0x7FC00000) +DO_REDUCE(sve_fminnmv_d, float64, , minnum, 0x7FF8000000000000ULL) + +DO_REDUCE(sve_fmaxnmv_h, float16, H1_2, maxnum, 0x7E00) +DO_REDUCE(sve_fmaxnmv_s, float32, H1_4, maxnum, 0x7FC00000) +DO_REDUCE(sve_fmaxnmv_d, float64, , maxnum, 0x7FF8000000000000ULL) + +DO_REDUCE(sve_fminv_h, float16, H1_2, min, float16_infinity) +DO_REDUCE(sve_fminv_s, float32, H1_4, min, float32_infinity) +DO_REDUCE(sve_fminv_d, float64, , min, float64_infinity) + +DO_REDUCE(sve_fmaxv_h, float16, H1_2, max, float16_chs(float16_infinity)) +DO_REDUCE(sve_fmaxv_s, float32, H1_4, max, float32_chs(float32_infinity)) +DO_REDUCE(sve_fmaxv_d, float64, , max, float64_chs(float64_infinity)) + +#undef DO_REDUCE + uint64_t HELPER(sve_fadda_h)(uint64_t nn, void *vm, void *vg, void *status, uint32_t desc) { diff --git a/qemu/target/arm/translate-sve.c b/qemu/target/arm/translate-sve.c index ef2b9811..dbf0511c 100644 --- a/qemu/target/arm/translate-sve.c +++ b/qemu/target/arm/translate-sve.c @@ -3595,6 +3595,64 @@ static bool trans_FMUL_zzx(DisasContext *s, arg_FMUL_zzx *a, uint32_t insn) return true; } +/* + *** SVE Floating Point Fast Reduction Group + */ + +typedef void gen_helper_fp_reduce(TCGContext *, TCGv_i64, TCGv_ptr, TCGv_ptr, + TCGv_ptr, TCGv_i32); + +static void do_reduce(DisasContext *s, arg_rpr_esz *a, + gen_helper_fp_reduce *fn) +{ + TCGContext *tcg_ctx = s->uc->tcg_ctx; + unsigned vsz = vec_full_reg_size(s); + unsigned p2vsz = pow2ceil(vsz); + TCGv_i32 t_desc = tcg_const_i32(tcg_ctx, simd_desc(vsz, p2vsz, 0)); + TCGv_ptr t_zn, t_pg, status; + TCGv_i64 temp; + + temp = tcg_temp_new_i64(tcg_ctx); + t_zn = tcg_temp_new_ptr(tcg_ctx); + t_pg = tcg_temp_new_ptr(tcg_ctx); + + tcg_gen_addi_ptr(tcg_ctx, t_zn, tcg_ctx->cpu_env, vec_full_reg_offset(s, a->rn)); + tcg_gen_addi_ptr(tcg_ctx, t_pg, tcg_ctx->cpu_env, pred_full_reg_offset(s, a->pg)); + status = get_fpstatus_ptr(tcg_ctx, a->esz == MO_16); + + fn(tcg_ctx, temp, t_zn, t_pg, status, t_desc); + tcg_temp_free_ptr(tcg_ctx, t_zn); + tcg_temp_free_ptr(tcg_ctx, t_pg); + tcg_temp_free_ptr(tcg_ctx, status); + tcg_temp_free_i32(tcg_ctx, t_desc); + + write_fp_dreg(s, a->rd, temp); + tcg_temp_free_i64(tcg_ctx, temp); +} + +#define DO_VPZ(NAME, name) \ +static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a, uint32_t insn) \ +{ \ + static gen_helper_fp_reduce * const fns[3] = { \ + gen_helper_sve_##name##_h, \ + gen_helper_sve_##name##_s, \ + gen_helper_sve_##name##_d, \ + }; \ + if (a->esz == 0) { \ + return false; \ + } \ + if (sve_access_check(s)) { \ + do_reduce(s, a, fns[a->esz - 1]); \ + } \ + return true; \ +} + +DO_VPZ(FADDV, faddv) +DO_VPZ(FMINNMV, fminnmv) +DO_VPZ(FMAXNMV, fmaxnmv) +DO_VPZ(FMINV, fminv) +DO_VPZ(FMAXV, fmaxv) + /* *** SVE Floating Point Accumulating Reduction Group */