cannam@154: /*********************************************************************** cannam@154: Copyright (c) 2017 Google Inc. cannam@154: Redistribution and use in source and binary forms, with or without cannam@154: modification, are permitted provided that the following conditions cannam@154: are met: cannam@154: - Redistributions of source code must retain the above copyright notice, cannam@154: this list of conditions and the following disclaimer. cannam@154: - Redistributions in binary form must reproduce the above copyright cannam@154: notice, this list of conditions and the following disclaimer in the cannam@154: documentation and/or other materials provided with the distribution. cannam@154: - Neither the name of Internet Society, IETF or IETF Trust, nor the cannam@154: names of specific contributors, may be used to endorse or promote cannam@154: products derived from this software without specific prior written cannam@154: permission. cannam@154: THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" cannam@154: AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE cannam@154: IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE cannam@154: ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE cannam@154: LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR cannam@154: CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF cannam@154: SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS cannam@154: INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN cannam@154: CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) cannam@154: ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE cannam@154: POSSIBILITY OF SUCH DAMAGE. cannam@154: ***********************************************************************/ cannam@154: cannam@154: #ifdef HAVE_CONFIG_H cannam@154: #include "config.h" cannam@154: #endif cannam@154: cannam@154: #include cannam@154: #include "pitch.h" cannam@154: cannam@154: #ifdef FIXED_POINT cannam@154: cannam@154: opus_val32 celt_inner_prod_neon(const opus_val16 *x, const opus_val16 *y, int N) cannam@154: { cannam@154: int i; cannam@154: opus_val32 xy; cannam@154: int16x8_t x_s16x8, y_s16x8; cannam@154: int32x4_t xy_s32x4 = vdupq_n_s32(0); cannam@154: int64x2_t xy_s64x2; cannam@154: int64x1_t xy_s64x1; cannam@154: cannam@154: for (i = 0; i < N - 7; i += 8) { cannam@154: x_s16x8 = vld1q_s16(&x[i]); cannam@154: y_s16x8 = vld1q_s16(&y[i]); cannam@154: xy_s32x4 = vmlal_s16(xy_s32x4, vget_low_s16 (x_s16x8), vget_low_s16 (y_s16x8)); cannam@154: xy_s32x4 = vmlal_s16(xy_s32x4, vget_high_s16(x_s16x8), vget_high_s16(y_s16x8)); cannam@154: } cannam@154: cannam@154: if (N - i >= 4) { cannam@154: const int16x4_t x_s16x4 = vld1_s16(&x[i]); cannam@154: const int16x4_t y_s16x4 = vld1_s16(&y[i]); cannam@154: xy_s32x4 = vmlal_s16(xy_s32x4, x_s16x4, y_s16x4); cannam@154: i += 4; cannam@154: } cannam@154: cannam@154: xy_s64x2 = vpaddlq_s32(xy_s32x4); cannam@154: xy_s64x1 = vadd_s64(vget_low_s64(xy_s64x2), vget_high_s64(xy_s64x2)); cannam@154: xy = vget_lane_s32(vreinterpret_s32_s64(xy_s64x1), 0); cannam@154: cannam@154: for (; i < N; i++) { cannam@154: xy = MAC16_16(xy, x[i], y[i]); cannam@154: } cannam@154: cannam@154: #ifdef OPUS_CHECK_ASM cannam@154: celt_assert(celt_inner_prod_c(x, y, N) == xy); cannam@154: #endif cannam@154: cannam@154: return xy; cannam@154: } cannam@154: cannam@154: void dual_inner_prod_neon(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, cannam@154: int N, opus_val32 *xy1, opus_val32 *xy2) cannam@154: { cannam@154: int i; cannam@154: opus_val32 xy01, xy02; cannam@154: int16x8_t x_s16x8, y01_s16x8, y02_s16x8; cannam@154: int32x4_t xy01_s32x4 = vdupq_n_s32(0); cannam@154: int32x4_t xy02_s32x4 = vdupq_n_s32(0); cannam@154: int64x2_t xy01_s64x2, xy02_s64x2; cannam@154: int64x1_t xy01_s64x1, xy02_s64x1; cannam@154: cannam@154: for (i = 0; i < N - 7; i += 8) { cannam@154: x_s16x8 = vld1q_s16(&x[i]); cannam@154: y01_s16x8 = vld1q_s16(&y01[i]); cannam@154: y02_s16x8 = vld1q_s16(&y02[i]); cannam@154: xy01_s32x4 = vmlal_s16(xy01_s32x4, vget_low_s16 (x_s16x8), vget_low_s16 (y01_s16x8)); cannam@154: xy02_s32x4 = vmlal_s16(xy02_s32x4, vget_low_s16 (x_s16x8), vget_low_s16 (y02_s16x8)); cannam@154: xy01_s32x4 = vmlal_s16(xy01_s32x4, vget_high_s16(x_s16x8), vget_high_s16(y01_s16x8)); cannam@154: xy02_s32x4 = vmlal_s16(xy02_s32x4, vget_high_s16(x_s16x8), vget_high_s16(y02_s16x8)); cannam@154: } cannam@154: cannam@154: if (N - i >= 4) { cannam@154: const int16x4_t x_s16x4 = vld1_s16(&x[i]); cannam@154: const int16x4_t y01_s16x4 = vld1_s16(&y01[i]); cannam@154: const int16x4_t y02_s16x4 = vld1_s16(&y02[i]); cannam@154: xy01_s32x4 = vmlal_s16(xy01_s32x4, x_s16x4, y01_s16x4); cannam@154: xy02_s32x4 = vmlal_s16(xy02_s32x4, x_s16x4, y02_s16x4); cannam@154: i += 4; cannam@154: } cannam@154: cannam@154: xy01_s64x2 = vpaddlq_s32(xy01_s32x4); cannam@154: xy02_s64x2 = vpaddlq_s32(xy02_s32x4); cannam@154: xy01_s64x1 = vadd_s64(vget_low_s64(xy01_s64x2), vget_high_s64(xy01_s64x2)); cannam@154: xy02_s64x1 = vadd_s64(vget_low_s64(xy02_s64x2), vget_high_s64(xy02_s64x2)); cannam@154: xy01 = vget_lane_s32(vreinterpret_s32_s64(xy01_s64x1), 0); cannam@154: xy02 = vget_lane_s32(vreinterpret_s32_s64(xy02_s64x1), 0); cannam@154: cannam@154: for (; i < N; i++) { cannam@154: xy01 = MAC16_16(xy01, x[i], y01[i]); cannam@154: xy02 = MAC16_16(xy02, x[i], y02[i]); cannam@154: } cannam@154: *xy1 = xy01; cannam@154: *xy2 = xy02; cannam@154: cannam@154: #ifdef OPUS_CHECK_ASM cannam@154: { cannam@154: opus_val32 xy1_c, xy2_c; cannam@154: dual_inner_prod_c(x, y01, y02, N, &xy1_c, &xy2_c); cannam@154: celt_assert(xy1_c == *xy1); cannam@154: celt_assert(xy2_c == *xy2); cannam@154: } cannam@154: #endif cannam@154: } cannam@154: cannam@154: #else /* !FIXED_POINT */ cannam@154: cannam@154: /* ========================================================================== */ cannam@154: cannam@154: #ifdef OPUS_CHECK_ASM cannam@154: cannam@154: /* This part of code simulates floating-point NEON operations. */ cannam@154: cannam@154: /* celt_inner_prod_neon_float_c_simulation() simulates the floating-point */ cannam@154: /* operations of celt_inner_prod_neon(), and both functions should have bit */ cannam@154: /* exact output. */ cannam@154: static opus_val32 celt_inner_prod_neon_float_c_simulation(const opus_val16 *x, const opus_val16 *y, int N) cannam@154: { cannam@154: int i; cannam@154: opus_val32 xy, xy0 = 0, xy1 = 0, xy2 = 0, xy3 = 0; cannam@154: for (i = 0; i < N - 3; i += 4) { cannam@154: xy0 = MAC16_16(xy0, x[i + 0], y[i + 0]); cannam@154: xy1 = MAC16_16(xy1, x[i + 1], y[i + 1]); cannam@154: xy2 = MAC16_16(xy2, x[i + 2], y[i + 2]); cannam@154: xy3 = MAC16_16(xy3, x[i + 3], y[i + 3]); cannam@154: } cannam@154: xy0 += xy2; cannam@154: xy1 += xy3; cannam@154: xy = xy0 + xy1; cannam@154: for (; i < N; i++) { cannam@154: xy = MAC16_16(xy, x[i], y[i]); cannam@154: } cannam@154: return xy; cannam@154: } cannam@154: cannam@154: /* dual_inner_prod_neon_float_c_simulation() simulates the floating-point */ cannam@154: /* operations of dual_inner_prod_neon(), and both functions should have bit */ cannam@154: /* exact output. */ cannam@154: static void dual_inner_prod_neon_float_c_simulation(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, cannam@154: int N, opus_val32 *xy1, opus_val32 *xy2) cannam@154: { cannam@154: int i; cannam@154: opus_val32 xy01, xy02, xy01_0 = 0, xy01_1 = 0, xy01_2 = 0, xy01_3 = 0, xy02_0 = 0, xy02_1 = 0, xy02_2 = 0, xy02_3 = 0; cannam@154: for (i = 0; i < N - 3; i += 4) { cannam@154: xy01_0 = MAC16_16(xy01_0, x[i + 0], y01[i + 0]); cannam@154: xy01_1 = MAC16_16(xy01_1, x[i + 1], y01[i + 1]); cannam@154: xy01_2 = MAC16_16(xy01_2, x[i + 2], y01[i + 2]); cannam@154: xy01_3 = MAC16_16(xy01_3, x[i + 3], y01[i + 3]); cannam@154: xy02_0 = MAC16_16(xy02_0, x[i + 0], y02[i + 0]); cannam@154: xy02_1 = MAC16_16(xy02_1, x[i + 1], y02[i + 1]); cannam@154: xy02_2 = MAC16_16(xy02_2, x[i + 2], y02[i + 2]); cannam@154: xy02_3 = MAC16_16(xy02_3, x[i + 3], y02[i + 3]); cannam@154: } cannam@154: xy01_0 += xy01_2; cannam@154: xy02_0 += xy02_2; cannam@154: xy01_1 += xy01_3; cannam@154: xy02_1 += xy02_3; cannam@154: xy01 = xy01_0 + xy01_1; cannam@154: xy02 = xy02_0 + xy02_1; cannam@154: for (; i < N; i++) { cannam@154: xy01 = MAC16_16(xy01, x[i], y01[i]); cannam@154: xy02 = MAC16_16(xy02, x[i], y02[i]); cannam@154: } cannam@154: *xy1 = xy01; cannam@154: *xy2 = xy02; cannam@154: } cannam@154: cannam@154: #endif /* OPUS_CHECK_ASM */ cannam@154: cannam@154: /* ========================================================================== */ cannam@154: cannam@154: opus_val32 celt_inner_prod_neon(const opus_val16 *x, const opus_val16 *y, int N) cannam@154: { cannam@154: int i; cannam@154: opus_val32 xy; cannam@154: float32x4_t xy_f32x4 = vdupq_n_f32(0); cannam@154: float32x2_t xy_f32x2; cannam@154: cannam@154: for (i = 0; i < N - 7; i += 8) { cannam@154: float32x4_t x_f32x4, y_f32x4; cannam@154: x_f32x4 = vld1q_f32(&x[i]); cannam@154: y_f32x4 = vld1q_f32(&y[i]); cannam@154: xy_f32x4 = vmlaq_f32(xy_f32x4, x_f32x4, y_f32x4); cannam@154: x_f32x4 = vld1q_f32(&x[i + 4]); cannam@154: y_f32x4 = vld1q_f32(&y[i + 4]); cannam@154: xy_f32x4 = vmlaq_f32(xy_f32x4, x_f32x4, y_f32x4); cannam@154: } cannam@154: cannam@154: if (N - i >= 4) { cannam@154: const float32x4_t x_f32x4 = vld1q_f32(&x[i]); cannam@154: const float32x4_t y_f32x4 = vld1q_f32(&y[i]); cannam@154: xy_f32x4 = vmlaq_f32(xy_f32x4, x_f32x4, y_f32x4); cannam@154: i += 4; cannam@154: } cannam@154: cannam@154: xy_f32x2 = vadd_f32(vget_low_f32(xy_f32x4), vget_high_f32(xy_f32x4)); cannam@154: xy_f32x2 = vpadd_f32(xy_f32x2, xy_f32x2); cannam@154: xy = vget_lane_f32(xy_f32x2, 0); cannam@154: cannam@154: for (; i < N; i++) { cannam@154: xy = MAC16_16(xy, x[i], y[i]); cannam@154: } cannam@154: cannam@154: #ifdef OPUS_CHECK_ASM cannam@154: celt_assert(ABS32(celt_inner_prod_neon_float_c_simulation(x, y, N) - xy) <= VERY_SMALL); cannam@154: #endif cannam@154: cannam@154: return xy; cannam@154: } cannam@154: cannam@154: void dual_inner_prod_neon(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, cannam@154: int N, opus_val32 *xy1, opus_val32 *xy2) cannam@154: { cannam@154: int i; cannam@154: opus_val32 xy01, xy02; cannam@154: float32x4_t xy01_f32x4 = vdupq_n_f32(0); cannam@154: float32x4_t xy02_f32x4 = vdupq_n_f32(0); cannam@154: float32x2_t xy01_f32x2, xy02_f32x2; cannam@154: cannam@154: for (i = 0; i < N - 7; i += 8) { cannam@154: float32x4_t x_f32x4, y01_f32x4, y02_f32x4; cannam@154: x_f32x4 = vld1q_f32(&x[i]); cannam@154: y01_f32x4 = vld1q_f32(&y01[i]); cannam@154: y02_f32x4 = vld1q_f32(&y02[i]); cannam@154: xy01_f32x4 = vmlaq_f32(xy01_f32x4, x_f32x4, y01_f32x4); cannam@154: xy02_f32x4 = vmlaq_f32(xy02_f32x4, x_f32x4, y02_f32x4); cannam@154: x_f32x4 = vld1q_f32(&x[i + 4]); cannam@154: y01_f32x4 = vld1q_f32(&y01[i + 4]); cannam@154: y02_f32x4 = vld1q_f32(&y02[i + 4]); cannam@154: xy01_f32x4 = vmlaq_f32(xy01_f32x4, x_f32x4, y01_f32x4); cannam@154: xy02_f32x4 = vmlaq_f32(xy02_f32x4, x_f32x4, y02_f32x4); cannam@154: } cannam@154: cannam@154: if (N - i >= 4) { cannam@154: const float32x4_t x_f32x4 = vld1q_f32(&x[i]); cannam@154: const float32x4_t y01_f32x4 = vld1q_f32(&y01[i]); cannam@154: const float32x4_t y02_f32x4 = vld1q_f32(&y02[i]); cannam@154: xy01_f32x4 = vmlaq_f32(xy01_f32x4, x_f32x4, y01_f32x4); cannam@154: xy02_f32x4 = vmlaq_f32(xy02_f32x4, x_f32x4, y02_f32x4); cannam@154: i += 4; cannam@154: } cannam@154: cannam@154: xy01_f32x2 = vadd_f32(vget_low_f32(xy01_f32x4), vget_high_f32(xy01_f32x4)); cannam@154: xy02_f32x2 = vadd_f32(vget_low_f32(xy02_f32x4), vget_high_f32(xy02_f32x4)); cannam@154: xy01_f32x2 = vpadd_f32(xy01_f32x2, xy01_f32x2); cannam@154: xy02_f32x2 = vpadd_f32(xy02_f32x2, xy02_f32x2); cannam@154: xy01 = vget_lane_f32(xy01_f32x2, 0); cannam@154: xy02 = vget_lane_f32(xy02_f32x2, 0); cannam@154: cannam@154: for (; i < N; i++) { cannam@154: xy01 = MAC16_16(xy01, x[i], y01[i]); cannam@154: xy02 = MAC16_16(xy02, x[i], y02[i]); cannam@154: } cannam@154: *xy1 = xy01; cannam@154: *xy2 = xy02; cannam@154: cannam@154: #ifdef OPUS_CHECK_ASM cannam@154: { cannam@154: opus_val32 xy1_c, xy2_c; cannam@154: dual_inner_prod_neon_float_c_simulation(x, y01, y02, N, &xy1_c, &xy2_c); cannam@154: celt_assert(ABS32(xy1_c - *xy1) <= VERY_SMALL); cannam@154: celt_assert(ABS32(xy2_c - *xy2) <= VERY_SMALL); cannam@154: } cannam@154: #endif cannam@154: } cannam@154: cannam@154: #endif /* FIXED_POINT */