Chris@69: /* Copyright (c) 2014-2015 Xiph.Org Foundation Chris@69: Written by Viswanath Puttagunta */ Chris@69: /** Chris@69: @file celt_neon_intr.c Chris@69: @brief ARM Neon Intrinsic optimizations for celt Chris@69: */ Chris@69: Chris@69: /* Chris@69: Redistribution and use in source and binary forms, with or without Chris@69: modification, are permitted provided that the following conditions Chris@69: are met: Chris@69: Chris@69: - Redistributions of source code must retain the above copyright Chris@69: notice, this list of conditions and the following disclaimer. Chris@69: Chris@69: - Redistributions in binary form must reproduce the above copyright Chris@69: notice, this list of conditions and the following disclaimer in the Chris@69: documentation and/or other materials provided with the distribution. Chris@69: Chris@69: THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS Chris@69: ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT Chris@69: LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR Chris@69: A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER Chris@69: OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, Chris@69: EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, Chris@69: PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR Chris@69: PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF Chris@69: LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING Chris@69: NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS Chris@69: SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Chris@69: */ Chris@69: Chris@69: #ifdef HAVE_CONFIG_H Chris@69: #include "config.h" Chris@69: #endif Chris@69: Chris@69: #include Chris@69: #include "../pitch.h" Chris@69: Chris@69: #if defined(FIXED_POINT) Chris@69: void xcorr_kernel_neon_fixed(const opus_val16 * x, const opus_val16 * y, opus_val32 sum[4], int len) Chris@69: { Chris@69: int j; Chris@69: int32x4_t a = vld1q_s32(sum); Chris@69: /* Load y[0...3] */ Chris@69: /* This requires len>0 to always be valid (which we assert in the C code). */ Chris@69: int16x4_t y0 = vld1_s16(y); Chris@69: y += 4; Chris@69: Chris@69: for (j = 0; j + 8 <= len; j += 8) Chris@69: { Chris@69: /* Load x[0...7] */ Chris@69: int16x8_t xx = vld1q_s16(x); Chris@69: int16x4_t x0 = vget_low_s16(xx); Chris@69: int16x4_t x4 = vget_high_s16(xx); Chris@69: /* Load y[4...11] */ Chris@69: int16x8_t yy = vld1q_s16(y); Chris@69: int16x4_t y4 = vget_low_s16(yy); Chris@69: int16x4_t y8 = vget_high_s16(yy); Chris@69: int32x4_t a0 = vmlal_lane_s16(a, y0, x0, 0); Chris@69: int32x4_t a1 = vmlal_lane_s16(a0, y4, x4, 0); Chris@69: Chris@69: int16x4_t y1 = vext_s16(y0, y4, 1); Chris@69: int16x4_t y5 = vext_s16(y4, y8, 1); Chris@69: int32x4_t a2 = vmlal_lane_s16(a1, y1, x0, 1); Chris@69: int32x4_t a3 = vmlal_lane_s16(a2, y5, x4, 1); Chris@69: Chris@69: int16x4_t y2 = vext_s16(y0, y4, 2); Chris@69: int16x4_t y6 = vext_s16(y4, y8, 2); Chris@69: int32x4_t a4 = vmlal_lane_s16(a3, y2, x0, 2); Chris@69: int32x4_t a5 = vmlal_lane_s16(a4, y6, x4, 2); Chris@69: Chris@69: int16x4_t y3 = vext_s16(y0, y4, 3); Chris@69: int16x4_t y7 = vext_s16(y4, y8, 3); Chris@69: int32x4_t a6 = vmlal_lane_s16(a5, y3, x0, 3); Chris@69: int32x4_t a7 = vmlal_lane_s16(a6, y7, x4, 3); Chris@69: Chris@69: y0 = y8; Chris@69: a = a7; Chris@69: x += 8; Chris@69: y += 8; Chris@69: } Chris@69: Chris@69: for (; j < len; j++) Chris@69: { Chris@69: int16x4_t x0 = vld1_dup_s16(x); /* load next x */ Chris@69: int32x4_t a0 = vmlal_s16(a, y0, x0); Chris@69: Chris@69: int16x4_t y4 = vld1_dup_s16(y); /* load next y */ Chris@69: y0 = vext_s16(y0, y4, 1); Chris@69: a = a0; Chris@69: x++; Chris@69: y++; Chris@69: } Chris@69: Chris@69: vst1q_s32(sum, a); Chris@69: } Chris@69: Chris@69: #else Chris@69: /* Chris@69: * Function: xcorr_kernel_neon_float Chris@69: * --------------------------------- Chris@69: * Computes 4 correlation values and stores them in sum[4] Chris@69: */ Chris@69: static void xcorr_kernel_neon_float(const float32_t *x, const float32_t *y, Chris@69: float32_t sum[4], int len) { Chris@69: float32x4_t YY[3]; Chris@69: float32x4_t YEXT[3]; Chris@69: float32x4_t XX[2]; Chris@69: float32x2_t XX_2; Chris@69: float32x4_t SUMM; Chris@69: const float32_t *xi = x; Chris@69: const float32_t *yi = y; Chris@69: Chris@69: celt_assert(len>0); Chris@69: Chris@69: YY[0] = vld1q_f32(yi); Chris@69: SUMM = vdupq_n_f32(0); Chris@69: Chris@69: /* Consume 8 elements in x vector and 12 elements in y Chris@69: * vector. However, the 12'th element never really gets Chris@69: * touched in this loop. So, if len == 8, then we only Chris@69: * must access y[0] to y[10]. y[11] must not be accessed Chris@69: * hence make sure len > 8 and not len >= 8 Chris@69: */ Chris@69: while (len > 8) { Chris@69: yi += 4; Chris@69: YY[1] = vld1q_f32(yi); Chris@69: yi += 4; Chris@69: YY[2] = vld1q_f32(yi); Chris@69: Chris@69: XX[0] = vld1q_f32(xi); Chris@69: xi += 4; Chris@69: XX[1] = vld1q_f32(xi); Chris@69: xi += 4; Chris@69: Chris@69: SUMM = vmlaq_lane_f32(SUMM, YY[0], vget_low_f32(XX[0]), 0); Chris@69: YEXT[0] = vextq_f32(YY[0], YY[1], 1); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[0], vget_low_f32(XX[0]), 1); Chris@69: YEXT[1] = vextq_f32(YY[0], YY[1], 2); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[1], vget_high_f32(XX[0]), 0); Chris@69: YEXT[2] = vextq_f32(YY[0], YY[1], 3); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[2], vget_high_f32(XX[0]), 1); Chris@69: Chris@69: SUMM = vmlaq_lane_f32(SUMM, YY[1], vget_low_f32(XX[1]), 0); Chris@69: YEXT[0] = vextq_f32(YY[1], YY[2], 1); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[0], vget_low_f32(XX[1]), 1); Chris@69: YEXT[1] = vextq_f32(YY[1], YY[2], 2); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[1], vget_high_f32(XX[1]), 0); Chris@69: YEXT[2] = vextq_f32(YY[1], YY[2], 3); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[2], vget_high_f32(XX[1]), 1); Chris@69: Chris@69: YY[0] = YY[2]; Chris@69: len -= 8; Chris@69: } Chris@69: Chris@69: /* Consume 4 elements in x vector and 8 elements in y Chris@69: * vector. However, the 8'th element in y never really gets Chris@69: * touched in this loop. So, if len == 4, then we only Chris@69: * must access y[0] to y[6]. y[7] must not be accessed Chris@69: * hence make sure len>4 and not len>=4 Chris@69: */ Chris@69: if (len > 4) { Chris@69: yi += 4; Chris@69: YY[1] = vld1q_f32(yi); Chris@69: Chris@69: XX[0] = vld1q_f32(xi); Chris@69: xi += 4; Chris@69: Chris@69: SUMM = vmlaq_lane_f32(SUMM, YY[0], vget_low_f32(XX[0]), 0); Chris@69: YEXT[0] = vextq_f32(YY[0], YY[1], 1); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[0], vget_low_f32(XX[0]), 1); Chris@69: YEXT[1] = vextq_f32(YY[0], YY[1], 2); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[1], vget_high_f32(XX[0]), 0); Chris@69: YEXT[2] = vextq_f32(YY[0], YY[1], 3); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YEXT[2], vget_high_f32(XX[0]), 1); Chris@69: Chris@69: YY[0] = YY[1]; Chris@69: len -= 4; Chris@69: } Chris@69: Chris@69: while (--len > 0) { Chris@69: XX_2 = vld1_dup_f32(xi++); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YY[0], XX_2, 0); Chris@69: YY[0]= vld1q_f32(++yi); Chris@69: } Chris@69: Chris@69: XX_2 = vld1_dup_f32(xi); Chris@69: SUMM = vmlaq_lane_f32(SUMM, YY[0], XX_2, 0); Chris@69: Chris@69: vst1q_f32(sum, SUMM); Chris@69: } Chris@69: Chris@69: void celt_pitch_xcorr_float_neon(const opus_val16 *_x, const opus_val16 *_y, Chris@69: opus_val32 *xcorr, int len, int max_pitch, int arch) { Chris@69: int i; Chris@69: (void)arch; Chris@69: celt_assert(max_pitch > 0); Chris@69: celt_sig_assert((((unsigned char *)_x-(unsigned char *)NULL)&3)==0); Chris@69: Chris@69: for (i = 0; i < (max_pitch-3); i += 4) { Chris@69: xcorr_kernel_neon_float((const float32_t *)_x, (const float32_t *)_y+i, Chris@69: (float32_t *)xcorr+i, len); Chris@69: } Chris@69: Chris@69: /* In case max_pitch isn't a multiple of 4, do non-unrolled version. */ Chris@69: for (; i < max_pitch; i++) { Chris@69: xcorr[i] = celt_inner_prod_neon(_x, _y+i, len); Chris@69: } Chris@69: } Chris@69: #endif