cannam@154: /* Copyright (c) 2014, Cisco Systems, INC
cannam@154:    Written by XiangMingZhu WeiZhou MinPeng YanWang
cannam@154: 
cannam@154:    Redistribution and use in source and binary forms, with or without
cannam@154:    modification, are permitted provided that the following conditions
cannam@154:    are met:
cannam@154: 
cannam@154:    - Redistributions of source code must retain the above copyright
cannam@154:    notice, this list of conditions and the following disclaimer.
cannam@154: 
cannam@154:    - Redistributions in binary form must reproduce the above copyright
cannam@154:    notice, this list of conditions and the following disclaimer in the
cannam@154:    documentation and/or other materials provided with the distribution.
cannam@154: 
cannam@154:    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
cannam@154:    ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
cannam@154:    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
cannam@154:    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
cannam@154:    OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
cannam@154:    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
cannam@154:    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
cannam@154:    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
cannam@154:    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
cannam@154:    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
cannam@154:    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cannam@154: */
cannam@154: 
cannam@154: #ifdef HAVE_CONFIG_H
cannam@154: #include "config.h"
cannam@154: #endif
cannam@154: 
cannam@154: #include "macros.h"
cannam@154: #include "celt_lpc.h"
cannam@154: #include "stack_alloc.h"
cannam@154: #include "mathops.h"
cannam@154: #include "pitch.h"
cannam@154: 
cannam@154: #if defined(OPUS_X86_MAY_HAVE_SSE) && !defined(FIXED_POINT)
cannam@154: 
cannam@154: #include <xmmintrin.h>
cannam@154: #include "arch.h"
cannam@154: 
cannam@154: void xcorr_kernel_sse(const opus_val16 *x, const opus_val16 *y, opus_val32 sum[4], int len)
cannam@154: {
cannam@154:    int j;
cannam@154:    __m128 xsum1, xsum2;
cannam@154:    xsum1 = _mm_loadu_ps(sum);
cannam@154:    xsum2 = _mm_setzero_ps();
cannam@154: 
cannam@154:    for (j = 0; j < len-3; j += 4)
cannam@154:    {
cannam@154:       __m128 x0 = _mm_loadu_ps(x+j);
cannam@154:       __m128 yj = _mm_loadu_ps(y+j);
cannam@154:       __m128 y3 = _mm_loadu_ps(y+j+3);
cannam@154: 
cannam@154:       xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x00),yj));
cannam@154:       xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x55),
cannam@154:                                           _mm_shuffle_ps(yj,y3,0x49)));
cannam@154:       xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xaa),
cannam@154:                                           _mm_shuffle_ps(yj,y3,0x9e)));
cannam@154:       xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xff),y3));
cannam@154:    }
cannam@154:    if (j < len)
cannam@154:    {
cannam@154:       xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j)));
cannam@154:       if (++j < len)
cannam@154:       {
cannam@154:          xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j)));
cannam@154:          if (++j < len)
cannam@154:          {
cannam@154:             xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j)));
cannam@154:          }
cannam@154:       }
cannam@154:    }
cannam@154:    _mm_storeu_ps(sum,_mm_add_ps(xsum1,xsum2));
cannam@154: }
cannam@154: 
cannam@154: 
cannam@154: void dual_inner_prod_sse(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02,
cannam@154:       int N, opus_val32 *xy1, opus_val32 *xy2)
cannam@154: {
cannam@154:    int i;
cannam@154:    __m128 xsum1, xsum2;
cannam@154:    xsum1 = _mm_setzero_ps();
cannam@154:    xsum2 = _mm_setzero_ps();
cannam@154:    for (i=0;i<N-3;i+=4)
cannam@154:    {
cannam@154:       __m128 xi = _mm_loadu_ps(x+i);
cannam@154:       __m128 y1i = _mm_loadu_ps(y01+i);
cannam@154:       __m128 y2i = _mm_loadu_ps(y02+i);
cannam@154:       xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(xi, y1i));
cannam@154:       xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(xi, y2i));
cannam@154:    }
cannam@154:    /* Horizontal sum */
cannam@154:    xsum1 = _mm_add_ps(xsum1, _mm_movehl_ps(xsum1, xsum1));
cannam@154:    xsum1 = _mm_add_ss(xsum1, _mm_shuffle_ps(xsum1, xsum1, 0x55));
cannam@154:    _mm_store_ss(xy1, xsum1);
cannam@154:    xsum2 = _mm_add_ps(xsum2, _mm_movehl_ps(xsum2, xsum2));
cannam@154:    xsum2 = _mm_add_ss(xsum2, _mm_shuffle_ps(xsum2, xsum2, 0x55));
cannam@154:    _mm_store_ss(xy2, xsum2);
cannam@154:    for (;i<N;i++)
cannam@154:    {
cannam@154:       *xy1 = MAC16_16(*xy1, x[i], y01[i]);
cannam@154:       *xy2 = MAC16_16(*xy2, x[i], y02[i]);
cannam@154:    }
cannam@154: }
cannam@154: 
cannam@154: opus_val32 celt_inner_prod_sse(const opus_val16 *x, const opus_val16 *y,
cannam@154:       int N)
cannam@154: {
cannam@154:    int i;
cannam@154:    float xy;
cannam@154:    __m128 sum;
cannam@154:    sum = _mm_setzero_ps();
cannam@154:    /* FIXME: We should probably go 8-way and use 2 sums. */
cannam@154:    for (i=0;i<N-3;i+=4)
cannam@154:    {
cannam@154:       __m128 xi = _mm_loadu_ps(x+i);
cannam@154:       __m128 yi = _mm_loadu_ps(y+i);
cannam@154:       sum = _mm_add_ps(sum,_mm_mul_ps(xi, yi));
cannam@154:    }
cannam@154:    /* Horizontal sum */
cannam@154:    sum = _mm_add_ps(sum, _mm_movehl_ps(sum, sum));
cannam@154:    sum = _mm_add_ss(sum, _mm_shuffle_ps(sum, sum, 0x55));
cannam@154:    _mm_store_ss(&xy, sum);
cannam@154:    for (;i<N;i++)
cannam@154:    {
cannam@154:       xy = MAC16_16(xy, x[i], y[i]);
cannam@154:    }
cannam@154:    return xy;
cannam@154: }
cannam@154: 
cannam@154: void comb_filter_const_sse(opus_val32 *y, opus_val32 *x, int T, int N,
cannam@154:       opus_val16 g10, opus_val16 g11, opus_val16 g12)
cannam@154: {
cannam@154:    int i;
cannam@154:    __m128 x0v;
cannam@154:    __m128 g10v, g11v, g12v;
cannam@154:    g10v = _mm_load1_ps(&g10);
cannam@154:    g11v = _mm_load1_ps(&g11);
cannam@154:    g12v = _mm_load1_ps(&g12);
cannam@154:    x0v = _mm_loadu_ps(&x[-T-2]);
cannam@154:    for (i=0;i<N-3;i+=4)
cannam@154:    {
cannam@154:       __m128 yi, yi2, x1v, x2v, x3v, x4v;
cannam@154:       const opus_val32 *xp = &x[i-T-2];
cannam@154:       yi = _mm_loadu_ps(x+i);
cannam@154:       x4v = _mm_loadu_ps(xp+4);
cannam@154: #if 0
cannam@154:       /* Slower version with all loads */
cannam@154:       x1v = _mm_loadu_ps(xp+1);
cannam@154:       x2v = _mm_loadu_ps(xp+2);
cannam@154:       x3v = _mm_loadu_ps(xp+3);
cannam@154: #else
cannam@154:       x2v = _mm_shuffle_ps(x0v, x4v, 0x4e);
cannam@154:       x1v = _mm_shuffle_ps(x0v, x2v, 0x99);
cannam@154:       x3v = _mm_shuffle_ps(x2v, x4v, 0x99);
cannam@154: #endif
cannam@154: 
cannam@154:       yi = _mm_add_ps(yi, _mm_mul_ps(g10v,x2v));
cannam@154: #if 0 /* Set to 1 to make it bit-exact with the non-SSE version */
cannam@154:       yi = _mm_add_ps(yi, _mm_mul_ps(g11v,_mm_add_ps(x3v,x1v)));
cannam@154:       yi = _mm_add_ps(yi, _mm_mul_ps(g12v,_mm_add_ps(x4v,x0v)));
cannam@154: #else
cannam@154:       /* Use partial sums */
cannam@154:       yi2 = _mm_add_ps(_mm_mul_ps(g11v,_mm_add_ps(x3v,x1v)),
cannam@154:                        _mm_mul_ps(g12v,_mm_add_ps(x4v,x0v)));
cannam@154:       yi = _mm_add_ps(yi, yi2);
cannam@154: #endif
cannam@154:       x0v=x4v;
cannam@154:       _mm_storeu_ps(y+i, yi);
cannam@154:    }
cannam@154: #ifdef CUSTOM_MODES
cannam@154:    for (;i<N;i++)
cannam@154:    {
cannam@154:       y[i] = x[i]
cannam@154:                + MULT16_32_Q15(g10,x[i-T])
cannam@154:                + MULT16_32_Q15(g11,ADD32(x[i-T+1],x[i-T-1]))
cannam@154:                + MULT16_32_Q15(g12,ADD32(x[i-T+2],x[i-T-2]));
cannam@154:    }
cannam@154: #endif
cannam@154: }
cannam@154: 
cannam@154: 
cannam@154: #endif