Chris@49: // Copyright (C) 2008-2013 NICTA (www.nicta.com.au)
Chris@49: // Copyright (C) 2008-2013 Conrad Sanderson
Chris@49: // 
Chris@49: // This Source Code Form is subject to the terms of the Mozilla Public
Chris@49: // License, v. 2.0. If a copy of the MPL was not distributed with this
Chris@49: // file, You can obtain one at http://mozilla.org/MPL/2.0/.
Chris@49: 
Chris@49: 
Chris@49: //! \addtogroup gemv
Chris@49: //! @{
Chris@49: 
Chris@49: 
Chris@49: 
Chris@49: //! for tiny square matrices, size <= 4x4
Chris@49: template<const bool do_trans_A=false, const bool use_alpha=false, const bool use_beta=false>
Chris@49: class gemv_emul_tinysq
Chris@49:   {
Chris@49:   public:
Chris@49:   
Chris@49:   
Chris@49:   template<const uword row, const uword col>
Chris@49:   struct pos
Chris@49:     {
Chris@49:     static const uword n2 = (do_trans_A == false) ? (row + col*2) : (col + row*2);
Chris@49:     static const uword n3 = (do_trans_A == false) ? (row + col*3) : (col + row*3);
Chris@49:     static const uword n4 = (do_trans_A == false) ? (row + col*4) : (col + row*4);
Chris@49:     };
Chris@49:   
Chris@49:   
Chris@49:   
Chris@49:   template<typename eT, const uword i>
Chris@49:   arma_hot
Chris@49:   arma_inline
Chris@49:   static
Chris@49:   void
Chris@49:   assign(eT* y, const eT acc, const eT alpha, const eT beta)
Chris@49:     {
Chris@49:     if(use_beta == false)
Chris@49:       {
Chris@49:       y[i] = (use_alpha == false) ? acc : alpha*acc;
Chris@49:       }
Chris@49:     else
Chris@49:       {
Chris@49:       const eT tmp = y[i];
Chris@49:       
Chris@49:       y[i] = beta*tmp + ( (use_alpha == false) ? acc : alpha*acc );
Chris@49:       }
Chris@49:     }
Chris@49:   
Chris@49:   
Chris@49: 
Chris@49:   template<typename eT, typename TA>
Chris@49:   arma_hot
Chris@49:   inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) )
Chris@49:     {
Chris@49:     arma_extra_debug_sigprint();
Chris@49:     
Chris@49:     const eT*  Am = A.memptr();
Chris@49:     
Chris@49:     switch(A.n_rows)
Chris@49:       {
Chris@49:       case 1:
Chris@49:         {
Chris@49:         const eT acc = Am[0] * x[0];
Chris@49:         
Chris@49:         assign<eT, 0>(y, acc, alpha, beta);
Chris@49:         }
Chris@49:         break;
Chris@49:       
Chris@49:       
Chris@49:       case 2:
Chris@49:         {
Chris@49:         const eT x0 = x[0];
Chris@49:         const eT x1 = x[1];
Chris@49:         
Chris@49:         const eT acc0 = Am[pos<0,0>::n2]*x0 + Am[pos<0,1>::n2]*x1;
Chris@49:         const eT acc1 = Am[pos<1,0>::n2]*x0 + Am[pos<1,1>::n2]*x1;
Chris@49:         
Chris@49:         assign<eT, 0>(y, acc0, alpha, beta);
Chris@49:         assign<eT, 1>(y, acc1, alpha, beta);
Chris@49:         }
Chris@49:         break;
Chris@49:       
Chris@49:         
Chris@49:       case 3:
Chris@49:         {
Chris@49:         const eT x0 = x[0];
Chris@49:         const eT x1 = x[1];
Chris@49:         const eT x2 = x[2];
Chris@49:         
Chris@49:         const eT acc0 = Am[pos<0,0>::n3]*x0 + Am[pos<0,1>::n3]*x1 + Am[pos<0,2>::n3]*x2;
Chris@49:         const eT acc1 = Am[pos<1,0>::n3]*x0 + Am[pos<1,1>::n3]*x1 + Am[pos<1,2>::n3]*x2;
Chris@49:         const eT acc2 = Am[pos<2,0>::n3]*x0 + Am[pos<2,1>::n3]*x1 + Am[pos<2,2>::n3]*x2;
Chris@49:         
Chris@49:         assign<eT, 0>(y, acc0, alpha, beta);
Chris@49:         assign<eT, 1>(y, acc1, alpha, beta);
Chris@49:         assign<eT, 2>(y, acc2, alpha, beta);
Chris@49:         }
Chris@49:         break;
Chris@49:       
Chris@49:       
Chris@49:       case 4:
Chris@49:         {
Chris@49:         const eT x0 = x[0];
Chris@49:         const eT x1 = x[1];
Chris@49:         const eT x2 = x[2];
Chris@49:         const eT x3 = x[3];
Chris@49:         
Chris@49:         const eT acc0 = Am[pos<0,0>::n4]*x0 + Am[pos<0,1>::n4]*x1 + Am[pos<0,2>::n4]*x2 + Am[pos<0,3>::n4]*x3;
Chris@49:         const eT acc1 = Am[pos<1,0>::n4]*x0 + Am[pos<1,1>::n4]*x1 + Am[pos<1,2>::n4]*x2 + Am[pos<1,3>::n4]*x3;
Chris@49:         const eT acc2 = Am[pos<2,0>::n4]*x0 + Am[pos<2,1>::n4]*x1 + Am[pos<2,2>::n4]*x2 + Am[pos<2,3>::n4]*x3;
Chris@49:         const eT acc3 = Am[pos<3,0>::n4]*x0 + Am[pos<3,1>::n4]*x1 + Am[pos<3,2>::n4]*x2 + Am[pos<3,3>::n4]*x3;
Chris@49:         
Chris@49:         assign<eT, 0>(y, acc0, alpha, beta);
Chris@49:         assign<eT, 1>(y, acc1, alpha, beta);
Chris@49:         assign<eT, 2>(y, acc2, alpha, beta);
Chris@49:         assign<eT, 3>(y, acc3, alpha, beta);
Chris@49:         }
Chris@49:         break;
Chris@49:       
Chris@49:       
Chris@49:       default:
Chris@49:         ;
Chris@49:       }
Chris@49:     }
Chris@49:     
Chris@49:   };
Chris@49: 
Chris@49: 
Chris@49: 
Chris@49: class gemv_emul_large_helper
Chris@49:   {
Chris@49:   public:
Chris@49:   
Chris@49:   template<typename eT, typename TA>
Chris@49:   arma_hot
Chris@49:   inline
Chris@49:   static
Chris@49:   typename arma_not_cx<eT>::result
Chris@49:   dot_row_col( const TA& A, const eT* x, const uword row, const uword N )
Chris@49:     {
Chris@49:     eT acc1 = eT(0);
Chris@49:     eT acc2 = eT(0);
Chris@49:     
Chris@49:     uword i,j;
Chris@49:     for(i=0, j=1; j < N; i+=2, j+=2)
Chris@49:       {
Chris@49:       const eT xi = x[i];
Chris@49:       const eT xj = x[j];
Chris@49:       
Chris@49:       acc1 += A.at(row,i) * xi;
Chris@49:       acc2 += A.at(row,j) * xj;
Chris@49:       }
Chris@49:     
Chris@49:     if(i < N)
Chris@49:       {
Chris@49:       acc1 += A.at(row,i) * x[i];
Chris@49:       }
Chris@49:     
Chris@49:     return (acc1 + acc2);
Chris@49:     }
Chris@49:   
Chris@49:   
Chris@49:   
Chris@49:   template<typename eT, typename TA>
Chris@49:   arma_hot
Chris@49:   inline
Chris@49:   static
Chris@49:   typename arma_cx_only<eT>::result
Chris@49:   dot_row_col( const TA& A, const eT* x, const uword row, const uword N )
Chris@49:     {
Chris@49:     typedef typename get_pod_type<eT>::result T;
Chris@49:     
Chris@49:     T val_real = T(0);
Chris@49:     T val_imag = T(0);
Chris@49:     
Chris@49:     for(uword i=0; i<N; ++i)
Chris@49:       {
Chris@49:       const std::complex<T>& Ai = A.at(row,i);
Chris@49:       const std::complex<T>& xi = x[i];
Chris@49:       
Chris@49:       const T a = Ai.real();
Chris@49:       const T b = Ai.imag();
Chris@49:       
Chris@49:       const T c = xi.real();
Chris@49:       const T d = xi.imag();
Chris@49:       
Chris@49:       val_real += (a*c) - (b*d);
Chris@49:       val_imag += (a*d) + (b*c);
Chris@49:       }
Chris@49:     
Chris@49:     return std::complex<T>(val_real, val_imag);
Chris@49:     }
Chris@49:   
Chris@49:   };
Chris@49: 
Chris@49: 
Chris@49: 
Chris@49: //! \brief
Chris@49: //! Partial emulation of ATLAS/BLAS gemv().
Chris@49: //! 'y' is assumed to have been set to the correct size (i.e. taking into account the transpose)
Chris@49: 
Chris@49: template<const bool do_trans_A=false, const bool use_alpha=false, const bool use_beta=false>
Chris@49: class gemv_emul_large
Chris@49:   {
Chris@49:   public:
Chris@49:   
Chris@49:   template<typename eT, typename TA>
Chris@49:   arma_hot
Chris@49:   inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) )
Chris@49:     {
Chris@49:     arma_extra_debug_sigprint();
Chris@49:     
Chris@49:     const uword A_n_rows = A.n_rows;
Chris@49:     const uword A_n_cols = A.n_cols;
Chris@49:     
Chris@49:     if(do_trans_A == false)
Chris@49:       {
Chris@49:       if(A_n_rows == 1)
Chris@49:         {
Chris@49:         const eT acc = op_dot::direct_dot_arma(A_n_cols, A.memptr(), x);
Chris@49:         
Chris@49:         if( (use_alpha == false) && (use_beta == false) )
Chris@49:           {
Chris@49:           y[0] = acc;
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == true) && (use_beta == false) )
Chris@49:           {
Chris@49:           y[0] = alpha * acc;
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == false) && (use_beta == true) )
Chris@49:           {
Chris@49:           y[0] = acc + beta*y[0];
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == true) && (use_beta == true) )
Chris@49:           {
Chris@49:           y[0] = alpha*acc + beta*y[0];
Chris@49:           }
Chris@49:         }
Chris@49:       else
Chris@49:       for(uword row=0; row < A_n_rows; ++row)
Chris@49:         {
Chris@49:         const eT acc = gemv_emul_large_helper::dot_row_col(A, x, row, A_n_cols);
Chris@49:         
Chris@49:         if( (use_alpha == false) && (use_beta == false) )
Chris@49:           {
Chris@49:           y[row] = acc;
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == true) && (use_beta == false) )
Chris@49:           {
Chris@49:           y[row] = alpha * acc;
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == false) && (use_beta == true) )
Chris@49:           {
Chris@49:           y[row] = acc + beta*y[row];
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == true) && (use_beta == true) )
Chris@49:           {
Chris@49:           y[row] = alpha*acc + beta*y[row];
Chris@49:           }
Chris@49:         }
Chris@49:       }
Chris@49:     else
Chris@49:     if(do_trans_A == true)
Chris@49:       {
Chris@49:       for(uword col=0; col < A_n_cols; ++col)
Chris@49:         {
Chris@49:         // col is interpreted as row when storing the results in 'y'
Chris@49:         
Chris@49:         
Chris@49:         // const eT* A_coldata = A.colptr(col);
Chris@49:         // 
Chris@49:         // eT acc = eT(0);
Chris@49:         // for(uword row=0; row < A_n_rows; ++row)
Chris@49:         //   {
Chris@49:         //   acc += A_coldata[row] * x[row];
Chris@49:         //   }
Chris@49:         
Chris@49:         const eT acc = op_dot::direct_dot_arma(A_n_rows, A.colptr(col), x);
Chris@49:         
Chris@49:         if( (use_alpha == false) && (use_beta == false) )
Chris@49:           {
Chris@49:           y[col] = acc;
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == true) && (use_beta == false) )
Chris@49:           {
Chris@49:           y[col] = alpha * acc;
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == false) && (use_beta == true) )
Chris@49:           {
Chris@49:           y[col] = acc + beta*y[col];
Chris@49:           }
Chris@49:         else
Chris@49:         if( (use_alpha == true) && (use_beta == true) )
Chris@49:           {
Chris@49:           y[col] = alpha*acc + beta*y[col];
Chris@49:           }
Chris@49:         
Chris@49:         }
Chris@49:       }
Chris@49:     }
Chris@49:   
Chris@49:   };
Chris@49: 
Chris@49: 
Chris@49: 
Chris@49: template<const bool do_trans_A=false, const bool use_alpha=false, const bool use_beta=false>
Chris@49: class gemv_emul
Chris@49:   {
Chris@49:   public:
Chris@49:   
Chris@49:   template<typename eT, typename TA>
Chris@49:   arma_hot
Chris@49:   inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0), const typename arma_not_cx<eT>::result* junk = 0 )
Chris@49:     {
Chris@49:     arma_extra_debug_sigprint();
Chris@49:     arma_ignore(junk);
Chris@49:     
Chris@49:     const uword A_n_rows = A.n_rows;
Chris@49:     const uword A_n_cols = A.n_cols;
Chris@49:     
Chris@49:     if( (A_n_rows <= 4) && (A_n_rows == A_n_cols) )
Chris@49:       {
Chris@49:       gemv_emul_tinysq<do_trans_A, use_alpha, use_beta>::apply(y, A, x, alpha, beta);
Chris@49:       }
Chris@49:     else
Chris@49:       {
Chris@49:       gemv_emul_large<do_trans_A, use_alpha, use_beta>::apply(y, A, x, alpha, beta);
Chris@49:       }
Chris@49:     }
Chris@49:   
Chris@49:   
Chris@49:   
Chris@49:   template<typename eT>
Chris@49:   arma_hot
Chris@49:   inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply( eT* y, const Mat<eT>& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0), const typename arma_cx_only<eT>::result* junk = 0 )
Chris@49:     {
Chris@49:     arma_extra_debug_sigprint();
Chris@49:     arma_ignore(junk);
Chris@49:     
Chris@49:     Mat<eT> tmp_A;
Chris@49:     
Chris@49:     if(do_trans_A)
Chris@49:       {
Chris@49:       op_htrans::apply_noalias(tmp_A, A);
Chris@49:       }
Chris@49:     
Chris@49:     const Mat<eT>& AA = (do_trans_A == false) ? A : tmp_A;
Chris@49:     
Chris@49:     const uword AA_n_rows = AA.n_rows;
Chris@49:     const uword AA_n_cols = AA.n_cols;
Chris@49:     
Chris@49:     if( (AA_n_rows <= 4) && (AA_n_rows == AA_n_cols) )
Chris@49:       {
Chris@49:       gemv_emul_tinysq<false, use_alpha, use_beta>::apply(y, AA, x, alpha, beta);
Chris@49:       }
Chris@49:     else
Chris@49:       {
Chris@49:       gemv_emul_large<false, use_alpha, use_beta>::apply(y, AA, x, alpha, beta);
Chris@49:       }
Chris@49:     }
Chris@49:   };
Chris@49: 
Chris@49: 
Chris@49: 
Chris@49: //! \brief
Chris@49: //! Wrapper for ATLAS/BLAS gemv function, using template arguments to control the arguments passed to gemv.
Chris@49: //! 'y' is assumed to have been set to the correct size (i.e. taking into account the transpose)
Chris@49: 
Chris@49: template<const bool do_trans_A=false, const bool use_alpha=false, const bool use_beta=false>
Chris@49: class gemv
Chris@49:   {
Chris@49:   public:
Chris@49:   
Chris@49:   template<typename eT, typename TA>
Chris@49:   inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply_blas_type( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) )
Chris@49:     {
Chris@49:     arma_extra_debug_sigprint();
Chris@49:     
Chris@49:     //const uword threshold = (is_complex<eT>::value == true) ? 16u : 64u;
Chris@49:     const uword threshold = (is_complex<eT>::value == true) ? 64u : 100u;
Chris@49:     
Chris@49:     if(A.n_elem <= threshold)
Chris@49:       {
Chris@49:       gemv_emul<do_trans_A, use_alpha, use_beta>::apply(y,A,x,alpha,beta);
Chris@49:       }
Chris@49:     else
Chris@49:       {
Chris@49:       #if defined(ARMA_USE_ATLAS)
Chris@49:         {
Chris@49:         if(is_complex<eT>::value == false)
Chris@49:           {
Chris@49:           // use gemm() instead of gemv() to work around a speed issue in Atlas 3.8.4
Chris@49:           
Chris@49:           arma_extra_debug_print("atlas::cblas_gemm()");
Chris@49:           
Chris@49:           atlas::cblas_gemm<eT>
Chris@49:             (
Chris@49:             atlas::CblasColMajor,
Chris@49:             (do_trans_A) ? ( is_complex<eT>::value ? CblasConjTrans : atlas::CblasTrans ) : atlas::CblasNoTrans,
Chris@49:             atlas::CblasNoTrans,
Chris@49:             (do_trans_A) ? A.n_cols : A.n_rows,
Chris@49:             1,
Chris@49:             (do_trans_A) ? A.n_rows : A.n_cols,
Chris@49:             (use_alpha) ? alpha : eT(1),
Chris@49:             A.mem,
Chris@49:             A.n_rows,
Chris@49:             x,
Chris@49:             (do_trans_A) ? A.n_rows : A.n_cols,
Chris@49:             (use_beta) ? beta : eT(0),
Chris@49:             y,
Chris@49:             (do_trans_A) ? A.n_cols : A.n_rows
Chris@49:             );
Chris@49:           }
Chris@49:         else
Chris@49:           {
Chris@49:           arma_extra_debug_print("atlas::cblas_gemv()");
Chris@49:           
Chris@49:           atlas::cblas_gemv<eT>
Chris@49:             (
Chris@49:             atlas::CblasColMajor,
Chris@49:             (do_trans_A) ? ( is_complex<eT>::value ? CblasConjTrans : atlas::CblasTrans ) : atlas::CblasNoTrans,
Chris@49:             A.n_rows,
Chris@49:             A.n_cols,
Chris@49:             (use_alpha) ? alpha : eT(1),
Chris@49:             A.mem,
Chris@49:             A.n_rows,
Chris@49:             x,
Chris@49:             1,
Chris@49:             (use_beta) ? beta : eT(0),
Chris@49:             y,
Chris@49:             1
Chris@49:             );
Chris@49:           }
Chris@49:         }
Chris@49:       #elif defined(ARMA_USE_BLAS)
Chris@49:         {
Chris@49:         arma_extra_debug_print("blas::gemv()");
Chris@49:         
Chris@49:         const char      trans_A     = (do_trans_A) ? ( is_complex<eT>::value ? 'C' : 'T' ) : 'N';
Chris@49:         const blas_int  m           = A.n_rows;
Chris@49:         const blas_int  n           = A.n_cols;
Chris@49:         const eT        local_alpha = (use_alpha) ? alpha : eT(1);
Chris@49:         //const blas_int  lda         = A.n_rows;
Chris@49:         const blas_int  inc         = 1;
Chris@49:         const eT        local_beta  = (use_beta) ? beta : eT(0);
Chris@49:         
Chris@49:         arma_extra_debug_print( arma_boost::format("blas::gemv(): trans_A = %c") % trans_A );
Chris@49:         
Chris@49:         blas::gemv<eT>
Chris@49:           (
Chris@49:           &trans_A,
Chris@49:           &m,
Chris@49:           &n,
Chris@49:           &local_alpha,
Chris@49:           A.mem,
Chris@49:           &m,  // lda
Chris@49:           x,
Chris@49:           &inc,
Chris@49:           &local_beta,
Chris@49:           y,
Chris@49:           &inc
Chris@49:           );
Chris@49:         }
Chris@49:       #else
Chris@49:         {
Chris@49:         gemv_emul<do_trans_A, use_alpha, use_beta>::apply(y,A,x,alpha,beta);
Chris@49:         }
Chris@49:       #endif
Chris@49:       }
Chris@49:     
Chris@49:     }
Chris@49:   
Chris@49:   
Chris@49:   
Chris@49:   template<typename eT, typename TA>
Chris@49:   arma_inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply( eT* y, const TA& A, const eT* x, const eT alpha = eT(1), const eT beta = eT(0) )
Chris@49:     {
Chris@49:     gemv_emul<do_trans_A, use_alpha, use_beta>::apply(y,A,x,alpha,beta);
Chris@49:     }
Chris@49:   
Chris@49:   
Chris@49:   
Chris@49:   template<typename TA>
Chris@49:   arma_inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply
Chris@49:     (
Chris@49:           float* y,
Chris@49:     const TA&    A,
Chris@49:     const float* x,
Chris@49:     const float  alpha = float(1),
Chris@49:     const float  beta  = float(0)
Chris@49:     )
Chris@49:     {
Chris@49:     gemv<do_trans_A, use_alpha, use_beta>::apply_blas_type(y,A,x,alpha,beta);
Chris@49:     }
Chris@49:   
Chris@49:   
Chris@49:   
Chris@49:   template<typename TA>
Chris@49:   arma_inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply
Chris@49:     (
Chris@49:           double* y,
Chris@49:     const TA&     A,
Chris@49:     const double* x,
Chris@49:     const double  alpha = double(1),
Chris@49:     const double  beta  = double(0)
Chris@49:     )
Chris@49:     {
Chris@49:     gemv<do_trans_A, use_alpha, use_beta>::apply_blas_type(y,A,x,alpha,beta);
Chris@49:     }
Chris@49:   
Chris@49:   
Chris@49:   
Chris@49:   template<typename TA>
Chris@49:   arma_inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply
Chris@49:     (
Chris@49:           std::complex<float>* y,
Chris@49:     const TA&                  A,
Chris@49:     const std::complex<float>* x,
Chris@49:     const std::complex<float>  alpha = std::complex<float>(1),
Chris@49:     const std::complex<float>  beta  = std::complex<float>(0)
Chris@49:     )
Chris@49:     {
Chris@49:     gemv<do_trans_A, use_alpha, use_beta>::apply_blas_type(y,A,x,alpha,beta);
Chris@49:     }
Chris@49: 
Chris@49: 
Chris@49:   
Chris@49:   template<typename TA>
Chris@49:   arma_inline
Chris@49:   static
Chris@49:   void
Chris@49:   apply
Chris@49:     (
Chris@49:           std::complex<double>* y,
Chris@49:     const TA&                   A,
Chris@49:     const std::complex<double>* x,
Chris@49:     const std::complex<double>  alpha = std::complex<double>(1),
Chris@49:     const std::complex<double>  beta  = std::complex<double>(0)
Chris@49:     )
Chris@49:     {
Chris@49:     gemv<do_trans_A, use_alpha, use_beta>::apply_blas_type(y,A,x,alpha,beta);
Chris@49:     }
Chris@49: 
Chris@49: 
Chris@49:   
Chris@49:   };
Chris@49: 
Chris@49: 
Chris@49: //! @}