blis/frame/3/trsm/3m/ukernels/bli_gemmtrsm3m_u_ukr_ref.c

   1 /*
   2
   3    BLIS
   4    An object-based framework for developing high-performance BLAS-like
   5    libraries.
   6
   7    Copyright (C) 2014, The University of Texas at Austin
   8
   9    Redistribution and use in source and binary forms, with or without
  10    modification, are permitted provided that the following conditions are
  11    met:
  12     - Redistributions of source code must retain the above copyright
  13       notice, this list of conditions and the following disclaimer.
  14     - Redistributions in binary form must reproduce the above copyright
  15       notice, this list of conditions and the following disclaimer in the
  16       documentation and/or other materials provided with the distribution.
  17     - Neither the name of The University of Texas at Austin nor the names
  18       of its contributors may be used to endorse or promote products
  19       derived from this software without specific prior written permission.
  20
  21    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29    THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32
  33 */
  34
  35 #include "blis.h"
  36
  37 #undef  GENTFUNCCO
  38 #define GENTFUNCCO( ctype, ctype_r, ch, chr, varname, gemmukr, trsmukr ) \
  39 \
  40 void PASTEMAC(ch,varname)( \
  41                            dim_t           k, \
  42                            ctype* restrict alpha, \
  43                            ctype* restrict a12, \
  44                            ctype* restrict a11, \
  45                            ctype* restrict b21, \
  46                            ctype* restrict b11, \
  47                            ctype* restrict c11, inc_t rs_c, inc_t cs_c, \
  48                            auxinfo_t*      data  \
  49                          ) \
  50 { \
  51         ctype_r           ab_r[ PASTEMAC(chr,mr) * \
  52                                 PASTEMAC(chr,nr) ] \
  53                            __attribute__((aligned(BLIS_STACK_BUF_ALIGN_SIZE))); \
  54         ctype_r           ab_i[ PASTEMAC(chr,mr) * \
  55                                 PASTEMAC(chr,nr) ] \
  56                            __attribute__((aligned(BLIS_STACK_BUF_ALIGN_SIZE))); \
  57         const inc_t       rs_ab      = 1; \
  58         const inc_t       cs_ab      = PASTEMAC(chr,mr); \
  59 \
  60 \
  61         const dim_t       m          = PASTEMAC(chr,mr); \
  62         const dim_t       n          = PASTEMAC(chr,nr); \
  63 \
  64         const inc_t       is_a       = bli_auxinfo_is_a( data ); \
  65         const inc_t       is_b       = bli_auxinfo_is_b( data ); \
  66 \
  67         ctype_r* restrict a11_r      = ( ctype_r* )a11; \
  68 \
  69         ctype_r* restrict a12_r      = ( ctype_r* )a12; \
  70         ctype_r* restrict a12_i      = ( ctype_r* )a12 +   is_a; \
  71         ctype_r* restrict a12_ri     = ( ctype_r* )a12 + 2*is_a; \
  72 \
  73         ctype_r* restrict b11_r      = ( ctype_r* )b11; \
  74         ctype_r* restrict b11_i      = ( ctype_r* )b11 +   is_b; \
  75         ctype_r* restrict b11_ri     = ( ctype_r* )b11 + 2*is_b; \
  76 \
  77         ctype_r* restrict b21_r      = ( ctype_r* )b21; \
  78         ctype_r* restrict b21_i      = ( ctype_r* )b21 +   is_b; \
  79         ctype_r* restrict b21_ri     = ( ctype_r* )b21 + 2*is_b; \
  80 \
  81         const inc_t       rs_b       = PASTEMAC(chr,packnr); \
  82         const inc_t       cs_b       = 1; \
  83 \
  84         ctype_r* restrict one_r       = PASTEMAC(chr,1); \
  85         ctype_r* restrict zero_r      = PASTEMAC(chr,0); \
  86         ctype_r* restrict minus_one_r = PASTEMAC(chr,m1); \
  87 \
  88         ctype_r           alpha_r     = PASTEMAC(ch,real)( *alpha ); \
  89         ctype_r           alpha_i     = PASTEMAC(ch,imag)( *alpha ); \
  90 \
  91         void*             a_next      = bli_auxinfo_next_a( data ); \
  92         void*             b_next      = bli_auxinfo_next_b( data ); \
  93 \
  94         dim_t             i, j; \
  95 \
  96 \
  97         /* Copy the contents of c to a temporary buffer ct. */ \
  98         if ( !PASTEMAC(chr,eq0)( alpha_i ) ) \
  99         { \
 100                 /* We can handle a non-zero imaginary component on alpha, but to do
 101                    so we have to manually scale b and then use alpha == 1 for the
 102                    micro-kernel calls. */ \
 103                 for ( i = 0; i < m; ++i ) \
 104                 for ( j = 0; j < n; ++j ) \
 105                 PASTEMAC(ch,scalris)( alpha_r, \
 106                                       alpha_i, \
 107                                       *(b11_r + i*rs_b + j*cs_b), \
 108                                       *(b11_i + i*rs_b + j*cs_b) ); \
 109 \
 110                 /* Use alpha.r == 1.0. */ \
 111                 alpha_r = *one_r; \
 112         } \
 113 \
 114 \
 115         /* b11.r = alpha.r * b11.r - (                 + a12.r * b21.r - a12.i * b21.i );
 116            b11.i = alpha.r * b11.i - ( a12.ri * b21.ri - a12.r * b21.r - a12.i * b21.i ); */ \
 117 \
 118         bli_auxinfo_set_next_ab( a12_i, b21_i, *data ); \
 119 \
 120         /* ab.r = a12.r * b21.r; */ \
 121         PASTEMAC(chr,gemmukr)( k, \
 122                                one_r, \
 123                                a12_r, \
 124                                b21_r, \
 125                                zero_r, \
 126                                ab_r, rs_ab, cs_ab, \
 127                                data ); \
 128 \
 129         bli_auxinfo_set_next_ab( a12_ri, b21_ri, *data ); \
 130 \
 131         /* ab.i = a12.i * b21.i; */ \
 132         PASTEMAC(chr,gemmukr)( k, \
 133                                one_r, \
 134                                a12_i, \
 135                                b21_i, \
 136                                zero_r, \
 137                                ab_i, rs_ab, cs_ab, \
 138                                data ); \
 139 \
 140         bli_auxinfo_set_next_ab( a_next, b_next, *data ); \
 141 \
 142         /* b11.i = alpha.r * b11.i - a12.ri * b21.ri; */ \
 143         PASTEMAC(chr,gemmukr)( k, \
 144                                minus_one_r, \
 145                                a12_ri, \
 146                                b21_ri, \
 147                                &alpha_r, \
 148                                b11_i, rs_b, cs_b, \
 149                                data ); \
 150 \
 151 \
 152         /* b11.r  = alpha.r * b11.r - ab.r;
 153            b11.r  =           b11.r + ab.i;
 154            b11.i  =           b11.i + ab.r;
 155            b11.i  =           b11.i + ab.i; */ \
 156         for ( j = 0; j < n; ++j ) \
 157         for ( i = 0; i < m; ++i ) \
 158         { \
 159                 ctype_r alphabeta_r = *(ab_r  + i*rs_ab + j*cs_ab); \
 160                 ctype_r alphabeta_i = *(ab_i  + i*rs_ab + j*cs_ab); \
 161                 ctype_r beta11_r    = *(b11_r + i*rs_b  + j*cs_b); \
 162                 ctype_r beta11_i    = *(b11_i + i*rs_b  + j*cs_b); \
 163 \
 164                 PASTEMAC(chr,scals)( alpha_r, beta11_r ); \
 165 \
 166                 PASTEMAC(chr,subs)( alphabeta_r, beta11_r ); \
 167                 PASTEMAC(chr,adds)( alphabeta_i, beta11_r ); \
 168                 PASTEMAC(chr,adds)( alphabeta_r, beta11_i ); \
 169                 PASTEMAC(chr,adds)( alphabeta_i, beta11_i ); \
 170 \
 171                 /* Store the local values back to b11. */ \
 172                 PASTEMAC(ch,copyris)( beta11_r, \
 173                                       beta11_i, \
 174                                       *(b11_r + i*rs_b + j*cs_b), \
 175                                       *(b11_i + i*rs_b + j*cs_b) ); \
 176 \
 177                 /* Update the ri part of b11. */ \
 178                 PASTEMAC(chr,add3s)( beta11_r, \
 179                                      beta11_i, \
 180                                      *(b11_ri + i*rs_b + j*cs_b) ); \
 181         } \
 182 \
 183 \
 184         /* b11 = inv(a11) * b11;
 185            c11 = b11; */ \
 186         PASTEMAC(ch,trsmukr)( a11_r, \
 187                               b11_r, \
 188                               c11, rs_c, cs_c, \
 189                               data ); \
 190 }
 191
 192 INSERT_GENTFUNCCO_BASIC2( gemmtrsm3m_u_ukr_ref, GEMM_UKERNEL, TRSM3M_U_UKERNEL )
 193