1 /*
3 BLIS
4 An object-based framework for developing high-performance BLAS-like
5 libraries.
7 Copyright (C) 2014, The University of Texas at Austin
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are
11 met:
12 - Redistributions of source code must retain the above copyright
13 notice, this list of conditions and the following disclaimer.
14 - Redistributions in binary form must reproduce the above copyright
15 notice, this list of conditions and the following disclaimer in the
16 documentation and/or other materials provided with the distribution.
17 - Neither the name of The University of Texas at Austin nor the names
18 of its contributors may be used to endorse or promote products
19 derived from this software without specific prior written permission.
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
35 #include "blis.h"
37 #define FUNCPTR_T hemv_fp
39 typedef void (*FUNCPTR_T)(
40 uplo_t uplo,
41 conj_t conja,
42 conj_t conjx,
43 conj_t conjh,
44 dim_t m,
45 void* alpha,
46 void* a, inc_t rs_a, inc_t cs_a,
47 void* x, inc_t incx,
48 void* beta,
49 void* y, inc_t incy
50 );
52 // If some mixed datatype functions will not be compiled, we initialize
53 // the corresponding elements of the function array to NULL.
54 #ifdef BLIS_ENABLE_MIXED_PRECISION_SUPPORT
55 static FUNCPTR_T GENARRAY3_ALL(ftypes,hemv_unf_var1);
56 #else
57 #ifdef BLIS_ENABLE_MIXED_DOMAIN_SUPPORT
58 static FUNCPTR_T GENARRAY3_EXT(ftypes,hemv_unf_var1);
59 #else
60 static FUNCPTR_T GENARRAY3_MIN(ftypes,hemv_unf_var1);
61 #endif
62 #endif
65 void bli_hemv_unf_var1( conj_t conjh,
66 obj_t* alpha,
67 obj_t* a,
68 obj_t* x,
69 obj_t* beta,
70 obj_t* y,
71 hemv_t* cntl )
72 {
73 num_t dt_a = bli_obj_datatype( *a );
74 num_t dt_x = bli_obj_datatype( *x );
75 num_t dt_y = bli_obj_datatype( *y );
77 uplo_t uplo = bli_obj_uplo( *a );
78 conj_t conja = bli_obj_conj_status( *a );
79 conj_t conjx = bli_obj_conj_status( *x );
81 dim_t m = bli_obj_length( *a );
83 void* buf_a = bli_obj_buffer_at_off( *a );
84 inc_t rs_a = bli_obj_row_stride( *a );
85 inc_t cs_a = bli_obj_col_stride( *a );
87 void* buf_x = bli_obj_buffer_at_off( *x );
88 inc_t incx = bli_obj_vector_inc( *x );
90 void* buf_y = bli_obj_buffer_at_off( *y );
91 inc_t incy = bli_obj_vector_inc( *y );
93 num_t dt_alpha;
94 void* buf_alpha;
96 num_t dt_beta;
97 void* buf_beta;
99 FUNCPTR_T f;
101 // The datatype of alpha MUST be the type union of a and x. This is to
102 // prevent any unnecessary loss of information during computation.
103 dt_alpha = bli_datatype_union( dt_a, dt_x );
104 buf_alpha = bli_obj_buffer_for_1x1( dt_alpha, *alpha );
106 // The datatype of beta MUST be the same as the datatype of y.
107 dt_beta = dt_y;
108 buf_beta = bli_obj_buffer_for_1x1( dt_beta, *beta );
110 // Index into the type combination array to extract the correct
111 // function pointer.
112 f = ftypes[dt_a][dt_x][dt_y];
114 // Invoke the function.
115 f( uplo,
116 conja,
117 conjx,
118 conjh,
119 m,
120 buf_alpha,
121 buf_a, rs_a, cs_a,
122 buf_x, incx,
123 buf_beta,
124 buf_y, incy );
125 }
128 #undef GENTFUNC3U12
129 #define GENTFUNC3U12( ctype_a, ctype_x, ctype_y, ctype_ax, cha, chx, chy, chax, varname, kername ) \
130 \
131 void PASTEMAC3(cha,chx,chy,varname)( \
132 uplo_t uplo, \
133 conj_t conja, \
134 conj_t conjx, \
135 conj_t conjh, \
136 dim_t m, \
137 void* alpha, \
138 void* a, inc_t rs_a, inc_t cs_a, \
139 void* x, inc_t incx, \
140 void* beta, \
141 void* y, inc_t incy \
142 ) \
143 { \
144 ctype_ax* alpha_cast = alpha; \
145 ctype_y* beta_cast = beta; \
146 ctype_a* a_cast = a; \
147 ctype_x* x_cast = x; \
148 ctype_y* y_cast = y; \
149 ctype_y* one = PASTEMAC(chy,1); \
150 ctype_y* zero = PASTEMAC(chy,0); \
151 ctype_a* A10; \
152 ctype_a* A11; \
153 ctype_a* a10t; \
154 ctype_a* alpha11; \
155 ctype_a* a21; \
156 ctype_x* x0; \
157 ctype_x* x1; \
158 ctype_x* chi11; \
159 ctype_y* y0; \
160 ctype_y* y1; \
161 ctype_y* y01; \
162 ctype_y* psi11; \
163 ctype_y* y21; \
164 ctype_x conjx_chi11; \
165 ctype_ax alpha_chi11; \
166 ctype_a alpha11_temp; \
167 dim_t i, k, j; \
168 dim_t b_fuse, f; \
169 dim_t n_behind; \
170 dim_t f_ahead, f_behind; \
171 inc_t rs_at, cs_at; \
172 conj_t conj0, conj1; \
173 \
174 if ( bli_zero_dim1( m ) ) return; \
175 \
176 /* The algorithm will be expressed in terms of the lower triangular case;
177 the upper triangular case is supported by swapping the row and column
178 strides of A and toggling some conj parameters. */ \
179 if ( bli_is_lower( uplo ) ) \
180 { \
181 rs_at = rs_a; \
182 cs_at = cs_a; \
183 \
184 conj0 = conja; \
185 conj1 = bli_apply_conj( conjh, conja ); \
186 } \
187 else /* if ( bli_is_upper( uplo ) ) */ \
188 { \
189 rs_at = cs_a; \
190 cs_at = rs_a; \
191 \
192 conj0 = bli_apply_conj( conjh, conja ); \
193 conj1 = conja; \
194 } \
195 \
196 /* If beta is zero, use setv. Otherwise, scale by beta. */ \
197 if ( PASTEMAC(chy,eq0)( *beta_cast ) ) \
198 { \
199 /* y = 0; */ \
200 PASTEMAC2(chy,chy,setv)( m, \
201 zero, \
202 y_cast, incy ); \
203 } \
204 else \
205 { \
206 /* y = beta * y; */ \
207 PASTEMAC2(chy,chy,scalv)( BLIS_NO_CONJUGATE, \
208 m, \
209 beta_cast, \
210 y_cast, incy ); \
211 } \
212 \
213 /* Query the fusing factor for the dotxaxpyf implementation. */ \
214 b_fuse = PASTEMAC(chax,dotxaxpyf_fusefac); \
215 \
216 for ( i = 0; i < m; i += f ) \
217 { \
218 f = bli_determine_blocksize_dim_f( i, m, b_fuse ); \
219 n_behind = i; \
220 A10 = a_cast + (i )*rs_at + (0 )*cs_at; \
221 A11 = a_cast + (i )*rs_at + (i )*cs_at; \
222 x0 = x_cast + (0 )*incx; \
223 x1 = x_cast + (i )*incx; \
224 y0 = y_cast + (0 )*incy; \
225 y1 = y_cast + (i )*incy; \
226 \
227 /* y1 = y1 + alpha * A10 * x0; (dotxf) */ \
228 /* y0 = y0 + alpha * A10' * x1; (axpyf) */ \
229 PASTEMAC3(cha,chx,chy,kername)( conj0, \
230 conj1, \
231 conjx, \
232 conjx, \
233 n_behind, \
234 f, \
235 alpha_cast, \
236 A10, cs_at, rs_at, \
237 x0, incx, \
238 x1, incx, \
239 one, \
240 y1, incy, \
241 y0, incy ); \
242 \
243 /* y1 = y1 + alpha * A11 * x1; (variant 4) */ \
244 for ( k = 0; k < f; ++k ) \
245 { \
246 f_behind = k; \
247 f_ahead = f - k - 1; \
248 a10t = A11 + (k )*rs_at + (0 )*cs_at; \
249 alpha11 = A11 + (k )*rs_at + (k )*cs_at; \
250 a21 = A11 + (k+1)*rs_at + (k )*cs_at; \
251 chi11 = x1 + (k )*incx; \
252 y01 = y1 + (0 )*incy; \
253 psi11 = y1 + (k )*incy; \
254 y21 = y1 + (k+1)*incy; \
255 \
256 /* y01 = y01 + alpha * a10t' * chi11; */ \
257 PASTEMAC2(chx,chx,copycjs)( conjx, *chi11, conjx_chi11 ); \
258 PASTEMAC3(chax,chx,chax,scal2s)( *alpha_cast, conjx_chi11, alpha_chi11 ); \
259 if ( bli_is_conj( conj1 ) ) \
260 { \
261 for ( j = 0; j < f_behind; ++j ) \
262 PASTEMAC3(chax,cha,chy,axpyjs)( alpha_chi11, *(a10t + j*cs_at), *(y01 + j*incy) ); \
263 } \
264 else \
265 { \
266 for ( j = 0; j < f_behind; ++j ) \
267 PASTEMAC3(chax,cha,chy,axpys)( alpha_chi11, *(a10t + j*cs_at), *(y01 + j*incy) ); \
268 } \
269 \
270 /* For hemv, explicitly set the imaginary component of alpha11 to
271 zero. */ \
272 PASTEMAC2(cha,cha,copycjs)( conja, *alpha11, alpha11_temp ); \
273 if ( bli_is_conj( conjh ) ) \
274 PASTEMAC(cha,seti0s)( alpha11_temp ); \
275 \
276 /* psi11 = psi11 + alpha * alpha11 * chi11; */ \
277 PASTEMAC3(chax,cha,chy,axpys)( alpha_chi11, alpha11_temp, *psi11 ); \
278 \
279 /* y21 = y21 + alpha * a21 * chi11; */ \
280 if ( bli_is_conj( conj0 ) ) \
281 { \
282 for ( j = 0; j < f_ahead; ++j ) \
283 PASTEMAC3(chax,cha,chy,axpyjs)( alpha_chi11, *(a21 + j*rs_at), *(y21 + j*incy) ); \
284 } \
285 else \
286 { \
287 for ( j = 0; j < f_ahead; ++j ) \
288 PASTEMAC3(chax,cha,chy,axpys)( alpha_chi11, *(a21 + j*rs_at), *(y21 + j*incy) ); \
289 } \
290 } \
291 } \
292 }
294 // Define the basic set of functions unconditionally, and then also some
295 // mixed datatype functions if requested.
296 INSERT_GENTFUNC3U12_BASIC( hemv_unf_var1, DOTXAXPYF_KERNEL )
298 #ifdef BLIS_ENABLE_MIXED_DOMAIN_SUPPORT
299 INSERT_GENTFUNC3U12_MIX_D( hemv_unf_var1, DOTXAXPYF_KERNEL )
300 #endif
302 #ifdef BLIS_ENABLE_MIXED_PRECISION_SUPPORT
303 INSERT_GENTFUNC3U12_MIX_P( hemv_unf_var1, DOTXAXPYF_KERNEL )
304 #endif