1 /*
3 BLIS
4 An object-based framework for developing high-performance BLAS-like
5 libraries.
7 Copyright (C) 2014, The University of Texas at Austin
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are
11 met:
12 - Redistributions of source code must retain the above copyright
13 notice, this list of conditions and the following disclaimer.
14 - Redistributions in binary form must reproduce the above copyright
15 notice, this list of conditions and the following disclaimer in the
16 documentation and/or other materials provided with the distribution.
17 - Neither the name of The University of Texas at Austin nor the names
18 of its contributors may be used to endorse or promote products
19 derived from this software without specific prior written permission.
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
35 #include "blis.h"
37 void bli_trmm_blk_var3f( obj_t* a,
38 obj_t* b,
39 obj_t* c,
40 gemm_t* cntl,
41 trmm_thrinfo_t* thread )
42 {
43 obj_t c_pack_s;
44 obj_t a1_pack_s, b1_pack_s;
46 obj_t a1, b1;
47 obj_t* a1_pack = NULL;
48 obj_t* b1_pack = NULL;
49 obj_t* c_pack = NULL;
51 dim_t i;
52 dim_t b_alg;
53 #ifdef BLIS_ENABLE_C66X_EDMA
54 dim_t b_alg_next;
55 #endif
56 dim_t k_trans;
58 #ifdef BLIS_ENABLE_C66X_EDMA
59 obj_t a1_dma_s, b1_dma_s;
60 obj_t *a1_dma = NULL;
61 obj_t *b1_dma = NULL;
62 #endif
64 if( thread_am_ochief( thread ) ){
65 // Initialize object for packing C
66 bli_obj_init_pack( &c_pack_s );
67 bli_packm_init( c, &c_pack_s,
68 cntl_sub_packm_c( cntl ) );
70 // Scale C by beta (if instructed).
71 bli_scalm_int( &BLIS_ONE,
72 c,
73 cntl_sub_scalm( cntl ) );
74 }
75 c_pack = thread_obroadcast( thread, &c_pack_s );
77 // Initialize pack objects for A and B that are passed into packm_init().
78 if( thread_am_ichief( thread ) ){
79 #ifdef BLIS_ENABLE_C66X_EDMA
80 bli_obj_init_dma( &a1_dma_s);
81 bli_obj_init_dma( &b1_dma_s );
82 #endif
83 bli_obj_init_pack( &a1_pack_s );
84 bli_obj_init_pack( &b1_pack_s );
85 }
86 #ifdef BLIS_ENABLE_C66X_EDMA
87 a1_dma = thread_ibroadcast( thread, &a1_dma_s);
88 b1_dma = thread_ibroadcast( thread, &b1_dma_s);
89 #endif
90 a1_pack = thread_ibroadcast( thread, &a1_pack_s );
91 b1_pack = thread_ibroadcast( thread, &b1_pack_s );
93 // Pack C (if instructed).
94 bli_packm_int( c, c_pack,
95 cntl_sub_packm_c( cntl ),
96 trmm_thread_sub_opackm( thread ) );
98 // Query dimension in partitioning direction.
99 k_trans = bli_obj_width_after_trans( *a );
101 // Partition along the k dimension.
102 for ( i = 0; i < k_trans; i += b_alg )
103 {
104 #ifdef BLIS_ENABLE_C66X_EDMA
105 if (i == 0)
106 {
107 // Determine the current algorithmic blocksize.
108 // NOTE: We call a trmm-specific function to determine the kc
109 // blocksize so that we can implement the "nudging" of kc to be
110 // a multiple of mr or nr, as needed.
111 b_alg = bli_trmm_determine_kc_f( i, k_trans, a, b,
112 cntl_blocksize( cntl ) );
114 // Acquire partitions for A1 and B1.
115 bli_acquire_mpart_l2r( BLIS_SUBPART1,
116 i, b_alg, a, &a1 );
117 bli_acquire_mpart_t2b( BLIS_SUBPART1,
118 i, b_alg, b, &b1 );
119 if( thread_am_ichief( thread ) )
120 {
121 // DMA control leaf unrolling for A
122 bli_dmam_init( &a1, a1_dma, cntl_sub_dmam_a( cntl ) );
123 bli_dmam_init( &b1, b1_dma, cntl_sub_dmam_b( cntl ) );
124 }
125 thread_ibarrier( thread );
126 bli_dmam_int( &a1, a1_dma, cntl_sub_dmam_a( cntl ), (dmam_thrinfo_t *) trmm_thread_sub_ipackm( thread ) );
127 bli_dmam_int( &b1, b1_dma, cntl_sub_dmam_b( cntl ), (dmam_thrinfo_t *) trmm_thread_sub_ipackm( thread ) );
129 }
130 else
131 {
132 b_alg = b_alg_next;
133 }
134 bli_dmam_wait(a1_dma, cntl_sub_dmam_a( cntl ), (dmam_thrinfo_t *) trmm_thread_sub_ipackm( thread ));
135 bli_dmam_wait(b1_dma, cntl_sub_dmam_b( cntl ), (dmam_thrinfo_t *) trmm_thread_sub_ipackm( thread ));
136 thread_ibarrier( thread );
138 // Initialize objects for packing A1 and B1.
139 if( thread_am_ichief( thread ) ) {
140 bli_packm_init( a1_dma, a1_pack,
141 cntl_sub_packm_a( cntl ) );
142 bli_packm_init( b1_dma, b1_pack,
143 cntl_sub_packm_b( cntl ) );
144 }
145 thread_ibarrier( thread );
147 // Pack A1 (if instructed).
148 bli_packm_int( a1_dma, a1_pack,
149 cntl_sub_packm_a( cntl ),
150 trmm_thread_sub_ipackm( thread ) );
152 // Pack B1 (if instructed).
153 bli_packm_int( b1_dma, b1_pack,
154 cntl_sub_packm_b( cntl ),
155 trmm_thread_sub_ipackm( thread ) );
157 //Get the next partition and start DMA
158 if( (i+b_alg) < k_trans ) //Making sure we aren't at the end of the matrix
159 {
160 // Determine the current algorithmic blocksize.
161 // NOTE: We call a trmm-specific function to determine the kc
162 // blocksize so that we can implement the "nudging" of kc to be
163 // a multiple of mr or nr, as needed.
164 b_alg_next = bli_trmm_determine_kc_f( i+b_alg, k_trans, a, b,
165 cntl_blocksize( cntl ) );
167 // Acquire partitions for A1 and B1.
168 bli_acquire_mpart_l2r( BLIS_SUBPART1,
169 i+b_alg, b_alg_next, a, &a1 );
170 bli_acquire_mpart_t2b( BLIS_SUBPART1,
171 i+b_alg, b_alg_next, b, &b1 );
172 if( thread_am_ichief( thread ) )
173 {
174 // DMA control leaf unrolling for A
175 //printf("Init DMA A2\n");
176 bli_dmam_init( &a1, a1_dma, cntl_sub_dmam_a( cntl ) );
177 bli_dmam_init( &b1, b1_dma, cntl_sub_dmam_b( cntl ) );
178 }
179 thread_ibarrier( thread );
180 bli_dmam_int( &a1, a1_dma, cntl_sub_dmam_a( cntl ), (dmam_thrinfo_t *) trmm_thread_sub_ipackm( thread ) );
181 bli_dmam_int( &b1, b1_dma, cntl_sub_dmam_b( cntl ), (dmam_thrinfo_t *) trmm_thread_sub_ipackm( thread ) );
184 }
187 #else
188 // Determine the current algorithmic blocksize.
189 // NOTE: We call a trmm-specific function to determine the kc
190 // blocksize so that we can implement the "nudging" of kc to be
191 // a multiple of mr or nr, as needed.
192 b_alg = bli_trmm_determine_kc_f( i, k_trans, a, b,
193 cntl_blocksize( cntl ) );
195 // Acquire partitions for A1 and B1.
196 bli_acquire_mpart_l2r( BLIS_SUBPART1,
197 i, b_alg, a, &a1 );
198 bli_acquire_mpart_t2b( BLIS_SUBPART1,
199 i, b_alg, b, &b1 );
201 // Initialize objects for packing A1 and B1.
202 if( thread_am_ichief( thread ) ) {
203 bli_packm_init( &a1, a1_pack,
204 cntl_sub_packm_a( cntl ) );
205 bli_packm_init( &b1, b1_pack,
206 cntl_sub_packm_b( cntl ) );
207 }
208 thread_ibarrier( thread );
210 // Pack A1 (if instructed).
211 bli_packm_int( &a1, a1_pack,
212 cntl_sub_packm_a( cntl ),
213 trmm_thread_sub_ipackm( thread ) );
215 // Pack B1 (if instructed).
216 bli_packm_int( &b1, b1_pack,
217 cntl_sub_packm_b( cntl ),
218 trmm_thread_sub_ipackm( thread ) );
219 #endif
221 // Perform trmm subproblem.
222 bli_trmm_int( &BLIS_ONE,
223 a1_pack,
224 b1_pack,
225 &BLIS_ONE,
226 c_pack,
227 cntl_sub_gemm( cntl ),
228 trmm_thread_sub_trmm( thread ) );
229 }
231 thread_obarrier( thread );
233 // Unpack C (if C was packed).
234 bli_unpackm_int( c_pack, c,
235 cntl_sub_unpackm_c( cntl ),
236 trmm_thread_sub_opackm( thread ) );
238 // If any packing buffers were acquired within packm, release them back
239 // to the memory manager.
240 if( thread_am_ochief( thread ) ){
241 bli_obj_release_pack( c_pack, cntl_sub_packm_c(cntl) );
242 }
243 if( thread_am_ichief( thread ) ){
244 #ifdef BLIS_ENABLE_C66X_EDMA
245 bli_obj_release_dma( a1_dma , cntl_sub_dmam_a(cntl) );
246 bli_obj_release_dma( b1_dma , cntl_sub_dmam_b(cntl) );
247 bli_obj_release_edma_handle( a1_dma );
248 bli_obj_release_edma_handle( b1_dma );
249 #endif
250 bli_obj_release_pack( a1_pack, cntl_sub_packm_a(cntl) );
251 bli_obj_release_pack( b1_pack, cntl_sub_packm_b(cntl) );
252 }
253 }