1 #ifndef INCLUDED_volk_32fc_x2_dot_prod_32fc_a_H
2 #define INCLUDED_volk_32fc_x2_dot_prod_32fc_a_H
10 #ifdef LV_HAVE_GENERIC
15 float * res = (
float*) result;
16 float * in = (
float*) input;
17 float * tp = (
float*) taps;
18 unsigned int n_2_ccomplex_blocks = num_bytes >> 4;
19 unsigned int isodd = (num_bytes >> 3) &1;
23 float sum0[2] = {0,0};
24 float sum1[2] = {0,0};
28 for(i = 0; i < n_2_ccomplex_blocks; ++i) {
31 sum0[0] += in[0] * tp[0] - in[1] * tp[1];
32 sum0[1] += in[0] * tp[1] + in[1] * tp[0];
33 sum1[0] += in[2] * tp[2] - in[3] * tp[3];
34 sum1[1] += in[2] * tp[3] + in[3] * tp[2];
43 res[0] = sum0[0] + sum1[0];
44 res[1] = sum0[1] + sum1[1];
48 for(i = 0; i < isodd; ++i) {
51 *result += input[(num_bytes >> 3) - 1] * taps[(num_bytes >> 3) - 1];
60 #if LV_HAVE_SSE && LV_HAVE_64
63 static inline void volk_32fc_x2_dot_prod_32fc_a_sse_64(
lv_32fc_t* result,
const lv_32fc_t* input,
const lv_32fc_t* taps,
unsigned int num_bytes) {
68 "# ccomplex_dotprod_generic (float* result, const float *input,\n\t"
69 "# const float *taps, unsigned num_bytes)\n\t"
70 "# float sum0 = 0;\n\t"
71 "# float sum1 = 0;\n\t"
72 "# float sum2 = 0;\n\t"
73 "# float sum3 = 0;\n\t"
75 "# sum0 += input[0] * taps[0] - input[1] * taps[1];\n\t"
76 "# sum1 += input[0] * taps[1] + input[1] * taps[0];\n\t"
77 "# sum2 += input[2] * taps[2] - input[3] * taps[3];\n\t"
78 "# sum3 += input[2] * taps[3] + input[3] * taps[2];\n\t"
81 "# } while (--n_2_ccomplex_blocks != 0);\n\t"
82 "# result[0] = sum0 + sum2;\n\t"
83 "# result[1] = sum1 + sum3;\n\t"
84 "# TODO: prefetch and better scheduling\n\t"
86 " xor %%r10, %%r10\n\t"
87 " movq %%rcx, %%rax\n\t"
88 " movq %%rcx, %%r8\n\t"
89 " movq %[rsi], %%r9\n\t"
90 " movq %[rdx], %%r10\n\t"
91 " xorps %%xmm6, %%xmm6 # zero accumulators\n\t"
92 " movaps 0(%%r9), %%xmm0\n\t"
93 " xorps %%xmm7, %%xmm7 # zero accumulators\n\t"
94 " movaps 0(%%r10), %%xmm2\n\t"
95 " shr $5, %%rax # rax = n_2_ccomplex_blocks / 2\n\t"
98 " # 4 taps / loop\n\t"
99 " # something like ?? cycles / loop\n\t"
101 "# complex prod: C += A * B, w/ temp Z & Y (or B), xmmPN=$0x8000000080000000\n\t"
102 "# movaps (%%r9), %%xmmA\n\t"
103 "# movaps (%%r10), %%xmmB\n\t"
104 "# movaps %%xmmA, %%xmmZ\n\t"
105 "# shufps $0xb1, %%xmmZ, %%xmmZ # swap internals\n\t"
106 "# mulps %%xmmB, %%xmmA\n\t"
107 "# mulps %%xmmZ, %%xmmB\n\t"
108 "# # SSE replacement for: pfpnacc %%xmmB, %%xmmA\n\t"
109 "# xorps %%xmmPN, %%xmmA\n\t"
110 "# movaps %%xmmA, %%xmmZ\n\t"
111 "# unpcklps %%xmmB, %%xmmA\n\t"
112 "# unpckhps %%xmmB, %%xmmZ\n\t"
113 "# movaps %%xmmZ, %%xmmY\n\t"
114 "# shufps $0x44, %%xmmA, %%xmmZ # b01000100\n\t"
115 "# shufps $0xee, %%xmmY, %%xmmA # b11101110\n\t"
116 "# addps %%xmmZ, %%xmmA\n\t"
117 "# addps %%xmmA, %%xmmC\n\t"
118 "# A=xmm0, B=xmm2, Z=xmm4\n\t"
119 "# A'=xmm1, B'=xmm3, Z'=xmm5\n\t"
120 " movaps 16(%%r9), %%xmm1\n\t"
121 " movaps %%xmm0, %%xmm4\n\t"
122 " mulps %%xmm2, %%xmm0\n\t"
123 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t"
124 " movaps 16(%%r10), %%xmm3\n\t"
125 " movaps %%xmm1, %%xmm5\n\t"
126 " addps %%xmm0, %%xmm6\n\t"
127 " mulps %%xmm3, %%xmm1\n\t"
128 " shufps $0xb1, %%xmm5, %%xmm5 # swap internals\n\t"
129 " addps %%xmm1, %%xmm6\n\t"
130 " mulps %%xmm4, %%xmm2\n\t"
131 " movaps 32(%%r9), %%xmm0\n\t"
132 " addps %%xmm2, %%xmm7\n\t"
133 " mulps %%xmm5, %%xmm3\n\t"
135 " movaps 32(%%r10), %%xmm2\n\t"
136 " addps %%xmm3, %%xmm7\n\t"
137 " add $32, %%r10\n\t"
141 " # We've handled the bulk of multiplies up to here.\n\t"
142 " # Let's sse if original n_2_ccomplex_blocks was odd.\n\t"
143 " # If so, we've got 2 more taps to do.\n\t"
146 " # The count was odd, do 2 more taps.\n\t"
147 " # Note that we've already got mm0/mm2 preloaded\n\t"
148 " # from the main loop.\n\t"
149 " movaps %%xmm0, %%xmm4\n\t"
150 " mulps %%xmm2, %%xmm0\n\t"
151 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t"
152 " addps %%xmm0, %%xmm6\n\t"
153 " mulps %%xmm4, %%xmm2\n\t"
154 " addps %%xmm2, %%xmm7\n\t"
156 " # neg inversor\n\t"
157 " xorps %%xmm1, %%xmm1\n\t"
158 " mov $0x80000000, %%r9\n\t"
159 " movd %%r9, %%xmm1\n\t"
160 " shufps $0x11, %%xmm1, %%xmm1 # b00010001 # 0 -0 0 -0\n\t"
162 " xorps %%xmm1, %%xmm6\n\t"
163 " movaps %%xmm6, %%xmm2\n\t"
164 " unpcklps %%xmm7, %%xmm6\n\t"
165 " unpckhps %%xmm7, %%xmm2\n\t"
166 " movaps %%xmm2, %%xmm3\n\t"
167 " shufps $0x44, %%xmm6, %%xmm2 # b01000100\n\t"
168 " shufps $0xee, %%xmm3, %%xmm6 # b11101110\n\t"
169 " addps %%xmm2, %%xmm6\n\t"
170 " # xmm6 = r1 i2 r3 i4\n\t"
171 " movhlps %%xmm6, %%xmm4 # xmm4 = r3 i4 ?? ??\n\t"
172 " addps %%xmm4, %%xmm6 # xmm6 = r1+r3 i2+i4 ?? ??\n\t"
173 " movlps %%xmm6, (%[rdi]) # store low 2x32 bits (complex) to memory\n\t"
175 :[rsi]
"r" (input), [rdx]
"r" (taps),
"c" (num_bytes), [rdi]
"r" (result)
176 :
"rax",
"r8",
"r9",
"r10"
180 int getem = num_bytes % 16;
183 for(; getem > 0; getem -= 8) {
186 *result += (input[(num_bytes >> 3) - 1] * taps[(num_bytes >> 3) - 1]);
196 #if LV_HAVE_SSE && LV_HAVE_32
198 static inline void volk_32fc_x2_dot_prod_32fc_a_sse_32(
lv_32fc_t* result,
const lv_32fc_t* input,
const lv_32fc_t* taps,
unsigned int num_bytes) {
203 " #movl %%esp, %%ebp\n\t"
204 " movl 12(%%ebp), %%eax # input\n\t"
205 " movl 16(%%ebp), %%edx # taps\n\t"
206 " movl 20(%%ebp), %%ecx # n_bytes\n\t"
207 " xorps %%xmm6, %%xmm6 # zero accumulators\n\t"
208 " movaps 0(%%eax), %%xmm0\n\t"
209 " xorps %%xmm7, %%xmm7 # zero accumulators\n\t"
210 " movaps 0(%%edx), %%xmm2\n\t"
211 " shrl $5, %%ecx # ecx = n_2_ccomplex_blocks / 2\n\t"
212 " jmp .%=L1_test\n\t"
213 " # 4 taps / loop\n\t"
214 " # something like ?? cycles / loop\n\t"
216 "# complex prod: C += A * B, w/ temp Z & Y (or B), xmmPN=$0x8000000080000000\n\t"
217 "# movaps (%%eax), %%xmmA\n\t"
218 "# movaps (%%edx), %%xmmB\n\t"
219 "# movaps %%xmmA, %%xmmZ\n\t"
220 "# shufps $0xb1, %%xmmZ, %%xmmZ # swap internals\n\t"
221 "# mulps %%xmmB, %%xmmA\n\t"
222 "# mulps %%xmmZ, %%xmmB\n\t"
223 "# # SSE replacement for: pfpnacc %%xmmB, %%xmmA\n\t"
224 "# xorps %%xmmPN, %%xmmA\n\t"
225 "# movaps %%xmmA, %%xmmZ\n\t"
226 "# unpcklps %%xmmB, %%xmmA\n\t"
227 "# unpckhps %%xmmB, %%xmmZ\n\t"
228 "# movaps %%xmmZ, %%xmmY\n\t"
229 "# shufps $0x44, %%xmmA, %%xmmZ # b01000100\n\t"
230 "# shufps $0xee, %%xmmY, %%xmmA # b11101110\n\t"
231 "# addps %%xmmZ, %%xmmA\n\t"
232 "# addps %%xmmA, %%xmmC\n\t"
233 "# A=xmm0, B=xmm2, Z=xmm4\n\t"
234 "# A'=xmm1, B'=xmm3, Z'=xmm5\n\t"
235 " movaps 16(%%eax), %%xmm1\n\t"
236 " movaps %%xmm0, %%xmm4\n\t"
237 " mulps %%xmm2, %%xmm0\n\t"
238 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t"
239 " movaps 16(%%edx), %%xmm3\n\t"
240 " movaps %%xmm1, %%xmm5\n\t"
241 " addps %%xmm0, %%xmm6\n\t"
242 " mulps %%xmm3, %%xmm1\n\t"
243 " shufps $0xb1, %%xmm5, %%xmm5 # swap internals\n\t"
244 " addps %%xmm1, %%xmm6\n\t"
245 " mulps %%xmm4, %%xmm2\n\t"
246 " movaps 32(%%eax), %%xmm0\n\t"
247 " addps %%xmm2, %%xmm7\n\t"
248 " mulps %%xmm5, %%xmm3\n\t"
249 " addl $32, %%eax\n\t"
250 " movaps 32(%%edx), %%xmm2\n\t"
251 " addps %%xmm3, %%xmm7\n\t"
252 " addl $32, %%edx\n\t"
256 " # We've handled the bulk of multiplies up to here.\n\t"
257 " # Let's sse if original n_2_ccomplex_blocks was odd.\n\t"
258 " # If so, we've got 2 more taps to do.\n\t"
259 " movl 20(%%ebp), %%ecx # n_2_ccomplex_blocks\n\t"
260 " shrl $4, %%ecx\n\t"
261 " andl $1, %%ecx\n\t"
263 " # The count was odd, do 2 more taps.\n\t"
264 " # Note that we've already got mm0/mm2 preloaded\n\t"
265 " # from the main loop.\n\t"
266 " movaps %%xmm0, %%xmm4\n\t"
267 " mulps %%xmm2, %%xmm0\n\t"
268 " shufps $0xb1, %%xmm4, %%xmm4 # swap internals\n\t"
269 " addps %%xmm0, %%xmm6\n\t"
270 " mulps %%xmm4, %%xmm2\n\t"
271 " addps %%xmm2, %%xmm7\n\t"
273 " # neg inversor\n\t"
274 " movl 8(%%ebp), %%eax \n\t"
275 " xorps %%xmm1, %%xmm1\n\t"
276 " movl $0x80000000, (%%eax)\n\t"
277 " movss (%%eax), %%xmm1\n\t"
278 " shufps $0x11, %%xmm1, %%xmm1 # b00010001 # 0 -0 0 -0\n\t"
280 " xorps %%xmm1, %%xmm6\n\t"
281 " movaps %%xmm6, %%xmm2\n\t"
282 " unpcklps %%xmm7, %%xmm6\n\t"
283 " unpckhps %%xmm7, %%xmm2\n\t"
284 " movaps %%xmm2, %%xmm3\n\t"
285 " shufps $0x44, %%xmm6, %%xmm2 # b01000100\n\t"
286 " shufps $0xee, %%xmm3, %%xmm6 # b11101110\n\t"
287 " addps %%xmm2, %%xmm6\n\t"
288 " # xmm6 = r1 i2 r3 i4\n\t"
289 " #movl 8(%%ebp), %%eax # @result\n\t"
290 " movhlps %%xmm6, %%xmm4 # xmm4 = r3 i4 ?? ??\n\t"
291 " addps %%xmm4, %%xmm6 # xmm6 = r1+r3 i2+i4 ?? ??\n\t"
292 " movlps %%xmm6, (%%eax) # store low 2x32 bits (complex) to memory\n\t"
296 :
"eax",
"ecx",
"edx"
300 int getem = num_bytes % 16;
302 for(; getem > 0; getem -= 8) {
305 *result += (input[(num_bytes >> 3) - 1] * taps[(num_bytes >> 3) - 1]);
322 #include <pmmintrin.h>
324 static inline void volk_32fc_x2_dot_prod_32fc_a_sse3(
lv_32fc_t* result,
const lv_32fc_t* input,
const lv_32fc_t* taps,
unsigned int num_bytes) {
328 memset(&dotProduct, 0x0, 2*
sizeof(
float));
330 unsigned int number = 0;
331 const unsigned int halfPoints = num_bytes >> 4;
333 __m128 x, y, yl, yh, z, tmp1, tmp2, dotProdVal;
338 dotProdVal = _mm_setzero_ps();
340 for(;number < halfPoints; number++){
342 x = _mm_load_ps((
float*)a);
343 y = _mm_load_ps((
float*)b);
345 yl = _mm_moveldup_ps(y);
346 yh = _mm_movehdup_ps(y);
348 tmp1 = _mm_mul_ps(x,yl);
350 x = _mm_shuffle_ps(x,x,0xB1);
352 tmp2 = _mm_mul_ps(x,yh);
354 z = _mm_addsub_ps(tmp1,tmp2);
356 dotProdVal = _mm_add_ps(dotProdVal, z);
364 _mm_store_ps((
float*)dotProductVector,dotProdVal);
366 dotProduct += ( dotProductVector[0] + dotProductVector[1] );
368 if((num_bytes >> 2) != 0) {
369 dotProduct += (*a) * (*b);
372 *result = dotProduct;
377 #ifdef LV_HAVE_SSE4_1
379 #include <smmintrin.h>
381 static inline void volk_32fc_x2_dot_prod_32fc_a_sse4_1(
lv_32fc_t* result,
const lv_32fc_t* input,
const lv_32fc_t* taps,
unsigned int num_bytes) {
382 volk_32fc_x2_dot_prod_32fc_a_sse3(result, input, taps, num_bytes);