Deepity
Ultra-fast, SIMD-accelerated Predictive Coding
Loading...
Searching...
No Matches
Tanh.h
Go to the documentation of this file.
1#pragma once
2#include <cassert>
3#include <cstddef>
4#include <immintrin.h>
5#include <omp.h>
6#include <sleef.h>
7#include <deepity/utils/activations/ActivationMacros.h>
8
14
15namespace Deep
16{
17
21 static inline void tanh(float *RESTRICT x, const size_t n) noexcept
22 {
23 assert(n != 0 && "n must not be 0.");
24 assert(x != nullptr && "x must not be null.");
25
26 size_t simd_end = 0;
27
28#if defined(__AVX512F__)
29
30 simd_end = n & ~(size_t)15;
31
32#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
33 for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(simd_end); i += 16)
34 {
35 __m512 v = _mm512_loadu_ps(x + i);
36 v = Sleef_tanhf16_u10avx512f(v);
37 _mm512_storeu_ps(x + i, v);
38 }
39
40#elif defined(__AVX2__)
41
42 simd_end = n & ~(size_t)7;
43
44#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
45 for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(simd_end); i += 8)
46 {
47 __m256 v = _mm256_loadu_ps(x + i);
48 v = Sleef_tanhf8_u10avx2(v);
49 _mm256_storeu_ps(x + i, v);
50 }
51
52#elif defined(__AVX__)
53
54 simd_end = n & ~(size_t)7;
55
56#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
57 for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(simd_end); i += 8)
58 {
59 __m256 v = _mm256_loadu_ps(x + i);
60 v = Sleef_tanhf8_u10avx(v);
61 _mm256_storeu_ps(x + i, v);
62 }
63
64#elif defined(__SSE4_1__) || defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64)
65
66 simd_end = n & ~(size_t)3;
67
68#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
69 for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(simd_end); i += 4)
70 {
71 __m128 v = _mm_loadu_ps(x + i);
72
73#if defined(__SSE4_1__)
74 v = Sleef_tanhf4_u10sse4(v);
75#else
76 v = Sleef_tanhf4_u10sse2(v);
77#endif
78
79 _mm_storeu_ps(x + i, v);
80 }
81
82#endif
83
84 for (size_t i = simd_end; i < n; ++i)
85 {
86 x[i] = Sleef_tanhf_u10(x[i]);
87 }
88 }
89
95 static inline void dTanh(float *RESTRICT x,
96 const size_t n,
97 const bool activated = false) noexcept
98 {
99 assert(n != 0 && "n must not be 0.");
100 assert(x != nullptr && "x must not be null.");
101
102 size_t i = 0;
103
104 if (activated)
105 {
106#if defined(__AVX512F__)
107
108 const __m512 ones = _mm512_set1_ps(1.0f);
109 const size_t simd_end = n & ~(size_t)15;
110
111 for (; i < simd_end; i += 16)
112 {
113 const __m512 t = _mm512_loadu_ps(x + i);
114
115#ifdef __FMA__
116 const __m512 res = _mm512_fnmadd_ps(t, t, ones);
117#else
118 const __m512 res = _mm512_sub_ps(
119 ones,
120 _mm512_mul_ps(t, t));
121#endif
122
123 _mm512_storeu_ps(x + i, res);
124 }
125
126#elif defined(__AVX2__)
127
128 const __m256 ones = _mm256_set1_ps(1.0f);
129 const size_t simd_end = n & ~(size_t)7;
130
131 for (; i < simd_end; i += 8)
132 {
133 const __m256 t = _mm256_loadu_ps(x + i);
134
135#ifdef __FMA__
136 const __m256 res = _mm256_fnmadd_ps(t, t, ones);
137#else
138 const __m256 res = _mm256_sub_ps(
139 ones,
140 _mm256_mul_ps(t, t));
141#endif
142
143 _mm256_storeu_ps(x + i, res);
144 }
145
146#elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64)
147
148 const __m128 ones = _mm_set1_ps(1.0f);
149 const size_t simd_end = n & ~(size_t)3;
150
151 for (; i < simd_end; i += 4)
152 {
153 const __m128 t = _mm_loadu_ps(x + i);
154
155#ifdef __FMA__
156 const __m128 res = _mm_fnmadd_ps(t, t, ones);
157#else
158 const __m128 res = _mm_sub_ps(
159 ones,
160 _mm_mul_ps(t, t));
161#endif
162
163 _mm_storeu_ps(x + i, res);
164 }
165
166#endif
167
168 for (; i < n; ++i)
169 {
170 const float t = x[i];
171 x[i] = 1.0f - t * t;
172 }
173
174 return;
175 }
176
177#if defined(__AVX512F__)
178
179 {
180 const __m512 ones = _mm512_set1_ps(1.0f);
181 const size_t simd_end = n & ~(size_t)15;
182
183#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
184 for (ptrdiff_t j = 0;
185 j < static_cast<ptrdiff_t>(simd_end);
186 j += 16)
187 {
188 __m512 t = _mm512_loadu_ps(x + j);
189 t = Sleef_tanhf16_u10avx512f(t);
190
191#ifdef __FMA__
192 t = _mm512_fnmadd_ps(t, t, ones);
193#else
194 t = _mm512_sub_ps(
195 ones,
196 _mm512_mul_ps(t, t));
197#endif
198
199 _mm512_storeu_ps(x + j, t);
200 }
201
202 i = simd_end;
203 }
204
205#elif defined(__AVX2__)
206
207 {
208 const __m256 ones = _mm256_set1_ps(1.0f);
209 const size_t simd_end = n & ~(size_t)7;
210
211#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
212 for (ptrdiff_t j = 0;
213 j < static_cast<ptrdiff_t>(simd_end);
214 j += 8)
215 {
216 __m256 t = _mm256_loadu_ps(x + j);
217 t = Sleef_tanhf8_u10avx2(t);
218
219#ifdef __FMA__
220 t = _mm256_fnmadd_ps(t, t, ones);
221#else
222 t = _mm256_sub_ps(
223 ones,
224 _mm256_mul_ps(t, t));
225#endif
226
227 _mm256_storeu_ps(x + j, t);
228 }
229
230 i = simd_end;
231 }
232
233#elif defined(__AVX__)
234
235 {
236 const __m256 ones = _mm256_set1_ps(1.0f);
237 const size_t simd_end = n & ~(size_t)7;
238
239#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
240 for (ptrdiff_t j = 0;
241 j < static_cast<ptrdiff_t>(simd_end);
242 j += 8)
243 {
244 __m256 t = _mm256_loadu_ps(x + j);
245 t = Sleef_tanhf8_u10avx(t);
246
247#ifdef __FMA__
248 t = _mm256_fnmadd_ps(t, t, ones);
249#else
250 t = _mm256_sub_ps(
251 ones,
252 _mm256_mul_ps(t, t));
253#endif
254
255 _mm256_storeu_ps(x + j, t);
256 }
257
258 i = simd_end;
259 }
260
261#elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64)
262
263 {
264 const __m128 ones = _mm_set1_ps(1.0f);
265 const size_t simd_end = n & ~(size_t)3;
266
267#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel())
268 for (ptrdiff_t j = 0;
269 j < static_cast<ptrdiff_t>(simd_end);
270 j += 4)
271 {
272 __m128 t = _mm_loadu_ps(x + j);
273
274#if defined(__SSE4_1__)
275 t = Sleef_tanhf4_u10sse4(t);
276#else
277 t = Sleef_tanhf4_u10sse2(t);
278#endif
279
280#ifdef __FMA__
281 t = _mm_fnmadd_ps(t, t, ones);
282#else
283 t = _mm_sub_ps(
284 ones,
285 _mm_mul_ps(t, t));
286#endif
287
288 _mm_storeu_ps(x + j, t);
289 }
290
291 i = simd_end;
292 }
293
294#else
295
296 i = 0;
297
298#endif
299
300 /*
301 * Scalar tail.
302 */
303 for (; i < n; ++i)
304 {
305 const float t = Sleef_tanhf_u10(x[i]);
306 x[i] = 1.0f - t * t;
307 }
308 }
309
317 static inline void dTanhInto(float *RESTRICT dst, const float *RESTRICT src, const size_t n) noexcept
318 {
319 assert(n != 0 && "n must not be 0.");
320 assert(dst != nullptr && "dst must not be null.");
321 assert(src != nullptr && "src must not be null.");
322
323 size_t i = 0;
324
325#if defined(__AVX512F__)
326 __m512 ones = _mm512_set1_ps(1.0f);
327 size_t simd_end = n - (n % 16);
328#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel())
329 for (ptrdiff_t j = 0; j < (ptrdiff_t)simd_end; j += 16)
330 {
331 __m512 s = _mm512_loadu_ps(src + j);
332 __m512 t = Sleef_tanhf16_u10avx512f(s);
333#ifdef __FMA__
334 __m512 res = _mm512_fnmadd_ps(t, t, ones); // 1 - t*t
335#else
336 __m512 res = _mm512_sub_ps(ones, _mm512_mul_ps(t, t));
337#endif
338 _mm512_storeu_ps(dst + j, res);
339 }
340 i = simd_end;
341#elif defined(__AVX2__)
342 __m256 ones = _mm256_set1_ps(1.0f);
343 size_t simd_end = n - (n % 8);
344#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel())
345 for (ptrdiff_t j = 0; j < (ptrdiff_t)simd_end; j += 8)
346 {
347 __m256 s = _mm256_loadu_ps(src + j);
348 __m256 t = Sleef_tanhf8_u10avx2(s);
349#ifdef __FMA__
350 __m256 res = _mm256_fnmadd_ps(t, t, ones); // 1 - t*t
351#else
352 __m256 res = _mm256_sub_ps(ones, _mm256_mul_ps(t, t));
353#endif
354 _mm256_storeu_ps(dst + j, res);
355 }
356 i = simd_end;
357#elif defined(__SSE4_1__) || defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64)
358 __m128 ones = _mm_set1_ps(1.0f);
359 size_t simd_end = n - (n % 4);
360#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel())
361 for (ptrdiff_t j = 0; j < (ptrdiff_t)simd_end; j += 4)
362 {
363 __m128 s = _mm_loadu_ps(src + j);
364#if defined(__SSE4_1__)
365 __m128 t = Sleef_tanhf4_u10sse4(s);
366#else
367 __m128 t = Sleef_tanhf4_u10sse2(s);
368#endif
369#ifdef __FMA__
370 __m128 res = _mm_fnmadd_ps(t, t, ones); // 1 - t*t
371#else
372 __m128 res = _mm_sub_ps(ones, _mm_mul_ps(t, t));
373#endif
374 _mm_storeu_ps(dst + j, res);
375 }
376 i = simd_end;
377#endif
378
379 for (; i < n; ++i)
380 {
381 float t = Sleef_tanhf_u10(src[i]);
382 dst[i] = 1.0f - t * t;
383 }
384 }
385
386} // namespace Deep
ActivationMacros.h
The RESTRICT portability macro shared by every activation header. Split out so each per-activation he...
RESTRICT
#define RESTRICT
Portability macro for the C++ restrict pointer qualifier (MSVC spells it differently from everyone el...
Definition ActivationMacros.h:16
Deep
This file exists to provide both Tensor.h and IComputeBackend.h with device types without introducing...
Definition Backend.cpp:10
Deep::tanh
static void tanh(float *, size_t) noexcept
See Tanh.h.
Deep::dTanhInto
static void dTanhInto(float *RESTRICT, const float *RESTRICT, size_t) noexcept
See Tanh.h.
Deep::dTanh
static void dTanh(float *, size_t, bool) noexcept
See Tanh.h.