Deepity
Ultra-fast, SIMD-accelerated Predictive Coding
Loading...
Searching...
No Matches
Gelu.h
Go to the documentation of this file.
1#pragma once
2#include <cassert>
3#include <cstddef>
4#include <immintrin.h>
5#include <sleef.h>
6#include <deepity/utils/activations/ActivationMacros.h>
7
13
14namespace Deep
15{
16
19 constexpr float MAGIC_GELU_1 = 0.7978845608028654f;
22 constexpr float MAGIC_GELU_2 = 0.044715f;
23
26 static inline void gelu(float *RESTRICT x, const size_t n) noexcept
27 {
28 assert(n != 0 && "n must not be 0.");
29 assert(x != nullptr && "x must not be null.");
30
31 ptrdiff_t simd_end = 0;
32
33#if defined(__AVX512F__)
34 const __m512 ones512 = _mm512_set1_ps(1.0f);
35 const __m512 half512 = _mm512_set1_ps(0.5f);
36 const __m512 sqrt2overpi512 = _mm512_set1_ps(MAGIC_GELU_1); // C1 = sqrt(2/pi)
37 const __m512 gelu_coeff512 = _mm512_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1); // C2 = C1 * 0.044715
38
39 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)15;
40 for (ptrdiff_t i = 0; i < simd_end; i += 16)
41 {
42 __m512 x512 = _mm512_loadu_ps(x + i);
43 __m512 cube512 = _mm512_mul_ps(x512, _mm512_mul_ps(x512, x512));
44 __m512 c1x512 = _mm512_mul_ps(sqrt2overpi512, x512); // C1*x
45 __m512 inner512 = _mm512_fmadd_ps(cube512, gelu_coeff512, c1x512); // x^3*C2 + C1*x
46 __m512 t512 = Sleef_tanhf16_u10(inner512);
47 __m512 res = _mm512_mul_ps(half512, _mm512_mul_ps(x512, _mm512_add_ps(t512, ones512)));
48 _mm512_storeu_ps(x + i, res);
49 }
50#elif defined(__AVX__)
51 const __m256 ones256 = _mm256_set1_ps(1.0f);
52 const __m256 half256 = _mm256_set1_ps(0.5f);
53 const __m256 sqrt2overpi256 = _mm256_set1_ps(MAGIC_GELU_1); // C1 = sqrt(2/pi)
54 const __m256 gelu_coeff256 = _mm256_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1); // C2 = C1 * 0.044715
55
56 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)7;
57 for (ptrdiff_t i = 0; i < simd_end; i += 8)
58 {
59 __m256 x256 = _mm256_loadu_ps(x + i);
60 __m256 cube256 = _mm256_mul_ps(x256, _mm256_mul_ps(x256, x256));
61 __m256 c1x256 = _mm256_mul_ps(sqrt2overpi256, x256); // C1*x
62#if defined(__AVX2__)
63 __m256 inner256 = _mm256_fmadd_ps(cube256, gelu_coeff256, c1x256); // x^3*C2 + C1*x
64#else
65 __m256 inner256 = _mm256_add_ps(_mm256_mul_ps(cube256, gelu_coeff256), c1x256);
66#endif
67 __m256 t256 = Sleef_tanhf8_u10(inner256);
68 __m256 res = _mm256_mul_ps(half256, _mm256_mul_ps(x256, _mm256_add_ps(t256, ones256)));
69 _mm256_storeu_ps(x + i, res);
70 }
71#elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64)
72 const __m128 ones128 = _mm_set1_ps(1.0f);
73 const __m128 half128 = _mm_set1_ps(0.5f);
74 const __m128 sqrt2overpi128 = _mm_set1_ps(MAGIC_GELU_1); // C1 = sqrt(2/pi)
75 const __m128 gelu_coeff128 = _mm_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1); // C2 = C1 * 0.044715
76
77 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)3;
78 for (ptrdiff_t i = 0; i < simd_end; i += 4)
79 {
80 __m128 x128 = _mm_loadu_ps(x + i);
81 __m128 cube128 = _mm_mul_ps(x128, _mm_mul_ps(x128, x128));
82 __m128 c1x128 = _mm_mul_ps(sqrt2overpi128, x128); // C1*x
83#ifdef __FMA__
84 __m128 inner128 = _mm_fmadd_ps(cube128, gelu_coeff128, c1x128); // x^3*C2 + C1*x
85#else
86 __m128 inner128 = _mm_add_ps(c1x128, _mm_mul_ps(cube128, gelu_coeff128));
87#endif
88 __m128 t128 = Sleef_tanhf4_u10(inner128);
89 __m128 res = _mm_mul_ps(half128, _mm_mul_ps(x128, _mm_add_ps(t128, ones128)));
90 _mm_storeu_ps(x + i, res);
91 }
92#endif
93 for (ptrdiff_t i = simd_end; i < n; ++i)
94 {
95 float xi = x[i];
96 float inner = MAGIC_GELU_1 * xi + (MAGIC_GELU_2 * MAGIC_GELU_1) * xi * xi * xi;
97 x[i] = 0.5f * xi * (1.0f + Sleef_tanhf_u10(inner));
98 }
99 }
100
110 static inline void dGelu(float *RESTRICT x, const size_t n, const bool activated) noexcept
111 {
112 assert(n != 0 && "n must not be 0.");
113 assert(x != nullptr && "x must not be null.");
114
115 ptrdiff_t simd_end = 0;
116
117#if defined(__AVX512F__)
118 const __m512 ones512 = _mm512_set1_ps(1.0f);
119 const __m512 half512 = _mm512_set1_ps(0.5f);
120 const __m512 c1_512 = _mm512_set1_ps(MAGIC_GELU_1); // C1
121 const __m512 c2_512 = _mm512_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1); // C2
122 const __m512 c2_x3_512 = _mm512_set1_ps(3.0f * MAGIC_GELU_2 * MAGIC_GELU_1); // 3 * C2
123
124 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)15;
125 for (ptrdiff_t i = 0; i < simd_end; i += 16)
126 {
127 __m512 x512 = _mm512_loadu_ps(x + i);
128 __m512 xsq512 = _mm512_mul_ps(x512, x512);
129 __m512 xcube512 = _mm512_mul_ps(xsq512, x512);
130 __m512 c1x512 = _mm512_mul_ps(c1_512, x512);
131 __m512 inner512 = _mm512_fmadd_ps(xcube512, c2_512, c1x512);
132
133 __m512 t512 = Sleef_tanhf16_u10(inner512);
134 __m512 tsq512 = _mm512_mul_ps(t512, t512);
135
136 __m512 one_minus_tsq512 = _mm512_sub_ps(ones512, tsq512);
137 __m512 gprime512 = _mm512_fmadd_ps(xsq512, c2_x3_512, c1_512);
138
139 __m512 term1 = _mm512_mul_ps(half512, _mm512_add_ps(ones512, t512));
140 __m512 term2 = _mm512_mul_ps(half512, _mm512_mul_ps(x512, _mm512_mul_ps(gprime512, one_minus_tsq512)));
141 __m512 res = _mm512_add_ps(term1, term2);
142 _mm512_storeu_ps(x + i, res);
143 }
144#elif defined(__AVX__)
145 const __m256 ones256 = _mm256_set1_ps(1.0f);
146 const __m256 half256 = _mm256_set1_ps(0.5f);
147 const __m256 c1_256 = _mm256_set1_ps(MAGIC_GELU_1);
148 const __m256 c2_256 = _mm256_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1);
149 const __m256 c2_x3_256 = _mm256_set1_ps(3.0f * MAGIC_GELU_2 * MAGIC_GELU_1);
150
151 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)7;
152 for (ptrdiff_t i = 0; i < simd_end; i += 8)
153 {
154 __m256 x256 = _mm256_loadu_ps(x + i);
155 __m256 xsq256 = _mm256_mul_ps(x256, x256);
156 __m256 xcube256 = _mm256_mul_ps(xsq256, x256);
157 __m256 c1x256 = _mm256_mul_ps(c1_256, x256);
158
159#if defined(__AVX2__)
160 __m256 inner256 = _mm256_fmadd_ps(xcube256, c2_256, c1x256);
161 __m256 gprime256 = _mm256_fmadd_ps(xsq256, c2_x3_256, c1_256);
162#else
163 __m256 inner256 = _mm256_add_ps(_mm256_mul_ps(xcube256, c2_256), c1x256);
164 __m256 gprime256 = _mm256_add_ps(_mm256_mul_ps(xsq256, c2_x3_256), c1_256);
165#endif
166
167 __m256 t256 = Sleef_tanhf8_u10(inner256);
168 __m256 tsq256 = _mm256_mul_ps(t256, t256);
169 __m256 one_minus_tsq256 = _mm256_sub_ps(ones256, tsq256);
170
171 __m256 term1 = _mm256_mul_ps(half256, _mm256_add_ps(ones256, t256));
172 __m256 term2 = _mm256_mul_ps(half256, _mm256_mul_ps(x256, _mm256_mul_ps(gprime256, one_minus_tsq256)));
173
174 _mm256_storeu_ps(x + i, _mm256_add_ps(term1, term2));
175 }
176#elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64)
177 const __m128 ones128 = _mm_set1_ps(1.0f);
178 const __m128 half128 = _mm_set1_ps(0.5f);
179 const __m128 c1_128 = _mm_set1_ps(MAGIC_GELU_1);
180 const __m128 c2_128 = _mm_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1);
181 const __m128 c2_x3_128 = _mm_set1_ps(3.0f * MAGIC_GELU_2 * MAGIC_GELU_1);
182
183 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)3;
184 for (ptrdiff_t i = 0; i < simd_end; i += 4)
185 {
186 __m128 x128 = _mm_loadu_ps(x + i);
187 __m128 xsq128 = _mm_mul_ps(x128, x128);
188 __m128 xcube128 = _mm_mul_ps(xsq128, x128);
189 __m128 c1x128 = _mm_mul_ps(c1_128, x128);
190
191#ifdef __FMA__
192 __m128 inner128 = _mm_fmadd_ps(xcube128, c2_128, c1x128);
193 __m128 gprime128 = _mm_fmadd_ps(xsq128, c2_x3_128, c1_128);
194#else
195 __m128 inner128 = _mm_add_ps(_mm_mul_ps(xcube128, c2_128), c1x128);
196 __m128 gprime128 = _mm_add_ps(_mm_mul_ps(xsq128, c2_x3_128), c1_128);
197#endif
198
199 __m128 t128 = Sleef_tanhf4_u10(inner128);
200 __m128 tsq128 = _mm_mul_ps(t128, t128);
201 __m128 one_minus_tsq128 = _mm_sub_ps(ones128, tsq128);
202
203 __m128 term1 = _mm_mul_ps(half128, _mm_add_ps(ones128, t128));
204 __m128 term2 = _mm_mul_ps(half128, _mm_mul_ps(x128, _mm_mul_ps(gprime128, one_minus_tsq128)));
205
206 _mm_storeu_ps(x + i, _mm_add_ps(term1, term2));
207 }
208#endif
209 for (ptrdiff_t i = simd_end; i < n; ++i)
210 {
211 float xi = x[i];
212 float xsq = xi * xi;
213
214 float inner = MAGIC_GELU_1 * xi + (MAGIC_GELU_2 * MAGIC_GELU_1) * xsq * xi;
215 float t = Sleef_tanhf_u10(inner);
216
217 float gprime = MAGIC_GELU_1 + (3.0f * MAGIC_GELU_2 * MAGIC_GELU_1) * xsq;
218
219 float term1 = 0.5f * (1.0f + t);
220 float term2 = 0.5f * xi * gprime * (1.0f - t * t);
221
222 x[i] = term1 + term2;
223 }
224 }
225
228 static inline void dGeluInto(float *RESTRICT dst, const float *RESTRICT src, const size_t n) noexcept
229 {
230
231 ptrdiff_t simd_end = 0;
232
233#if defined(__AVX512F__)
234 const __m512 ones512 = _mm512_set1_ps(1.0f);
235 const __m512 half512 = _mm512_set1_ps(0.5f);
236 const __m512 c1_512 = _mm512_set1_ps(MAGIC_GELU_1); // C1
237 const __m512 c2_512 = _mm512_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1); // C2
238 const __m512 c2_x3_512 = _mm512_set1_ps(3.0f * MAGIC_GELU_2 * MAGIC_GELU_1); // 3 * C2
239
240 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)15;
241 for (ptrdiff_t i = 0; i < simd_end; i += 16)
242 {
243 __m512 x512 = _mm512_loadu_ps(src + i);
244 __m512 xsq512 = _mm512_mul_ps(x512, x512);
245 __m512 xcube512 = _mm512_mul_ps(xsq512, x512);
246 __m512 c1x512 = _mm512_mul_ps(c1_512, x512);
247 __m512 inner512 = _mm512_fmadd_ps(xcube512, c2_512, c1x512);
248
249 __m512 t512 = Sleef_tanhf16_u10(inner512);
250 __m512 tsq512 = _mm512_mul_ps(t512, t512);
251
252 __m512 one_minus_tsq512 = _mm512_sub_ps(ones512, tsq512);
253 __m512 gprime512 = _mm512_fmadd_ps(xsq512, c2_x3_512, c1_512);
254
255 __m512 term1 = _mm512_mul_ps(half512, _mm512_add_ps(ones512, t512));
256 __m512 term2 = _mm512_mul_ps(half512, _mm512_mul_ps(x512, _mm512_mul_ps(gprime512, one_minus_tsq512)));
257 __m512 res = _mm512_add_ps(term1, term2);
258 _mm512_storeu_ps(dst + i, res);
259 }
260#elif defined(__AVX__)
261 const __m256 ones256 = _mm256_set1_ps(1.0f);
262 const __m256 half256 = _mm256_set1_ps(0.5f);
263 const __m256 c1_256 = _mm256_set1_ps(MAGIC_GELU_1);
264 const __m256 c2_256 = _mm256_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1);
265 const __m256 c2_x3_256 = _mm256_set1_ps(3.0f * MAGIC_GELU_2 * MAGIC_GELU_1);
266
267 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)7;
268 for (ptrdiff_t i = 0; i < simd_end; i += 8)
269 {
270 __m256 x256 = _mm256_loadu_ps(src + i);
271 __m256 xsq256 = _mm256_mul_ps(x256, x256);
272 __m256 xcube256 = _mm256_mul_ps(xsq256, x256);
273 __m256 c1x256 = _mm256_mul_ps(c1_256, x256);
274
275#if defined(__AVX2__)
276 __m256 inner256 = _mm256_fmadd_ps(xcube256, c2_256, c1x256);
277 __m256 gprime256 = _mm256_fmadd_ps(xsq256, c2_x3_256, c1_256);
278#else
279 __m256 inner256 = _mm256_add_ps(_mm256_mul_ps(xcube256, c2_256), c1x256);
280 __m256 gprime256 = _mm256_add_ps(_mm256_mul_ps(xsq256, c2_x3_256), c1_256);
281#endif
282
283 __m256 t256 = Sleef_tanhf8_u10(inner256);
284 __m256 tsq256 = _mm256_mul_ps(t256, t256);
285 __m256 one_minus_tsq256 = _mm256_sub_ps(ones256, tsq256);
286
287 __m256 term1 = _mm256_mul_ps(half256, _mm256_add_ps(ones256, t256));
288 __m256 term2 = _mm256_mul_ps(half256, _mm256_mul_ps(x256, _mm256_mul_ps(gprime256, one_minus_tsq256)));
289
290 _mm256_storeu_ps(dst + i, _mm256_add_ps(term1, term2));
291 }
292#elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64)
293 const __m128 ones128 = _mm_set1_ps(1.0f);
294 const __m128 half128 = _mm_set1_ps(0.5f);
295 const __m128 c1_128 = _mm_set1_ps(MAGIC_GELU_1);
296 const __m128 c2_128 = _mm_set1_ps(MAGIC_GELU_2 * MAGIC_GELU_1);
297 const __m128 c2_x3_128 = _mm_set1_ps(3.0f * MAGIC_GELU_2 * MAGIC_GELU_1);
298
299 simd_end = (ptrdiff_t)n & ~(ptrdiff_t)3;
300 for (ptrdiff_t i = 0; i < simd_end; i += 4)
301 {
302 __m128 x128 = _mm_loadu_ps(src + i);
303 __m128 xsq128 = _mm_mul_ps(x128, x128);
304 __m128 xcube128 = _mm_mul_ps(xsq128, x128);
305 __m128 c1x128 = _mm_mul_ps(c1_128, x128);
306
307#ifdef __FMA__
308 __m128 inner128 = _mm_fmadd_ps(xcube128, c2_128, c1x128);
309 __m128 gprime128 = _mm_fmadd_ps(xsq128, c2_x3_128, c1_128);
310#else
311 __m128 inner128 = _mm_add_ps(_mm_mul_ps(xcube128, c2_128), c1x128);
312 __m128 gprime128 = _mm_add_ps(_mm_mul_ps(xsq128, c2_x3_128), c1_128);
313#endif
314
315 __m128 t128 = Sleef_tanhf4_u10(inner128);
316 __m128 tsq128 = _mm_mul_ps(t128, t128);
317 __m128 one_minus_tsq128 = _mm_sub_ps(ones128, tsq128);
318
319 __m128 term1 = _mm_mul_ps(half128, _mm_add_ps(ones128, t128));
320 __m128 term2 = _mm_mul_ps(half128, _mm_mul_ps(x128, _mm_mul_ps(gprime128, one_minus_tsq128)));
321
322 _mm_storeu_ps(dst + i, _mm_add_ps(term1, term2));
323 }
324#endif
325 for (ptrdiff_t i = simd_end; i < n; ++i)
326 {
327 float xi = src[i];
328 float xsq = xi * xi;
329
330 float inner = MAGIC_GELU_1 * xi + (MAGIC_GELU_2 * MAGIC_GELU_1) * xsq * xi;
331 float t = Sleef_tanhf_u10(inner);
332
333 float gprime = MAGIC_GELU_1 + (3.0f * MAGIC_GELU_2 * MAGIC_GELU_1) * xsq;
334
335 float term1 = 0.5f * (1.0f + t);
336 float term2 = 0.5f * xi * gprime * (1.0f - t * t);
337
338 dst[i] = term1 + term2;
339 }
340 }
341
342} // namespace Deep
ActivationMacros.h
The RESTRICT portability macro shared by every activation header. Split out so each per-activation he...
RESTRICT
#define RESTRICT
Portability macro for the C++ restrict pointer qualifier (MSVC spells it differently from everyone el...
Definition ActivationMacros.h:16
Deep
This file exists to provide both Tensor.h and IComputeBackend.h with device types without introducing...
Definition Backend.cpp:10
Deep::dGelu
static void dGelu(float *, size_t, bool) noexcept
See Gelu.h.
Deep::gelu
static void gelu(float *, size_t) noexcept
See Gelu.h.
Deep::MAGIC_GELU_1
constexpr float MAGIC_GELU_1
sqrt(2/pi), the leading constant in the tanh approximation of GELU. Also used directly by CUDABackend...
Definition Gelu.h:19
Deep::MAGIC_GELU_2
constexpr float MAGIC_GELU_2
The tanh approximation's cubic-term coefficient (0.044715), paired with MAGIC_GELU_1.
Definition Gelu.h:22
Deep::dGeluInto
static void dGeluInto(float *RESTRICT, const float *RESTRICT, size_t) noexcept
See Gelu.h.