32 lines
776 B
Plaintext
32 lines
776 B
Plaintext
#ifndef GELU_KERNEL_CUH_
|
|
#define GELU_KERNEL_CUH_
|
|
|
|
#include <cuda_runtime.h>
|
|
#include "../utils/cuda_utils.cuh"
|
|
|
|
#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
|
|
|
|
__global__ void gelu_forward_kernel(float* __restrict__ out, const float* __restrict__ inp, int N) {
|
|
// Implement this
|
|
|
|
int x = blockDim.x * blockIdx.x + threadIdx.x;
|
|
|
|
if(x < N){
|
|
float val = inp[x];
|
|
float cube = 0.044715f * val * val * val;
|
|
out[x] = 0.5f * val * (1.0f + tanhf(GELU_SCALING_FACTOR * (val + cube)));
|
|
}
|
|
|
|
}
|
|
|
|
// Launch kernel here
|
|
void gelu_forward(float* out, const float* inp, int N) {
|
|
// Implement this
|
|
dim3 block(32,1,1);
|
|
dim3 grid((N + block.x -1) / block.x);
|
|
|
|
gelu_forward_kernel<<<grid, block>>>(out, inp, N);
|
|
|
|
}
|
|
|
|
#endif // GELU_KERNEL_CUH_ |