ECE 408 sem project

kernels_op_9/gelu.cuh

@@ -0,0 +1,32 @@
+#ifndef GELU_KERNEL_CUH_
+#define GELU_KERNEL_CUH_
+
+#include <cuda_runtime.h>
+#include "../utils/cuda_utils.cuh"
+
+#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
+
+__global__ void gelu_forward_kernel(float* __restrict__ out, const float* __restrict__ inp, int N) {
+    // Implement this
+
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+
+    if(x < N){
+        float val = inp[x];
+        float cube = 0.044715f * val * val * val;
+        out[x] = 0.5f * val * (1.0f + tanhf(GELU_SCALING_FACTOR * (val + cube)));
+    }
+
+}
+
+// Launch kernel here
+void gelu_forward(float* out, const float* inp, int N) {
+    // Implement this
+    dim3 block(32,1,1);
+    dim3 grid((N + block.x -1) / block.x);
+
+    gelu_forward_kernel<<<grid, block>>>(out, inp, N);
+
+}
+
+#endif // GELU_KERNEL_CUH_