comp files

gpt2.cuh

@@ -150,22 +150,23 @@ void fill_in_activation_sizes(size_t* act_sizes, int B, int T, GPT2Config config
     size_t C = config.channels;
     act_sizes[0] = B * T * C; // encoded
     act_sizes[1] = L * B * T * C; // ln1
-    act_sizes[2] = 1; // ln1_mean, reserved for future use
-    act_sizes[3] = 1; // ln1_rstd, reserved for future use
+    //to align making 1 to 4
+    act_sizes[2] = 4; // ln1_mean, reserved for future use
+    act_sizes[3] = 4; // ln1_rstd, reserved for future use
     act_sizes[4] = L * B * T * C; // atty
     act_sizes[5] = L * B * NH * T * T; // att
     act_sizes[6] = L * B * T * C; // attproj
     act_sizes[7] = L * B * T * C; // residual2
     act_sizes[8] = L * B * T * C; // ln2
-    act_sizes[9] = 1; // ln2_mean, reserved for future use
-    act_sizes[10] = 1; // ln2_rstd, reserved for future use
+    act_sizes[9] = 4; // ln2_mean, reserved for future use
+    act_sizes[10] = 4; // ln2_rstd, reserved for future use
     act_sizes[11] = L * B * T * 4*C; // fch
     act_sizes[12] = L * B * T * 4*C; // fch_gelu
     act_sizes[13] = L * B * T * C; // fcproj
     act_sizes[14] = L * B * T * C; // residual3
     act_sizes[15] = B * T * C; // lnf
-    act_sizes[16] = 1; // lnf_mean, reserved for future use
-    act_sizes[17] = 1; // lnf_rstd, reserved for future use
+    act_sizes[16] = 4; // lnf_mean, reserved for future use
+    act_sizes[17] = 4; // lnf_rstd, reserved for future use
     act_sizes[18] = B * T; // losses
     act_sizes[19] = L * B * T * 3*C; // qkvr
     act_sizes[20] = B * T * max(3*C, max(NH*T, Vp)); // output / scratch
@@ -179,19 +180,23 @@ typedef struct {
     float* residual3; // (B, T, C)
 } GradActTensors;
 
+// op_13: 256-byte aligned activations.
+// Each activation tensor's stride is rounded up to a multiple of 64 floats (256 bytes).
+#define align_to_256_bytes(x) \
+    (x + (256/sizeof(float) - (x % (256/sizeof(float)))) % (256/sizeof(float)))
+
 float* malloc_and_point(float** targets[], const size_t* act_sizes, int n) {
     size_t num_activations = 0;
     for (size_t i = 0; i < n; i++) {
-        num_activations += act_sizes[i];
+        num_activations += align_to_256_bytes(act_sizes[i]);
     }
     float* acts_memory;
     cudaCheck(cudaMalloc((void**)&acts_memory, num_activations * sizeof(float)));
-    // clear the allocated memory since cudaMalloc does not guarantee zeroed memory
     cudaCheck(cudaMemset(acts_memory, 0, num_activations * sizeof(float)));
     float* acts_memory_iterator = acts_memory;
     for (size_t i = 0; i < n; i++) {
         *(targets[i]) = acts_memory_iterator;
-        acts_memory_iterator += act_sizes[i];
+        acts_memory_iterator += align_to_256_bytes(act_sizes[i]);
     }
     return acts_memory;
 }
@@ -385,8 +390,10 @@ void gpt2_forward(GPT2 *model, int* inputs, int B, int T) {
         matmul_forward(l_attproj, l_atty, l_attprojw, l_attprojb, B, T, C, C);
         residual_forward(l_residual2, residual, l_attproj, B*T*C);
         layernorm_forward(l_ln2, l_ln2_mean, l_ln2_rstd, l_residual2, l_ln2w, l_ln2b, B, T, C);
-        matmul_forward(l_fch, l_ln2, l_fcw, l_fcb, B, T, C, 4*C);
-        gelu_forward(l_fch_gelu, l_fch, B*T*4*C);
+        //op16 matmul + bias + GeLU fusion
+        // matmul_forward(l_fch, l_ln2, l_fcw, l_fcb, B, T, C, 4*C);
+        // gelu_forward(l_fch_gelu, l_fch, B*T*4*C);
+        matmul_forward_fused_gelu(l_fch_gelu, l_ln2, l_fcw, l_fcb, B, T, C, 4*C);
         matmul_forward(l_fcproj, l_fch_gelu, l_fcprojw, l_fcprojb, B, T, 4*C, C);
         residual_forward(l_residual3, l_residual2, l_fcproj, B*T*C);
     }