ECE 408 sem project
This commit is contained in:
Aryan Singh committed
1 parent
1d79ccd2e3
commit
fa3efcaf07
139 files changed
+11634
-1
No files matched your search
@@ -0,0 +1,139 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifndef ATTENTION_CUH
|
||||
#define ATTENTION_CUH
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
|
||||
void permute_kernel_cpu(float* q, float* k, float* v,
|
||||
const float* inp,
|
||||
int B, int T, int NH, int d) {
|
||||
for (int b = 0; b < B; ++b) {
|
||||
for (int nh_idx = 0; nh_idx < NH; ++nh_idx) {
|
||||
for (int n = 0; n < T; ++n) {
|
||||
for (int d_ = 0; d_ < d; ++d_) {
|
||||
int inp_idx = (b * T * 3 * NH * d) + (n * 3 * NH * d) + (0 * NH * d) + (nh_idx * d) + d_;
|
||||
q[b * NH * T * d + nh_idx * T * d + n * d + d_] = inp[inp_idx];
|
||||
k[b * NH * T * d + nh_idx * T * d + n * d + d_] = inp[inp_idx + NH * d];
|
||||
v[b * NH * T * d + nh_idx * T * d + n * d + d_] = inp[inp_idx + 2 * (NH * d)];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void unpermute_kernel_cpu(float* inp, float* out, int B, int T, int NH, int d) {
|
||||
for (int b = 0; b < B; ++b) {
|
||||
for (int n = 0; n < T; ++n) {
|
||||
for (int nh_idx = 0; nh_idx < NH; ++nh_idx) {
|
||||
for (int d_ = 0; d_ < d; ++d_) {
|
||||
int other_idx = (b * NH * T * d) + (n * NH * d) + (nh_idx * d) + d_;
|
||||
out[other_idx] = inp[b * NH * T * d + nh_idx * T * d + n * d + d_];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void softmax_forward_cpu(float* out, float inv_temperature, const float* inp, int N, int T) {
|
||||
for (int idx = 0; idx < N * T; ++idx) {
|
||||
// The thread's position in its current row
|
||||
int own_pos = idx % T;
|
||||
|
||||
// Pointer to the current row
|
||||
const float* x = inp + idx * T;
|
||||
|
||||
float maxval = -FLT_MAX;
|
||||
// Think about what this for loop condition is doing (Note the significance of i <= own_pos)
|
||||
for (int i = 0; i <= own_pos; ++i) {
|
||||
maxval = fmaxf(maxval, x[i]);
|
||||
}
|
||||
|
||||
// Compute softmax
|
||||
float sumval = 0.0f;
|
||||
for (int i = 0; i <= own_pos; ++i) {
|
||||
float ev = expf(inv_temperature * (x[i] - maxval));
|
||||
sumval += ev;
|
||||
out[idx * T + i] = ev;
|
||||
}
|
||||
|
||||
// Normalize
|
||||
float norm = 1.0f / sumval;
|
||||
for (int i = 0; i <= own_pos; ++i) {
|
||||
out[idx * T + i] *= norm;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void attention_forward_cpu(float* out, float* qkvr, float* att,
|
||||
float* inp,
|
||||
int B, int T, int C, int NH) {
|
||||
// Note: `inp` is re-used as a scratch buffer.
|
||||
// Its contents will be overwritten by this function.
|
||||
|
||||
// inp is (B, T, 3C) QKV
|
||||
// preatt, att are (B, NH, T, T)
|
||||
// output is (B, T, C)
|
||||
int HS = C / NH; // head size
|
||||
|
||||
// permute and separate inp from (B, T, 3, NH, HS) to 3X (B, NH, T, HS)
|
||||
float *q, *k, *v;
|
||||
q = qkvr + 0 * B * T * C;
|
||||
k = qkvr + 1 * B * T * C;
|
||||
v = qkvr + 2 * B * T * C;
|
||||
|
||||
permute_kernel_cpu(q, k, v, inp, B, T, NH, HS);
|
||||
|
||||
// Attention matmul: Q @ K^T
|
||||
// Compute pre-attention scores (B, NH, T, T)
|
||||
float* preatt = inp;
|
||||
for (int b = 0; b < B; b++) {
|
||||
for (int nh = 0; nh < NH; nh++) {
|
||||
for (int t1 = 0; t1 < T; t1++) {
|
||||
for (int t2 = 0; t2 < T; t2++) {
|
||||
float sum = 0.0f;
|
||||
for (int hs = 0; hs < HS; hs++) {
|
||||
sum += k[b * NH * T * HS + nh * T * HS + t2 * HS + hs] *
|
||||
q[b * NH * T * HS + nh * T * HS + t1 * HS + hs];
|
||||
}
|
||||
preatt[b * NH * T * T + nh * T * T + t1 * T + t2] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute the softmax
|
||||
float scale = 1.0 / sqrtf(HS);
|
||||
softmax_forward_cpu(att, scale, preatt, B * NH, T);
|
||||
|
||||
|
||||
float* vaccum = inp;
|
||||
// Attention matmul: P @ V, where P holds the attention probabilities
|
||||
// (B, NH, T, T) @ (B, NH, T, HS) -> (B, NH, T, HS)
|
||||
for (int b = 0; b < B; ++b) {
|
||||
for (int nh = 0; nh < NH; ++nh) {
|
||||
for (int t = 0; t < T; ++t) {
|
||||
for (int hs = 0; hs < HS; ++hs) {
|
||||
float sum = 0.0f;
|
||||
for (int t2 = 0; t2 < T; ++t2) {
|
||||
sum += att[b * NH * T * T + nh * T * T + t * T + t2] *
|
||||
v[b * NH * T * HS + nh * T * HS + t2 * HS + hs];
|
||||
}
|
||||
vaccum[b * NH * T * HS + nh * T * HS + t * HS + hs] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unpermute_kernel_cpu(vaccum, out, B, T, NH, HS);
|
||||
}
|
||||
|
||||
#endif // ATTENTION_CUH
|
||||
@@ -0,0 +1,27 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifndef ENCODER_CUH
|
||||
#define ENCODER_CUH
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
|
||||
void encoder_forward_cpu(float* out, const int* inp, const float* wte, const float* wpe,
|
||||
int B, int T, int C) {
|
||||
for (int b = 0; b < B; ++b) {
|
||||
for (int token_pos = 0; token_pos < T; ++token_pos) {
|
||||
for (int c = 0; c < C; ++c) {
|
||||
int token = inp[b * T + token_pos];
|
||||
out[b * T * C + token_pos * C + c] = wte[token * C + c] + wpe[token_pos * C + c];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif // ENCODER_CUH
|
||||
@@ -0,0 +1,26 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifndef GELU_CUH
|
||||
#define GELU_CUH
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
|
||||
#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
|
||||
|
||||
void gelu_forward_cpu(float* out, const float* inp, int N) {
|
||||
for (int i = 0; i < N; ++i) {
|
||||
float val = inp[i];
|
||||
float cube = 0.044715f * val * val * val;
|
||||
// Think about what GELU is, are we computing the exact gelu function here?
|
||||
out[i] = 0.5f * val * (1.0f + tanhf(GELU_SCALING_FACTOR * (val + cube)));
|
||||
}
|
||||
}
|
||||
|
||||
#endif // GELU_CUH
|
||||
@@ -0,0 +1,43 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifndef LAYERNORM_CUH
|
||||
#define LAYERNORM_CUH
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
|
||||
void layernorm_forward_cpu(float* out, float* mean, float* rstd, float* inp, float* weight, float* bias,
|
||||
int B, int T, int C) {
|
||||
for (int b = 0; b < B; ++b) {
|
||||
for (int t = 0; t < T; ++t) {
|
||||
// For each token in the batch, normalize across the channel dimension
|
||||
float sum = 0.0f;
|
||||
for (int c = 0; c < C; ++c) {
|
||||
sum += inp[b * T * C + t * C + c];
|
||||
}
|
||||
float m = sum / C;
|
||||
|
||||
sum = 0.0f;
|
||||
for (int c = 0; c < C; ++c) {
|
||||
float diff = inp[b * T * C + t * C + c] - m;
|
||||
sum += diff * diff;
|
||||
}
|
||||
// Standard deviation of the channel values
|
||||
// but think about why we take the reciprocal
|
||||
float s = rsqrtf(sum / C + 1e-5f);
|
||||
|
||||
for (int c = 0; c < C; ++c) {
|
||||
float n = s * (inp[b * T * C + t * C + c] - m);
|
||||
out[b * T * C + t * C + c] = n * weight[c] + bias[c];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif // LAYERNORM_CUH
|
||||
@@ -0,0 +1,76 @@
|
||||
#ifndef LOCAL_ATTENTION_CUH
|
||||
#define LOCAL_ATTENTION_CUH
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
#include "attention.cuh"
|
||||
|
||||
#define WINDOW_SIZE 128
|
||||
|
||||
void local_attention_forward_cpu(float* out, float* qkvr, float* inp,
|
||||
int B, int T, int NH, int HS) {
|
||||
|
||||
float *q, *k, *v;
|
||||
q = qkvr + 0 * B * T * NH * HS;
|
||||
k = qkvr + 1 * B * T * NH * HS;
|
||||
v = qkvr + 2 * B * T * NH * HS;
|
||||
|
||||
// permute and separate inp from (B, T, 3, NH, HS) to 3X (B, NH, T, HS)
|
||||
permute_kernel_cpu(q, k, v, inp, B, T, NH, HS);
|
||||
|
||||
float scale = 1.0 / sqrtf(HS);
|
||||
float* preatt = inp;
|
||||
for(int b = 0; b < B; b++){
|
||||
for(int nh = 0; nh < NH; nh++){
|
||||
for(int q_t = 0; q_t < T; q_t++){
|
||||
// weights buffer for the current window
|
||||
float* weights = (float*) calloc(WINDOW_SIZE, sizeof(float));
|
||||
|
||||
// causal window start for keys
|
||||
int k_start = max(0, q_t - WINDOW_SIZE);
|
||||
|
||||
// raw attention scores within the window
|
||||
for(int k_t = k_start; k_t <= q_t; k_t++){
|
||||
float sum = 0.0f;
|
||||
for(int hs = 0; hs < HS; hs++){
|
||||
sum += k[b * NH * T * HS + nh * T * HS + k_t * HS + hs] *
|
||||
q[b * NH * T * HS + nh * T * HS + q_t * HS + hs];
|
||||
}
|
||||
weights[q_t - k_t] = sum;
|
||||
}
|
||||
|
||||
// softmax over the window
|
||||
float maxval = -FLT_MAX;
|
||||
for(int k_t = k_start; k_t <= q_t; k_t++){
|
||||
maxval = fmaxf(maxval, weights[q_t - k_t]);
|
||||
}
|
||||
float sumval = 0.0f;
|
||||
for(int k_t = k_start; k_t <= q_t; k_t++){
|
||||
float ev = expf(scale * (weights[q_t - k_t] - maxval));
|
||||
sumval += ev;
|
||||
weights[q_t - k_t] = ev;
|
||||
}
|
||||
|
||||
for(int k_t = k_start; k_t <= q_t; k_t++){
|
||||
weights[q_t - k_t] /= sumval;
|
||||
}
|
||||
|
||||
// weighted sum of values
|
||||
for(int hs = 0; hs < HS; hs++){
|
||||
float sum = 0;
|
||||
for(int k_t = k_start; k_t <= q_t; k_t++){
|
||||
sum += weights[q_t - k_t] * v[b * NH * T * HS + nh * T * HS + k_t * HS + hs];
|
||||
}
|
||||
preatt[b * NH * T * HS + nh * T * HS + q_t * HS + hs] = sum;
|
||||
}
|
||||
|
||||
free(weights);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unpermute_kernel_cpu(preatt, out, B, T, NH, HS);
|
||||
}
|
||||
|
||||
#endif // LOCAL_ATTENTION_CUH
|
||||
@@ -0,0 +1,35 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifndef MATMUL_CUH
|
||||
#define MATMUL_CUH
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
|
||||
void matmul_forward_cpu(float* out, const float* inp, const float* weight, const float* bias,
|
||||
int B, int T, int C, int OC) {
|
||||
// Input (B,T,C) @ Weight (OC, C) + Bias is (OC) = Output (B,T,OC)
|
||||
for (int b = 0; b < B; b++) {
|
||||
for (int t = 0; t < T; t++) {
|
||||
float* out_token_ptr = out + b * T * OC + t * OC;
|
||||
const float* inp_token_ptr = inp + b * T * C + t * C;
|
||||
for (int o = 0; o < OC; o++) {
|
||||
float acc = (bias != NULL) ? bias[o] : 0.0f;
|
||||
const float* weight_row_ptr = weight + o*C;
|
||||
// Dot product of the input token values and corresponding row of weight values
|
||||
for (int i = 0; i < C; i++) {
|
||||
acc += inp_token_ptr[i] * weight_row_ptr[i];
|
||||
}
|
||||
out_token_ptr[o] = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif // MATMUL_CUH
|
||||
@@ -0,0 +1,22 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifndef RESIDUAL_CUH
|
||||
#define RESIDUAL_CUH
|
||||
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
|
||||
void residual_forward_cpu(float* out, const float* inp1, const float* inp2, int N) {
|
||||
// Element-wise addition for residual connections
|
||||
for (int i = 0; i < N; ++i) {
|
||||
out[i] = inp1[i] + inp2[i];
|
||||
}
|
||||
}
|
||||
|
||||
#endif // RESIDUAL_CUH
|
||||
Reference in new issue
Block a user