ECE 408 sem project
This commit is contained in:
Aryan Singh committed
1 parent
1d79ccd2e3
commit
fa3efcaf07
139 files changed
+11634
-1
No files matched your search
@@ -0,0 +1,498 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <assert.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <math.h>
|
||||
#include <float.h>
|
||||
#include <cublas_v2.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <iostream>
|
||||
|
||||
// Include CPU kernels
|
||||
#include "cpu_kernels/attention.cuh"
|
||||
#include "cpu_kernels/encoder.cuh"
|
||||
#include "cpu_kernels/gelu.cuh"
|
||||
#include "cpu_kernels/layernorm.cuh"
|
||||
#include "cpu_kernels/matmul.cuh"
|
||||
#include "cpu_kernels/residual.cuh"
|
||||
|
||||
// Include GPU kernels
|
||||
#include "kernels/attention.cuh"
|
||||
#include "kernels/encoder.cuh"
|
||||
#include "kernels/gelu.cuh"
|
||||
#include "kernels/layernorm.cuh"
|
||||
#include "kernels/matmul.cuh"
|
||||
#include "kernels/residual.cuh"
|
||||
#include "kernels/softmax.cuh"
|
||||
|
||||
// Include CUDA utilities
|
||||
#include "utils/cuda_utils.cuh"
|
||||
|
||||
int main() {
|
||||
|
||||
// set up the device
|
||||
int deviceIdx = 0;
|
||||
cudaCheck(cudaSetDevice(deviceIdx));
|
||||
cudaDeviceProp deviceProp;
|
||||
cudaGetDeviceProperties(&deviceProp, deviceIdx);
|
||||
|
||||
// setup cuBLAS and cuBLASLt
|
||||
cublasCheck(cublasCreate(&cublas_handle));
|
||||
// TF32 precision is equivalent to torch.set_float32_matmul_precision('high')
|
||||
int enable_tf32 = 0;
|
||||
cublas_compute_type = enable_tf32 ? CUBLAS_COMPUTE_32F_FAST_TF32 : CUBLAS_COMPUTE_32F;
|
||||
cublasMath_t cublas_math_mode = enable_tf32 ? CUBLAS_TF32_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH;
|
||||
cublasCheck(cublasSetMathMode(cublas_handle, cublas_math_mode));
|
||||
|
||||
const int B = 4; // Batch size
|
||||
const int T = 128; // Sequence length
|
||||
const int C = 128; // Input channels
|
||||
const int OC = 4 * C; // Output channels
|
||||
|
||||
const int NH = 12; // Number of heads
|
||||
|
||||
const float error_threshold = 0; // TODO: Set the error threshold for GPU vs CPU output comparisons
|
||||
// This is in unit of percentage difference in output values
|
||||
// Change this to a small value (like 0.01) for your own testing
|
||||
|
||||
const float ep = 0.0001; // Epsilon value for percentage calculation to avoid getting infinity
|
||||
|
||||
// used in multi-head self-attention
|
||||
const int HS = C / NH; // head size
|
||||
|
||||
// Generate random input data
|
||||
srand(time(NULL));
|
||||
|
||||
/*------------------------ encoder_forward test ------------------------*/
|
||||
|
||||
// Allocate memory for input data
|
||||
int* enc_inp_cpu = (int*)malloc(B * T * sizeof(int));
|
||||
float* wte_enc = (float*)malloc(C * C * sizeof(float));
|
||||
float* wpe_enc = (float*)malloc(T * C * sizeof(float));
|
||||
|
||||
for (int i = 0; i < B * T; ++i) {
|
||||
enc_inp_cpu[i] = (int)rand() / RAND_MAX;
|
||||
}
|
||||
for (int i = 0; i < C * C; ++i) {
|
||||
wte_enc[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
for (int i = 0; i < T * C; ++i) {
|
||||
wpe_enc[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
|
||||
// copy data to GPU
|
||||
int* enc_inp_gpu_d;
|
||||
float* wte_enc_d;
|
||||
float* wpe_enc_d;
|
||||
cudaMalloc(&enc_inp_gpu_d, B * T * sizeof(int));
|
||||
cudaMalloc(&wte_enc_d, C * C * sizeof(float));
|
||||
cudaMalloc(&wpe_enc_d, T * C * sizeof(float));
|
||||
cudaMemcpy(enc_inp_gpu_d, enc_inp_cpu, B * T * sizeof(int), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(wte_enc_d, wte_enc, C * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(wpe_enc_d, wpe_enc, T * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
|
||||
|
||||
// Allocate memory for output
|
||||
float* enc_out_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* enc_out_gpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* enc_out_gpu_d;
|
||||
cudaMalloc(&enc_out_gpu_d, B * T * C * sizeof(float));
|
||||
|
||||
// Call CPU function for reference output
|
||||
encoder_forward_cpu(enc_out_cpu, enc_inp_cpu, wte_enc, wpe_enc, B, T, C);
|
||||
|
||||
// TODO: Call your GPU function
|
||||
// encoder_forward(enc_out_gpu_d, enc_inp_gpu_d, wte_enc_d, wpe_enc_d, B, T, C);
|
||||
|
||||
// TODO: Copy output from your GPU function back to CPU
|
||||
// cudaMemcpy(..., cudaMemcpyDeviceToHost);
|
||||
|
||||
int enc_totElements = B * T * C;
|
||||
int enc_error = 0;
|
||||
for (int i = 0; i < enc_totElements; i++) {
|
||||
if (fabs(enc_out_cpu[i] - enc_out_gpu[i]) / (enc_out_cpu[i] + ep) > error_threshold) {
|
||||
enc_error++;
|
||||
}
|
||||
}
|
||||
|
||||
if (enc_error > 0) {
|
||||
printf("+++++++++++++++ encoder_forward test failed with %d differing values +++++++++++++++\n", enc_error);
|
||||
printf("+++++++++++++++ Check your implementation or error_threshold value +++++++++++++++\n");
|
||||
} else {
|
||||
printf("--------------------encoder_forward test passed!--------------------\n");
|
||||
}
|
||||
|
||||
/*------------------------ layernorm_forward test ------------------------*/
|
||||
|
||||
// Allocate memory for input data
|
||||
float* lay_inp_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* weight_ln = (float*)malloc(C * sizeof(float));
|
||||
float* bias_ln = (float*)malloc(C * sizeof(float));
|
||||
float* rstd_ln = (float*)malloc(B * T * sizeof(float));
|
||||
float* mean_ln = (float*)malloc(B * T * sizeof(float));
|
||||
|
||||
// Generate random input data
|
||||
for (int i = 0; i < B * T * C; ++i) {
|
||||
lay_inp_cpu[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
for (int i = 0; i < B * T; ++i) {
|
||||
rstd_ln[i] = (float)rand() / RAND_MAX;
|
||||
mean_ln[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
for (int i = 0; i < C; ++i) {
|
||||
weight_ln[i] = (float)rand() / RAND_MAX;
|
||||
bias_ln[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
|
||||
// copy data to GPU
|
||||
float* lay_inp_gpu_d;
|
||||
float* weight_ln_d;
|
||||
float* bias_ln_d;
|
||||
float* rstd_ln_d;
|
||||
float* mean_ln_d;
|
||||
cudaMalloc(&lay_inp_gpu_d, B * T * C * sizeof(float));
|
||||
cudaMalloc(&weight_ln_d, C * sizeof(float));
|
||||
cudaMalloc(&bias_ln_d, C * sizeof(float));
|
||||
cudaMalloc(&rstd_ln_d, B * T * sizeof(float));
|
||||
cudaMalloc(&mean_ln_d, B * T * sizeof(float));
|
||||
cudaMemcpy(lay_inp_gpu_d, lay_inp_cpu, B * T * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(weight_ln_d, weight_ln, C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(bias_ln_d, bias_ln, C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(rstd_ln_d, rstd_ln, B * T * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(mean_ln_d, mean_ln, B * T * sizeof(float), cudaMemcpyHostToDevice);
|
||||
|
||||
// Allocate memory for output
|
||||
float* lay_out_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* lay_out_gpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* lay_out_gpu_d;
|
||||
cudaMalloc(&lay_out_gpu_d, B * T * C * sizeof(float));
|
||||
|
||||
// Call CPU function for reference output
|
||||
layernorm_forward_cpu(lay_out_cpu, mean_ln, rstd_ln, lay_inp_cpu, weight_ln, bias_ln, B, T, C);
|
||||
|
||||
// TODO: Call your GPU function
|
||||
// layernorm_forward(lay_out_gpu_d, mean_ln_d, rstd_ln_d, lay_inp_gpu_d, weight_ln_d, bias_ln_d, B, T, C);
|
||||
|
||||
// TODO: Copy output from your GPU function back to CPU
|
||||
// cudaMemcpy(..., cudaMemcpyDeviceToHost);
|
||||
|
||||
int lay_totElements = B * T * C;
|
||||
int lay_error = 0;
|
||||
for (int i = 0; i < lay_totElements; i++) {
|
||||
if (fabs(lay_out_cpu[i] - lay_out_gpu[i])/(lay_out_cpu[i] + ep) > error_threshold) {
|
||||
lay_error++;
|
||||
}
|
||||
}
|
||||
|
||||
if (lay_error > 0) {
|
||||
printf("+++++++++++++++ layernorm_forward test failed with %d differing values +++++++++++++++\n", lay_error);
|
||||
printf("+++++++++++++++ Check your implementation or error_threshold value +++++++++++++++\n");
|
||||
} else {
|
||||
printf("--------------------layernorm_forward test passed!--------------------\n");
|
||||
}
|
||||
|
||||
/*------------------------ attention_forward test ------------------------*/
|
||||
|
||||
// Allocate memory for input data
|
||||
int inp_size = B * T * max(3 * C, NH * T);
|
||||
float* att_inp_cpu = (float*)malloc(inp_size * sizeof(float));
|
||||
float* qkvr = (float*)malloc(3 * B * T * C * sizeof(float));
|
||||
float* att = (float*)malloc(B * NH * NH * T * T * sizeof(float));
|
||||
// create separate copy of input for GPU to avoid input memory being overwritten
|
||||
float* att_inp_gpu = (float*)malloc(inp_size * sizeof(float));
|
||||
|
||||
// Generate random input data
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
att_inp_cpu[i] = (float)rand() / RAND_MAX;
|
||||
att_inp_gpu[i] = att_inp_cpu[i];
|
||||
}
|
||||
|
||||
// copy data to GPU
|
||||
float* att_inp_gpu_d;
|
||||
float* qkvr_d;
|
||||
float* att_d;
|
||||
cudaMalloc(&att_inp_gpu_d, inp_size * sizeof(float));
|
||||
cudaMalloc(&qkvr_d, 3 * B * T * C * sizeof(float));
|
||||
cudaMalloc(&att_d, B * NH * NH * T * T * sizeof(float));
|
||||
cudaMemcpy(att_inp_gpu_d, att_inp_gpu, inp_size * sizeof(float), cudaMemcpyHostToDevice);
|
||||
|
||||
// Allocate memory for output
|
||||
float* att_out_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* att_out_gpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* att_out_gpu_d;
|
||||
cudaMalloc(&att_out_gpu_d, B * T * C * sizeof(float));
|
||||
|
||||
// Call CPU function for reference output
|
||||
attention_forward_cpu(att_out_cpu, qkvr, att, att_inp_cpu, B, T, C, NH);
|
||||
|
||||
// TODO: Call your GPU function
|
||||
// attention_forward(att_out_gpu_d, qkvr_d, att_d, att_inp_gpu_d, B, T, C, NH);
|
||||
|
||||
// TODO: Copy output from your GPU function back to CPU
|
||||
// cudaMemcpy(..., cudaMemcpyDeviceToHost);
|
||||
|
||||
int att_totElements = B * T * C;
|
||||
int att_diff = 0;
|
||||
// Loop through the results to count differences
|
||||
for (int i = 0; i < att_totElements; ++i) {
|
||||
// Check if the result differs by more than the tolerance
|
||||
if (fabs(att_out_cpu[i] - att_out_gpu[i])/(att_out_cpu[i] + ep) > error_threshold) {
|
||||
++att_diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (att_diff > 0) {
|
||||
printf("+++++++++++++++ attention_forward test failed with %d differing values +++++++++++++++\n", att_diff);
|
||||
printf("+++++++++++++++ Check your implementation or error_threshold value +++++++++++++++\n");
|
||||
printf("++++++++ Try testing each attention component separately for easier debugging ++++++++\n");
|
||||
} else {
|
||||
printf("--------------------attention_forward test passed!--------------------\n");
|
||||
}
|
||||
|
||||
/*------------------------ residual_forward test ------------------------*/
|
||||
|
||||
// Allocate memory for input data
|
||||
float* res_inp1_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* res_inp2_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* res_out_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* res_out_gpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
|
||||
// Generate random input data
|
||||
for (int i = 0; i < B * T * C; ++i) {
|
||||
res_inp1_cpu[i] = (float)rand() / RAND_MAX;
|
||||
res_inp2_cpu[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
|
||||
// copy data to GPU
|
||||
float* res_inp1_gpu_d;
|
||||
float* res_inp2_gpu_d;
|
||||
float* res_out_gpu_d;
|
||||
cudaMalloc(&res_inp1_gpu_d, B * T * C * sizeof(float));
|
||||
cudaMalloc(&res_inp2_gpu_d, B * T * C * sizeof(float));
|
||||
cudaMalloc(&res_out_gpu_d, B * T * C * sizeof(float));
|
||||
cudaMemcpy(res_inp1_gpu_d, res_inp1_cpu, B * T * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(res_inp2_gpu_d, res_inp2_cpu, B * T * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
|
||||
// Call CPU function for reference output
|
||||
residual_forward_cpu(res_out_cpu, res_inp1_cpu, res_inp2_cpu, B * T * C);
|
||||
|
||||
// TODO: Call your GPU function
|
||||
// residual_forward(res_out_gpu_d, res_inp1_gpu_d, res_inp2_gpu_d, B * T * C);
|
||||
|
||||
// TODO: Copy output from your GPU function back to CPU
|
||||
// cudaMemcpy(..., cudaMemcpyDeviceToHost);
|
||||
|
||||
int res_totElements = B * T * C;
|
||||
int res_diff = 0;
|
||||
// Loop through the results to count differences
|
||||
for (int i = 0; i < res_totElements; ++i) {
|
||||
// Check if the result differs by more than the tolerance
|
||||
if (fabs(res_out_cpu[i] - res_out_gpu[i])/ (res_out_cpu[i] + ep) > error_threshold) {
|
||||
++res_diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (res_diff > 0) {
|
||||
printf("+++++++++++++++ residual_forward test failed with %d differing values +++++++++++++++\n", res_diff);
|
||||
printf("+++++++++++++++ Check your implementation or error_threshold value +++++++++++++++\n");
|
||||
} else {
|
||||
printf("--------------------residual_forward test passed!--------------------\n");
|
||||
}
|
||||
|
||||
/*------------------------ gelu_forward test ------------------------*/
|
||||
|
||||
// Allocate memory for input data
|
||||
float* gelu_inp_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* gelu_out_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* gelu_out_gpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
|
||||
// Generate random input data
|
||||
for (int i = 0; i < B * T * C; ++i) {
|
||||
gelu_inp_cpu[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
|
||||
// copy data to GPU
|
||||
float* gelu_inp_gpu_d;
|
||||
float* gelu_out_gpu_d;
|
||||
cudaMalloc(&gelu_inp_gpu_d, B * T * C * sizeof(float));
|
||||
cudaMalloc(&gelu_out_gpu_d, B * T * C * sizeof(float));
|
||||
cudaMemcpy(gelu_inp_gpu_d, gelu_inp_cpu, B * T * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
|
||||
// Call CPU function for reference output
|
||||
gelu_forward_cpu(gelu_out_cpu, gelu_inp_cpu, B * T * C);
|
||||
|
||||
// TODO: Call your GPU function
|
||||
// gelu_forward(gelu_out_gpu_d, gelu_inp_gpu_d, B * T * C);
|
||||
|
||||
// TODO: Copy output from your GPU function back to CPU
|
||||
// cudaMemcpy(..., cudaMemcpyDeviceToHost);
|
||||
|
||||
int gelu_totElements = B * T * C;
|
||||
int gelu_diff = 0;
|
||||
for (int i = 0; i < gelu_totElements; ++i) {
|
||||
if (fabs(gelu_out_cpu[i] - gelu_out_gpu[i])/(gelu_out_cpu[i] + ep) > error_threshold) {
|
||||
++gelu_diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (gelu_diff > 0) {
|
||||
printf("+++++++++++++++ gelu_forward test failed with %d differing values +++++++++++++++\n", gelu_diff);
|
||||
printf("+++++++++++++++ Check your implementation or error_threshold value +++++++++++++++\n");
|
||||
} else {
|
||||
printf("--------------------gelu_forward test passed!--------------------\n");
|
||||
}
|
||||
|
||||
/*------------------------ matmul_forward test ------------------------*/
|
||||
|
||||
// Allocate memory for input data
|
||||
float* mat_inp_cpu = (float*)malloc(B * T * C * sizeof(float));
|
||||
float* matmul_weight = (float*)malloc(4 * C * C * sizeof(float));
|
||||
float* matmul_bias = (float*)malloc(4 * C * sizeof(float));
|
||||
float* mat_out_cpu = (float*)malloc(B * T * 4 * C * sizeof(float));
|
||||
|
||||
// Generate random input data
|
||||
for (int i = 0; i < B * T * C; ++i) {
|
||||
mat_inp_cpu[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
for (int i = 0; i < 4 * C * C; ++i) {
|
||||
matmul_weight[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
for (int i = 0; i < 4 * C; ++i) {
|
||||
matmul_bias[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
|
||||
// copy data to GPU
|
||||
float* mat_inp_gpu_d;
|
||||
float* matmul_weight_d;
|
||||
float* matmul_bias_d;
|
||||
float* mat_out_gpu_d;
|
||||
float* mat_out_gpu = (float*)malloc(B * T * 4 * C * sizeof(float));
|
||||
cudaMalloc(&mat_inp_gpu_d, B * T * C * sizeof(float));
|
||||
cudaMalloc(&matmul_weight_d, 4 * C * C * sizeof(float));
|
||||
cudaMalloc(&matmul_bias_d, 4 * C * sizeof(float));
|
||||
cudaMalloc(&mat_out_gpu_d, B * T * 4 * C * sizeof(float));
|
||||
cudaMemcpy(mat_inp_gpu_d, mat_inp_cpu, B * T * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(matmul_weight_d, matmul_weight, 4 * C * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
cudaMemcpy(matmul_bias_d, matmul_bias, 4 * C * sizeof(float), cudaMemcpyHostToDevice);
|
||||
|
||||
// Call CPU function for reference output
|
||||
matmul_forward_cpu(mat_out_cpu, mat_inp_cpu, matmul_weight, matmul_bias, B, T, C, OC);
|
||||
|
||||
// TODO: Call your GPU function
|
||||
// matmul_forward(mat_out_gpu_d, mat_inp_gpu_d, matmul_weight_d, matmul_bias_d, B, T, C, OC);
|
||||
|
||||
// TODO: Copy output from your GPU function back to CPU
|
||||
// cudaMemcpy(..., cudaMemcpyDeviceToHost);
|
||||
|
||||
int mat_totElements = B * T * 4 * C;
|
||||
int mat_diff = 0;
|
||||
for (int i = 0; i < mat_totElements; ++i) {
|
||||
if (fabs(mat_out_cpu[i] - mat_out_gpu[i])/(mat_out_cpu[i] + ep) > error_threshold) {
|
||||
++mat_diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (mat_diff > 0) {
|
||||
printf("+++++++++++++++ matmul_forward test failed with %d differing values +++++++++++++++\n", mat_diff);
|
||||
printf("+++++++++++++++ Check your implementation or error_threshold value +++++++++++++++\n");
|
||||
} else {
|
||||
printf("--------------------matmul_forward test passed!--------------------\n");
|
||||
}
|
||||
|
||||
printf("********************* All Random Tests Finished! *********************\n");
|
||||
|
||||
// Free allocated memory
|
||||
|
||||
free(enc_inp_cpu);
|
||||
free(enc_out_cpu);
|
||||
free(wte_enc);
|
||||
free(wpe_enc);
|
||||
free(enc_out_gpu);
|
||||
cudaFree(enc_inp_gpu_d);
|
||||
cudaFree(wte_enc_d);
|
||||
cudaFree(wpe_enc_d);
|
||||
cudaFree(enc_out_gpu_d);
|
||||
|
||||
free(lay_inp_cpu);
|
||||
free(lay_out_cpu);
|
||||
free(weight_ln);
|
||||
free(bias_ln);
|
||||
free(rstd_ln);
|
||||
free(mean_ln);
|
||||
free(lay_out_gpu);
|
||||
cudaFree(lay_inp_gpu_d);
|
||||
cudaFree(weight_ln_d);
|
||||
cudaFree(bias_ln_d);
|
||||
cudaFree(rstd_ln_d);
|
||||
cudaFree(mean_ln_d);
|
||||
cudaFree(lay_out_gpu_d);
|
||||
|
||||
free(perm_inp_cpu);
|
||||
free(perm_q_cpu);
|
||||
free(perm_k_cpu);
|
||||
free(perm_v_cpu);
|
||||
free(perm_q_gpu);
|
||||
free(perm_k_gpu);
|
||||
free(perm_v_gpu);
|
||||
cudaFree(perm_inp_gpu_d);
|
||||
cudaFree(perm_q_d);
|
||||
cudaFree(perm_k_d);
|
||||
cudaFree(perm_v_d);
|
||||
|
||||
free(unperm_inp_cpu);
|
||||
free(unperm_out_cpu);
|
||||
free(unperm_out_gpu);
|
||||
cudaFree(unperm_inp_gpu_d);
|
||||
cudaFree(unperm_out_gpu_d);
|
||||
|
||||
free(softmax_inp_cpu);
|
||||
free(softmax_inp_gpu);
|
||||
free(softmax_out_cpu);
|
||||
free(softmax_out_gpu);
|
||||
cudaFree(softmax_inp_gpu_d);
|
||||
cudaFree(softmax_out_gpu_d);
|
||||
|
||||
free(att_inp_cpu);
|
||||
free(att_out_cpu);
|
||||
free(att_inp_gpu);
|
||||
free(att_out_gpu);
|
||||
free(qkvr);
|
||||
free(att);
|
||||
cudaFree(att_inp_gpu_d);
|
||||
cudaFree(qkvr_d);
|
||||
cudaFree(att_d);
|
||||
cudaFree(att_out_gpu_d);
|
||||
|
||||
free(res_inp1_cpu);
|
||||
free(res_inp2_cpu);
|
||||
free(res_out_cpu);
|
||||
free(res_out_gpu);
|
||||
cudaFree(res_inp1_gpu_d);
|
||||
cudaFree(res_inp2_gpu_d);
|
||||
cudaFree(res_out_gpu_d);
|
||||
|
||||
free(gelu_inp_cpu);
|
||||
free(gelu_out_cpu);
|
||||
free(gelu_out_gpu);
|
||||
cudaFree(gelu_inp_gpu_d);
|
||||
cudaFree(gelu_out_gpu_d);
|
||||
|
||||
free(mat_inp_cpu);
|
||||
free(mat_out_gpu);
|
||||
free(matmul_weight);
|
||||
free(matmul_bias);
|
||||
cudaFree(mat_inp_gpu_d);
|
||||
cudaFree(matmul_weight_d);
|
||||
cudaFree(matmul_bias_d);
|
||||
cudaFree(mat_out_gpu_d);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user