114 lines
17 KiB
Plaintext
114 lines
17 KiB
Plaintext
job 17968510 starting on gpub019.delta.ncsa.illinois.edu
|
|
|
|
|
|
test-gpt2-kernels - TF32 Mode is turned OFF.
|
|
test-gpt2 - [State]
|
|
Batch Size: 4
|
|
Sequence Length: 64
|
|
GPT2 - Allocated 221 MiB for activations
|
|
test-gpt2 - Printing 10 Expected vs Actual Logits:
|
|
-43.431618, -43.431740
|
|
-39.836346, -39.836452
|
|
-43.065910, -43.066021
|
|
-42.828045, -42.828144
|
|
-43.529541, -43.529644
|
|
-44.318398, -44.318497
|
|
-41.227425, -41.227528
|
|
-41.270760, -41.270863
|
|
-42.541393, -42.541523
|
|
-42.394997, -42.395119
|
|
---------------- Logits OK! (max err 0.000043, RMSE 0.000002) ----------------
|
|
_________________________ All tests passed! _________________________
|
|
Collecting data...
|
|
Generating '/tmp/nsys-report-8179.qdstrm'
|
|
|
|
[1/8] [0% ] report_op_8.nsys-rep
|
|
[1/8] [0% ] report_op_8.nsys-rep
|
|
[1/8] [=16% ] report_op_8.nsys-rep
|
|
[1/8] [=17% ] report_op_8.nsys-rep
|
|
[1/8] [==19% ] report_op_8.nsys-rep
|
|
[1/8] [=======36% ] report_op_8.nsys-rep
|
|
[1/8] [===========53% ] report_op_8.nsys-rep
|
|
[1/8] [============54% ] report_op_8.nsys-rep
|
|
[1/8] [============55% ] report_op_8.nsys-rep
|
|
[1/8] [============56% ] report_op_8.nsys-rep
|
|
[1/8] [============57% ] report_op_8.nsys-rep
|
|
[1/8] [==============62% ] report_op_8.nsys-rep
|
|
[1/8] [==================78% ] report_op_8.nsys-rep
|
|
[1/8] [======================92% ] report_op_8.nsys-rep
|
|
[1/8] [========================100%] report_op_8.nsys-rep
|
|
[1/8] [========================100%] report_op_8.nsys-rep
|
|
|
|
[2/8] [0% ] report_op_8.sqlite
|
|
[2/8] [1% ] report_op_8.sqlite
|
|
[2/8] [2% ] report_op_8.sqlite
|
|
[2/8] [3% ] report_op_8.sqlite
|
|
[2/8] [4% ] report_op_8.sqlite
|
|
[2/8] [5% ] report_op_8.sqlite
|
|
[2/8] [6% ] report_op_8.sqlite
|
|
[2/8] [7% ] report_op_8.sqlite
|
|
[2/8] [8% ] report_op_8.sqlite
|
|
[2/8] [9% ] report_op_8.sqlite
|
|
[2/8] [10% ] report_op_8.sqlite
|
|
[2/8] [11% ] report_op_8.sqlite
|
|
[2/8] [12% ] report_op_8.sqlite
|
|
[2/8] [13% ] report_op_8.sqlite
|
|
[2/8] [14% ] report_op_8.sqlite
|
|
[2/8] [=15% ] report_op_8.sqlite
|
|
[2/8] [=16% ] report_op_8.sqlite
|
|
[2/8] [=17% ] report_op_8.sqlite
|
|
[2/8] [==18% ] report_op_8.sqlite
|
|
[2/8] [==19% ] report_op_8.sqlite
|
|
[2/8] [==20% ] report_op_8.sqlite
|
|
[2/8] [==21% ] report_op_8.sqlite
|
|
[2/8] [===22% ] report_op_8.sqlite
|
|
[2/8] [===23% ] report_op_8.sqlite
|
|
[2/8] [===24% ] report_op_8.sqlite
|
|
[2/8] [====25% ] report_op_8.sqlite
|
|
[2/8] [====26% ] report_op_8.sqlite
|
|
[2/8] [====27% ] report_op_8.sqlite
|
|
[2/8] [====28% ] report_op_8.sqlite
|
|
[2/8] [=====29% ] report_op_8.sqlite
|
|
[2/8] [=====30% ] report_op_8.sqlite
|
|
[2/8] [=====31% ] report_op_8.sqlite
|
|
[2/8] [=====32% ] report_op_8.sqlite
|
|
[2/8] [======33% ] report_op_8.sqlite
|
|
[2/8] [======34% ] report_op_8.sqlite
|
|
[2/8] [======35% ] report_op_8.sqlite
|
|
[2/8] [=======36% ] report_op_8.sqlite
|
|
[2/8] [=======37% ] report_op_8.sqlite
|
|
[2/8] [=======38% ] report_op_8.sqlite
|
|
[2/8] [=======39% ] report_op_8.sqlite
|
|
[2/8] [========40% ] report_op_8.sqlite
|
|
[2/8] [========41% ] report_op_8.sqlite
|
|
[2/8] [========42% ] report_op_8.sqlite
|
|
[2/8] [=========43% ] report_op_8.sqlite
|
|
[2/8] [=========44% ] report_op_8.sqlite
|
|
[2/8] [=========45% ] report_op_8.sqlite
|
|
[2/8] [=========46% ] report_op_8.sqlite
|
|
[2/8] [==========47% ] report_op_8.sqlite
|
|
[2/8] [==========48% ] report_op_8.sqlite
|
|
[2/8] [==========49% ] report_op_8.sqlite
|
|
[2/8] [===========50% ] report_op_8.sqlite
|
|
[2/8] [===========51% ] report_op_8.sqlite
|
|
[2/8] [===========52% ] report_op_8.sqlite
|
|
[2/8] [===========53% ] report_op_8.sqlite
|
|
[2/8] [============54% ] report_op_8.sqlite
|
|
[2/8] [============55% ] report_op_8.sqlite
|
|
[2/8] [============56% ] report_op_8.sqlite
|
|
[2/8] [============57% ] report_op_8.sqlite
|
|
[2/8] [=============58% ] report_op_8.sqlite
|
|
[2/8] [=============59% ] report_op_8.sqlite
|
|
[2/8] [=============60% ] report_op_8.sqlite
|
|
[2/8] [==============61% ] report_op_8.sqlite
|
|
[2/8] [==============62% ] report_op_8.sqlite
|
|
[2/8] [==============63% ] report_op_8.sqlite
|
|
[2/8] [==============64% ] report_op_8.sqlite
|
|
[2/8] [===============65% ] report_op_8.sqlite
|
|
[2/8] [===============66% ] report_op_8.sqlite
|
|
[2/8] [===============67% ] report_op_8.sqlite
|
|
[2/8] [================68% ] report_op_8.sqlite
|
|
[2/8] [================69% ] report_op_8.sqlite
|
|
[2/8] [================70% ] report_op_8.sqlite
|
|
[2/8] [================71% ] report_op_8.sqlite
|
|
[2/8] [=================72% ] report_op_8.sqlite
|
|
[2/8] [=================73% ] report_op_8.sqlite
|
|
[2/8] [=================74% ] report_op_8.sqlite
|
|
[2/8] [==================75% ] report_op_8.sqlite
|
|
[2/8] [==================76% ] report_op_8.sqlite
|
|
[2/8] [==================77% ] report_op_8.sqlite
|
|
[2/8] [==================78% ] report_op_8.sqlite
|
|
[2/8] [===================79% ] report_op_8.sqlite
|
|
[2/8] [===================80% ] report_op_8.sqlite
|
|
[2/8] [===================81% ] report_op_8.sqlite
|
|
[2/8] [===================82% ] report_op_8.sqlite
|
|
[2/8] [====================83% ] report_op_8.sqlite
|
|
[2/8] [====================84% ] report_op_8.sqlite
|
|
[2/8] [====================85% ] report_op_8.sqlite
|
|
[2/8] [=====================86% ] report_op_8.sqlite
|
|
[2/8] [=====================87% ] report_op_8.sqlite
|
|
[2/8] [=====================88% ] report_op_8.sqlite
|
|
[2/8] [=====================89% ] report_op_8.sqlite
|
|
[2/8] [======================90% ] report_op_8.sqlite
|
|
[2/8] [======================91% ] report_op_8.sqlite
|
|
[2/8] [======================92% ] report_op_8.sqlite
|
|
[2/8] [=======================93% ] report_op_8.sqlite
|
|
[2/8] [=======================94% ] report_op_8.sqlite
|
|
[2/8] [=======================95% ] report_op_8.sqlite
|
|
[2/8] [=======================96% ] report_op_8.sqlite
|
|
[2/8] [========================97% ] report_op_8.sqlite
|
|
[2/8] [========================98% ] report_op_8.sqlite
|
|
[2/8] [========================99% ] report_op_8.sqlite
|
|
[2/8] [========================100%] report_op_8.sqlite
|
|
[2/8] [========================100%] report_op_8.sqlite
|
|
[3/8] Executing 'nvtx_sum' stats report
|
|
[4/8] Executing 'osrt_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ----------------------
|
|
67.3 1022283375 20 51114168.8 27054794.0 1583 120736053 48517322.9 poll
|
|
20.5 310542000 6 51757000.0 64761.5 7243 218143736 89469059.2 fread
|
|
11.5 175188820 581 301529.8 23153.0 1042 36499184 1835892.4 ioctl
|
|
0.2 2594881 11 235898.3 31449.0 1071 1069584 376213.4 write
|
|
0.2 2357747 25 94309.9 9197.0 7644 1637859 324412.9 mmap64
|
|
0.1 1729901 38 45523.7 6337.0 3606 635861 134756.5 fopen
|
|
0.0 749602 32 23425.1 2199.5 1352 361046 82772.6 fclose
|
|
0.0 589738 17 34690.5 7565.0 1883 392946 93966.9 mmap
|
|
0.0 578494 1 578494.0 578494.0 578494 578494 0.0 pthread_cond_wait
|
|
0.0 374161 2 187080.5 187080.5 171201 202960 22457.0 pthread_create
|
|
0.0 331501 55 6027.3 5911.0 2294 15840 2589.0 open64
|
|
0.0 207579 1 207579.0 207579.0 207579 207579 0.0 fflush
|
|
0.0 124864 1 124864.0 124864.0 124864 124864 0.0 pthread_cond_broadcast
|
|
0.0 76323 1 76323.0 76323.0 76323 76323 0.0 fgets
|
|
0.0 44675 10 4467.5 3977.0 2856 7935 1692.1 munmap
|
|
0.0 39913 6 6652.2 6271.5 3116 9868 2660.1 open
|
|
0.0 25270 9 2807.8 2014.0 1022 7725 2162.0 read
|
|
0.0 21710 3 7236.7 8766.0 3636 9308 3130.0 pipe2
|
|
0.0 18114 2 9057.0 9057.0 5981 12133 4350.1 socket
|
|
0.0 12684 1 12684.0 12684.0 12684 12684 0.0 connect
|
|
0.0 6432 1 6432.0 6432.0 6432 6432 0.0 fopen64
|
|
0.0 3236 2 1618.0 1618.0 1453 1783 233.3 fwrite
|
|
0.0 3096 1 3096.0 3096.0 3096 3096 0.0 bind
|
|
0.0 1273 1 1273.0 1273.0 1273 1273 0.0 listen
|
|
0.0 1032 1 1032.0 1032.0 1032 1032 0.0 fcntl
|
|
|
|
[5/8] Executing 'cuda_api_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ---------------------------------
|
|
85.6 293521673 3 97840557.7 36810988.0 22002 256688683 138790859.1 cudaMemcpy
|
|
10.7 36666998 1 36666998.0 36666998.0 36666998 36666998 0.0 cudaMallocHost
|
|
1.9 6357156 12 529763.0 5770.5 191 2259966 809011.1 cudaFree
|
|
0.7 2461734 1 2461734.0 2461734.0 2461734 2461734 0.0 cudaFreeHost
|
|
0.5 1640935 1 1640935.0 1640935.0 1640935 1640935 0.0 cudaGetDeviceProperties_v2_v12000
|
|
0.3 1027820 171 6010.6 2836.0 2705 408266 31163.4 cudaLaunchKernel
|
|
0.2 808456 6 134742.7 121007.0 5671 361818 133007.2 cudaMalloc
|
|
0.1 236221 50 4724.4 4078.0 3597 14758 1955.6 cudaMemcpyToSymbolAsync
|
|
0.1 183980 838 219.5 180.0 80 5500 215.8 cuGetProcAddress_v2
|
|
0.0 35717 1 35717.0 35717.0 35717 35717 0.0 cudaMemset
|
|
0.0 22080 18 1226.7 265.0 250 11311 2818.0 cudaEventCreateWithFlags
|
|
0.0 13526 4 3381.5 2960.5 1123 6482 2246.3 cudaDeviceSynchronize
|
|
0.0 6731 18 373.9 240.0 210 1874 397.0 cudaEventDestroy
|
|
0.0 5430 4 1357.5 1362.5 1192 1513 131.5 cuInit
|
|
0.0 1493 3 497.7 120.0 120 1253 654.1 cuModuleGetLoadingMode
|
|
0.0 1251 2 625.5 625.5 280 971 488.6 cudaGetDriverEntryPoint_v11030
|
|
|
|
[6/8] Executing 'cuda_gpu_kern_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- --------- --------- -------- -------- ----------- -----------------------------------------------------------------------------------------------
|
|
97.8 243408998 49 4967530.6 4539333.0 1227137 73740560 10131683.4 matmul_forward_kernel(float *, const float *, const float *, const float *, int, int, int, int)
|
|
1.3 3217858 25 128714.3 128672.0 123872 131969 2081.9 layernorm_forward_kernel(float *, float *, float *, const float *, int, int, int)
|
|
0.5 1152128 12 96010.7 96016.0 95840 96192 115.2 preattention_scores_kernel(const float *, const float *, int, int, int, int, float *)
|
|
0.1 346496 12 28874.7 28800.0 28160 29824 545.5 attention_v_kernel(float *, const float *, const float *, int, int, int, int)
|
|
0.1 226400 12 18866.7 18880.0 18656 19008 83.5 gelu_forward_kernel(float *, const float *, int)
|
|
0.1 209731 12 17477.6 17488.5 17280 17696 134.1 softmax_forward_kernel(float *, float, const float *, int, int)
|
|
0.1 145122 24 6046.8 6064.0 5632 6560 361.4 residual_forward_kernel(float *, const float *, const float *, int)
|
|
0.0 96960 12 8080.0 8016.0 7936 8288 135.4 permute_kernel(float *, float *, float *, const float *, int, int, int, int)
|
|
0.0 79648 1 79648.0 79648.0 79648 79648 0.0 encoder_forward_kernel(float *, const int *, const float *, const float *, int, int, int)
|
|
0.0 70560 12 5880.0 5888.0 5856 5888 14.5 unpermute_kernel(const float *, float *, int, int, int, int)
|
|
|
|
[7/8] Executing 'cuda_gpu_mem_time_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation
|
|
-------- --------------- ----- ---------- ---------- -------- -------- ----------- ------------------------------
|
|
80.7 36707943 2 18353971.5 18353971.5 864 36707079 25955213.5 [CUDA memcpy Host-to-Device]
|
|
18.2 8280169 1 8280169.0 8280169.0 8280169 8280169 0.0 [CUDA memcpy Device-to-Host]
|
|
0.9 407233 1 407233.0 407233.0 407233 407233 0.0 [CUDA memset]
|
|
0.2 76416 50 1528.3 1472.0 1376 1952 143.8 [CUDA memcpy Device-to-Device]
|
|
|
|
[8/8] Executing 'cuda_gpu_mem_size_sum' stats report
|
|
|
|
Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation
|
|
---------- ----- -------- -------- -------- -------- ----------- ------------------------------
|
|
497.905 2 248.952 248.952 0.001 497.904 352.070 [CUDA memcpy Host-to-Device]
|
|
232.392 1 232.392 232.392 232.392 232.392 0.000 [CUDA memset]
|
|
51.511 1 51.511 51.511 51.511 51.511 0.000 [CUDA memcpy Device-to-Host]
|
|
0.154 50 0.003 0.003 0.003 0.003 0.000 [CUDA memcpy Device-to-Device]
|
|
|
|
Generated:
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_8.nsys-rep
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_8.sqlite
|