108 lines
16 KiB
Plaintext
108 lines
16 KiB
Plaintext
job 18002676 starting on gpub022.delta.ncsa.illinois.edu
|
|
|
|
|
|
test-gpt2-kernels - TF32 Mode is turned OFF.
|
|
test-gpt2 - [State]
|
|
Batch Size: 4
|
|
Sequence Length: 64
|
|
GPT2 - Allocated 221 MiB for activations
|
|
test-gpt2 - Printing 10 Expected vs Actual Logits:
|
|
-43.431618, -43.431740
|
|
-39.836346, -39.836452
|
|
-43.065910, -43.066021
|
|
-42.828045, -42.828144
|
|
-43.529541, -43.529644
|
|
-44.318398, -44.318497
|
|
-41.227425, -41.227528
|
|
-41.270760, -41.270863
|
|
-42.541393, -42.541523
|
|
-42.394997, -42.395119
|
|
---------------- Logits OK! (max err 0.000069, RMSE 0.000002) ----------------
|
|
_________________________ All tests passed! _________________________
|
|
Collecting data...
|
|
Generating '/tmp/nsys-report-4b2b.qdstrm'
|
|
|
|
[1/8] [0% ] nsys-report-d663.nsys-rep
|
|
[1/8] [0% ] nsys-report-d663.nsys-rep
|
|
[1/8] [6% ] nsys-report-d663.nsys-rep
|
|
[1/8] [=====29% ] nsys-report-d663.nsys-rep
|
|
[1/8] [===========52% ] nsys-report-d663.nsys-rep
|
|
[1/8] [===========53% ] nsys-report-d663.nsys-rep
|
|
[1/8] [=====32% ] nsys-report-d663.nsys-rep
|
|
[1/8] [===========52% ] nsys-report-d663.nsys-rep
|
|
[1/8] [===========53% ] nsys-report-d663.nsys-rep
|
|
[1/8] [============54% ] nsys-report-d663.nsys-rep
|
|
[1/8] [============57% ] nsys-report-d663.nsys-rep
|
|
[1/8] [=============58% ] nsys-report-d663.nsys-rep
|
|
[1/8] [=============59% ] nsys-report-d663.nsys-rep
|
|
[1/8] [=================74% ] nsys-report-d663.nsys-rep
|
|
[1/8] [======================92% ] nsys-report-d663.nsys-rep
|
|
[1/8] [========================100%] nsys-report-d663.nsys-rep
|
|
[1/8] [========================100%] nsys-report-d663.nsys-rep
|
|
|
|
[2/8] [0% ] nsys-report-472f.sqlite
|
|
[2/8] [1% ] nsys-report-472f.sqlite
|
|
[2/8] [2% ] nsys-report-472f.sqlite
|
|
[2/8] [3% ] nsys-report-472f.sqlite
|
|
[2/8] [4% ] nsys-report-472f.sqlite
|
|
[2/8] [5% ] nsys-report-472f.sqlite
|
|
[2/8] [6% ] nsys-report-472f.sqlite
|
|
[2/8] [7% ] nsys-report-472f.sqlite
|
|
[2/8] [8% ] nsys-report-472f.sqlite
|
|
[2/8] [9% ] nsys-report-472f.sqlite
|
|
[2/8] [10% ] nsys-report-472f.sqlite
|
|
[2/8] [11% ] nsys-report-472f.sqlite
|
|
[2/8] [12% ] nsys-report-472f.sqlite
|
|
[2/8] [13% ] nsys-report-472f.sqlite
|
|
[2/8] [14% ] nsys-report-472f.sqlite
|
|
[2/8] [=15% ] nsys-report-472f.sqlite
|
|
[2/8] [=16% ] nsys-report-472f.sqlite
|
|
[2/8] [=17% ] nsys-report-472f.sqlite
|
|
[2/8] [==18% ] nsys-report-472f.sqlite
|
|
[2/8] [==19% ] nsys-report-472f.sqlite
|
|
[2/8] [==20% ] nsys-report-472f.sqlite
|
|
[2/8] [==21% ] nsys-report-472f.sqlite
|
|
[2/8] [===22% ] nsys-report-472f.sqlite
|
|
[2/8] [===23% ] nsys-report-472f.sqlite
|
|
[2/8] [===24% ] nsys-report-472f.sqlite
|
|
[2/8] [====25% ] nsys-report-472f.sqlite
|
|
[2/8] [====26% ] nsys-report-472f.sqlite
|
|
[2/8] [====27% ] nsys-report-472f.sqlite
|
|
[2/8] [====28% ] nsys-report-472f.sqlite
|
|
[2/8] [=====29% ] nsys-report-472f.sqlite
|
|
[2/8] [=====30% ] nsys-report-472f.sqlite
|
|
[2/8] [=====31% ] nsys-report-472f.sqlite
|
|
[2/8] [=====32% ] nsys-report-472f.sqlite
|
|
[2/8] [======33% ] nsys-report-472f.sqlite
|
|
[2/8] [======34% ] nsys-report-472f.sqlite
|
|
[2/8] [======35% ] nsys-report-472f.sqlite
|
|
[2/8] [=======36% ] nsys-report-472f.sqlite
|
|
[2/8] [=======37% ] nsys-report-472f.sqlite
|
|
[2/8] [=======38% ] nsys-report-472f.sqlite
|
|
[2/8] [=======39% ] nsys-report-472f.sqlite
|
|
[2/8] [========40% ] nsys-report-472f.sqlite
|
|
[2/8] [========41% ] nsys-report-472f.sqlite
|
|
[2/8] [========42% ] nsys-report-472f.sqlite
|
|
[2/8] [=========43% ] nsys-report-472f.sqlite
|
|
[2/8] [=========44% ] nsys-report-472f.sqlite
|
|
[2/8] [=========45% ] nsys-report-472f.sqlite
|
|
[2/8] [=========46% ] nsys-report-472f.sqlite
|
|
[2/8] [==========47% ] nsys-report-472f.sqlite
|
|
[2/8] [==========48% ] nsys-report-472f.sqlite
|
|
[2/8] [==========49% ] nsys-report-472f.sqlite
|
|
[2/8] [===========50% ] nsys-report-472f.sqlite
|
|
[2/8] [===========51% ] nsys-report-472f.sqlite
|
|
[2/8] [===========52% ] nsys-report-472f.sqlite
|
|
[2/8] [===========53% ] nsys-report-472f.sqlite
|
|
[2/8] [============54% ] nsys-report-472f.sqlite
|
|
[2/8] [============55% ] nsys-report-472f.sqlite
|
|
[2/8] [============56% ] nsys-report-472f.sqlite
|
|
[2/8] [============57% ] nsys-report-472f.sqlite
|
|
[2/8] [=============58% ] nsys-report-472f.sqlite
|
|
[2/8] [=============59% ] nsys-report-472f.sqlite
|
|
[2/8] [=============60% ] nsys-report-472f.sqlite
|
|
[2/8] [==============61% ] nsys-report-472f.sqlite
|
|
[2/8] [==============62% ] nsys-report-472f.sqlite
|
|
[2/8] [==============63% ] nsys-report-472f.sqlite
|
|
[2/8] [==============64% ] nsys-report-472f.sqlite
|
|
[2/8] [===============65% ] nsys-report-472f.sqlite
|
|
[2/8] [===============66% ] nsys-report-472f.sqlite
|
|
[2/8] [===============67% ] nsys-report-472f.sqlite
|
|
[2/8] [================68% ] nsys-report-472f.sqlite
|
|
[2/8] [================69% ] nsys-report-472f.sqlite
|
|
[2/8] [================70% ] nsys-report-472f.sqlite
|
|
[2/8] [================71% ] nsys-report-472f.sqlite
|
|
[2/8] [=================72% ] nsys-report-472f.sqlite
|
|
[2/8] [=================73% ] nsys-report-472f.sqlite
|
|
[2/8] [=================74% ] nsys-report-472f.sqlite
|
|
[2/8] [==================75% ] nsys-report-472f.sqlite
|
|
[2/8] [==================76% ] nsys-report-472f.sqlite
|
|
[2/8] [==================77% ] nsys-report-472f.sqlite
|
|
[2/8] [==================78% ] nsys-report-472f.sqlite
|
|
[2/8] [===================79% ] nsys-report-472f.sqlite
|
|
[2/8] [===================80% ] nsys-report-472f.sqlite
|
|
[2/8] [===================81% ] nsys-report-472f.sqlite
|
|
[2/8] [===================82% ] nsys-report-472f.sqlite
|
|
[2/8] [====================83% ] nsys-report-472f.sqlite
|
|
[2/8] [====================84% ] nsys-report-472f.sqlite
|
|
[2/8] [====================85% ] nsys-report-472f.sqlite
|
|
[2/8] [=====================86% ] nsys-report-472f.sqlite
|
|
[2/8] [=====================87% ] nsys-report-472f.sqlite
|
|
[2/8] [=====================88% ] nsys-report-472f.sqlite
|
|
[2/8] [=====================89% ] nsys-report-472f.sqlite
|
|
[2/8] [======================90% ] nsys-report-472f.sqlite
|
|
[2/8] [======================91% ] nsys-report-472f.sqlite
|
|
[2/8] [======================92% ] nsys-report-472f.sqlite
|
|
[2/8] [=======================93% ] nsys-report-472f.sqlite
|
|
[2/8] [=======================94% ] nsys-report-472f.sqlite
|
|
[2/8] [=======================95% ] nsys-report-472f.sqlite
|
|
[2/8] [=======================96% ] nsys-report-472f.sqlite
|
|
[2/8] [========================97% ] nsys-report-472f.sqlite
|
|
[2/8] [========================98% ] nsys-report-472f.sqlite
|
|
[2/8] [========================99% ] nsys-report-472f.sqlite
|
|
[2/8] [========================100%] nsys-report-472f.sqlite
|
|
[2/8] [========================100%] nsys-report-472f.sqlite
|
|
[3/8] Executing 'nvtx_sum' stats report
|
|
[4/8] Executing 'osrt_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ----------------------
|
|
65.5 1046221746 20 52311087.3 36977154.5 1343 144034477 47744624.9 poll
|
|
25.3 403984081 582 694130.7 495337.5 1012 19219062 1320637.1 ioctl
|
|
7.8 124190546 6 20698424.3 67121.0 7685 115338096 46492858.4 fread
|
|
0.5 7610123 1 7610123.0 7610123.0 7610123 7610123 0.0 pthread_cond_wait
|
|
0.4 7042952 1 7042952.0 7042952.0 7042952 7042952 0.0 pthread_cond_broadcast
|
|
0.1 2350546 25 94021.8 9598.0 8175 1635631 323841.6 mmap64
|
|
0.1 1607267 38 42296.5 6812.5 3627 684381 126942.5 fopen
|
|
0.1 872311 11 79301.0 27341.0 1052 372016 110728.5 write
|
|
0.0 662102 32 20690.7 2364.0 1362 311784 71411.8 fclose
|
|
0.0 599354 17 35256.1 4849.0 2225 375633 91066.5 mmap
|
|
0.0 363971 2 181985.5 181985.5 164879 199092 24192.2 pthread_create
|
|
0.0 355093 55 6456.2 6452.0 2405 12613 2601.2 open64
|
|
0.0 207257 1 207257.0 207257.0 207257 207257 0.0 fflush
|
|
0.0 77074 1 77074.0 77074.0 77074 77074 0.0 fgets
|
|
0.0 39155 8 4894.4 5225.0 2806 6762 1570.3 munmap
|
|
0.0 36088 6 6014.7 5986.5 3056 8997 2029.8 open
|
|
0.0 21803 3 7267.7 8657.0 3928 9218 2905.8 pipe2
|
|
0.0 18696 2 9348.0 9348.0 5150 13546 5936.9 socket
|
|
0.0 17072 8 2134.0 1773.5 1132 3908 1047.0 read
|
|
0.0 10660 1 10660.0 10660.0 10660 10660 0.0 connect
|
|
0.0 6462 1 6462.0 6462.0 6462 6462 0.0 fopen64
|
|
0.0 3516 2 1758.0 1758.0 1412 2104 489.3 fwrite
|
|
0.0 3275 3 1091.7 1082.0 1071 1122 26.8 fcntl
|
|
0.0 1634 1 1634.0 1634.0 1634 1634 0.0 bind
|
|
|
|
[5/8] Executing 'cuda_api_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ---------------------------------
|
|
85.9 292488145 3 97496048.3 36522068.0 19767 255946310 138430374.4 cudaMemcpy
|
|
5.8 19849256 147 135029.0 2775.0 2665 18870790 1556382.3 cudaLaunchKernel
|
|
2.9 9971082 1 9971082.0 9971082.0 9971082 9971082 0.0 cudaGetDeviceProperties_v2_v12000
|
|
2.6 8726842 6 1454473.7 1517760.0 3446 2841288 1477680.0 cudaMalloc
|
|
1.6 5319195 12 443266.3 5159.5 171 1876362 633355.6 cudaFree
|
|
0.7 2439536 1 2439536.0 2439536.0 2439536 2439536 0.0 cudaFreeHost
|
|
0.4 1329358 1 1329358.0 1329358.0 1329358 1329358 0.0 cudaMallocHost
|
|
0.1 173678 838 207.3 190.0 80 1042 105.9 cuGetProcAddress_v2
|
|
0.0 35757 1 35757.0 35757.0 35757 35757 0.0 cudaMemset
|
|
0.0 19408 18 1078.2 276.0 250 7765 1931.9 cudaEventCreateWithFlags
|
|
0.0 12392 4 3098.0 2845.0 1002 5700 2009.4 cudaDeviceSynchronize
|
|
0.0 7253 18 402.9 285.0 200 1854 379.7 cudaEventDestroy
|
|
0.0 5671 4 1417.8 1428.0 1263 1552 141.7 cuInit
|
|
0.0 1483 3 494.3 151.0 110 1222 630.5 cuModuleGetLoadingMode
|
|
0.0 1262 2 631.0 631.0 461 801 240.4 cudaGetDriverEntryPoint_v11030
|
|
|
|
[6/8] Executing 'cuda_gpu_kern_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- --------- --------- -------- -------- ----------- ----------------------------------------------------------------------------------------------------
|
|
97.6 243397681 49 4967299.6 4539935.0 1227263 73746344 10132529.6 matmul_forward_kernel(float *, const float *, const float *, const float *, int, int, int, int)
|
|
1.4 3576415 25 143056.6 142880.0 141472 146976 1318.1 layernorm_forward_kernel(float *, float *, float *, const float *, const float *, const float *, in…
|
|
0.8 1879295 12 156607.9 156656.0 155584 157120 383.9 void flash_attention_forward_kernel<(int)64, (int)64, (int)64>(const float *, const float *, const …
|
|
0.1 228672 12 19056.0 19040.0 18880 19264 90.0 gelu_forward_kernel(float *, const float *, int)
|
|
0.1 146208 24 6092.0 6080.0 5728 6560 355.1 residual_forward_kernel(float *, const float *, const float *, int)
|
|
0.0 80768 1 80768.0 80768.0 80768 80768 0.0 encoder_forward_kernel(float *, const int *, const float *, const float *, int, int, int)
|
|
0.0 78560 12 6546.7 6480.0 6400 6912 158.2 permute_kernel(float *, float *, float *, const float *, int, int, int, int)
|
|
0.0 37952 12 3162.7 3152.0 3040 3296 108.2 unpermute_kernel(const float *, float *, int, int, int, int)
|
|
|
|
[7/8] Executing 'cuda_gpu_mem_time_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation
|
|
-------- --------------- ----- ---------- ---------- -------- -------- ----------- ----------------------------
|
|
82.5 36437716 2 18218858.0 18218858.0 832 36436884 25764179.4 [CUDA memcpy Host-to-Device]
|
|
16.5 7304990 1 7304990.0 7304990.0 7304990 7304990 0.0 [CUDA memcpy Device-to-Host]
|
|
0.9 406816 1 406816.0 406816.0 406816 406816 0.0 [CUDA memset]
|
|
|
|
[8/8] Executing 'cuda_gpu_mem_size_sum' stats report
|
|
|
|
Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation
|
|
---------- ----- -------- -------- -------- -------- ----------- ----------------------------
|
|
497.905 2 248.952 248.952 0.001 497.904 352.070 [CUDA memcpy Host-to-Device]
|
|
232.392 1 232.392 232.392 232.392 232.392 0.000 [CUDA memset]
|
|
51.511 1 51.511 51.511 51.511 51.511 0.000 [CUDA memcpy Device-to-Host]
|
|
|
|
Generated:
|
|
/tmp/nsys-report-d663.nsys-rep
|
|
/tmp/nsys-report-472f.sqlite
|