110 lines
16 KiB
Plaintext
110 lines
16 KiB
Plaintext
job 17968383 starting on gpub013.delta.ncsa.illinois.edu
|
|
|
|
|
|
test-gpt2-kernels - TF32 Mode is turned OFF.
|
|
test-gpt2 - [State]
|
|
Batch Size: 4
|
|
Sequence Length: 64
|
|
GPT2 - Allocated 221 MiB for activations
|
|
test-gpt2 - Printing 10 Expected vs Actual Logits:
|
|
-43.431618, -43.431740
|
|
-39.836346, -39.836452
|
|
-43.065910, -43.066021
|
|
-42.828045, -42.828144
|
|
-43.529541, -43.529644
|
|
-44.318398, -44.318497
|
|
-41.227425, -41.227528
|
|
-41.270760, -41.270863
|
|
-42.541393, -42.541523
|
|
-42.394997, -42.395119
|
|
---------------- Logits OK! (max err 0.000043, RMSE 0.000002) ----------------
|
|
_________________________ All tests passed! _________________________
|
|
Collecting data...
|
|
Generating '/tmp/nsys-report-ec00.qdstrm'
|
|
|
|
[1/8] [0% ] report_op_9.nsys-rep
|
|
[1/8] [0% ] report_op_9.nsys-rep
|
|
[1/8] [5% ] report_op_9.nsys-rep
|
|
[1/8] [====28% ] report_op_9.nsys-rep
|
|
[1/8] [===========51% ] report_op_9.nsys-rep
|
|
[1/8] [======34% ] report_op_9.nsys-rep
|
|
[1/8] [===========51% ] report_op_9.nsys-rep
|
|
[1/8] [===========52% ] report_op_9.nsys-rep
|
|
[1/8] [===========53% ] report_op_9.nsys-rep
|
|
[1/8] [============54% ] report_op_9.nsys-rep
|
|
[1/8] [============55% ] report_op_9.nsys-rep
|
|
[1/8] [============56% ] report_op_9.nsys-rep
|
|
[1/8] [============57% ] report_op_9.nsys-rep
|
|
[1/8] [=============58% ] report_op_9.nsys-rep
|
|
[1/8] [=============59% ] report_op_9.nsys-rep
|
|
[1/8] [==================77% ] report_op_9.nsys-rep
|
|
[1/8] [======================91% ] report_op_9.nsys-rep
|
|
[1/8] [========================100%] report_op_9.nsys-rep
|
|
[1/8] [========================100%] report_op_9.nsys-rep
|
|
|
|
[2/8] [0% ] report_op_9.sqlite
|
|
[2/8] [1% ] report_op_9.sqlite
|
|
[2/8] [2% ] report_op_9.sqlite
|
|
[2/8] [3% ] report_op_9.sqlite
|
|
[2/8] [4% ] report_op_9.sqlite
|
|
[2/8] [5% ] report_op_9.sqlite
|
|
[2/8] [6% ] report_op_9.sqlite
|
|
[2/8] [7% ] report_op_9.sqlite
|
|
[2/8] [8% ] report_op_9.sqlite
|
|
[2/8] [9% ] report_op_9.sqlite
|
|
[2/8] [10% ] report_op_9.sqlite
|
|
[2/8] [11% ] report_op_9.sqlite
|
|
[2/8] [12% ] report_op_9.sqlite
|
|
[2/8] [13% ] report_op_9.sqlite
|
|
[2/8] [14% ] report_op_9.sqlite
|
|
[2/8] [=15% ] report_op_9.sqlite
|
|
[2/8] [=16% ] report_op_9.sqlite
|
|
[2/8] [=17% ] report_op_9.sqlite
|
|
[2/8] [==18% ] report_op_9.sqlite
|
|
[2/8] [==19% ] report_op_9.sqlite
|
|
[2/8] [==20% ] report_op_9.sqlite
|
|
[2/8] [==21% ] report_op_9.sqlite
|
|
[2/8] [===22% ] report_op_9.sqlite
|
|
[2/8] [===23% ] report_op_9.sqlite
|
|
[2/8] [===24% ] report_op_9.sqlite
|
|
[2/8] [====25% ] report_op_9.sqlite
|
|
[2/8] [====26% ] report_op_9.sqlite
|
|
[2/8] [====27% ] report_op_9.sqlite
|
|
[2/8] [====28% ] report_op_9.sqlite
|
|
[2/8] [=====29% ] report_op_9.sqlite
|
|
[2/8] [=====30% ] report_op_9.sqlite
|
|
[2/8] [=====31% ] report_op_9.sqlite
|
|
[2/8] [=====32% ] report_op_9.sqlite
|
|
[2/8] [======33% ] report_op_9.sqlite
|
|
[2/8] [======34% ] report_op_9.sqlite
|
|
[2/8] [======35% ] report_op_9.sqlite
|
|
[2/8] [=======36% ] report_op_9.sqlite
|
|
[2/8] [=======37% ] report_op_9.sqlite
|
|
[2/8] [=======38% ] report_op_9.sqlite
|
|
[2/8] [=======39% ] report_op_9.sqlite
|
|
[2/8] [========40% ] report_op_9.sqlite
|
|
[2/8] [========41% ] report_op_9.sqlite
|
|
[2/8] [========42% ] report_op_9.sqlite
|
|
[2/8] [=========43% ] report_op_9.sqlite
|
|
[2/8] [=========44% ] report_op_9.sqlite
|
|
[2/8] [=========45% ] report_op_9.sqlite
|
|
[2/8] [=========46% ] report_op_9.sqlite
|
|
[2/8] [==========47% ] report_op_9.sqlite
|
|
[2/8] [==========48% ] report_op_9.sqlite
|
|
[2/8] [==========49% ] report_op_9.sqlite
|
|
[2/8] [===========50% ] report_op_9.sqlite
|
|
[2/8] [===========51% ] report_op_9.sqlite
|
|
[2/8] [===========52% ] report_op_9.sqlite
|
|
[2/8] [===========53% ] report_op_9.sqlite
|
|
[2/8] [============54% ] report_op_9.sqlite
|
|
[2/8] [============55% ] report_op_9.sqlite
|
|
[2/8] [============56% ] report_op_9.sqlite
|
|
[2/8] [============57% ] report_op_9.sqlite
|
|
[2/8] [=============58% ] report_op_9.sqlite
|
|
[2/8] [=============59% ] report_op_9.sqlite
|
|
[2/8] [=============60% ] report_op_9.sqlite
|
|
[2/8] [==============61% ] report_op_9.sqlite
|
|
[2/8] [==============62% ] report_op_9.sqlite
|
|
[2/8] [==============63% ] report_op_9.sqlite
|
|
[2/8] [==============64% ] report_op_9.sqlite
|
|
[2/8] [===============65% ] report_op_9.sqlite
|
|
[2/8] [===============66% ] report_op_9.sqlite
|
|
[2/8] [===============67% ] report_op_9.sqlite
|
|
[2/8] [================68% ] report_op_9.sqlite
|
|
[2/8] [================69% ] report_op_9.sqlite
|
|
[2/8] [================70% ] report_op_9.sqlite
|
|
[2/8] [================71% ] report_op_9.sqlite
|
|
[2/8] [=================72% ] report_op_9.sqlite
|
|
[2/8] [=================73% ] report_op_9.sqlite
|
|
[2/8] [=================74% ] report_op_9.sqlite
|
|
[2/8] [==================75% ] report_op_9.sqlite
|
|
[2/8] [==================76% ] report_op_9.sqlite
|
|
[2/8] [==================77% ] report_op_9.sqlite
|
|
[2/8] [==================78% ] report_op_9.sqlite
|
|
[2/8] [===================79% ] report_op_9.sqlite
|
|
[2/8] [===================80% ] report_op_9.sqlite
|
|
[2/8] [===================81% ] report_op_9.sqlite
|
|
[2/8] [===================82% ] report_op_9.sqlite
|
|
[2/8] [====================83% ] report_op_9.sqlite
|
|
[2/8] [====================84% ] report_op_9.sqlite
|
|
[2/8] [====================85% ] report_op_9.sqlite
|
|
[2/8] [=====================86% ] report_op_9.sqlite
|
|
[2/8] [=====================87% ] report_op_9.sqlite
|
|
[2/8] [=====================88% ] report_op_9.sqlite
|
|
[2/8] [=====================89% ] report_op_9.sqlite
|
|
[2/8] [======================90% ] report_op_9.sqlite
|
|
[2/8] [======================91% ] report_op_9.sqlite
|
|
[2/8] [======================92% ] report_op_9.sqlite
|
|
[2/8] [=======================93% ] report_op_9.sqlite
|
|
[2/8] [=======================94% ] report_op_9.sqlite
|
|
[2/8] [=======================95% ] report_op_9.sqlite
|
|
[2/8] [=======================96% ] report_op_9.sqlite
|
|
[2/8] [========================97% ] report_op_9.sqlite
|
|
[2/8] [========================98% ] report_op_9.sqlite
|
|
[2/8] [========================99% ] report_op_9.sqlite
|
|
[2/8] [========================100%] report_op_9.sqlite
|
|
[2/8] [========================100%] report_op_9.sqlite
|
|
[3/8] Executing 'nvtx_sum' stats report
|
|
[4/8] Executing 'osrt_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ----------------------
|
|
72.6 792125713 18 44006984.1 22081891.5 1563 100126637 45893643.6 poll
|
|
12.9 141279538 6 23546589.7 15352175.0 7734 76623723 30441839.7 fread
|
|
12.4 135292558 580 233263.0 22111.5 1042 20047399 980798.2 ioctl
|
|
0.8 8667478 38 228091.5 6211.5 3536 7839121 1269544.8 fopen
|
|
0.6 7020827 1 7020827.0 7020827.0 7020827 7020827 0.0 pthread_cond_broadcast
|
|
0.2 2351517 25 94060.7 8677.0 7273 1636246 324069.4 mmap64
|
|
0.2 1904648 10 190464.8 28103.0 21490 803937 283644.9 write
|
|
0.1 655628 32 20488.4 2274.0 1373 286927 69823.0 fclose
|
|
0.1 586901 17 34523.6 4980.0 2384 383689 91890.5 mmap
|
|
0.0 336080 2 168040.0 168040.0 147897 188183 28486.5 pthread_create
|
|
0.0 322394 55 5861.7 5971.0 2254 13936 2322.7 open64
|
|
0.0 222768 1 222768.0 222768.0 222768 222768 0.0 fflush
|
|
0.0 80561 1 80561.0 80561.0 80561 80561 0.0 pthread_cond_wait
|
|
0.0 77105 1 77105.0 77105.0 77105 77105 0.0 fgets
|
|
0.0 38674 6 6445.7 6247.0 3537 9528 2196.9 open
|
|
0.0 36928 8 4616.0 4478.0 2985 6963 1491.7 munmap
|
|
0.0 25617 2 12808.5 12808.5 11451 14166 1919.8 socket
|
|
0.0 20939 3 6979.7 8236.0 4388 8315 2244.8 pipe2
|
|
0.0 14717 7 2102.4 1914.0 1032 3597 1008.0 read
|
|
0.0 10630 1 10630.0 10630.0 10630 10630 0.0 connect
|
|
0.0 6502 1 6502.0 6502.0 6502 6502 0.0 fopen64
|
|
0.0 3517 2 1758.5 1758.5 1343 2174 587.6 fwrite
|
|
0.0 1983 1 1983.0 1983.0 1983 1983 0.0 bind
|
|
0.0 1202 1 1202.0 1202.0 1202 1202 0.0 fcntl
|
|
|
|
[5/8] Executing 'cuda_api_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ---------------------------------
|
|
95.2 292665484 3 97555161.3 36209486.0 21961 256434037 138777725.5 cudaMemcpy
|
|
1.7 5116658 12 426388.2 6582.0 170 1779706 610228.6 cudaFree
|
|
1.0 3123114 171 18263.8 2885.0 2755 2481560 189524.2 cudaLaunchKernel
|
|
0.8 2449450 1 2449450.0 2449450.0 2449450 2449450 0.0 cudaFreeHost
|
|
0.5 1532732 1 1532732.0 1532732.0 1532732 1532732 0.0 cudaGetDeviceProperties_v2_v12000
|
|
0.5 1478801 1 1478801.0 1478801.0 1478801 1478801 0.0 cudaMallocHost
|
|
0.3 816982 6 136163.7 117796.0 3527 382948 140753.7 cudaMalloc
|
|
0.1 187447 838 223.7 151.0 70 32301 1113.0 cuGetProcAddress_v2
|
|
0.0 34615 1 34615.0 34615.0 34615 34615 0.0 cudaMemset
|
|
0.0 16342 18 907.9 305.0 250 6673 1587.6 cudaEventCreateWithFlags
|
|
0.0 13825 4 3456.3 3000.0 1032 6793 2455.7 cudaDeviceSynchronize
|
|
0.0 7285 18 404.7 260.5 200 2224 469.9 cudaEventDestroy
|
|
0.0 5220 4 1305.0 1287.5 1152 1493 140.7 cuInit
|
|
0.0 1262 3 420.7 140.0 100 1022 521.2 cuModuleGetLoadingMode
|
|
0.0 831 2 415.5 415.5 250 581 234.1 cudaGetDriverEntryPoint_v11030
|
|
|
|
[6/8] Executing 'cuda_gpu_kern_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- --------- --------- -------- -------- ----------- ----------------------------------------------------------------------------------------------------
|
|
97.6 243432571 49 4968011.7 4540305.0 1227228 73745324 10132336.9 matmul_forward_kernel(float *, const float *, const float *, const float *, int, int, int, int)
|
|
1.4 3577301 25 143092.0 143039.0 140960 146752 1309.3 layernorm_forward_kernel(float *, float *, float *, const float *, const float *, const float *, in…
|
|
0.5 1152060 12 96005.0 95984.0 95872 96128 81.5 preattention_scores_kernel(const float *, const float *, int, int, int, int, float *)
|
|
0.1 347807 12 28983.9 28816.0 28575 29984 463.4 attention_v_kernel(float *, const float *, const float *, int, int, int, int)
|
|
0.1 227615 12 18967.9 18960.0 18752 19072 88.6 gelu_forward_kernel(float *, const float *, int)
|
|
0.1 209791 12 17482.6 17488.0 17280 17888 184.8 softmax_forward_kernel(float *, float, const float *, int, int)
|
|
0.1 146176 24 6090.7 6144.0 5696 6560 350.5 residual_forward_kernel(float *, const float *, const float *, int)
|
|
0.0 97408 12 8117.3 8112.0 7872 8320 116.0 permute_kernel(float *, float *, float *, const float *, int, int, int, int)
|
|
0.0 79679 1 79679.0 79679.0 79679 79679 0.0 encoder_forward_kernel(float *, const int *, const float *, const float *, int, int, int)
|
|
0.0 70528 12 5877.3 5888.0 5856 5888 15.8 unpermute_kernel(const float *, float *, int, int, int, int)
|
|
|
|
[7/8] Executing 'cuda_gpu_mem_time_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation
|
|
-------- --------------- ----- ---------- ---------- -------- -------- ----------- ----------------------------
|
|
82.0 36088520 2 18044260.0 18044260.0 864 36087656 25517215.3 [CUDA memcpy Host-to-Device]
|
|
17.0 7499463 1 7499463.0 7499463.0 7499463 7499463 0.0 [CUDA memcpy Device-to-Host]
|
|
0.9 407838 1 407838.0 407838.0 407838 407838 0.0 [CUDA memset]
|
|
|
|
[8/8] Executing 'cuda_gpu_mem_size_sum' stats report
|
|
|
|
Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation
|
|
---------- ----- -------- -------- -------- -------- ----------- ----------------------------
|
|
497.905 2 248.952 248.952 0.001 497.904 352.070 [CUDA memcpy Host-to-Device]
|
|
232.392 1 232.392 232.392 232.392 232.392 0.000 [CUDA memset]
|
|
51.511 1 51.511 51.511 51.511 51.511 0.000 [CUDA memcpy Device-to-Host]
|
|
|
|
Generated:
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_9.nsys-rep
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_9.sqlite
|