encode-chrombpnet-ATAC-GM18909-ENCSR917WJS-ENCSR731VRA / fold_3 /logs.models.fold_3.ENCSR917WJS /logfile.modelling.fold_3.ENCSR917WJS.stderr.txt
| 2022-09-20 14:17:26.860340: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:22:33.790878: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 14:22:33.793093: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-20 14:22:33.840886: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 14:22:33.840979: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:22:33.867406: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 14:22:33.867473: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 14:22:33.881613: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 14:22:33.887861: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 14:22:33.911421: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 14:22:33.917816: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 14:22:33.919313: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 14:22:33.922282: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 14:22:33.922642: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-20 14:22:33.922723: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 14:22:33.923012: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 14:22:33.923050: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:22:33.923077: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 14:22:33.923102: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 14:22:33.923126: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 14:22:33.923149: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 14:22:33.923173: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 14:22:33.923197: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 14:22:33.923221: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 14:22:33.923654: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 14:22:33.925253: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:22:35.627880: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-20 14:22:35.627982: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-20 14:22:35.628000: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-20 14:22:35.632496: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-09-20 14:22:37.282641: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-20 14:22:37.291380: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz | |
| 2022-09-20 14:22:37.495831: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 14:22:38.581002: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 14:22:38.589599: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 14:23:25.035828: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:23:26.763020: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 14:23:26.763858: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-20 14:23:26.804391: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 14:23:26.804462: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:23:26.807185: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 14:23:26.807238: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 14:23:26.808392: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 14:23:26.808597: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 14:23:26.811381: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 14:23:26.811965: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 14:23:26.812108: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 14:23:26.813187: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 14:23:26.813511: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-20 14:23:26.813595: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 14:23:26.814153: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 14:23:26.814203: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:23:26.814230: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 14:23:26.814253: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 14:23:26.814274: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 14:23:26.814295: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 14:23:26.814317: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 14:23:26.814338: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 14:23:26.814359: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 14:23:26.814791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 14:23:26.814838: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 14:23:27.336360: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-20 14:23:27.336452: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-20 14:23:27.336469: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-20 14:23:27.337538: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-09-20 14:28:51.910710: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-20 14:28:51.911204: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz | |
| 2022-09-20 14:28:53.703409: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 14:28:53.964573: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 14:28:53.982236: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2080s vs `on_train_batch_end` time: 0.2976s). Check your callbacks. | |
| 2022-09-20 19:49:41.941502: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:49:44.735450: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 19:49:44.736431: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-20 19:49:44.770470: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 19:49:44.770534: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:49:44.773335: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 19:49:44.773390: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 19:49:44.774554: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 19:49:44.774783: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 19:49:44.777854: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 19:49:44.778531: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 19:49:44.778679: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 19:49:44.779191: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 19:49:44.779516: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-20 19:49:44.779594: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 19:49:44.779902: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 19:49:44.779934: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:49:44.779963: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 19:49:44.779989: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 19:49:44.780013: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 19:49:44.780038: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 19:49:44.780063: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 19:49:44.780087: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 19:49:44.780112: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 19:49:44.780576: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 19:49:44.780611: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:49:45.341675: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-20 19:49:45.341764: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-20 19:49:45.341781: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-20 19:49:45.343445: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-09-20 19:51:04.159422: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-20 19:51:04.163245: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz | |
| 2022-09-20 19:51:04.260621: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 19:51:04.509381: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 19:51:04.511425: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors. | |
| , UserWarning) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-09-20 19:56:01.593740: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:56:04.157409: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 19:56:04.158436: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-20 19:56:04.198580: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 19:56:04.198649: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:56:04.201506: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 19:56:04.201562: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 19:56:04.202789: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 19:56:04.203009: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 19:56:04.205923: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 19:56:04.206535: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 19:56:04.206683: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 19:56:04.207220: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 19:56:04.207543: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-20 19:56:04.207636: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 19:56:04.207959: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 19:56:04.207990: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:56:04.208021: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 19:56:04.208049: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 19:56:04.208076: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 19:56:04.208102: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 19:56:04.208144: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 19:56:04.208170: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 19:56:04.208196: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 19:56:04.208652: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 19:56:04.208688: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 19:56:04.759918: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-20 19:56:04.760007: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-20 19:56:04.760024: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-20 19:56:04.760916: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-09-20 19:57:19.383310: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-20 19:57:19.385887: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz | |
| 2022-09-20 19:57:19.450580: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 19:57:19.693250: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 19:57:19.694778: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-09-20 20:01:57.110025: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:01:59.954962: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 20:01:59.955935: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-20 20:01:59.993699: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 20:01:59.993765: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:01:59.996600: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:01:59.996654: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:01:59.997827: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 20:01:59.998048: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 20:02:00.000930: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 20:02:00.001548: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 20:02:00.001697: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 20:02:00.002225: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 20:02:00.002548: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-20 20:02:00.002627: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 20:02:00.002923: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 20:02:00.002953: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:02:00.002979: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:02:00.003004: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:02:00.003046: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 20:02:00.003072: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 20:02:00.003096: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 20:02:00.003134: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 20:02:00.003157: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 20:02:00.003607: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 20:02:00.003644: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:02:00.539122: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-20 20:02:00.539211: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-20 20:02:00.539228: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-20 20:02:00.540176: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-09-20 20:03:14.278315: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-20 20:03:14.280258: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz | |
| 2022-09-20 20:03:14.320712: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:03:14.552213: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:03:14.553753: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-09-20 20:04:32.344012: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:04:33.652701: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 20:04:33.653629: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-20 20:04:33.687937: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 20:04:33.688017: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:04:33.690963: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:04:33.691022: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:04:33.692208: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 20:04:33.692420: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 20:04:33.695308: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 20:04:33.695902: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 20:04:33.696047: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 20:04:33.696544: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 20:04:33.696886: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-20 20:04:33.696968: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 20:04:33.697255: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 20:04:33.697295: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:04:33.697324: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:04:33.697351: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:04:33.697377: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 20:04:33.697403: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 20:04:33.697428: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 20:04:33.697454: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 20:04:33.697480: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 20:04:33.697897: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 20:04:33.697942: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:04:34.250619: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-20 20:04:34.250724: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-20 20:04:34.250744: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-20 20:04:34.251610: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-09-20 20:04:51.246839: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-20 20:04:51.247381: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz | |
| 2022-09-20 20:04:51.484782: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:04:51.788343: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:04:51.790324: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 20:04:56.818586: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-09-20 20:04:56.819069: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-09-20 20:04:57.315847: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-09-20 20:04:57.316218: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/ATAC/ENCSR917WJS//chrombpnet_model_feb15_fold_3//footprints’: File exists | |
| 2022-09-20 20:14:01.809503: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:14:03.103379: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 20:14:03.104416: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-20 20:14:03.139304: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 20:14:03.139383: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:14:03.142144: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:14:03.142201: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:14:03.143443: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 20:14:03.143672: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 20:14:03.146629: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 20:14:03.147274: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 20:14:03.147421: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 20:14:03.148271: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 20:14:03.148586: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-20 20:14:03.148664: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-20 20:14:03.148950: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:03:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-20 20:14:03.148987: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:14:03.149012: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:14:03.149037: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:14:03.149060: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-20 20:14:03.149084: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-20 20:14:03.149107: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-20 20:14:03.149130: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-20 20:14:03.149153: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-20 20:14:03.151157: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-20 20:14:03.151205: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-20 20:14:03.696376: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-20 20:14:03.696463: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-20 20:14:03.696479: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-20 20:14:03.697350: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:03:00.0, compute capability: 7.0) | |
| 2022-09-20 20:14:20.318977: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-20 20:14:20.319517: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz | |
| 2022-09-20 20:14:20.480261: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-20 20:14:20.786460: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-20 20:14:20.788320: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |