encode-chrombpnet-DNASE-H9-ENCSR964WHY-ENCSR878NYN / fold_0 /logs.models.fold_0.ENCSR964WHY /logfile.modelling.fold_0.ENCSR964WHY.stderr.txt
| INFO: underlay of /etc/localtime required more than 50 (88) bind mounts | |
| INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts | |
| INFO: underlay of /etc/localtime required more than 50 (88) bind mounts | |
| INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts | |
| 2023-03-25 01:03:05.639246: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:14:08.859934: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 01:14:08.866205: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2023-03-25 01:14:08.912589: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 01:14:08.912702: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:14:09.266921: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 01:14:09.267125: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 01:14:09.287993: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 01:14:09.433277: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 01:14:09.526485: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 01:14:09.550694: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 01:14:09.556246: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 01:14:09.559131: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 01:14:09.559626: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2023-03-25 01:14:09.561004: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 01:14:09.561701: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 01:14:09.561759: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:14:09.561796: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 01:14:09.561822: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 01:14:09.561847: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 01:14:09.561871: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 01:14:09.561894: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 01:14:09.561918: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 01:14:09.561942: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 01:14:09.563415: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 01:14:09.565529: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:14:13.079154: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2023-03-25 01:14:13.079258: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2023-03-25 01:14:13.079275: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2023-03-25 01:14:13.084472: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30120 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:44:00.0, compute capability: 7.0) | |
| 2023-03-25 01:14:14.604722: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2023-03-25 01:14:14.628135: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2500050000 Hz | |
| 2023-03-25 01:14:14.915677: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 01:14:17.729469: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 01:14:17.742439: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 01:15:04.200771: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:15:06.563609: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 01:15:06.564693: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2023-03-25 01:15:06.622791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 01:15:06.622897: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:15:06.626426: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 01:15:06.626556: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 01:15:06.628223: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 01:15:06.628719: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 01:15:06.631556: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 01:15:06.632439: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 01:15:06.632861: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 01:15:06.633791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 01:15:06.634170: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2023-03-25 01:15:06.634282: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 01:15:06.634857: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 01:15:06.634891: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:15:06.634915: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 01:15:06.634932: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 01:15:06.634950: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 01:15:06.634967: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 01:15:06.634984: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 01:15:06.635001: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 01:15:06.635017: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 01:15:06.635766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 01:15:06.635811: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 01:15:07.181133: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2023-03-25 01:15:07.181241: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2023-03-25 01:15:07.181257: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2023-03-25 01:15:07.182718: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30120 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:44:00.0, compute capability: 7.0) | |
| 2023-03-25 01:21:34.098110: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2023-03-25 01:21:34.098653: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2500050000 Hz | |
| 2023-03-25 01:21:36.000213: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 01:21:36.291207: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 01:21:36.310240: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1751s vs `on_train_batch_end` time: 0.2527s). Check your callbacks. | |
| 2023-03-25 06:43:37.537800: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:43:41.678315: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 06:43:41.679391: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2023-03-25 06:43:41.734406: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 06:43:41.734509: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:43:41.737764: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:43:41.737839: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:43:41.739145: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 06:43:41.739516: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 06:43:41.742227: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 06:43:41.742965: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 06:43:41.743386: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 06:43:41.745430: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 06:43:41.745822: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2023-03-25 06:43:41.745931: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 06:43:41.746389: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 06:43:41.746421: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:43:41.746455: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:43:41.746474: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:43:41.746493: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 06:43:41.746511: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 06:43:41.746529: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 06:43:41.746547: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 06:43:41.746566: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 06:43:41.747293: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 06:43:41.747355: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:43:42.348890: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2023-03-25 06:43:42.348998: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2023-03-25 06:43:42.349014: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2023-03-25 06:43:42.353076: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30120 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:44:00.0, compute capability: 7.0) | |
| 2023-03-25 06:46:36.537720: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2023-03-25 06:46:36.542438: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2500050000 Hz | |
| 2023-03-25 06:46:36.664265: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:46:37.007163: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:46:37.010175: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors. | |
| , UserWarning) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2023-03-25 06:51:51.503624: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:51:54.778243: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 06:51:54.779402: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2023-03-25 06:51:54.839761: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 06:51:54.839856: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:51:54.842741: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:51:54.842820: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:51:54.844186: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 06:51:54.844561: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 06:51:54.847453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 06:51:54.848240: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 06:51:54.848661: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 06:51:54.849620: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 06:51:54.849997: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2023-03-25 06:51:54.850150: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 06:51:54.850631: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 06:51:54.850662: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:51:54.850685: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:51:54.850704: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:51:54.850722: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 06:51:54.850740: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 06:51:54.850758: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 06:51:54.850776: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 06:51:54.850794: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 06:51:54.851549: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 06:51:54.851593: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:51:55.450451: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2023-03-25 06:51:55.450561: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2023-03-25 06:51:55.450577: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2023-03-25 06:51:55.451996: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30120 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:44:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2023-03-25 06:53:39.424894: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2023-03-25 06:53:39.428035: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2500050000 Hz | |
| 2023-03-25 06:53:39.504507: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:53:39.834970: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:53:39.836750: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2023-03-25 06:58:36.416021: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:58:39.731604: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 06:58:39.732751: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2023-03-25 06:58:39.794799: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 06:58:39.794897: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:58:39.797817: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:58:39.797917: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:58:39.799563: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 06:58:39.800003: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 06:58:39.803463: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 06:58:39.804402: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 06:58:39.804913: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 06:58:39.805926: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 06:58:39.806356: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2023-03-25 06:58:39.806482: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 06:58:39.807036: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 06:58:39.807073: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:58:39.807120: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 06:58:39.807146: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 06:58:39.807169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 06:58:39.807190: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 06:58:39.807212: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 06:58:39.807233: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 06:58:39.807255: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 06:58:39.814387: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 06:58:39.814442: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 06:58:40.434202: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2023-03-25 06:58:40.434320: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2023-03-25 06:58:40.434337: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2023-03-25 06:58:40.435797: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30120 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:44:00.0, compute capability: 7.0) | |
| 2023-03-25 07:00:25.478647: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2023-03-25 07:00:25.480881: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2500050000 Hz | |
| 2023-03-25 07:00:25.530980: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 07:00:25.828995: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 07:00:25.830849: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2023-03-25 07:02:05.936120: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:02:07.666513: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 07:02:07.667646: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2023-03-25 07:02:07.723337: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 07:02:07.723461: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:02:07.726880: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 07:02:07.727032: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 07:02:07.728697: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 07:02:07.729077: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 07:02:07.732004: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 07:02:07.732891: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 07:02:07.733332: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 07:02:07.735532: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 07:02:07.735897: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2023-03-25 07:02:07.736035: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 07:02:07.736530: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 07:02:07.736575: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:02:07.736602: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 07:02:07.736623: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 07:02:07.736643: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 07:02:07.736664: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 07:02:07.736684: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 07:02:07.736703: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 07:02:07.736724: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 07:02:07.737506: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 07:02:07.737562: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:02:08.300603: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2023-03-25 07:02:08.300712: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2023-03-25 07:02:08.300728: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2023-03-25 07:02:08.302082: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30120 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:44:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2023-03-25 07:02:24.816879: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2023-03-25 07:02:24.817541: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2500050000 Hz | |
| 2023-03-25 07:02:25.088054: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 07:02:25.439662: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 07:02:25.441883: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| mkdir: cannot create directory ‘/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR964WHY//chrombpnet_model_feb15_fold_0//footprints’: File exists | |
| 2023-03-25 07:05:57.394324: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:05:59.100741: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 07:05:59.101871: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2023-03-25 07:05:59.165147: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 07:05:59.165257: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:05:59.168299: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 07:05:59.168400: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 07:05:59.170012: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 07:05:59.170441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 07:05:59.173808: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 07:05:59.174777: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 07:05:59.175348: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 07:05:59.176337: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 07:05:59.176734: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2023-03-25 07:05:59.176893: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2023-03-25 07:05:59.177410: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:44:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.74GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2023-03-25 07:05:59.177455: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:05:59.177485: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 07:05:59.177509: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 07:05:59.177533: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2023-03-25 07:05:59.177556: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2023-03-25 07:05:59.177579: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2023-03-25 07:05:59.177599: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2023-03-25 07:05:59.177617: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2023-03-25 07:05:59.178339: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2023-03-25 07:05:59.178389: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2023-03-25 07:05:59.779272: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2023-03-25 07:05:59.803128: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2023-03-25 07:05:59.803154: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2023-03-25 07:05:59.804598: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30120 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:44:00.0, compute capability: 7.0) | |
| 2023-03-25 07:06:15.862286: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2023-03-25 07:06:15.862928: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2500050000 Hz | |
| 2023-03-25 07:06:16.060652: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2023-03-25 07:06:16.427671: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2023-03-25 07:06:16.429696: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |