encode-chrombpnet-ATAC-HG03280-ENCSR554WGQ-ENCSR845EWD / fold_1 /logs.models.fold_1.ENCSR554WGQ /logfile.modelling.fold_1.ENCSR554WGQ.stderr.txt
| 2022-09-21 16:11:37.349593: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:15:55.430606: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 16:15:55.432257: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-21 16:15:55.501286: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 16:15:55.501361: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:15:55.529426: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 16:15:55.529504: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 16:15:55.544605: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 16:15:55.551487: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 16:15:55.576375: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 16:15:55.583127: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 16:15:55.584651: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 16:15:55.587184: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 16:15:55.587583: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-21 16:15:55.587677: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 16:15:55.590675: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 16:15:55.590722: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:15:55.590752: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 16:15:55.590777: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 16:15:55.590801: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 16:15:55.590824: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 16:15:55.590847: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 16:15:55.590871: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 16:15:55.590895: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 16:15:55.594472: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 16:15:55.596157: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:15:57.573942: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-21 16:15:57.574071: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-21 16:15:57.574092: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-21 16:15:57.579435: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:88:00.0, compute capability: 7.0) | |
| 2022-09-21 16:15:59.010780: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-21 16:15:59.019988: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz | |
| 2022-09-21 16:15:59.241255: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 16:16:00.418735: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 16:16:00.427459: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 16:16:40.413125: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:16:42.359473: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 16:16:42.360507: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-21 16:16:42.429989: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 16:16:42.430089: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:16:42.433194: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 16:16:42.433256: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 16:16:42.434573: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 16:16:42.434821: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 16:16:42.438030: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 16:16:42.438692: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 16:16:42.438859: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 16:16:42.439810: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 16:16:42.440164: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-21 16:16:42.440248: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 16:16:42.440731: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 16:16:42.440784: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:16:42.440818: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 16:16:42.440847: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 16:16:42.440874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 16:16:42.440900: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 16:16:42.440927: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 16:16:42.440953: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 16:16:42.440993: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 16:16:42.441872: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 16:16:42.441916: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 16:16:43.119973: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-21 16:16:43.120137: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-21 16:16:43.120157: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-21 16:16:43.121646: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:88:00.0, compute capability: 7.0) | |
| 2022-09-21 16:21:10.144447: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-21 16:21:10.144913: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz | |
| 2022-09-21 16:21:12.070101: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 16:21:12.343699: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 16:21:12.361359: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2089s vs `on_train_batch_end` time: 0.2968s). Check your callbacks. | |
| 2022-09-21 20:37:54.409609: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:37:57.489000: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:37:57.490144: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-21 20:37:57.549302: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:37:57.549386: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:37:57.552494: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:37:57.552555: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:37:57.553879: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:37:57.554168: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:37:57.557378: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:37:57.558029: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:37:57.558188: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:37:57.559167: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:37:57.559509: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-21 20:37:57.559595: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:37:57.560081: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:37:57.560113: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:37:57.560141: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:37:57.560165: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:37:57.560190: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:37:57.560214: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:37:57.560238: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:37:57.560262: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:37:57.560287: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:37:57.561131: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:37:57.561173: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:37:58.236709: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-21 20:37:58.236810: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-21 20:37:58.236830: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-21 20:37:58.240109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:88:00.0, compute capability: 7.0) | |
| 2022-09-21 20:39:10.279958: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-21 20:39:10.284126: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz | |
| 2022-09-21 20:39:10.391636: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:39:10.679752: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:39:10.681874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors. | |
| , UserWarning) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-09-21 20:43:23.217563: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:43:26.041873: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:43:26.043022: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-21 20:43:26.111243: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:43:26.111324: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:43:26.114410: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:43:26.114470: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:43:26.115828: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:43:26.116060: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:43:26.119273: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:43:26.119968: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:43:26.120131: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:43:26.124352: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:43:26.124708: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-21 20:43:26.124819: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:43:26.125307: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:43:26.125338: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:43:26.125366: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:43:26.125392: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:43:26.125416: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:43:26.125441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:43:26.125465: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:43:26.125489: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:43:26.125514: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:43:26.128796: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:43:26.128848: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:43:26.816080: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-21 20:43:26.816182: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-21 20:43:26.816200: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-21 20:43:26.817741: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:88:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-09-21 20:44:36.265050: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-21 20:44:36.267748: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz | |
| 2022-09-21 20:44:36.340893: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:44:36.647812: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:44:36.649453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-09-21 20:48:32.991270: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:48:35.846536: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:48:35.847690: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-21 20:48:35.912741: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:48:35.912819: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:48:35.915906: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:48:35.915970: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:48:35.917238: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:48:35.917470: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:48:35.920593: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:48:35.921241: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:48:35.921404: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:48:35.922338: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:48:35.922682: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-21 20:48:35.922766: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:48:35.923239: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:48:35.923270: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:48:35.923297: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:48:35.923322: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:48:35.923368: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:48:35.923394: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:48:35.923418: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:48:35.923441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:48:35.923465: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:48:35.924324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:48:35.924364: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:48:36.595124: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-21 20:48:36.595226: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-21 20:48:36.595246: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-21 20:48:36.601850: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:88:00.0, compute capability: 7.0) | |
| 2022-09-21 20:49:43.985623: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-21 20:49:43.987641: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz | |
| 2022-09-21 20:49:44.032920: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:49:44.322193: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:49:44.323811: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-09-21 20:50:54.532863: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:50:55.951174: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:50:55.952265: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-21 20:50:56.019492: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:50:56.019589: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:50:56.022641: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:50:56.022721: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:50:56.024017: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:50:56.024245: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:50:56.027417: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:50:56.028119: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:50:56.028298: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:50:56.029586: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:50:56.029947: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-21 20:50:56.030036: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:50:56.030507: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:50:56.030549: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:50:56.030581: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:50:56.030610: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:50:56.030655: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:50:56.030691: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:50:56.030722: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:50:56.030751: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:50:56.030780: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:50:56.031592: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:50:56.031644: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:50:56.703262: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-21 20:50:56.703401: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-21 20:50:56.703422: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-21 20:50:56.705426: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:88:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-09-21 20:51:14.305022: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-21 20:51:14.305614: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz | |
| 2022-09-21 20:51:14.564730: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:51:14.989270: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:51:14.991210: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:51:20.178185: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-09-21 20:51:20.178630: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-09-21 20:51:20.675339: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| 2022-09-21 20:51:20.675762: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available. | |
| mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/ATAC/ENCSR554WGQ//chrombpnet_model_feb15_fold_1//footprints’: File exists | |
| 2022-09-21 20:58:56.129035: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:58:57.537285: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:58:57.538393: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-09-21 20:58:57.587040: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:58:57.587128: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:58:57.590163: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:58:57.590225: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:58:57.591501: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:58:57.591733: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:58:57.594864: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:58:57.595503: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:58:57.595662: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:58:57.596588: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:58:57.596933: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-09-21 20:58:57.597023: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-09-21 20:58:57.598211: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:88:00.0 name: NVIDIA TITAN V computeCapability: 7.0 | |
| coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s | |
| 2022-09-21 20:58:57.598262: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:58:57.598294: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:58:57.598322: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:58:57.598349: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-09-21 20:58:57.598376: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-09-21 20:58:57.598418: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-09-21 20:58:57.598444: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-09-21 20:58:57.598471: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-09-21 20:58:57.600295: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-09-21 20:58:57.600345: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-09-21 20:58:58.287511: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-09-21 20:58:58.517343: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-09-21 20:58:58.517412: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-09-21 20:58:58.521463: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:88:00.0, compute capability: 7.0) | |
| 2022-09-21 20:59:16.208767: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-09-21 20:59:16.209389: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400085000 Hz | |
| 2022-09-21 20:59:16.395440: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-09-21 20:59:16.756893: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-09-21 20:59:16.758772: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |