encode-chrombpnet-DNASE-limb-ENCSR818JGZ-ENCSR138OCE / fold_3 /logs.models.fold_3.ENCSR818JGZ /logfile.modelling.fold_3.ENCSR818JGZ.stderr.txt
| INFO: underlay of /etc/localtime required more than 50 (88) bind mounts | |
| INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts | |
| INFO: underlay of /etc/localtime required more than 50 (88) bind mounts | |
| INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts | |
| 2022-10-15 15:40:27.293551: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:04.312511: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 15:52:04.317475: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-15 15:52:04.386605: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 15:52:04.386705: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:04.414998: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 15:52:04.415128: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 15:52:04.429794: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 15:52:04.436617: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 15:52:04.460655: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 15:52:04.466984: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 15:52:04.468432: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 15:52:04.470803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 15:52:04.471208: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-15 15:52:04.471312: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 15:52:04.471761: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 15:52:04.471805: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:04.471828: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 15:52:04.471847: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 15:52:04.471865: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 15:52:04.471883: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 15:52:04.471901: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 15:52:04.471919: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 15:52:04.471936: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 15:52:04.472706: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 15:52:04.475219: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:06.466312: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-15 15:52:06.466411: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-15 15:52:06.466427: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-15 15:52:06.471477: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30130 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:89:00.0, compute capability: 7.0) | |
| 2022-10-15 15:52:08.030887: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-15 15:52:08.048878: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2499905000 Hz | |
| 2022-10-15 15:52:08.303693: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 15:52:09.588179: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 15:52:09.599329: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 15:52:50.826294: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:53.007831: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 15:52:53.008799: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-15 15:52:53.065565: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 15:52:53.065667: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:53.068661: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 15:52:53.068723: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 15:52:53.070097: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 15:52:53.070453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 15:52:53.073316: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 15:52:53.074091: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 15:52:53.074446: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 15:52:53.075323: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 15:52:53.075709: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-15 15:52:53.075797: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 15:52:53.076227: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 15:52:53.076258: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:53.076280: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 15:52:53.076298: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 15:52:53.076314: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 15:52:53.076331: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 15:52:53.076348: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 15:52:53.076367: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 15:52:53.076386: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 15:52:53.077075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 15:52:53.077114: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 15:52:53.618232: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-15 15:52:53.618330: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-15 15:52:53.618345: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-15 15:52:53.619650: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30130 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:89:00.0, compute capability: 7.0) | |
| 2022-10-15 15:59:32.789380: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-15 15:59:32.789933: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2499905000 Hz | |
| 2022-10-15 15:59:34.659876: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 15:59:34.990622: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 15:59:35.012229: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1770s vs `on_train_batch_end` time: 0.2548s). Check your callbacks. | |
| 2022-10-15 20:23:21.888976: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:23:25.675035: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:23:25.676071: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-15 20:23:25.737011: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:23:25.737108: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:23:25.740052: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:23:25.740117: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:23:25.741490: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:23:25.741855: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:23:25.744723: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:23:25.745485: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:23:25.745902: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:23:25.746779: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:23:25.747151: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-15 20:23:25.747234: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:23:25.747665: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:23:25.747696: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:23:25.747722: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:23:25.747744: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:23:25.747764: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:23:25.747795: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:23:25.747818: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:23:25.747838: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:23:25.747858: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:23:25.748584: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:23:25.748621: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:23:26.310714: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-15 20:23:26.310819: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-15 20:23:26.310835: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-15 20:23:26.321794: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30130 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:89:00.0, compute capability: 7.0) | |
| 2022-10-15 20:26:10.234778: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-15 20:26:10.239396: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2499905000 Hz | |
| 2022-10-15 20:26:10.354926: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:26:10.661041: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:26:10.663299: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors. | |
| , UserWarning) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-10-15 20:31:20.138013: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:31:23.263191: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:31:23.264257: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-15 20:31:23.342943: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:31:23.343039: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:31:23.345959: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:31:23.346023: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:31:23.347408: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:31:23.347756: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:31:23.350641: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:31:23.351407: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:31:23.351818: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:31:23.352709: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:31:23.353076: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-15 20:31:23.353158: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:31:23.353593: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:31:23.353622: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:31:23.353645: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:31:23.353664: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:31:23.353683: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:31:23.353701: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:31:23.353719: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:31:23.353737: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:31:23.353755: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:31:23.354509: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:31:23.354549: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:31:23.894483: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-15 20:31:23.894582: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-15 20:31:23.894599: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-15 20:31:23.896006: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30130 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:89:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-10-15 20:33:00.602902: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-15 20:33:00.606020: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2499905000 Hz | |
| 2022-10-15 20:33:00.681535: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:33:00.962731: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:33:00.964430: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-10-15 20:37:50.442775: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:37:53.587819: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:37:53.588884: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-15 20:37:53.645319: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:37:53.645415: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:37:53.648313: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:37:53.648375: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:37:53.649744: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:37:53.650108: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:37:53.652969: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:37:53.653712: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:37:53.654132: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:37:53.655403: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:37:53.655767: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-15 20:37:53.655858: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:37:53.656276: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:37:53.656307: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:37:53.656351: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:37:53.656375: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:37:53.656396: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:37:53.656417: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:37:53.656437: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:37:53.656457: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:37:53.656477: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:37:53.657203: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:37:53.657242: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:37:54.227686: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-15 20:37:54.227802: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-15 20:37:54.227819: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-15 20:37:54.229202: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30130 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:89:00.0, compute capability: 7.0) | |
| 2022-10-15 20:39:30.686973: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-15 20:39:30.689194: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2499905000 Hz | |
| 2022-10-15 20:39:30.738142: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:39:31.030613: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:39:31.032250: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide | |
| cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:]) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide | |
| profile_prob = profile / np.sum(profile) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide | |
| shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels) | |
| /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide | |
| curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob) | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| findfont: Font family ['normal'] not found. Falling back to DejaVu Sans. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument. | |
| 2022-10-15 20:41:26.491026: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:41:28.041281: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:41:28.042264: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-15 20:41:28.093849: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:41:28.093952: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:41:28.096845: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:41:28.096912: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:41:28.098302: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:41:28.098658: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:41:28.101537: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:41:28.102302: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:41:28.102710: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:41:28.103572: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:41:28.103942: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-15 20:41:28.104025: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:41:28.104450: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:41:28.104486: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:41:28.104507: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:41:28.104526: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:41:28.104545: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:41:28.104563: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:41:28.104582: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:41:28.104600: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:41:28.104618: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:41:28.105365: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:41:28.105409: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:41:28.680162: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-15 20:41:28.680262: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-15 20:41:28.680277: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-15 20:41:28.681596: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30130 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:89:00.0, compute capability: 7.0) | |
| WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually. | |
| 2022-10-15 20:41:43.603822: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-15 20:41:43.604436: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2499905000 Hz | |
| 2022-10-15 20:41:43.867396: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:41:44.200195: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:41:44.202081: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR818JGZ//chrombppnet_model_encsr880cub_bias_fold_3//footprints’: File exists | |
| 2022-10-15 20:45:01.593770: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:45:03.130135: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:45:03.131137: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1 | |
| 2022-10-15 20:45:03.186842: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:45:03.186944: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:45:03.189844: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:45:03.189910: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:45:03.191283: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:45:03.191640: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:45:03.194522: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:45:03.195284: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:45:03.195693: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:45:03.196565: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:45:03.196926: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA | |
| To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. | |
| 2022-10-15 20:45:03.197026: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set | |
| 2022-10-15 20:45:03.197446: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: | |
| pciBusID: 0000:89:00.0 name: Tesla V100S-PCIE-32GB computeCapability: 7.0 | |
| coreClock: 1.597GHz coreCount: 80 deviceMemorySize: 31.75GiB deviceMemoryBandwidth: 1.03TiB/s | |
| 2022-10-15 20:45:03.197481: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:45:03.197503: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:45:03.197523: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:45:03.197543: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10 | |
| 2022-10-15 20:45:03.197562: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10 | |
| 2022-10-15 20:45:03.197581: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10 | |
| 2022-10-15 20:45:03.197599: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11 | |
| 2022-10-15 20:45:03.197618: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |
| 2022-10-15 20:45:03.198315: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0 | |
| 2022-10-15 20:45:03.198359: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 | |
| 2022-10-15 20:45:03.759799: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 2022-10-15 20:45:03.759912: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0 | |
| 2022-10-15 20:45:03.759931: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N | |
| 2022-10-15 20:45:03.761390: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30130 MB memory) -> physical GPU (device: 0, name: Tesla V100S-PCIE-32GB, pci bus id: 0000:89:00.0, compute capability: 7.0) | |
| 2022-10-15 20:45:18.276426: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2) | |
| 2022-10-15 20:45:18.277047: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2499905000 Hz | |
| 2022-10-15 20:45:18.455455: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11 | |
| 2022-10-15 20:45:18.802675: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11 | |
| 2022-10-15 20:45:18.804510: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8 | |