Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +10 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.args.json +23 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.batch_loss.tsv +0 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.bias_formatting.stderr.txt +38 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.bias_formatting.stdout.txt +1 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet.params.json +11 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_data_params.tsv +3 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_formatting.stderr.txt +40 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_formatting.stdout.txt +1 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_model_params.tsv +9 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_no_bias_formatting.stderr.txt +1 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_no_bias_formatting.stdout.txt +1 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.epoch_loss.csv +20 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stderr.txt +328 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stdout.txt +3 -0
- fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stdout_v1.txt +3 -0
- fold_0/model.bias_scaled.fold_0.ENCSR622HTS.h5 +3 -0
- fold_0/model.bias_scaled.fold_0.ENCSR622HTS.tar +3 -0
- fold_0/model.chrombpnet.fold_0.ENCSR622HTS.h5 +3 -0
- fold_0/model.chrombpnet.fold_0.ENCSR622HTS.tar +3 -0
- fold_0/model.chrombpnet_nobias.fold_0.ENCSR622HTS.h5 +3 -0
- fold_0/model.chrombpnet_nobias.fold_0.ENCSR622HTS.tar +3 -0
- fold_1/logs.models.fold_1.ENCSR622HTS/logfile.modelling.fold_1.ENCSR622HTS.stdout.txt +3 -0
- fold_1/logs.models.fold_1.ENCSR622HTS/logfile.modelling.fold_1.ENCSR622HTS.stdout_v1.txt +3 -0
- fold_1/model.bias_scaled.fold_1.ENCSR622HTS.h5 +3 -0
- fold_1/model.bias_scaled.fold_1.ENCSR622HTS.tar +3 -0
- fold_1/model.chrombpnet.fold_1.ENCSR622HTS.h5 +3 -0
- fold_1/model.chrombpnet.fold_1.ENCSR622HTS.tar +3 -0
- fold_1/model.chrombpnet_nobias.fold_1.ENCSR622HTS.h5 +3 -0
- fold_1/model.chrombpnet_nobias.fold_1.ENCSR622HTS.tar +3 -0
- fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.args.json +23 -0
- fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.chrombpnet_no_bias_formatting.stderr.txt +1 -0
- fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.chrombpnet_no_bias_formatting.stdout.txt +1 -0
- fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.stdout.txt +3 -0
- fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.stdout_v1.txt +3 -0
- fold_2/model.bias_scaled.fold_2.ENCSR622HTS.h5 +3 -0
- fold_2/model.bias_scaled.fold_2.ENCSR622HTS.tar +3 -0
- fold_2/model.chrombpnet.fold_2.ENCSR622HTS.h5 +3 -0
- fold_2/model.chrombpnet.fold_2.ENCSR622HTS.tar +3 -0
- fold_2/model.chrombpnet_nobias.fold_2.ENCSR622HTS.h5 +3 -0
- fold_2/model.chrombpnet_nobias.fold_2.ENCSR622HTS.tar +3 -0
- fold_3/logs.models.fold_3.ENCSR622HTS/logfile.modelling.fold_3.ENCSR622HTS.stdout.txt +3 -0
- fold_3/logs.models.fold_3.ENCSR622HTS/logfile.modelling.fold_3.ENCSR622HTS.stdout_v1.txt +3 -0
- fold_3/model.bias_scaled.fold_3.ENCSR622HTS.h5 +3 -0
- fold_3/model.bias_scaled.fold_3.ENCSR622HTS.tar +3 -0
- fold_3/model.chrombpnet.fold_3.ENCSR622HTS.h5 +3 -0
- fold_3/model.chrombpnet.fold_3.ENCSR622HTS.tar +3 -0
- fold_3/model.chrombpnet_nobias.fold_3.ENCSR622HTS.h5 +3 -0
- fold_3/model.chrombpnet_nobias.fold_3.ENCSR622HTS.tar +3 -0
- fold_4/logs.models.fold_4.ENCSR622HTS/logfile.modelling.fold_4.ENCSR622HTS.stdout.txt +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stdout.txt filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.stdout.txt filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
fold_3/logs.models.fold_3.ENCSR622HTS/logfile.modelling.fold_3.ENCSR622HTS.stdout.txt filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
fold_3/logs.models.fold_3.ENCSR622HTS/logfile.modelling.fold_3.ENCSR622HTS.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
fold_1/logs.models.fold_1.ENCSR622HTS/logfile.modelling.fold_1.ENCSR622HTS.stdout.txt filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
fold_1/logs.models.fold_1.ENCSR622HTS/logfile.modelling.fold_1.ENCSR622HTS.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
fold_4/logs.models.fold_4.ENCSR622HTS/logfile.modelling.fold_4.ENCSR622HTS.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
fold_4/logs.models.fold_4.ENCSR622HTS/logfile.modelling.fold_4.ENCSR622HTS.stdout.txt filter=lfs diff=lfs merge=lfs -text
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.args.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
|
| 3 |
+
"bigwig": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//preprocessing/bigWigs/ENCSR622HTS.bigWig",
|
| 4 |
+
"peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombpnet_model_encsr880cub_bias//filtered.peaks.bed",
|
| 5 |
+
"nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombpnet_model_encsr880cub_bias//filtered.nonpeaks.bed",
|
| 6 |
+
"output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombpnet_model_encsr880cub_bias//chrombpnet",
|
| 7 |
+
"chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json",
|
| 8 |
+
"trackables": [
|
| 9 |
+
"logcount_predictions_loss",
|
| 10 |
+
"loss",
|
| 11 |
+
"logits_profile_predictions_loss",
|
| 12 |
+
"val_logcount_predictions_loss",
|
| 13 |
+
"val_loss",
|
| 14 |
+
"val_logits_profile_predictions_loss"
|
| 15 |
+
],
|
| 16 |
+
"epochs": 50,
|
| 17 |
+
"early_stop": 5,
|
| 18 |
+
"batch_size": 64,
|
| 19 |
+
"learning_rate": 0.001,
|
| 20 |
+
"params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombpnet_model_encsr880cub_bias//chrombpnet_model_params.tsv",
|
| 21 |
+
"seed": 1234,
|
| 22 |
+
"architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
|
| 23 |
+
}
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.batch_loss.tsv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.bias_formatting.stderr.txt
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
|
| 2 |
+
INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
|
| 3 |
+
2023-07-15 02:24:33.880227: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 4 |
+
2023-07-15 02:24:36.819198: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 5 |
+
2023-07-15 02:24:36.824628: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 6 |
+
2023-07-15 02:24:36.859973: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 7 |
+
pciBusID: 0000:83:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
|
| 8 |
+
coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
|
| 9 |
+
2023-07-15 02:24:36.860055: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 10 |
+
2023-07-15 02:24:36.889101: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 11 |
+
2023-07-15 02:24:36.889169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 12 |
+
2023-07-15 02:24:36.903043: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 13 |
+
2023-07-15 02:24:36.909159: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 14 |
+
2023-07-15 02:24:36.931691: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 15 |
+
2023-07-15 02:24:36.937090: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 16 |
+
2023-07-15 02:24:36.938242: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 17 |
+
2023-07-15 02:24:36.947923: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 18 |
+
2023-07-15 02:24:36.948290: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 19 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 20 |
+
2023-07-15 02:24:36.949156: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 21 |
+
2023-07-15 02:24:36.949415: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 22 |
+
pciBusID: 0000:83:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
|
| 23 |
+
coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
|
| 24 |
+
2023-07-15 02:24:36.949446: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 25 |
+
2023-07-15 02:24:36.949472: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 26 |
+
2023-07-15 02:24:36.949494: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 27 |
+
2023-07-15 02:24:36.949516: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 28 |
+
2023-07-15 02:24:36.949537: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 29 |
+
2023-07-15 02:24:36.949559: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 30 |
+
2023-07-15 02:24:36.949580: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 31 |
+
2023-07-15 02:24:36.949602: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 32 |
+
2023-07-15 02:24:36.949976: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 33 |
+
2023-07-15 02:24:36.951318: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 34 |
+
2023-07-15 02:24:38.745722: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 35 |
+
2023-07-15 02:24:38.745822: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 36 |
+
2023-07-15 02:24:38.745840: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 37 |
+
2023-07-15 02:24:38.748812: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:83:00.0, compute capability: 6.0)
|
| 38 |
+
2023-07-15 02:24:39.592142: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.bias_formatting.stdout.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/new_model_formats/bias_model_scaled
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet.params.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"counts_loss_weight": "20.6",
|
| 3 |
+
"filters": "512",
|
| 4 |
+
"n_dil_layers": "8",
|
| 5 |
+
"bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5",
|
| 6 |
+
"inputlen": "2114",
|
| 7 |
+
"outputlen": "1000",
|
| 8 |
+
"max_jitter": "500",
|
| 9 |
+
"chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json",
|
| 10 |
+
"negative_sampling_ratio": "0.1"
|
| 11 |
+
}
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_data_params.tsv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
counts_sum_min_thresh 4.0
|
| 2 |
+
counts_sum_max_thresh 6362.48
|
| 3 |
+
trainings_pts_post_thresh 170760
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_formatting.stderr.txt
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
|
| 2 |
+
INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
|
| 3 |
+
2023-07-15 01:20:13.762982: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 4 |
+
2023-07-15 01:20:16.161236: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 5 |
+
2023-07-15 01:20:16.165012: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 6 |
+
2023-07-15 01:20:17.338218: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 7 |
+
pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
|
| 8 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
|
| 9 |
+
2023-07-15 01:20:17.338319: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 10 |
+
2023-07-15 01:20:17.356578: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 11 |
+
2023-07-15 01:20:17.356717: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 12 |
+
2023-07-15 01:20:17.366275: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 13 |
+
2023-07-15 01:20:17.370966: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 14 |
+
2023-07-15 01:20:17.386646: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 15 |
+
2023-07-15 01:20:17.390789: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 16 |
+
2023-07-15 01:20:17.391698: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 17 |
+
2023-07-15 01:20:17.406819: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 18 |
+
2023-07-15 01:20:17.407207: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 19 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 20 |
+
2023-07-15 01:20:17.408398: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 21 |
+
2023-07-15 01:20:17.430084: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 22 |
+
pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
|
| 23 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
|
| 24 |
+
2023-07-15 01:20:17.430114: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 25 |
+
2023-07-15 01:20:17.430134: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 26 |
+
2023-07-15 01:20:17.430149: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 27 |
+
2023-07-15 01:20:17.430163: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 28 |
+
2023-07-15 01:20:17.430176: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 29 |
+
2023-07-15 01:20:17.430188: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 30 |
+
2023-07-15 01:20:17.430200: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 31 |
+
2023-07-15 01:20:17.430213: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 32 |
+
2023-07-15 01:20:17.472546: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 33 |
+
2023-07-15 01:20:17.473865: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 34 |
+
2023-07-15 01:20:19.777216: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 35 |
+
2023-07-15 01:20:19.777387: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 36 |
+
2023-07-15 01:20:19.777400: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 37 |
+
2023-07-15 01:20:19.871282: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:07:00.0, compute capability: 8.0)
|
| 38 |
+
2023-07-15 01:20:22.279934: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
|
| 39 |
+
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors.
|
| 40 |
+
, UserWarning)
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_formatting.stdout.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_model_params.tsv
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
counts_loss_weight 20.6
|
| 2 |
+
filters 512
|
| 3 |
+
n_dil_layers 8
|
| 4 |
+
bias_model_path /scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5
|
| 5 |
+
inputlen 2114
|
| 6 |
+
outputlen 1000
|
| 7 |
+
max_jitter 500
|
| 8 |
+
chr_fold_path /scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json
|
| 9 |
+
negative_sampling_ratio 0.1
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_no_bias_formatting.stderr.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet_wo_bias
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.chrombpnet_no_bias_formatting.stdout.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet_wo_bias
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.epoch_loss.csv
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
|
| 2 |
+
0,3.007693290710449,842.727783203125,904.6864013671875,0.8334112763404846,786.574462890625,803.74267578125
|
| 3 |
+
1,0.8227980136871338,778.1151123046875,795.0650024414062,0.7304946184158325,764.9931030273438,780.0409545898438
|
| 4 |
+
2,0.7502110004425049,761.330810546875,776.7850952148438,0.6677016615867615,750.3544311523438,764.109619140625
|
| 5 |
+
3,0.7048489451408386,752.5178833007812,767.038330078125,0.6758238673210144,752.1743774414062,766.0961303710938
|
| 6 |
+
4,0.6693955063819885,745.4822387695312,759.2711791992188,0.6246047616004944,748.4168090820312,761.283935546875
|
| 7 |
+
5,0.6376872062683105,738.2029418945312,751.3397827148438,0.562009871006012,747.6002197265625,759.1774291992188
|
| 8 |
+
6,0.6143935322761536,733.5409545898438,746.1971435546875,0.6088924407958984,739.8034057617188,752.3462524414062
|
| 9 |
+
7,0.5954197645187378,730.1849365234375,742.4503173828125,0.6120826601982117,744.8975830078125,757.506591796875
|
| 10 |
+
8,0.5821287631988525,725.0712890625,737.0619506835938,0.5537213087081909,745.1658325195312,756.5720825195312
|
| 11 |
+
9,0.5632614493370056,721.4506225585938,733.0538940429688,0.5547593832015991,737.7760009765625,749.2039794921875
|
| 12 |
+
10,0.5501141548156738,720.0061645507812,731.3387451171875,0.5491799712181091,740.6593627929688,751.9721069335938
|
| 13 |
+
11,0.5430052280426025,717.202880859375,728.38916015625,0.5680010318756104,745.7555541992188,757.4566040039062
|
| 14 |
+
12,0.5235801339149475,713.8707885742188,724.6550903320312,0.5688911080360413,746.2537841796875,757.9725341796875
|
| 15 |
+
13,0.48242977261543274,701.0297241210938,710.9675903320312,0.5226465463638306,737.020751953125,747.7872924804688
|
| 16 |
+
14,0.46213996410369873,693.9562377929688,703.4769897460938,0.5214941501617432,742.6201782226562,753.3624877929688
|
| 17 |
+
15,0.44867628812789917,688.89794921875,698.1398315429688,0.5970543026924133,741.0379028320312,753.337646484375
|
| 18 |
+
16,0.4337303340435028,686.6624755859375,695.5971069335938,0.5259267687797546,743.0822143554688,753.9161376953125
|
| 19 |
+
17,0.41040557622909546,678.6127319335938,687.066162109375,0.5241705179214478,744.9776000976562,755.7755126953125
|
| 20 |
+
18,0.39812371134757996,674.9517822265625,683.1533203125,0.5378451943397522,746.9336547851562,758.0137329101562
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stderr.txt
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2022-03-23 18:48:46.332785: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 2 |
+
2022-03-23 18:54:26.441283: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 3 |
+
2022-03-23 18:54:26.443582: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 4 |
+
2022-03-23 18:54:26.902258: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 5 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 6 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 7 |
+
2022-03-23 18:54:26.902385: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 8 |
+
2022-03-23 18:54:26.927469: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 9 |
+
2022-03-23 18:54:26.927640: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 10 |
+
2022-03-23 18:54:26.941457: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 11 |
+
2022-03-23 18:54:26.947728: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 12 |
+
2022-03-23 18:54:26.969724: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 13 |
+
2022-03-23 18:54:26.976065: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 14 |
+
2022-03-23 18:54:26.977525: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 15 |
+
2022-03-23 18:54:26.983810: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 16 |
+
2022-03-23 18:54:26.984181: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 17 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 18 |
+
2022-03-23 18:54:26.984261: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 19 |
+
2022-03-23 18:54:26.986603: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 20 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 21 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 22 |
+
2022-03-23 18:54:26.986634: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 23 |
+
2022-03-23 18:54:26.986655: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 24 |
+
2022-03-23 18:54:26.986671: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 25 |
+
2022-03-23 18:54:26.986686: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 26 |
+
2022-03-23 18:54:26.986699: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 27 |
+
2022-03-23 18:54:26.986713: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 28 |
+
2022-03-23 18:54:26.986726: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 29 |
+
2022-03-23 18:54:26.986740: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 30 |
+
2022-03-23 18:54:26.991742: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 31 |
+
2022-03-23 18:54:26.993380: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 32 |
+
2022-03-23 18:54:28.743934: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 33 |
+
2022-03-23 18:54:28.744061: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 34 |
+
2022-03-23 18:54:28.744073: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 35 |
+
2022-03-23 18:54:28.753021: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37401 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:8d:00.0, compute capability: 8.0)
|
| 36 |
+
2022-03-23 18:54:30.024677: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
|
| 37 |
+
2022-03-23 18:54:30.033695: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999945000 Hz
|
| 38 |
+
2022-03-23 18:54:30.220359: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 39 |
+
2022-03-23 18:54:31.863279: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 40 |
+
2022-03-23 18:54:31.872794: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 41 |
+
2022-03-23 18:55:04.102723: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 42 |
+
2022-03-23 18:55:05.777406: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 43 |
+
2022-03-23 18:55:05.778444: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 44 |
+
2022-03-23 18:55:06.077808: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 45 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 46 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 47 |
+
2022-03-23 18:55:06.077952: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 48 |
+
2022-03-23 18:55:06.080067: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 49 |
+
2022-03-23 18:55:06.080128: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 50 |
+
2022-03-23 18:55:06.081100: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 51 |
+
2022-03-23 18:55:06.081281: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 52 |
+
2022-03-23 18:55:06.083477: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 53 |
+
2022-03-23 18:55:06.083986: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 54 |
+
2022-03-23 18:55:06.084115: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 55 |
+
2022-03-23 18:55:06.086846: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 56 |
+
2022-03-23 18:55:06.087176: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 57 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 58 |
+
2022-03-23 18:55:06.087254: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 59 |
+
2022-03-23 18:55:06.088586: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 60 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 61 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 62 |
+
2022-03-23 18:55:06.088634: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 63 |
+
2022-03-23 18:55:06.088653: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 64 |
+
2022-03-23 18:55:06.088667: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 65 |
+
2022-03-23 18:55:06.088680: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 66 |
+
2022-03-23 18:55:06.088693: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 67 |
+
2022-03-23 18:55:06.088707: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 68 |
+
2022-03-23 18:55:06.088720: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 69 |
+
2022-03-23 18:55:06.088733: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 70 |
+
2022-03-23 18:55:06.091302: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 71 |
+
2022-03-23 18:55:06.091339: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 72 |
+
2022-03-23 18:55:06.591334: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 73 |
+
2022-03-23 18:55:06.591470: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 74 |
+
2022-03-23 18:55:06.591481: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 75 |
+
2022-03-23 18:55:06.595870: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37401 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:8d:00.0, compute capability: 8.0)
|
| 76 |
+
2022-03-23 19:00:20.268691: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
|
| 77 |
+
2022-03-23 19:00:20.269189: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999945000 Hz
|
| 78 |
+
2022-03-23 19:00:21.754767: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 79 |
+
2022-03-23 19:00:22.275985: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 80 |
+
2022-03-23 19:00:22.323458: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 81 |
+
2022-03-23 19:00:25.359376: I tensorflow/stream_executor/cuda/cuda_blas.cc:1838] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
|
| 82 |
+
2022-03-23 21:07:22.979769: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 83 |
+
2022-03-23 21:07:25.617893: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 84 |
+
2022-03-23 21:07:25.618951: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 85 |
+
2022-03-23 21:07:26.050314: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 86 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 87 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 88 |
+
2022-03-23 21:07:26.050441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 89 |
+
2022-03-23 21:07:26.052536: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 90 |
+
2022-03-23 21:07:26.052606: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 91 |
+
2022-03-23 21:07:26.053582: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 92 |
+
2022-03-23 21:07:26.053766: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 93 |
+
2022-03-23 21:07:26.055865: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 94 |
+
2022-03-23 21:07:26.056343: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 95 |
+
2022-03-23 21:07:26.056474: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 96 |
+
2022-03-23 21:07:26.060726: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 97 |
+
2022-03-23 21:07:26.061099: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 98 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 99 |
+
2022-03-23 21:07:26.061188: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 100 |
+
2022-03-23 21:07:26.063333: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 101 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 102 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 103 |
+
2022-03-23 21:07:26.063366: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 104 |
+
2022-03-23 21:07:26.063397: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 105 |
+
2022-03-23 21:07:26.063413: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 106 |
+
2022-03-23 21:07:26.063428: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 107 |
+
2022-03-23 21:07:26.063443: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 108 |
+
2022-03-23 21:07:26.063457: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 109 |
+
2022-03-23 21:07:26.063470: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 110 |
+
2022-03-23 21:07:26.063485: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 111 |
+
2022-03-23 21:07:26.067511: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 112 |
+
2022-03-23 21:07:26.067546: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 113 |
+
2022-03-23 21:07:26.580938: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 114 |
+
2022-03-23 21:07:26.581074: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 115 |
+
2022-03-23 21:07:26.581086: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 116 |
+
2022-03-23 21:07:26.586185: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37401 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:8d:00.0, compute capability: 8.0)
|
| 117 |
+
2022-03-23 21:08:55.485267: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
|
| 118 |
+
2022-03-23 21:08:55.488902: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999945000 Hz
|
| 119 |
+
2022-03-23 21:08:55.577220: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 120 |
+
2022-03-23 21:08:56.103418: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 121 |
+
2022-03-23 21:08:56.105976: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 122 |
+
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors.
|
| 123 |
+
, UserWarning)
|
| 124 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
|
| 125 |
+
cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
|
| 126 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
|
| 127 |
+
profile_prob = profile / np.sum(profile)
|
| 128 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
|
| 129 |
+
shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
|
| 130 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
|
| 131 |
+
curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
|
| 132 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 133 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 134 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 135 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 136 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 137 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 138 |
+
2022-03-23 21:11:25.448003: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 139 |
+
2022-03-23 21:11:27.864976: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 140 |
+
2022-03-23 21:11:27.866050: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 141 |
+
2022-03-23 21:11:28.170967: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 142 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 143 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 144 |
+
2022-03-23 21:11:28.171091: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 145 |
+
2022-03-23 21:11:28.173178: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 146 |
+
2022-03-23 21:11:28.173240: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 147 |
+
2022-03-23 21:11:28.174183: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 148 |
+
2022-03-23 21:11:28.174358: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 149 |
+
2022-03-23 21:11:28.176501: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 150 |
+
2022-03-23 21:11:28.176979: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 151 |
+
2022-03-23 21:11:28.177115: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 152 |
+
2022-03-23 21:11:28.180344: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 153 |
+
2022-03-23 21:11:28.180678: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 154 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 155 |
+
2022-03-23 21:11:28.180781: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 156 |
+
2022-03-23 21:11:28.182403: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 157 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 158 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 159 |
+
2022-03-23 21:11:28.182428: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 160 |
+
2022-03-23 21:11:28.182446: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 161 |
+
2022-03-23 21:11:28.182460: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 162 |
+
2022-03-23 21:11:28.182474: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 163 |
+
2022-03-23 21:11:28.182487: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 164 |
+
2022-03-23 21:11:28.182508: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 165 |
+
2022-03-23 21:11:28.182521: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 166 |
+
2022-03-23 21:11:28.182534: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 167 |
+
2022-03-23 21:11:28.185579: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 168 |
+
2022-03-23 21:11:28.185609: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 169 |
+
2022-03-23 21:11:28.700331: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 170 |
+
2022-03-23 21:11:28.700459: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 171 |
+
2022-03-23 21:11:28.700471: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 172 |
+
2022-03-23 21:11:28.705699: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37401 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:8d:00.0, compute capability: 8.0)
|
| 173 |
+
WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
|
| 174 |
+
2022-03-23 21:12:52.742026: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
|
| 175 |
+
2022-03-23 21:12:52.744533: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999945000 Hz
|
| 176 |
+
2022-03-23 21:12:52.803388: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 177 |
+
2022-03-23 21:12:53.293244: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 178 |
+
2022-03-23 21:12:53.295151: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 179 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
|
| 180 |
+
cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
|
| 181 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
|
| 182 |
+
profile_prob = profile / np.sum(profile)
|
| 183 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
|
| 184 |
+
shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
|
| 185 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
|
| 186 |
+
curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
|
| 187 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 188 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 189 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 190 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 191 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 192 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 193 |
+
2022-03-23 21:15:12.876907: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 194 |
+
2022-03-23 21:15:15.314663: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 195 |
+
2022-03-23 21:15:15.315742: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 196 |
+
2022-03-23 21:15:15.631354: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 197 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 198 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 199 |
+
2022-03-23 21:15:15.631504: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 200 |
+
2022-03-23 21:15:15.633590: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 201 |
+
2022-03-23 21:15:15.633654: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 202 |
+
2022-03-23 21:15:15.634600: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 203 |
+
2022-03-23 21:15:15.634783: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 204 |
+
2022-03-23 21:15:15.636949: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 205 |
+
2022-03-23 21:15:15.637441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 206 |
+
2022-03-23 21:15:15.637583: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 207 |
+
2022-03-23 21:15:15.640803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 208 |
+
2022-03-23 21:15:15.641130: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 209 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 210 |
+
2022-03-23 21:15:15.641204: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 211 |
+
2022-03-23 21:15:15.642828: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 212 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 213 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 214 |
+
2022-03-23 21:15:15.642856: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 215 |
+
2022-03-23 21:15:15.642878: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 216 |
+
2022-03-23 21:15:15.642894: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 217 |
+
2022-03-23 21:15:15.642934: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 218 |
+
2022-03-23 21:15:15.642951: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 219 |
+
2022-03-23 21:15:15.642966: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 220 |
+
2022-03-23 21:15:15.642981: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 221 |
+
2022-03-23 21:15:15.642996: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 222 |
+
2022-03-23 21:15:15.646034: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 223 |
+
2022-03-23 21:15:15.646064: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 224 |
+
2022-03-23 21:15:16.155545: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 225 |
+
2022-03-23 21:15:16.155679: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 226 |
+
2022-03-23 21:15:16.155690: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 227 |
+
2022-03-23 21:15:16.160836: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37401 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:8d:00.0, compute capability: 8.0)
|
| 228 |
+
2022-03-23 21:16:37.986399: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
|
| 229 |
+
2022-03-23 21:16:37.988242: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999945000 Hz
|
| 230 |
+
2022-03-23 21:16:38.027663: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 231 |
+
2022-03-23 21:16:38.532251: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 232 |
+
2022-03-23 21:16:38.534194: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 233 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
|
| 234 |
+
cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
|
| 235 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
|
| 236 |
+
profile_prob = profile / np.sum(profile)
|
| 237 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
|
| 238 |
+
shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
|
| 239 |
+
/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
|
| 240 |
+
curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
|
| 241 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 242 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 243 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 244 |
+
findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
|
| 245 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 246 |
+
No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
|
| 247 |
+
2022-03-23 21:17:53.910671: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 248 |
+
2022-03-23 21:17:55.103267: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 249 |
+
2022-03-23 21:17:55.104342: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 250 |
+
2022-03-23 21:17:55.406922: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 251 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 252 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 253 |
+
2022-03-23 21:17:55.407055: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 254 |
+
2022-03-23 21:17:55.409222: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 255 |
+
2022-03-23 21:17:55.409275: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 256 |
+
2022-03-23 21:17:55.410270: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 257 |
+
2022-03-23 21:17:55.410460: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 258 |
+
2022-03-23 21:17:55.412661: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 259 |
+
2022-03-23 21:17:55.413138: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 260 |
+
2022-03-23 21:17:55.413265: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 261 |
+
2022-03-23 21:17:55.416061: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 262 |
+
2022-03-23 21:17:55.416393: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 263 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 264 |
+
2022-03-23 21:17:55.416466: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 265 |
+
2022-03-23 21:17:55.418965: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 266 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 267 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 268 |
+
2022-03-23 21:17:55.418993: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 269 |
+
2022-03-23 21:17:55.419009: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 270 |
+
2022-03-23 21:17:55.419024: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 271 |
+
2022-03-23 21:17:55.419037: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 272 |
+
2022-03-23 21:17:55.419051: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 273 |
+
2022-03-23 21:17:55.419064: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 274 |
+
2022-03-23 21:17:55.419077: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 275 |
+
2022-03-23 21:17:55.419091: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 276 |
+
2022-03-23 21:17:55.421752: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 277 |
+
2022-03-23 21:17:55.421788: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 278 |
+
2022-03-23 21:17:55.922869: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 279 |
+
2022-03-23 21:17:55.923028: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 280 |
+
2022-03-23 21:17:55.923040: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 281 |
+
2022-03-23 21:17:55.928123: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37401 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:8d:00.0, compute capability: 8.0)
|
| 282 |
+
WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
|
| 283 |
+
2022-03-23 21:18:10.837775: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
|
| 284 |
+
2022-03-23 21:18:10.838394: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999945000 Hz
|
| 285 |
+
2022-03-23 21:18:11.080076: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 286 |
+
2022-03-23 21:18:11.728968: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 287 |
+
2022-03-23 21:18:11.731339: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 288 |
+
mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombpnet_model_encsr880cub_bias//footprints’: File exists
|
| 289 |
+
2022-03-23 21:20:11.526000: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 290 |
+
2022-03-23 21:20:12.703430: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 291 |
+
2022-03-23 21:20:12.704530: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
|
| 292 |
+
2022-03-23 21:20:13.002374: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 293 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 294 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 295 |
+
2022-03-23 21:20:13.002510: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 296 |
+
2022-03-23 21:20:13.004694: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 297 |
+
2022-03-23 21:20:13.004757: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 298 |
+
2022-03-23 21:20:13.005743: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 299 |
+
2022-03-23 21:20:13.005930: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 300 |
+
2022-03-23 21:20:13.008132: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 301 |
+
2022-03-23 21:20:13.008615: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 302 |
+
2022-03-23 21:20:13.008745: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 303 |
+
2022-03-23 21:20:13.011477: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 304 |
+
2022-03-23 21:20:13.011819: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
|
| 305 |
+
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
|
| 306 |
+
2022-03-23 21:20:13.011893: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
|
| 307 |
+
2022-03-23 21:20:13.013226: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
|
| 308 |
+
pciBusID: 0000:8d:00.0 name: NVIDIA A100-SXM4-40GB computeCapability: 8.0
|
| 309 |
+
coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 39.41GiB deviceMemoryBandwidth: 1.41TiB/s
|
| 310 |
+
2022-03-23 21:20:13.013279: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 311 |
+
2022-03-23 21:20:13.013297: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 312 |
+
2022-03-23 21:20:13.013312: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 313 |
+
2022-03-23 21:20:13.013326: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
|
| 314 |
+
2022-03-23 21:20:13.013339: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
|
| 315 |
+
2022-03-23 21:20:13.013353: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
|
| 316 |
+
2022-03-23 21:20:13.013366: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
|
| 317 |
+
2022-03-23 21:20:13.013380: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
| 318 |
+
2022-03-23 21:20:13.015933: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
|
| 319 |
+
2022-03-23 21:20:13.015966: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
|
| 320 |
+
2022-03-23 21:20:13.517332: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
|
| 321 |
+
2022-03-23 21:20:13.517471: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
|
| 322 |
+
2022-03-23 21:20:13.517482: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
|
| 323 |
+
2022-03-23 21:20:13.522517: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 37401 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:8d:00.0, compute capability: 8.0)
|
| 324 |
+
2022-03-23 21:20:30.393340: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
|
| 325 |
+
2022-03-23 21:20:30.393943: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 1999945000 Hz
|
| 326 |
+
2022-03-23 21:20:30.537517: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
|
| 327 |
+
2022-03-23 21:20:31.074552: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
|
| 328 |
+
2022-03-23 21:20:31.076949: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stdout.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1958db949ce339e7c96f85befe4f9e4e8bf86dbbd7bf2759d315bbb6972c8f92
|
| 3 |
+
size 14161910
|
fold_0/logs.models.fold_0.ENCSR622HTS/logfile.modelling.fold_0.ENCSR622HTS.stdout_v1.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6dd20056b9839cd1804b08c18aa27df9ffa54602945d137c965c43e9fec8eab8
|
| 3 |
+
size 14160304
|
fold_0/model.bias_scaled.fold_0.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0800e98cd12eba1e963396fa508fb8a3327eade2cd2a761b43fc8e9d88d3edd7
|
| 3 |
+
size 2688440
|
fold_0/model.bias_scaled.fold_0.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7e75f0294aa3425fbe14b340597b6fdb5e93be54ca52c02a7bb24f8e51193614
|
| 3 |
+
size 1198080
|
fold_0/model.chrombpnet.fold_0.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f935792e5a1788a26ca7af94dd12530ac895535efc02999a5689543b1795e00b
|
| 3 |
+
size 26447928
|
fold_0/model.chrombpnet.fold_0.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:07a4ee304e2da995134d3a9f7049ac39055e740171b24fa78d10828c3501f87e
|
| 3 |
+
size 27607040
|
fold_0/model.chrombpnet_nobias.fold_0.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7fdf1d8862d4e69e04b75559b26ba4317dccc2757c227a4a0d8ffa387633c5a7
|
| 3 |
+
size 25583536
|
fold_0/model.chrombpnet_nobias.fold_0.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:78fd4a927133f88c6dff0ac044373ee243578fd4dc5af66e2307bd70e4c9b6bb
|
| 3 |
+
size 26081280
|
fold_1/logs.models.fold_1.ENCSR622HTS/logfile.modelling.fold_1.ENCSR622HTS.stdout.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc7a1ae87e9d8cc71f6d648f2f624ace2399c178aaf76318caff7c43ea7a2bdd
|
| 3 |
+
size 12409438
|
fold_1/logs.models.fold_1.ENCSR622HTS/logfile.modelling.fold_1.ENCSR622HTS.stdout_v1.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c26e3d70cd8b2d4dd01e6e5296ad60b776d8df63156e0a7e49c623894f8b5672
|
| 3 |
+
size 12407765
|
fold_1/model.bias_scaled.fold_1.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f6413b31f52e31a6cc835503fbf4bcc9bacf9e6ee9758e24916a9f916b517155
|
| 3 |
+
size 2688440
|
fold_1/model.bias_scaled.fold_1.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:be0a932ca0360e801a80824821c58c8be8b042f1c4bed4b805ba9da27f52ef95
|
| 3 |
+
size 1198080
|
fold_1/model.chrombpnet.fold_1.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f99dacd3a54f2c1fea6bc1e2469c9a0e1a47ecd318ee670b6546063f5e8cbb4d
|
| 3 |
+
size 26447928
|
fold_1/model.chrombpnet.fold_1.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:43c4031590e00d9b28161a4ee68d5811c25007d4de2d81b7a131af1260d90fc3
|
| 3 |
+
size 27607040
|
fold_1/model.chrombpnet_nobias.fold_1.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8d217d7abfeecb086afb2617ddb90cd33e5026057c9c5e359a03af8c6600a049
|
| 3 |
+
size 25583536
|
fold_1/model.chrombpnet_nobias.fold_1.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1741953dde2d2c7cfcb46988a1419da9949608158ab7fc1b9c9487e67045de6c
|
| 3 |
+
size 26081280
|
fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.args.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
|
| 3 |
+
"bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//preprocessing/bigWigs/ENCSR622HTS.bigWig",
|
| 4 |
+
"peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2//filtered.peaks.bed",
|
| 5 |
+
"nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2//filtered.nonpeaks.bed",
|
| 6 |
+
"output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet",
|
| 7 |
+
"chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
|
| 8 |
+
"trackables": [
|
| 9 |
+
"logcount_predictions_loss",
|
| 10 |
+
"loss",
|
| 11 |
+
"logits_profile_predictions_loss",
|
| 12 |
+
"val_logcount_predictions_loss",
|
| 13 |
+
"val_loss",
|
| 14 |
+
"val_logits_profile_predictions_loss"
|
| 15 |
+
],
|
| 16 |
+
"epochs": 50,
|
| 17 |
+
"early_stop": 5,
|
| 18 |
+
"batch_size": 64,
|
| 19 |
+
"learning_rate": 0.001,
|
| 20 |
+
"params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet_model_params.tsv",
|
| 21 |
+
"seed": 1234,
|
| 22 |
+
"architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
|
| 23 |
+
}
|
fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.chrombpnet_no_bias_formatting.stderr.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/chrombpnet_wo_bias
|
fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.chrombpnet_no_bias_formatting.stdout.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR622HTS//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/chrombpnet_wo_bias
|
fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.stdout.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:28dbc31e4d21ef1bcd3400478b40b98f69f73d4356456cd244df104e3291340a
|
| 3 |
+
size 12676992
|
fold_2/logs.models.fold_2.ENCSR622HTS/logfile.modelling.fold_2.ENCSR622HTS.stdout_v1.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9227e26b4cddef277d08114a21feee1060aa6754156fa712e84ca33e7dd3152a
|
| 3 |
+
size 19911117
|
fold_2/model.bias_scaled.fold_2.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ef383b134e7c0194cc706a3738bc59161845a688f5fbcb05db78390b60245cb5
|
| 3 |
+
size 2688440
|
fold_2/model.bias_scaled.fold_2.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e72052c7a3d0cfb97b84ff735ac67756896a15bf1e659bf3cfa0dd779d444285
|
| 3 |
+
size 1198080
|
fold_2/model.chrombpnet.fold_2.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:51d39bc1a3f8d9a5658a5acec34e8d543db2452882b1762431b7a3854df0ef2a
|
| 3 |
+
size 26447928
|
fold_2/model.chrombpnet.fold_2.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:662031e144d6346a80a662c7fb032661f5782bca6c7810002ccd2dea27a76750
|
| 3 |
+
size 27607040
|
fold_2/model.chrombpnet_nobias.fold_2.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:442539d557d2cb6e39eea939576c43ebe4a174e02a5d3fbecfb4dee8e11fccd3
|
| 3 |
+
size 25583536
|
fold_2/model.chrombpnet_nobias.fold_2.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:73eb5dd0fcade13a6b95dc8afbe26acffe542657805d86632f9358901f6087ad
|
| 3 |
+
size 26081280
|
fold_3/logs.models.fold_3.ENCSR622HTS/logfile.modelling.fold_3.ENCSR622HTS.stdout.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:96c88b6b9d73f2e6724e6ca20348b3c6602723313f791202aee19eb8d08f464a
|
| 3 |
+
size 12720520
|
fold_3/logs.models.fold_3.ENCSR622HTS/logfile.modelling.fold_3.ENCSR622HTS.stdout_v1.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c069eca58c66ea04c1c01e86de008efc493a3d6547db54787a69368995515b4b
|
| 3 |
+
size 12718847
|
fold_3/model.bias_scaled.fold_3.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:636f4343fa4e03a1fcf862d885ae9ed13840b2dde9be425df639c5ba28ea9d78
|
| 3 |
+
size 2688440
|
fold_3/model.bias_scaled.fold_3.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ba15951fdd7e893d310d719073555d6387ab24a97840d1c98727e163ecf7bc1f
|
| 3 |
+
size 1198080
|
fold_3/model.chrombpnet.fold_3.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fd99819873e9bd0a2d96a4070c7626a7d9b31a48f2a93acbacfd0f3094305772
|
| 3 |
+
size 26447928
|
fold_3/model.chrombpnet.fold_3.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6a0219ecdebf99849bc3e1cd4fa25067e119c81751c9c655765881d8dded063d
|
| 3 |
+
size 27607040
|
fold_3/model.chrombpnet_nobias.fold_3.ENCSR622HTS.h5
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b784c0195b4dcdcd355b2daf021db513403320d9ac9dda0c3a6576fff11e1674
|
| 3 |
+
size 25583536
|
fold_3/model.chrombpnet_nobias.fold_3.ENCSR622HTS.tar
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:87ed0ecc2be973300758a6fa04c13ef34c02d8d8899ee60bacf1a9b4706e08d9
|
| 3 |
+
size 26081280
|
fold_4/logs.models.fold_4.ENCSR622HTS/logfile.modelling.fold_4.ENCSR622HTS.stdout.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bd11541458df00e210a651504ab4926211edd08598aa087c0d43ad1390dfe7c4
|
| 3 |
+
size 13462091
|