Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_3/logs.models.fold_3.ENCSR499IFY/logfile.modelling.fold_3.ENCSR499IFY.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_3/logs.models.fold_3.ENCSR499IFY/logfile.modelling.fold_3.ENCSR499IFY.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text
+fold_4/logs.models.fold_4.ENCSR499IFY/logfile.modelling.fold_4.ENCSR499IFY.stdout.txt filter=lfs diff=lfs merge=lfs -text
+fold_4/logs.models.fold_4.ENCSR499IFY/logfile.modelling.fold_4.ENCSR499IFY.stdout_v1.txt filter=lfs diff=lfs merge=lfs -text

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.args.json

@@ -0,0 +1,23 @@
+{
+    "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
+    "bigwig": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//preprocessing/bigWigs/ENCSR499IFY.bigWig",
+    "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombpnet_model_encsr880cub_bias//filtered.peaks.bed",
+    "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombpnet_model_encsr880cub_bias//filtered.nonpeaks.bed",
+    "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombpnet_model_encsr880cub_bias//chrombpnet",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json",
+    "trackables": [
+        "logcount_predictions_loss",
+        "loss",
+        "logits_profile_predictions_loss",
+        "val_logcount_predictions_loss",
+        "val_loss",
+        "val_logits_profile_predictions_loss"
+    ],
+    "epochs": 50,
+    "early_stop": 5,
+    "batch_size": 64,
+    "learning_rate": 0.001,
+    "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombpnet_model_encsr880cub_bias//chrombpnet_model_params.tsv",
+    "seed": 1234,
+    "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
+}

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.bias_formatting.stderr.txt

@@ -0,0 +1,38 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 02:25:46.751445: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.791741: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:25:49.797649: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 02:25:49.844591: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:03:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:25:49.844692: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.874809: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:25:49.874970: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:25:49.890426: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:25:49.897041: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:25:49.920248: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:25:49.926083: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:25:49.927277: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:25:49.929064: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:25:49.929425: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 02:25:49.930330: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:25:49.930731: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:03:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:25:49.930763: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.930788: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:25:49.930810: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:25:49.930831: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:25:49.930852: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:25:49.930872: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:25:49.930893: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:25:49.930914: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:25:49.931302: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:25:49.932670: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:51.813068: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 02:25:51.813166: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-15 02:25:51.813184: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-15 02:25:51.816212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:03:00.0, compute capability: 6.0)
+2023-07-15 02:25:52.677252: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/new_model_formats/bias_model_scaled

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.chrombpnet.params.json

@@ -0,0 +1,11 @@
+{
+    "counts_loss_weight": "72.2",
+    "filters": "512",
+    "n_dil_layers": "8",
+    "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5",
+    "inputlen": "2114",
+    "outputlen": "1000",
+    "max_jitter": "500",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json",
+    "negative_sampling_ratio": "0.1"
+}

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.chrombpnet_data_params.tsv

@@ -0,0 +1,3 @@
+counts_sum_min_thresh	46.0
+counts_sum_max_thresh	7879.0
+trainings_pts_post_thresh	167254

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.chrombpnet_formatting.stderr.txt

@@ -0,0 +1,40 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 01:22:57.054289: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:22:59.459287: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 01:22:59.463312: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 01:23:00.450675: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4b:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-15 01:23:00.450764: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:00.469025: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 01:23:00.469081: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 01:23:00.478659: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 01:23:00.483368: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 01:23:00.499268: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 01:23:00.503531: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 01:23:00.504460: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 01:23:00.514810: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 01:23:00.515090: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 01:23:00.515843: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 01:23:00.522522: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:4b:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-15 01:23:00.522567: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:00.522599: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 01:23:00.522624: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 01:23:00.522646: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 01:23:00.522667: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 01:23:00.522688: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 01:23:00.522709: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 01:23:00.522738: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 01:23:00.566308: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 01:23:00.568711: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:03.150781: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 01:23:03.150895: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-15 01:23:03.150926: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-15 01:23:03.227424: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:4b:00.0, compute capability: 8.0)
+2023-07-15 01:23:05.405253: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.chrombpnet_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.chrombpnet_model_params.tsv

@@ -0,0 +1,9 @@
+counts_loss_weight	72.2
+filters	512
+n_dil_layers	8
+bias_model_path	/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5
+inputlen	2114
+outputlen	1000
+max_jitter	500
+chr_fold_path	/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json
+negative_sampling_ratio	0.1

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.chrombpnet_no_bias_formatting.stderr.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet_wo_bias

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.chrombpnet_no_bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet_wo_bias

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.epoch_loss.csv

@@ -0,0 +1,19 @@
+epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
+0,2.4609220027923584,1567.404052734375,1745.08447265625,0.5545490980148315,1535.661376953125,1575.7001953125
+1,0.5555264353752136,1457.946044921875,1498.0535888671875,0.5098289847373962,1489.5947265625,1526.4039306640625
+2,0.5123683214187622,1428.850341796875,1465.8448486328125,0.47503775358200073,1484.7197265625,1519.0166015625
+3,0.4829057455062866,1412.720947265625,1447.5885009765625,0.4594004452228546,1447.5213623046875,1480.68994140625
+4,0.46193745732307434,1400.1368408203125,1433.4896240234375,0.46995651721954346,1463.15625,1497.0872802734375
+5,0.4410182535648346,1389.33447265625,1421.175048828125,0.43430012464523315,1444.315673828125,1475.67138671875
+6,0.4297390580177307,1379.6566162109375,1410.6826171875,0.4268477261066437,1462.916015625,1493.7340087890625
+7,0.41504839062690735,1374.0078125,1403.975830078125,0.47369056940078735,1457.5947265625,1491.79541015625
+8,0.40357181429862976,1369.57080078125,1398.70947265625,0.41153740882873535,1456.9620361328125,1486.675048828125
+9,0.3581252694129944,1346.19580078125,1372.051513671875,0.4067368507385254,1445.547607421875,1474.9140625
+10,0.34317252039909363,1335.447509765625,1360.2261962890625,0.44583311676979065,1441.0877685546875,1473.2763671875
+11,0.32857444882392883,1328.456298828125,1352.1778564453125,0.4068053364753723,1444.43798828125,1473.80810546875
+12,0.31517356634140015,1324.0521240234375,1346.8094482421875,0.41220733523368835,1440.9853515625,1470.746337890625
+13,0.3026072084903717,1319.542724609375,1341.389892578125,0.4227643311023712,1451.1624755859375,1481.686767578125
+14,0.2932106554508209,1314.4273681640625,1335.5946044921875,0.4312989413738251,1441.189453125,1472.3294677734375
+15,0.2852897346019745,1309.218994140625,1329.8154296875,0.4159727692604065,1458.042724609375,1488.074951171875
+16,0.25657919049263,1299.13623046875,1317.6610107421875,0.4348949193954468,1452.6429443359375,1484.0423583984375
+17,0.24612373113632202,1295.00341796875,1312.7720947265625,0.4299258887767792,1452.241943359375,1483.28271484375

fold_0/logs.models.fold_0.ENCSR499IFY/logfile.modelling.fold_0.ENCSR499IFY.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5373a98fe4ea3f0448e25e33ecb2cdf715a7aec99c7102c5791d50c7d88c9d2
+size 13437852

fold_0/model.bias_scaled.fold_0.ENCSR499IFY.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f58e87ed577cfa7dde15b6164200915b8b33feceed658abd179cf30d74349b0
+size 2688440

fold_0/model.bias_scaled.fold_0.ENCSR499IFY.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b4fb339319acaa4f817948c4205712bdbf5dfc791e24c09ca0dea2938e026e
+size 1198080

fold_0/model.chrombpnet.fold_0.ENCSR499IFY.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df71495421adb87a9449ff81fd563ddab7bc8f0c2c53c6a0efc4e6beb81b1e3e
+size 26447928

fold_0/model.chrombpnet.fold_0.ENCSR499IFY.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:506948e9b4eba0881c6bff9fdcd052cf4d12ce0f915037ad71f4357b0450fb54
+size 27607040

fold_0/model.chrombpnet_nobias.fold_0.ENCSR499IFY.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96b5182e861ffc3532b19a9c7c5c841a24e2c69cbf187a5fd793e5ff6a40927c
+size 25583536

fold_0/model.chrombpnet_nobias.fold_0.ENCSR499IFY.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef918f378e41f2dc5053be472b0d5d9305597e2fe06a67cd72fad126e81eaf7
+size 26081280

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.args.json

@@ -0,0 +1,23 @@
+{
+    "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
+    "bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//preprocessing/bigWigs/ENCSR499IFY.bigWig",
+    "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1//filtered.peaks.bed",
+    "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1//filtered.nonpeaks.bed",
+    "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1//chrombpnet",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_1.json",
+    "trackables": [
+        "logcount_predictions_loss",
+        "loss",
+        "logits_profile_predictions_loss",
+        "val_logcount_predictions_loss",
+        "val_loss",
+        "val_logits_profile_predictions_loss"
+    ],
+    "epochs": 50,
+    "early_stop": 5,
+    "batch_size": 64,
+    "learning_rate": 0.001,
+    "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1//chrombpnet_model_params.tsv",
+    "seed": 1234,
+    "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
+}

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.bias_formatting.stderr.txt

@@ -0,0 +1,38 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 02:25:46.754832: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.734669: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:25:49.739557: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 02:25:49.772034: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:82:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:25:49.772105: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.799598: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:25:49.799681: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:25:49.812936: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:25:49.818880: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:25:49.840453: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:25:49.845815: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:25:49.846954: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:25:49.848701: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:25:49.849154: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 02:25:49.850083: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:25:49.850369: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:82:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:25:49.850404: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.850432: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:25:49.850458: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:25:49.850483: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:25:49.850508: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:25:49.850533: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:25:49.850558: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:25:49.850583: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:25:49.850936: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:25:49.852225: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:51.724442: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 02:25:51.724552: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-15 02:25:51.724574: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-15 02:25:51.727575: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:82:00.0, compute capability: 6.0)
+2023-07-15 02:25:52.611884: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/bias_model_scaled

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.chrombpnet.params.json

@@ -0,0 +1,11 @@
+{
+    "counts_loss_weight": "72.4",
+    "filters": "512",
+    "n_dil_layers": "8",
+    "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/bias_model_scaled.h5",
+    "inputlen": "2114",
+    "outputlen": "1000",
+    "max_jitter": "500",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_1.json",
+    "negative_sampling_ratio": "0.1"
+}

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.chrombpnet_data_params.tsv

@@ -0,0 +1,3 @@
+counts_sum_min_thresh	47.0
+counts_sum_max_thresh	7895.14
+trainings_pts_post_thresh	173142

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.chrombpnet_formatting.stderr.txt

@@ -0,0 +1,40 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 01:22:57.145755: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:22:59.523709: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 01:22:59.527799: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 01:23:01.014664: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:0a:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-15 01:23:01.014887: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:01.035986: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 01:23:01.036033: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 01:23:01.045389: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 01:23:01.049959: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 01:23:01.065446: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 01:23:01.069591: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 01:23:01.070526: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 01:23:01.205378: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 01:23:01.205875: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 01:23:01.207527: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 01:23:01.350135: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:0a:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-15 01:23:01.350212: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:01.350258: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 01:23:01.350286: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 01:23:01.350311: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 01:23:01.350335: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 01:23:01.350359: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 01:23:01.350382: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 01:23:01.350406: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 01:23:01.450145: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 01:23:01.452669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:03.357648: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 01:23:03.357814: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-15 01:23:03.357829: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-15 01:23:03.449394: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:0a:00.0, compute capability: 8.0)
+2023-07-15 01:23:05.480575: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.chrombpnet_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/chrombpnet

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.chrombpnet_model_params.tsv

@@ -0,0 +1,9 @@
+counts_loss_weight	72.4
+filters	512
+n_dil_layers	8
+bias_model_path	/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/bias_model_scaled.h5
+inputlen	2114
+outputlen	1000
+max_jitter	500
+chr_fold_path	/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_1.json
+negative_sampling_ratio	0.1

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.chrombpnet_no_bias_formatting.stderr.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/chrombpnet_wo_bias

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.chrombpnet_no_bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/chrombpnet_wo_bias

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.epoch_loss.csv

@@ -0,0 +1,24 @@
+epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
+0,2.36102294921875,1558.7039794921875,1729.6456298828125,0.5429004430770874,1619.5616455078125,1658.867919921875
+1,0.5462691783905029,1445.3580322265625,1484.90771484375,0.5112063884735107,1599.5716552734375,1636.583740234375
+2,0.5094148516654968,1417.509521484375,1454.38818359375,0.4536455571651459,1574.40771484375,1607.2509765625
+3,0.4815596342086792,1402.145263671875,1437.0093994140625,0.44641828536987305,1551.8565673828125,1584.1767578125
+4,0.4622742533683777,1391.1876220703125,1424.65234375,0.43815121054649353,1547.203369140625,1578.9256591796875
+5,0.4472609758377075,1381.8077392578125,1414.1878662109375,0.4727908968925476,1543.4520263671875,1577.6824951171875
+6,0.43304434418678284,1372.348388671875,1403.7017822265625,0.43399858474731445,1547.8665771484375,1579.288330078125
+7,0.4197702407836914,1366.2169189453125,1396.6080322265625,0.47027456760406494,1548.876708984375,1582.924560546875
+8,0.4077554941177368,1360.246337890625,1389.7681884765625,0.41783252358436584,1545.528564453125,1575.7801513671875
+9,0.40093347430229187,1356.8082275390625,1385.839111328125,0.41566503047943115,1532.31494140625,1562.409912109375
+10,0.390603631734848,1350.90185546875,1379.1812744140625,0.445639431476593,1555.5218505859375,1587.7862548828125
+11,0.38173359632492065,1347.790283203125,1375.426513671875,0.4057251811027527,1553.2713623046875,1582.6463623046875
+12,0.37493330240249634,1343.3453369140625,1370.4869384765625,0.4033301770687103,1553.360595703125,1582.5614013671875
+13,0.332925409078598,1325.7425537109375,1349.8468017578125,0.40408840775489807,1533.10009765625,1562.3555908203125
+14,0.3147265613079071,1315.8179931640625,1338.6036376953125,0.3976442515850067,1531.3411865234375,1560.13037109375
+15,0.2994890809059143,1311.033935546875,1332.717041015625,0.4067270755767822,1534.906982421875,1564.3548583984375
+16,0.28836625814437866,1305.07861328125,1325.958251953125,0.4051903486251831,1552.3682861328125,1581.7041015625
+17,0.2776637077331543,1302.80419921875,1322.9068603515625,0.40621134638786316,1530.273681640625,1559.68359375
+18,0.2688189148902893,1298.2769775390625,1317.740478515625,0.4051782190799713,1547.256103515625,1576.5909423828125
+19,0.2603435218334198,1295.4129638671875,1314.2623291015625,0.4118784964084625,1543.35205078125,1573.1724853515625
+20,0.2524522542953491,1291.6123046875,1309.8887939453125,0.42233338952064514,1548.03271484375,1578.6099853515625
+21,0.22852419316768646,1282.42822265625,1298.973876953125,0.42139557003974915,1549.4940185546875,1580.0030517578125
+22,0.2205371856689453,1276.53125,1292.4991455078125,0.43633055686950684,1556.949951171875,1588.5401611328125

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.stderr.txt

@@ -0,0 +1,336 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2022-10-15 15:19:51.112385: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:32:22.006162: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 15:32:22.012108: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 15:32:22.061680: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 15:32:22.061815: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:32:22.095351: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 15:32:22.095534: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 15:32:22.111723: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 15:32:22.119999: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 15:32:22.148364: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 15:32:22.156159: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 15:32:22.157913: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 15:32:22.160119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 15:32:22.160562: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 15:32:22.161653: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 15:32:22.162937: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 15:32:22.162998: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:32:22.163039: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 15:32:22.163066: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 15:32:22.163091: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 15:32:22.163116: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 15:32:22.163142: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 15:32:22.163167: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 15:32:22.163192: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 15:32:22.163716: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 15:32:22.165500: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:32:24.377119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 15:32:24.377236: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2022-10-15 15:32:24.377256: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2022-10-15 15:32:24.381965: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:83:00.0, compute capability: 7.0)
+2022-10-15 15:32:26.359763: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 15:32:26.378958: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz
+2022-10-15 15:32:26.664128: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 15:32:28.043179: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 15:32:28.054319: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 15:33:21.677211: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:33:24.248456: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 15:33:24.249769: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-15 15:33:24.288824: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 15:33:24.288935: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:33:24.292328: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 15:33:24.292409: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 15:33:24.293968: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 15:33:24.294375: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 15:33:24.297904: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 15:33:24.299720: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 15:33:24.300199: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 15:33:24.300890: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 15:33:24.301258: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-15 15:33:24.301362: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-15 15:33:24.301709: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-15 15:33:24.301745: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:33:24.301777: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 15:33:24.301811: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 15:33:24.301839: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-15 15:33:24.301865: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-15 15:33:24.301889: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-15 15:33:24.301914: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-15 15:33:24.301938: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-15 15:33:24.302433: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-15 15:33:24.302479: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-15 15:33:24.967684: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-15 15:33:24.967807: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2022-10-15 15:33:24.967830: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2022-10-15 15:33:24.968825: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:83:00.0, compute capability: 7.0)
+2022-10-15 15:42:49.304993: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-15 15:42:49.305589: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz
+2022-10-15 15:42:51.534493: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-15 15:42:51.862789: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-15 15:42:51.886250: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2147s vs `on_train_batch_end` time: 0.2928s). Check your callbacks.
+2022-10-16 00:58:20.062777: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 00:58:23.337394: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 00:58:23.338620: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-16 00:58:23.377732: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 00:58:23.377839: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 00:58:23.381016: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 00:58:23.381101: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 00:58:23.382466: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 00:58:23.382814: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 00:58:23.385946: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 00:58:23.386696: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 00:58:23.387088: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 00:58:23.387706: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 00:58:23.388069: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-16 00:58:23.388161: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 00:58:23.388467: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 00:58:23.388497: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 00:58:23.388524: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 00:58:23.388549: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 00:58:23.388573: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 00:58:23.388597: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 00:58:23.388621: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 00:58:23.388645: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 00:58:23.388669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 00:58:23.389158: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 00:58:23.389197: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 00:58:24.001902: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-16 00:58:24.002008: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2022-10-16 00:58:24.002027: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2022-10-16 00:58:24.003070: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:83:00.0, compute capability: 7.0)
+2022-10-16 01:00:42.373851: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-16 01:00:42.377949: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz
+2022-10-16 01:00:42.548760: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:00:42.790436: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:00:42.792583: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2022-10-16 01:06:06.795679: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:06:09.718499: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:06:09.719775: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-16 01:06:09.754465: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:06:09.754570: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:06:09.757696: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:06:09.757766: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:06:09.759136: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:06:09.759470: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:06:09.762547: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:06:09.763306: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:06:09.763684: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:06:09.764333: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:06:09.764679: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-16 01:06:09.764769: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:06:09.765073: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:06:09.765104: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:06:09.765137: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:06:09.765168: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:06:09.765197: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:06:09.765226: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:06:09.765256: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:06:09.765285: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:06:09.765314: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:06:09.765790: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:06:09.765837: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:06:10.337886: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-16 01:06:10.338017: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2022-10-16 01:06:10.338035: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2022-10-16 01:06:10.338976: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:83:00.0, compute capability: 7.0)
+WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
+2022-10-16 01:07:58.074885: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-16 01:07:58.077637: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz
+2022-10-16 01:07:58.147219: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:07:58.397600: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:07:58.399201: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2022-10-16 01:13:01.887573: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:13:04.846701: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:13:04.847933: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-16 01:13:04.884306: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:13:04.884424: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:13:04.887521: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:13:04.887585: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:13:04.888924: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:13:04.889256: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:13:04.892275: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:13:04.893006: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:13:04.893377: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:13:04.893960: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:13:04.894301: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-16 01:13:04.894396: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:13:04.894691: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:13:04.894722: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:13:04.894774: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:13:04.894810: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:13:04.894837: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:13:04.894866: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:13:04.894894: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:13:04.894920: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:13:04.894946: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:13:04.895426: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:13:04.895467: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:13:05.483534: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-16 01:13:05.483639: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2022-10-16 01:13:05.483659: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2022-10-16 01:13:05.485128: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:83:00.0, compute capability: 7.0)
+2022-10-16 01:14:52.326063: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-16 01:14:52.328089: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz
+2022-10-16 01:14:52.376058: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:14:52.629559: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:14:52.631188: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
+  cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
+  profile_prob = profile / np.sum(profile)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
+  shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
+/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
+  curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
+2022-10-16 01:16:32.426981: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:16:34.309431: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:16:34.310771: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-16 01:16:34.352239: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:16:34.352362: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:16:34.355835: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:16:34.355952: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:16:34.357529: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:16:34.357946: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:16:34.361468: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:16:34.362418: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:16:34.362912: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:16:34.363602: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:16:34.363969: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-16 01:16:34.364074: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:16:34.364399: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:16:34.364444: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:16:34.364474: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:16:34.364502: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:16:34.364530: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:16:34.364557: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:16:34.364584: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:16:34.364611: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:16:34.364638: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:16:34.365143: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:16:34.365218: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:16:35.060669: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-16 01:16:35.060790: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2022-10-16 01:16:35.060821: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2022-10-16 01:16:35.061885: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:83:00.0, compute capability: 7.0)
+WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
+2022-10-16 01:16:58.035361: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-16 01:16:58.036058: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz
+2022-10-16 01:16:58.344864: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:16:58.743398: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:16:58.745594: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:17:04.214306: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+2022-10-16 01:17:04.214890: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.96GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+2022-10-16 01:17:04.711718: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+2022-10-16 01:17:04.712266: W tensorflow/core/common_runtime/bfc_allocator.cc:248] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.77GiB with freed_by_count=0. The caller indicates that this is not a failure, but may mean that there could be performance gains if more memory were available.
+mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_1//footprints’: File exists
+2022-10-16 01:21:00.854785: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:21:02.726259: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:21:02.727632: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2022-10-16 01:21:02.767875: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:21:02.768000: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:21:02.771512: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:21:02.771694: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:21:02.773385: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:21:02.773869: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:21:02.777392: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:21:02.778419: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:21:02.778903: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:21:02.779595: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:21:02.779986: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2022-10-16 01:21:02.780088: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2022-10-16 01:21:02.780433: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:83:00.0 name: NVIDIA TITAN V computeCapability: 7.0
+coreClock: 1.455GHz coreCount: 80 deviceMemorySize: 11.78GiB deviceMemoryBandwidth: 607.97GiB/s
+2022-10-16 01:21:02.780479: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:21:02.780515: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:21:02.780546: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:21:02.780576: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2022-10-16 01:21:02.780606: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2022-10-16 01:21:02.780635: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2022-10-16 01:21:02.780664: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2022-10-16 01:21:02.780693: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2022-10-16 01:21:02.781212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2022-10-16 01:21:02.781264: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2022-10-16 01:21:03.471151: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2022-10-16 01:21:03.471270: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2022-10-16 01:21:03.471290: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2022-10-16 01:21:03.472316: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10912 MB memory) -> physical GPU (device: 0, name: NVIDIA TITAN V, pci bus id: 0000:83:00.0, compute capability: 7.0)
+2022-10-16 01:21:27.239867: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
+2022-10-16 01:21:27.240569: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2400050000 Hz
+2022-10-16 01:21:27.460128: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2022-10-16 01:21:27.861331: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2022-10-16 01:21:27.863411: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.stdout.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77b69d0174f1298e7318d1d384a866e94c87e4fd760de6004cbd9420c03293ce
+size 17300611

fold_1/logs.models.fold_1.ENCSR499IFY/logfile.modelling.fold_1.ENCSR499IFY.stdout_v1.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3bb0e728db878581efa75e71cf4b1847742d39709f73d5c8ad8eb006be19f04
+size 17298938

fold_1/model.bias_scaled.fold_1.ENCSR499IFY.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf806993b519bd492e293b3154cf2e7b6cf8655ffcfcf81f6a3f7c797ed5af7f
+size 2688440

fold_1/model.bias_scaled.fold_1.ENCSR499IFY.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d291d022b2244b0197d462da8cd1c7d83fc99658990fdcd2e91909313ed36ef4
+size 1198080

fold_1/model.chrombpnet.fold_1.ENCSR499IFY.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6a24d88651ad44a7d9284fe042c81fa14124ec03369435f57624acd25edc725
+size 26447928

fold_1/model.chrombpnet.fold_1.ENCSR499IFY.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a122d025329f7350d9b47985e8f60d6c1369d28e96b38cd879c64a2b6569ab6
+size 27607040

fold_1/model.chrombpnet_nobias.fold_1.ENCSR499IFY.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:331f4e9c16bcda455ce54677a96824924af8ef6a6aa1c047aea6d37018fd25e7
+size 25583536

fold_1/model.chrombpnet_nobias.fold_1.ENCSR499IFY.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fb9e8afa84076cff0d6e428456befbdc8b5bdfea1ae27f56b78d77d129ca415
+size 26081280

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.args.json

@@ -0,0 +1,23 @@
+{
+    "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
+    "bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//preprocessing/bigWigs/ENCSR499IFY.bigWig",
+    "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2//filtered.peaks.bed",
+    "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2//filtered.nonpeaks.bed",
+    "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "trackables": [
+        "logcount_predictions_loss",
+        "loss",
+        "logits_profile_predictions_loss",
+        "val_logcount_predictions_loss",
+        "val_loss",
+        "val_logits_profile_predictions_loss"
+    ],
+    "epochs": 50,
+    "early_stop": 5,
+    "batch_size": 64,
+    "learning_rate": 0.001,
+    "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet_model_params.tsv",
+    "seed": 1234,
+    "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
+}

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.bias_formatting.stderr.txt

@@ -0,0 +1,38 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 02:25:46.797157: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.813675: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:25:49.818890: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 02:25:49.852845: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:04:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:25:49.852907: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.882554: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:25:49.882669: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:25:49.897901: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:25:49.904111: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:25:49.926854: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:25:49.932536: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:25:49.933721: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:25:49.950106: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:25:49.950547: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 02:25:49.951473: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 02:25:49.970397: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:04:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
+2023-07-15 02:25:49.970506: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:49.970565: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 02:25:49.970588: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 02:25:49.970609: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 02:25:49.970630: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 02:25:49.970650: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 02:25:49.970670: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 02:25:49.970691: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 02:25:49.971127: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 02:25:49.972708: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 02:25:51.830433: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 02:25:51.830492: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-15 02:25:51.830509: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-15 02:25:51.833535: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:04:00.0, compute capability: 6.0)
+2023-07-15 02:25:52.703035: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.bias_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/bias_model_scaled

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.chrombpnet.params.json

@@ -0,0 +1,11 @@
+{
+    "counts_loss_weight": "72.5",
+    "filters": "512",
+    "n_dil_layers": "8",
+    "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5",
+    "inputlen": "2114",
+    "outputlen": "1000",
+    "max_jitter": "500",
+    "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
+    "negative_sampling_ratio": "0.1"
+}

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.chrombpnet_data_params.tsv

@@ -0,0 +1,3 @@
+counts_sum_min_thresh	49.0
+counts_sum_max_thresh	7866.62
+trainings_pts_post_thresh	176614

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.chrombpnet_formatting.stderr.txt

@@ -0,0 +1,40 @@
+INFO:    underlay of /etc/localtime required more than 50 (88) bind mounts
+INFO:    underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
+2023-07-15 01:22:57.125245: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:22:59.428707: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 01:22:59.432057: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
+2023-07-15 01:23:00.251780: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:c3:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-15 01:23:00.251886: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:00.273169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 01:23:00.273219: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 01:23:00.282454: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 01:23:00.286849: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 01:23:00.302199: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 01:23:00.306339: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 01:23:00.307246: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 01:23:00.319162: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 01:23:00.319438: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2023-07-15 01:23:00.320131: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
+2023-07-15 01:23:00.328170: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties: 
+pciBusID: 0000:c3:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
+coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
+2023-07-15 01:23:00.328213: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:00.328242: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
+2023-07-15 01:23:00.328264: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
+2023-07-15 01:23:00.328285: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
+2023-07-15 01:23:00.328304: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
+2023-07-15 01:23:00.328323: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
+2023-07-15 01:23:00.328342: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
+2023-07-15 01:23:00.328361: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
+2023-07-15 01:23:00.372296: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
+2023-07-15 01:23:00.374673: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
+2023-07-15 01:23:03.024583: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
+2023-07-15 01:23:03.024684: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267]      0 
+2023-07-15 01:23:03.024697: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0:   N 
+2023-07-15 01:23:03.031083: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:c3:00.0, compute capability: 8.0)
+2023-07-15 01:23:05.076851: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
+/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning:  is not loaded, but a Lambda layer uses it. It may cause errors.
+  , UserWarning)

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.chrombpnet_formatting.stdout.txt

@@ -0,0 +1 @@
+singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/chrombpnet

fold_2/logs.models.fold_2.ENCSR499IFY/logfile.modelling.fold_2.ENCSR499IFY.chrombpnet_model_params.tsv

@@ -0,0 +1,9 @@
+counts_loss_weight	72.5
+filters	512
+n_dil_layers	8
+bias_model_path	/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR499IFY//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5
+inputlen	2114
+outputlen	1000
+max_jitter	500
+chr_fold_path	/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json
+negative_sampling_ratio	0.1