Hugging Face's logo

kundajelab
/
encode-chrombpnet-DNASE-dorsolateral-prefrontal-cortex-ENCSR904LMG-ENCSR095YZQ

chrombpnet
encode
chromatin-accessibility
DNASE
prefrontal
hg38
Model card Files
xet
 Community
chang-m-yun commited on
Commit
966aac2
·
verified ·
1 Parent(s): b0defb7

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.args.json +23 -0
  2. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.batch_loss.tsv +0 -0
  3. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.bias_formatting.stderr.txt +38 -0
  4. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.bias_formatting.stdout.txt +1 -0
  5. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet.params.json +11 -0
  6. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_data_params.tsv +3 -0
  7. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_formatting.stderr.txt +40 -0
  8. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_formatting.stdout.txt +1 -0
  9. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_model_params.tsv +9 -0
  10. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_no_bias_formatting.stderr.txt +1 -0
  11. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_no_bias_formatting.stdout.txt +1 -0
  12. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.epoch_loss.csv +15 -0
  13. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.stderr.txt +0 -0
  14. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.stdout.txt +0 -0
  15. fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.stdout_v1.txt +0 -0
  16. fold_0/model.bias_scaled.fold_0.ENCSR904LMG.h5 +3 -0
  17. fold_0/model.bias_scaled.fold_0.ENCSR904LMG.tar +3 -0
  18. fold_0/model.chrombpnet.fold_0.ENCSR904LMG.h5 +3 -0
  19. fold_0/model.chrombpnet.fold_0.ENCSR904LMG.tar +3 -0
  20. fold_0/model.chrombpnet_nobias.fold_0.ENCSR904LMG.h5 +3 -0
  21. fold_0/model.chrombpnet_nobias.fold_0.ENCSR904LMG.tar +3 -0
  22. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.args.json +23 -0
  23. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.batch_loss.tsv +0 -0
  24. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.bias_formatting.stderr.txt +38 -0
  25. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.bias_formatting.stdout.txt +1 -0
  26. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet.params.json +11 -0
  27. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_data_params.tsv +3 -0
  28. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_formatting.stderr.txt +40 -0
  29. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_formatting.stdout.txt +1 -0
  30. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_model_params.tsv +9 -0
  31. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_no_bias_formatting.stderr.txt +1 -0
  32. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_no_bias_formatting.stdout.txt +1 -0
  33. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.epoch_loss.csv +13 -0
  34. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.stderr.txt +332 -0
  35. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.stdout.txt +0 -0
  36. fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.stdout_v1.txt +0 -0
  37. fold_1/model.bias_scaled.fold_1.ENCSR904LMG.h5 +3 -0
  38. fold_1/model.bias_scaled.fold_1.ENCSR904LMG.tar +3 -0
  39. fold_1/model.chrombpnet.fold_1.ENCSR904LMG.h5 +3 -0
  40. fold_1/model.chrombpnet.fold_1.ENCSR904LMG.tar +3 -0
  41. fold_1/model.chrombpnet_nobias.fold_1.ENCSR904LMG.h5 +3 -0
  42. fold_1/model.chrombpnet_nobias.fold_1.ENCSR904LMG.tar +3 -0
  43. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.args.json +23 -0
  44. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.batch_loss.tsv +0 -0
  45. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.bias_formatting.stderr.txt +38 -0
  46. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.bias_formatting.stdout.txt +1 -0
  47. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet.params.json +11 -0
  48. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet_data_params.tsv +3 -0
  49. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet_formatting.stderr.txt +40 -0
  50. fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet_formatting.stdout.txt +1 -0
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.args.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
3
+ "bigwig": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//preprocessing/bigWigs/ENCSR904LMG.bigWig",
4
+ "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombpnet_model_encsr880cub_bias//filtered.peaks.bed",
5
+ "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombpnet_model_encsr880cub_bias//filtered.nonpeaks.bed",
6
+ "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombpnet_model_encsr880cub_bias//chrombpnet",
7
+ "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json",
8
+ "trackables": [
9
+ "logcount_predictions_loss",
10
+ "loss",
11
+ "logits_profile_predictions_loss",
12
+ "val_logcount_predictions_loss",
13
+ "val_loss",
14
+ "val_logits_profile_predictions_loss"
15
+ ],
16
+ "epochs": 50,
17
+ "early_stop": 5,
18
+ "batch_size": 64,
19
+ "learning_rate": 0.001,
20
+ "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombpnet_model_encsr880cub_bias//chrombpnet_model_params.tsv",
21
+ "seed": 1234,
22
+ "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
23
+ }
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.batch_loss.tsv ADDED
The diff for this file is too large to render. See raw diff
 
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.bias_formatting.stderr.txt ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
2
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
3
+ 2023-07-15 02:18:31.926105: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
4
+ 2023-07-15 02:18:34.256296: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
5
+ 2023-07-15 02:18:34.259750: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
6
+ 2023-07-15 02:18:35.012607: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
7
+ pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
8
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
9
+ 2023-07-15 02:18:35.012754: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
10
+ 2023-07-15 02:18:35.031109: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
11
+ 2023-07-15 02:18:35.031220: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
12
+ 2023-07-15 02:18:35.040812: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
13
+ 2023-07-15 02:18:35.045441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
14
+ 2023-07-15 02:18:35.061317: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
15
+ 2023-07-15 02:18:35.065556: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
16
+ 2023-07-15 02:18:35.066502: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
17
+ 2023-07-15 02:18:35.098772: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
18
+ 2023-07-15 02:18:35.099087: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
19
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
20
+ 2023-07-15 02:18:35.099957: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
21
+ 2023-07-15 02:18:35.119828: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
22
+ pciBusID: 0000:07:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
23
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
24
+ 2023-07-15 02:18:35.119860: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
25
+ 2023-07-15 02:18:35.119881: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
26
+ 2023-07-15 02:18:35.119895: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
27
+ 2023-07-15 02:18:35.119908: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
28
+ 2023-07-15 02:18:35.119921: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
29
+ 2023-07-15 02:18:35.119933: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
30
+ 2023-07-15 02:18:35.119945: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
31
+ 2023-07-15 02:18:35.119958: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
32
+ 2023-07-15 02:18:35.145429: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
33
+ 2023-07-15 02:18:35.146746: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
34
+ 2023-07-15 02:18:37.536284: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
35
+ 2023-07-15 02:18:37.536453: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
36
+ 2023-07-15 02:18:37.536468: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
37
+ 2023-07-15 02:18:37.543314: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:07:00.0, compute capability: 8.0)
38
+ 2023-07-15 02:18:38.665743: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.bias_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/new_model_formats/bias_model_scaled
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet.params.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "counts_loss_weight": "13.9",
3
+ "filters": "512",
4
+ "n_dil_layers": "8",
5
+ "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5",
6
+ "inputlen": "2114",
7
+ "outputlen": "1000",
8
+ "max_jitter": "500",
9
+ "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json",
10
+ "negative_sampling_ratio": "0.1"
11
+ }
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_data_params.tsv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ counts_sum_min_thresh 14.0
2
+ counts_sum_max_thresh 1581.0
3
+ trainings_pts_post_thresh 170834
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_formatting.stderr.txt ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
2
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
3
+ 2023-07-15 01:13:20.765117: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
4
+ 2023-07-15 01:13:24.018472: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
5
+ 2023-07-15 01:13:24.023372: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
6
+ 2023-07-15 01:13:24.084056: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
7
+ pciBusID: 0000:03:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
8
+ coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
9
+ 2023-07-15 01:13:24.084165: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
10
+ 2023-07-15 01:13:24.113043: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
11
+ 2023-07-15 01:13:24.113202: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
12
+ 2023-07-15 01:13:24.126821: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
13
+ 2023-07-15 01:13:24.132962: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
14
+ 2023-07-15 01:13:24.157095: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
15
+ 2023-07-15 01:13:24.163021: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
16
+ 2023-07-15 01:13:24.164224: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
17
+ 2023-07-15 01:13:24.171543: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
18
+ 2023-07-15 01:13:24.171944: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
19
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
20
+ 2023-07-15 01:13:24.172881: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
21
+ 2023-07-15 01:13:24.173936: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
22
+ pciBusID: 0000:03:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
23
+ coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
24
+ 2023-07-15 01:13:24.173995: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
25
+ 2023-07-15 01:13:24.174034: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
26
+ 2023-07-15 01:13:24.174057: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
27
+ 2023-07-15 01:13:24.174078: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
28
+ 2023-07-15 01:13:24.174100: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
29
+ 2023-07-15 01:13:24.174121: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
30
+ 2023-07-15 01:13:24.174142: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
31
+ 2023-07-15 01:13:24.174164: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
32
+ 2023-07-15 01:13:24.174881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
33
+ 2023-07-15 01:13:24.176310: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
34
+ 2023-07-15 01:13:26.052366: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
35
+ 2023-07-15 01:13:26.052467: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
36
+ 2023-07-15 01:13:26.052486: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
37
+ 2023-07-15 01:13:26.055496: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:03:00.0, compute capability: 6.0)
38
+ 2023-07-15 01:13:28.408118: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
39
+ /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors.
40
+ , UserWarning)
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_model_params.tsv ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ counts_loss_weight 13.9
2
+ filters 512
3
+ n_dil_layers 8
4
+ bias_model_path /scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombpnet_model_encsr880cub_bias/bias_model_scaled.h5
5
+ inputlen 2114
6
+ outputlen 1000
7
+ max_jitter 500
8
+ chr_fold_path /scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_0.json
9
+ negative_sampling_ratio 0.1
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_no_bias_formatting.stderr.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet_wo_bias
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.chrombpnet_no_bias_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombpnet_model_encsr880cub_bias/new_model_formats/chrombpnet_wo_bias
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.epoch_loss.csv ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
2
+ 0,0.37358251214027405,477.523193359375,482.71636962890625,0.21345578134059906,478.3991394042969,481.3660888671875
3
+ 1,0.2447594404220581,464.92987060546875,468.33258056640625,0.19440695643424988,477.8491516113281,480.5513916015625
4
+ 2,0.2239121049642563,461.99798583984375,465.1103210449219,0.19802577793598175,471.88037109375,474.6328125
5
+ 3,0.20998676121234894,460.25830078125,463.1770324707031,0.17061738669872284,471.87347412109375,474.244873046875
6
+ 4,0.2003222554922104,458.994140625,461.7794189453125,0.1671324521303177,473.1056823730469,475.4284362792969
7
+ 5,0.19365045428276062,457.7388000488281,460.4306640625,0.1600153148174286,472.7225036621094,474.9468688964844
8
+ 6,0.18870779871940613,456.4270935058594,459.0494384765625,0.16617469489574432,471.8562927246094,474.165771484375
9
+ 7,0.18495289981365204,456.1794738769531,458.7496643066406,0.16049350798130035,471.8617858886719,474.09246826171875
10
+ 8,0.18123096227645874,455.028076171875,457.5464172363281,0.16054847836494446,470.7580871582031,472.9897155761719
11
+ 9,0.17599160969257355,454.4422912597656,456.8890075683594,0.15482574701309204,475.06866455078125,477.22076416015625
12
+ 10,0.17191052436828613,453.72991943359375,456.1200256347656,0.1583748459815979,471.4794616699219,473.6807556152344
13
+ 11,0.17172862589359283,452.9721374511719,455.3593444824219,0.1593923568725586,474.40142822265625,476.6170654296875
14
+ 12,0.15634171664714813,448.9422607421875,451.1151428222656,0.15076783299446106,472.7935791015625,474.88909912109375
15
+ 13,0.15057694911956787,446.5379333496094,448.63140869140625,0.1646168977022171,472.81781005859375,475.1057434082031
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.stderr.txt ADDED
The diff for this file is too large to render. See raw diff
 
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.stdout.txt ADDED
The diff for this file is too large to render. See raw diff
 
fold_0/logs.models.fold_0.ENCSR904LMG/logfile.modelling.fold_0.ENCSR904LMG.stdout_v1.txt ADDED
The diff for this file is too large to render. See raw diff
 
fold_0/model.bias_scaled.fold_0.ENCSR904LMG.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04eeec0224054c2262287accb830754da23a65ad38e5fec4b72694ec8f6b8257
3
+ size 2688440
fold_0/model.bias_scaled.fold_0.ENCSR904LMG.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:241293194edf40840f3dc49dd0d5bc33cbfce10005fc277153e9b20935828bd5
3
+ size 1198080
fold_0/model.chrombpnet.fold_0.ENCSR904LMG.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5647958cf5ce901e7fe798cb8be628f95d5e64c34b85346ab1f5d50383deefa1
3
+ size 26447928
fold_0/model.chrombpnet.fold_0.ENCSR904LMG.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2affd441769731d5e53aab9d94f5e12d10b3b889b9d6695d58f9814198b0a64
3
+ size 27607040
fold_0/model.chrombpnet_nobias.fold_0.ENCSR904LMG.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2c6ddccd3eed3e86ea476bc17cfb1a2489708bfab4ad8e64db994e97cbe0465
3
+ size 25583536
fold_0/model.chrombpnet_nobias.fold_0.ENCSR904LMG.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a5fb706215acad8b761cd1cc57f7361d89d456ea1f126b192ad89f21173fc60
3
+ size 26081280
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.args.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
3
+ "bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//preprocessing/bigWigs/ENCSR904LMG.bigWig",
4
+ "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1//filtered.peaks.bed",
5
+ "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1//filtered.nonpeaks.bed",
6
+ "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1//chrombpnet",
7
+ "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_1.json",
8
+ "trackables": [
9
+ "logcount_predictions_loss",
10
+ "loss",
11
+ "logits_profile_predictions_loss",
12
+ "val_logcount_predictions_loss",
13
+ "val_loss",
14
+ "val_logits_profile_predictions_loss"
15
+ ],
16
+ "epochs": 50,
17
+ "early_stop": 5,
18
+ "batch_size": 64,
19
+ "learning_rate": 0.001,
20
+ "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1//chrombpnet_model_params.tsv",
21
+ "seed": 1234,
22
+ "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
23
+ }
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.batch_loss.tsv ADDED
The diff for this file is too large to render. See raw diff
 
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.bias_formatting.stderr.txt ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
2
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
3
+ 2023-07-15 02:18:31.285718: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
4
+ 2023-07-15 02:18:33.556239: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
5
+ 2023-07-15 02:18:33.559561: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
6
+ 2023-07-15 02:18:34.026337: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
7
+ pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
8
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
9
+ 2023-07-15 02:18:34.026452: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
10
+ 2023-07-15 02:18:34.046220: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
11
+ 2023-07-15 02:18:34.046357: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
12
+ 2023-07-15 02:18:34.055536: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
13
+ 2023-07-15 02:18:34.059804: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
14
+ 2023-07-15 02:18:34.075212: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
15
+ 2023-07-15 02:18:34.079275: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
16
+ 2023-07-15 02:18:34.080172: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
17
+ 2023-07-15 02:18:34.199376: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
18
+ 2023-07-15 02:18:34.199870: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
19
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
20
+ 2023-07-15 02:18:34.201302: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
21
+ 2023-07-15 02:18:34.213971: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
22
+ pciBusID: 0000:c0:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
23
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
24
+ 2023-07-15 02:18:34.214020: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
25
+ 2023-07-15 02:18:34.214050: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
26
+ 2023-07-15 02:18:34.214073: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
27
+ 2023-07-15 02:18:34.214092: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
28
+ 2023-07-15 02:18:34.214112: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
29
+ 2023-07-15 02:18:34.214130: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
30
+ 2023-07-15 02:18:34.214149: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
31
+ 2023-07-15 02:18:34.214167: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
32
+ 2023-07-15 02:18:34.228559: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
33
+ 2023-07-15 02:18:34.230802: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
34
+ 2023-07-15 02:18:37.407104: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
35
+ 2023-07-15 02:18:37.407188: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
36
+ 2023-07-15 02:18:37.407201: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
37
+ 2023-07-15 02:18:37.413397: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:c0:00.0, compute capability: 8.0)
38
+ 2023-07-15 02:18:38.423160: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.bias_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/bias_model_scaled
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet.params.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "counts_loss_weight": "13.9",
3
+ "filters": "512",
4
+ "n_dil_layers": "8",
5
+ "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/bias_model_scaled.h5",
6
+ "inputlen": "2114",
7
+ "outputlen": "1000",
8
+ "max_jitter": "500",
9
+ "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_1.json",
10
+ "negative_sampling_ratio": "0.1"
11
+ }
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_data_params.tsv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ counts_sum_min_thresh 14.0
2
+ counts_sum_max_thresh 1582.18
3
+ trainings_pts_post_thresh 170564
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_formatting.stderr.txt ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
2
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
3
+ 2023-07-15 01:13:20.968348: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
4
+ 2023-07-15 01:13:23.996137: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
5
+ 2023-07-15 01:13:24.000662: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
6
+ 2023-07-15 01:13:24.043563: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
7
+ pciBusID: 0000:82:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
8
+ coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
9
+ 2023-07-15 01:13:24.043680: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
10
+ 2023-07-15 01:13:24.069626: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
11
+ 2023-07-15 01:13:24.069716: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
12
+ 2023-07-15 01:13:24.083847: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
13
+ 2023-07-15 01:13:24.089940: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
14
+ 2023-07-15 01:13:24.112854: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
15
+ 2023-07-15 01:13:24.118314: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
16
+ 2023-07-15 01:13:24.119477: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
17
+ 2023-07-15 01:13:24.127510: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
18
+ 2023-07-15 01:13:24.127864: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
19
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
20
+ 2023-07-15 01:13:24.128907: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
21
+ 2023-07-15 01:13:24.130736: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
22
+ pciBusID: 0000:82:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
23
+ coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
24
+ 2023-07-15 01:13:24.130772: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
25
+ 2023-07-15 01:13:24.130803: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
26
+ 2023-07-15 01:13:24.130831: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
27
+ 2023-07-15 01:13:24.130857: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
28
+ 2023-07-15 01:13:24.130884: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
29
+ 2023-07-15 01:13:24.130910: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
30
+ 2023-07-15 01:13:24.130936: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
31
+ 2023-07-15 01:13:24.130972: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
32
+ 2023-07-15 01:13:24.131341: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
33
+ 2023-07-15 01:13:24.132744: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
34
+ 2023-07-15 01:13:25.998572: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
35
+ 2023-07-15 01:13:25.998679: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
36
+ 2023-07-15 01:13:25.998700: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
37
+ 2023-07-15 01:13:26.001805: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:82:00.0, compute capability: 6.0)
38
+ 2023-07-15 01:13:28.465525: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
39
+ /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors.
40
+ , UserWarning)
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/chrombpnet
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_model_params.tsv ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ counts_loss_weight 13.9
2
+ filters 512
3
+ n_dil_layers 8
4
+ bias_model_path /scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/bias_model_scaled.h5
5
+ inputlen 2114
6
+ outputlen 1000
7
+ max_jitter 500
8
+ chr_fold_path /scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_1.json
9
+ negative_sampling_ratio 0.1
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_no_bias_formatting.stderr.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/chrombpnet_wo_bias
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.chrombpnet_no_bias_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/chrombpnet_wo_bias.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1/new_model_formats/chrombpnet_wo_bias
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.epoch_loss.csv ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ epoch,logcount_predictions_loss,logits_profile_predictions_loss,loss,val_logcount_predictions_loss,val_logits_profile_predictions_loss,val_loss
2
+ 0,0.7592453360557556,473.06158447265625,483.6145935058594,0.26493188738822937,524.0032958984375,527.6857299804688
3
+ 1,0.2576517164707184,461.7140808105469,465.29547119140625,0.22284428775310516,520.547607421875,523.645263671875
4
+ 2,0.23335988819599152,458.6826171875,461.9261169433594,0.20125892758369446,516.4982299804688,519.2952270507812
5
+ 3,0.21795572340488434,456.2723083496094,459.3017578125,0.19429877400398254,515.0861206054688,517.7869262695312
6
+ 4,0.2049148976802826,455.2070007324219,458.0549621582031,0.20070594549179077,512.42578125,515.215576171875
7
+ 5,0.19805048406124115,453.4013366699219,456.1537780761719,0.18465450406074524,514.4502563476562,517.016845703125
8
+ 6,0.19229824841022491,452.6071472167969,455.2795104980469,0.20425425469875336,510.68475341796875,513.5234375
9
+ 7,0.18561072647571564,451.78094482421875,454.3614501953125,0.17776840925216675,512.3582763671875,514.8294067382812
10
+ 8,0.18404100835323334,450.48358154296875,453.04132080078125,0.18669606745243073,514.5365600585938,517.131591796875
11
+ 9,0.1785842627286911,449.68328857421875,452.1656494140625,0.19740094244480133,515.6992797851562,518.4429321289062
12
+ 10,0.1630580872297287,446.2938537597656,448.5606689453125,0.16735924780368805,512.8272094726562,515.1539306640625
13
+ 11,0.1581270843744278,444.5502624511719,446.7477111816406,0.1656743884086609,515.2924194335938,517.5955200195312
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.stderr.txt ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
2
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
3
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
4
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
5
+ 2022-10-16 20:58:09.370539: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
6
+ 2022-10-16 21:05:28.346460: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
7
+ 2022-10-16 21:05:28.351034: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
8
+ 2022-10-16 21:05:28.843718: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
9
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
10
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
11
+ 2022-10-16 21:05:28.843799: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
12
+ 2022-10-16 21:05:28.862220: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
13
+ 2022-10-16 21:05:28.862309: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
14
+ 2022-10-16 21:05:28.874330: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
15
+ 2022-10-16 21:05:28.878956: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
16
+ 2022-10-16 21:05:28.894445: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
17
+ 2022-10-16 21:05:28.899980: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
18
+ 2022-10-16 21:05:28.901083: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
19
+ 2022-10-16 21:05:28.905483: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
20
+ 2022-10-16 21:05:28.905862: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
21
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
22
+ 2022-10-16 21:05:28.905998: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
23
+ 2022-10-16 21:05:28.907473: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
24
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
25
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
26
+ 2022-10-16 21:05:28.907509: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
27
+ 2022-10-16 21:05:28.907530: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
28
+ 2022-10-16 21:05:28.907548: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
29
+ 2022-10-16 21:05:28.907565: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
30
+ 2022-10-16 21:05:28.907581: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
31
+ 2022-10-16 21:05:28.907609: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
32
+ 2022-10-16 21:05:28.907626: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
33
+ 2022-10-16 21:05:28.907642: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
34
+ 2022-10-16 21:05:28.910600: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
35
+ 2022-10-16 21:05:28.911904: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
36
+ 2022-10-16 21:05:30.549371: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
37
+ 2022-10-16 21:05:30.549543: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
38
+ 2022-10-16 21:05:30.549561: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
39
+ 2022-10-16 21:05:30.556371: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75712 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
40
+ 2022-10-16 21:05:32.104529: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
41
+ 2022-10-16 21:05:32.118796: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2449945000 Hz
42
+ 2022-10-16 21:05:32.309196: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
43
+ 2022-10-16 21:05:33.778915: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
44
+ 2022-10-16 21:05:33.788979: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
45
+ 2022-10-16 21:06:02.728367: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
46
+ 2022-10-16 21:06:04.417321: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
47
+ 2022-10-16 21:06:04.418168: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
48
+ 2022-10-16 21:06:04.709014: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
49
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
50
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
51
+ 2022-10-16 21:06:04.709167: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
52
+ 2022-10-16 21:06:04.711663: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
53
+ 2022-10-16 21:06:04.711743: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
54
+ 2022-10-16 21:06:04.712951: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
55
+ 2022-10-16 21:06:04.713281: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
56
+ 2022-10-16 21:06:04.715786: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
57
+ 2022-10-16 21:06:04.716430: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
58
+ 2022-10-16 21:06:04.716738: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
59
+ 2022-10-16 21:06:04.718444: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
60
+ 2022-10-16 21:06:04.718778: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
61
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
62
+ 2022-10-16 21:06:04.718891: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
63
+ 2022-10-16 21:06:04.719814: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
64
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
65
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
66
+ 2022-10-16 21:06:04.719846: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
67
+ 2022-10-16 21:06:04.719869: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
68
+ 2022-10-16 21:06:04.719887: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
69
+ 2022-10-16 21:06:04.719904: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
70
+ 2022-10-16 21:06:04.719921: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
71
+ 2022-10-16 21:06:04.719938: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
72
+ 2022-10-16 21:06:04.719954: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
73
+ 2022-10-16 21:06:04.719971: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
74
+ 2022-10-16 21:06:04.721597: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
75
+ 2022-10-16 21:06:04.721632: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
76
+ 2022-10-16 21:06:05.204092: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
77
+ 2022-10-16 21:06:05.204276: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
78
+ 2022-10-16 21:06:05.204291: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
79
+ 2022-10-16 21:06:05.207147: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75712 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
80
+ 2022-10-16 21:12:01.273159: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
81
+ 2022-10-16 21:12:01.273702: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2449945000 Hz
82
+ 2022-10-16 21:12:02.670515: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
83
+ 2022-10-16 21:12:03.150155: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
84
+ 2022-10-16 21:12:03.166543: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
85
+ 2022-10-16 21:12:06.232086: I tensorflow/stream_executor/cuda/cuda_blas.cc:1838] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
86
+ 2022-10-16 22:24:45.858726: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
87
+ 2022-10-16 22:24:48.725036: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
88
+ 2022-10-16 22:24:48.726197: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
89
+ 2022-10-16 22:24:49.045963: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
90
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
91
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
92
+ 2022-10-16 22:24:49.046032: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
93
+ 2022-10-16 22:24:49.049254: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
94
+ 2022-10-16 22:24:49.049327: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
95
+ 2022-10-16 22:24:49.051135: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
96
+ 2022-10-16 22:24:49.051549: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
97
+ 2022-10-16 22:24:49.055035: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
98
+ 2022-10-16 22:24:49.055888: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
99
+ 2022-10-16 22:24:49.056329: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
100
+ 2022-10-16 22:24:49.058148: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
101
+ 2022-10-16 22:24:49.058502: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
102
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
103
+ 2022-10-16 22:24:49.058657: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
104
+ 2022-10-16 22:24:49.059561: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
105
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
106
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
107
+ 2022-10-16 22:24:49.059663: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
108
+ 2022-10-16 22:24:49.059695: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
109
+ 2022-10-16 22:24:49.059721: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
110
+ 2022-10-16 22:24:49.059745: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
111
+ 2022-10-16 22:24:49.059768: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
112
+ 2022-10-16 22:24:49.059792: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
113
+ 2022-10-16 22:24:49.059815: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
114
+ 2022-10-16 22:24:49.059839: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
115
+ 2022-10-16 22:24:49.061519: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
116
+ 2022-10-16 22:24:49.061558: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
117
+ 2022-10-16 22:24:49.553324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
118
+ 2022-10-16 22:24:49.553493: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
119
+ 2022-10-16 22:24:49.553507: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
120
+ 2022-10-16 22:24:49.556425: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75712 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
121
+ 2022-10-16 22:26:46.403834: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
122
+ 2022-10-16 22:26:46.407023: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2449945000 Hz
123
+ 2022-10-16 22:26:46.486196: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
124
+ 2022-10-16 22:26:46.946402: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
125
+ 2022-10-16 22:26:46.948519: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
126
+ /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors.
127
+ , UserWarning)
128
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
129
+ cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
130
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
131
+ profile_prob = profile / np.sum(profile)
132
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
133
+ shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
134
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
135
+ curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
136
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
137
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
138
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
139
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
140
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
141
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
142
+ 2022-10-16 22:29:05.005016: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
143
+ 2022-10-16 22:29:07.474648: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
144
+ 2022-10-16 22:29:07.475833: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
145
+ 2022-10-16 22:29:07.787953: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
146
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
147
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
148
+ 2022-10-16 22:29:07.788040: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
149
+ 2022-10-16 22:29:07.791045: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
150
+ 2022-10-16 22:29:07.791108: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
151
+ 2022-10-16 22:29:07.792483: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
152
+ 2022-10-16 22:29:07.792941: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
153
+ 2022-10-16 22:29:07.795607: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
154
+ 2022-10-16 22:29:07.796321: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
155
+ 2022-10-16 22:29:07.796711: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
156
+ 2022-10-16 22:29:07.798441: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
157
+ 2022-10-16 22:29:07.798778: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
158
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
159
+ 2022-10-16 22:29:07.798899: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
160
+ 2022-10-16 22:29:07.799771: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
161
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
162
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
163
+ 2022-10-16 22:29:07.799798: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
164
+ 2022-10-16 22:29:07.799821: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
165
+ 2022-10-16 22:29:07.799842: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
166
+ 2022-10-16 22:29:07.799862: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
167
+ 2022-10-16 22:29:07.799881: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
168
+ 2022-10-16 22:29:07.799900: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
169
+ 2022-10-16 22:29:07.799919: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
170
+ 2022-10-16 22:29:07.799938: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
171
+ 2022-10-16 22:29:07.801560: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
172
+ 2022-10-16 22:29:07.801608: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
173
+ 2022-10-16 22:29:08.300086: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
174
+ 2022-10-16 22:29:08.300301: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
175
+ 2022-10-16 22:29:08.300316: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
176
+ 2022-10-16 22:29:08.303239: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75712 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
177
+ WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
178
+ 2022-10-16 22:30:42.456659: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
179
+ 2022-10-16 22:30:42.458917: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2449945000 Hz
180
+ 2022-10-16 22:30:42.510867: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
181
+ 2022-10-16 22:30:42.982214: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
182
+ 2022-10-16 22:30:42.984050: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
183
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
184
+ cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
185
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
186
+ profile_prob = profile / np.sum(profile)
187
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
188
+ shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
189
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
190
+ curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
191
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
192
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
193
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
194
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
195
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
196
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
197
+ 2022-10-16 22:32:49.558928: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
198
+ 2022-10-16 22:32:52.084491: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
199
+ 2022-10-16 22:32:52.085832: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
200
+ 2022-10-16 22:32:52.367477: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
201
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
202
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
203
+ 2022-10-16 22:32:52.367557: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
204
+ 2022-10-16 22:32:52.370836: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
205
+ 2022-10-16 22:32:52.370894: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
206
+ 2022-10-16 22:32:52.372268: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
207
+ 2022-10-16 22:32:52.372708: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
208
+ 2022-10-16 22:32:52.375315: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
209
+ 2022-10-16 22:32:52.376017: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
210
+ 2022-10-16 22:32:52.376415: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
211
+ 2022-10-16 22:32:52.378129: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
212
+ 2022-10-16 22:32:52.378449: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
213
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
214
+ 2022-10-16 22:32:52.378569: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
215
+ 2022-10-16 22:32:52.379465: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
216
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
217
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
218
+ 2022-10-16 22:32:52.379492: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
219
+ 2022-10-16 22:32:52.379534: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
220
+ 2022-10-16 22:32:52.379555: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
221
+ 2022-10-16 22:32:52.379573: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
222
+ 2022-10-16 22:32:52.379631: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
223
+ 2022-10-16 22:32:52.379650: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
224
+ 2022-10-16 22:32:52.379668: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
225
+ 2022-10-16 22:32:52.379685: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
226
+ 2022-10-16 22:32:52.381303: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
227
+ 2022-10-16 22:32:52.381338: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
228
+ 2022-10-16 22:32:52.874567: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
229
+ 2022-10-16 22:32:52.874759: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
230
+ 2022-10-16 22:32:52.874773: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
231
+ 2022-10-16 22:32:52.877732: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75712 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
232
+ 2022-10-16 22:34:26.077585: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
233
+ 2022-10-16 22:34:26.079254: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2449945000 Hz
234
+ 2022-10-16 22:34:26.112681: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
235
+ 2022-10-16 22:34:26.575357: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
236
+ 2022-10-16 22:34:26.576999: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
237
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:69: RuntimeWarning: invalid value encountered in true_divide
238
+ cur_jsd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),pred_probs[idx,:])
239
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/utils/metrics_utils.py:196: RuntimeWarning: invalid value encountered in true_divide
240
+ profile_prob = profile / np.sum(profile)
241
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:78: RuntimeWarning: invalid value encountered in true_divide
242
+ shuffled_labels_prob=shuffled_labels/np.nansum(shuffled_labels)
243
+ /home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/metrics.py:88: RuntimeWarning: invalid value encountered in true_divide
244
+ curr_jsd_rnd=jensenshannon(true_counts[idx,:]/np.nansum(true_counts[idx,:]),shuffled_labels_prob)
245
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
246
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
247
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
248
+ findfont: Font family ['normal'] not found. Falling back to DejaVu Sans.
249
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
250
+ No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
251
+ 2022-10-16 22:35:35.699250: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
252
+ 2022-10-16 22:35:36.896066: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
253
+ 2022-10-16 22:35:36.896960: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
254
+ 2022-10-16 22:35:37.175171: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
255
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
256
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
257
+ 2022-10-16 22:35:37.175243: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
258
+ 2022-10-16 22:35:37.178045: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
259
+ 2022-10-16 22:35:37.178108: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
260
+ 2022-10-16 22:35:37.179382: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
261
+ 2022-10-16 22:35:37.179841: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
262
+ 2022-10-16 22:35:37.182446: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
263
+ 2022-10-16 22:35:37.183214: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
264
+ 2022-10-16 22:35:37.183737: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
265
+ 2022-10-16 22:35:37.185429: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
266
+ 2022-10-16 22:35:37.185735: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
267
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
268
+ 2022-10-16 22:35:37.185867: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
269
+ 2022-10-16 22:35:37.186836: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
270
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
271
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
272
+ 2022-10-16 22:35:37.186882: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
273
+ 2022-10-16 22:35:37.186914: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
274
+ 2022-10-16 22:35:37.186941: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
275
+ 2022-10-16 22:35:37.186965: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
276
+ 2022-10-16 22:35:37.186989: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
277
+ 2022-10-16 22:35:37.187013: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
278
+ 2022-10-16 22:35:37.187037: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
279
+ 2022-10-16 22:35:37.187061: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
280
+ 2022-10-16 22:35:37.188754: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
281
+ 2022-10-16 22:35:37.188810: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
282
+ 2022-10-16 22:35:37.679126: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
283
+ 2022-10-16 22:35:37.679253: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
284
+ 2022-10-16 22:35:37.679267: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
285
+ 2022-10-16 22:35:37.682008: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75712 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
286
+ WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.
287
+ 2022-10-16 22:35:53.796104: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
288
+ 2022-10-16 22:35:53.796823: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2449945000 Hz
289
+ 2022-10-16 22:35:53.993882: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
290
+ 2022-10-16 22:35:54.528439: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
291
+ 2022-10-16 22:35:54.530428: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
292
+ mkdir: cannot create directory ‘/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_1//footprints’: File exists
293
+ 2022-10-16 22:37:34.441645: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
294
+ 2022-10-16 22:37:35.666549: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
295
+ 2022-10-16 22:37:35.667468: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
296
+ 2022-10-16 22:37:35.977109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
297
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
298
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
299
+ 2022-10-16 22:37:35.977226: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
300
+ 2022-10-16 22:37:35.980527: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
301
+ 2022-10-16 22:37:35.980620: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
302
+ 2022-10-16 22:37:35.981918: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
303
+ 2022-10-16 22:37:35.982254: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
304
+ 2022-10-16 22:37:35.984832: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
305
+ 2022-10-16 22:37:35.985489: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
306
+ 2022-10-16 22:37:35.985932: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
307
+ 2022-10-16 22:37:35.987718: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
308
+ 2022-10-16 22:37:35.988010: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
309
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
310
+ 2022-10-16 22:37:35.988170: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
311
+ 2022-10-16 22:37:35.989154: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
312
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
313
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.21GiB deviceMemoryBandwidth: 1.85TiB/s
314
+ 2022-10-16 22:37:35.989196: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
315
+ 2022-10-16 22:37:35.989225: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
316
+ 2022-10-16 22:37:35.989249: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
317
+ 2022-10-16 22:37:35.989271: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
318
+ 2022-10-16 22:37:35.989292: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
319
+ 2022-10-16 22:37:35.989313: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
320
+ 2022-10-16 22:37:35.989334: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
321
+ 2022-10-16 22:37:35.989355: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
322
+ 2022-10-16 22:37:35.991074: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
323
+ 2022-10-16 22:37:35.991125: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
324
+ 2022-10-16 22:37:36.482605: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
325
+ 2022-10-16 22:37:36.482745: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
326
+ 2022-10-16 22:37:36.482759: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
327
+ 2022-10-16 22:37:36.485697: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75712 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
328
+ 2022-10-16 22:37:53.878061: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)
329
+ 2022-10-16 22:37:53.878829: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2449945000 Hz
330
+ 2022-10-16 22:37:54.016185: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
331
+ 2022-10-16 22:37:54.550977: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
332
+ 2022-10-16 22:37:54.552815: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.stdout.txt ADDED
The diff for this file is too large to render. See raw diff
 
fold_1/logs.models.fold_1.ENCSR904LMG/logfile.modelling.fold_1.ENCSR904LMG.stdout_v1.txt ADDED
The diff for this file is too large to render. See raw diff
 
fold_1/model.bias_scaled.fold_1.ENCSR904LMG.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5497f925feccc4ec47d2ca6ff329372b460f1647d66666e8b473425719441798
3
+ size 2688440
fold_1/model.bias_scaled.fold_1.ENCSR904LMG.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:740c1792dc5c3dc768069d4f4b7728e066f8de039512ee4e7caa188464c96661
3
+ size 1198080
fold_1/model.chrombpnet.fold_1.ENCSR904LMG.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e29c7d61db97a2c37c75816fc5ade92e690ec91800b25c16619db0fb78091ebb
3
+ size 26447928
fold_1/model.chrombpnet.fold_1.ENCSR904LMG.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b0126a6094c82f8edb0f4faa0c4c5b495e40ac0c076aa57ac6329ded1e9f38b
3
+ size 27607040
fold_1/model.chrombpnet_nobias.fold_1.ENCSR904LMG.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3d771ca9fd49c5e8cc78849161b5a2e652c57693e0af85d024b945b7ebe18e4
3
+ size 25583536
fold_1/model.chrombpnet_nobias.fold_1.ENCSR904LMG.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f23a17a7da7e8ad3370247a458683e1983927939a12f00edd4376b0bda9631e1
3
+ size 26081280
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.args.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "genome": "/scratch/groups/akundaje/anusri/chromatin_atlas/reference/hg38.genome.fa",
3
+ "bigwig": "/oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//preprocessing/bigWigs/ENCSR904LMG.bigWig",
4
+ "peaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2//filtered.peaks.bed",
5
+ "nonpeaks": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2//filtered.nonpeaks.bed",
6
+ "output_prefix": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet",
7
+ "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
8
+ "trackables": [
9
+ "logcount_predictions_loss",
10
+ "loss",
11
+ "logits_profile_predictions_loss",
12
+ "val_logcount_predictions_loss",
13
+ "val_loss",
14
+ "val_logits_profile_predictions_loss"
15
+ ],
16
+ "epochs": 50,
17
+ "early_stop": 5,
18
+ "batch_size": 64,
19
+ "learning_rate": 0.001,
20
+ "params": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2//chrombpnet_model_params.tsv",
21
+ "seed": 1234,
22
+ "architecture_from_file": "/home/users/anusri/chromatin-atlas-anvil/sherlock/chrombpnet/src/training/models/chrombpnet_with_bias_model.py"
23
+ }
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.batch_loss.tsv ADDED
The diff for this file is too large to render. See raw diff
 
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.bias_formatting.stderr.txt ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
2
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
3
+ 2023-07-15 02:18:31.789847: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
4
+ 2023-07-15 02:18:34.049869: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
5
+ 2023-07-15 02:18:34.053144: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
6
+ 2023-07-15 02:18:34.588720: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
7
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
8
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
9
+ 2023-07-15 02:18:34.588772: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
10
+ 2023-07-15 02:18:34.611755: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
11
+ 2023-07-15 02:18:34.611896: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
12
+ 2023-07-15 02:18:34.621137: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
13
+ 2023-07-15 02:18:34.625410: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
14
+ 2023-07-15 02:18:34.640555: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
15
+ 2023-07-15 02:18:34.644590: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
16
+ 2023-07-15 02:18:34.645473: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
17
+ 2023-07-15 02:18:34.678134: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
18
+ 2023-07-15 02:18:34.678602: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
19
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
20
+ 2023-07-15 02:18:34.680152: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
21
+ 2023-07-15 02:18:34.689459: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
22
+ pciBusID: 0000:84:00.0 name: NVIDIA A100-SXM4-80GB computeCapability: 8.0
23
+ coreClock: 1.41GHz coreCount: 108 deviceMemorySize: 79.15GiB deviceMemoryBandwidth: 1.85TiB/s
24
+ 2023-07-15 02:18:34.689507: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
25
+ 2023-07-15 02:18:34.689535: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
26
+ 2023-07-15 02:18:34.689556: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
27
+ 2023-07-15 02:18:34.689576: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
28
+ 2023-07-15 02:18:34.689595: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
29
+ 2023-07-15 02:18:34.689613: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
30
+ 2023-07-15 02:18:34.689631: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
31
+ 2023-07-15 02:18:34.689649: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
32
+ 2023-07-15 02:18:34.701567: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
33
+ 2023-07-15 02:18:34.703340: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
34
+ 2023-07-15 02:18:37.536766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
35
+ 2023-07-15 02:18:37.536882: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
36
+ 2023-07-15 02:18:37.536899: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
37
+ 2023-07-15 02:18:37.543695: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 75650 MB memory) -> physical GPU (device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:84:00.0, compute capability: 8.0)
38
+ 2023-07-15 02:18:38.588366: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.bias_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/bias_model_scaled
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet.params.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "counts_loss_weight": "13.9",
3
+ "filters": "512",
4
+ "n_dil_layers": "8",
5
+ "bias_model_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/DNASE/ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2/bias_model_scaled.h5",
6
+ "inputlen": "2114",
7
+ "outputlen": "1000",
8
+ "max_jitter": "500",
9
+ "chr_fold_path": "/scratch/groups/akundaje/anusri/chromatin_atlas/splits/fold_2.json",
10
+ "negative_sampling_ratio": "0.1"
11
+ }
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet_data_params.tsv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ counts_sum_min_thresh 15.0
2
+ counts_sum_max_thresh 1580.0
3
+ trainings_pts_post_thresh 175751
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet_formatting.stderr.txt ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ INFO: underlay of /etc/localtime required more than 50 (88) bind mounts
2
+ INFO: underlay of /usr/bin/nvidia-smi required more than 50 (355) bind mounts
3
+ 2023-07-15 01:13:20.854259: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
4
+ 2023-07-15 01:13:23.938955: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set
5
+ 2023-07-15 01:13:23.944970: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1
6
+ 2023-07-15 01:13:23.982196: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
7
+ pciBusID: 0000:04:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
8
+ coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
9
+ 2023-07-15 01:13:23.982294: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
10
+ 2023-07-15 01:13:24.012356: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
11
+ 2023-07-15 01:13:24.012505: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
12
+ 2023-07-15 01:13:24.026540: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
13
+ 2023-07-15 01:13:24.032689: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
14
+ 2023-07-15 01:13:24.055206: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
15
+ 2023-07-15 01:13:24.060832: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
16
+ 2023-07-15 01:13:24.062078: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
17
+ 2023-07-15 01:13:24.065324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
18
+ 2023-07-15 01:13:24.065683: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
19
+ To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
20
+ 2023-07-15 01:13:24.066644: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set
21
+ 2023-07-15 01:13:24.066897: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720] Found device 0 with properties:
22
+ pciBusID: 0000:04:00.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
23
+ coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.89GiB deviceMemoryBandwidth: 681.88GiB/s
24
+ 2023-07-15 01:13:24.066932: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
25
+ 2023-07-15 01:13:24.066964: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11
26
+ 2023-07-15 01:13:24.067003: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11
27
+ 2023-07-15 01:13:24.067025: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10
28
+ 2023-07-15 01:13:24.067047: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10
29
+ 2023-07-15 01:13:24.067068: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.10
30
+ 2023-07-15 01:13:24.067089: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11
31
+ 2023-07-15 01:13:24.067110: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8
32
+ 2023-07-15 01:13:24.067487: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1862] Adding visible gpu devices: 0
33
+ 2023-07-15 01:13:24.068942: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0
34
+ 2023-07-15 01:13:25.932093: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1261] Device interconnect StreamExecutor with strength 1 edge matrix:
35
+ 2023-07-15 01:13:25.932199: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1267] 0
36
+ 2023-07-15 01:13:25.932216: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1280] 0: N
37
+ 2023-07-15 01:13:25.935260: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1406] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 14957 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:04:00.0, compute capability: 6.0)
38
+ 2023-07-15 01:13:28.292474: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.
39
+ /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:1059: UserWarning: is not loaded, but a Lambda layer uses it. It may cause errors.
40
+ , UserWarning)
fold_2/logs.models.fold_2.ENCSR904LMG/logfile.modelling.fold_2.ENCSR904LMG.chrombpnet_formatting.stdout.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ singularity exec --nv /home/groups/akundaje/anusri/simg/tf-atlas_gcp-modeling.sif python get_new_tf_model_format.py -i /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2/chrombpnet.h5 -o /oak/stanford/groups/akundaje/projects/chromatin-atlas-2022/DNASE//ENCSR904LMG//chrombppnet_model_encsr880cub_bias_fold_2/new_model_formats/chrombpnet