Import from Quazim0t0/neural-cd-preserve; repoint refs to NeuralVerified
Browse files- GF256.pt +3 -0
- LICENSE +21 -0
- README.md +71 -0
- cdpreserve/__init__.py +0 -0
- cdpreserve/archive.py +120 -0
- cdpreserve/common.py +56 -0
- cdpreserve/gf256.py +80 -0
- cdpreserve/rs.py +104 -0
- cdpreserve/scan.py +57 -0
- cli.py +58 -0
- requirements.txt +1 -0
- step_cd.py +80 -0
GF256.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:95a3bd56efcb21b046ccb4b400a0965d720bd4828e883cb543c584f2e9dca77d
|
| 3 |
+
size 564882
|
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Dean Byrne (Quazim0t0)
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
README.md
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
library_name: pytorch
|
| 4 |
+
tags:
|
| 5 |
+
- verified-units
|
| 6 |
+
- reed-solomon
|
| 7 |
+
- archival
|
| 8 |
+
- preservation
|
| 9 |
+
- self-healing
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# neural-cd-preserve — self-healing optical-disc archives
|
| 13 |
+
|
| 14 |
+
**Repositories:** [GitHub](https://github.com/quzi93/neural-cd-preserve) · [🤗 HuggingFace](https://huggingface.co/NeuralVerified/neural-cd-preserve)
|
| 15 |
+
|
| 16 |
+
Scan an optical disc into a self-healing `.pt` archive that **detects and repairs
|
| 17 |
+
bit-rot** and reconstructs the disc even from a damaged copy. Built on the same
|
| 18 |
+
neural-verified Reed-Solomon core as
|
| 19 |
+
[neural-storage](https://huggingface.co/NeuralVerified/neural-storage): each unique
|
| 20 |
+
chunk is RS-split into shards, every shard carries its own SHA-256, so silent
|
| 21 |
+
corruption is detected, marked as an erasure, and repaired from survivors (as long
|
| 22 |
+
as ≥ *k* of *n* shards remain).
|
| 23 |
+
|
| 24 |
+
> **Honest by design:** RS self-healing protects the *archive* against future
|
| 25 |
+
> bit-rot. It cannot recover disc sectors that were already unreadable at scan
|
| 26 |
+
> time — those are recorded, and re-reads are merged. No entropy is beaten.
|
| 27 |
+
|
| 28 |
+
## Use
|
| 29 |
+
|
| 30 |
+
```bash
|
| 31 |
+
pip install torch
|
| 32 |
+
python step_cd.py # full self-healing demo
|
| 33 |
+
|
| 34 |
+
python cli.py archive \\.\D: mydisc.pt --label MY_CD # drive or .iso
|
| 35 |
+
python cli.py verify mydisc.pt
|
| 36 |
+
python cli.py heal mydisc.pt
|
| 37 |
+
python cli.py restore mydisc.pt out.iso
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
The demo injects 149 silent bit-flips; `verify` detects all, `heal` repairs all,
|
| 41 |
+
`restore` returns bit-exact; corrupting beyond RS capacity is flagged **LOST**,
|
| 42 |
+
never silently wrong.
|
| 43 |
+
|
| 44 |
+
Weights: `GF256.pt` (verified GF(256) LOG/EXP).
|
| 45 |
+
|
| 46 |
+
**Create your own verified unit** (template: `cdpreserve/gf256.py`): write the
|
| 47 |
+
exact golden finite function → enumerate the domain (decompose big/linear ones
|
| 48 |
+
into bit/byte slices, see `cdpreserve/rs.py`) → `common.train` → `common.verify`
|
| 49 |
+
must be bit-exact on 100% of inputs → compose.
|
| 50 |
+
|
| 51 |
+
## Citation
|
| 52 |
+
|
| 53 |
+
```bibtex
|
| 54 |
+
@misc{byrne2026neuralcdpreserve,
|
| 55 |
+
title = {neural-cd-preserve: Self-Healing Optical-Disc Archival with Verified Erasure Coding},
|
| 56 |
+
author = {Byrne, Dean (Quazim0t0)},
|
| 57 |
+
year = {2026},
|
| 58 |
+
howpublished = {\url{https://huggingface.co/NeuralVerified/neural-cd-preserve}}
|
| 59 |
+
}
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
**Dean Byrne (Quazim0t0)** · 2026
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
---
|
| 66 |
+
|
| 67 |
+
<!-- neuralverified-relocation-note -->
|
| 68 |
+
> **Now hosted by [NeuralVerified](https://huggingface.co/NeuralVerified).**
|
| 69 |
+
>
|
| 70 |
+
> This repo was moved into the NeuralVerified organization to help organize my profile.
|
| 71 |
+
> Originally published at [`Quazim0t0/neural-cd-preserve`](https://huggingface.co/Quazim0t0/neural-cd-preserve).
|
cdpreserve/__init__.py
ADDED
|
File without changes
|
cdpreserve/archive.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Self-healing .pt disc archive.
|
| 3 |
+
|
| 4 |
+
A disc image is chunked (fixed size), duplicate chunks deduped, and each unique
|
| 5 |
+
chunk Reed-Solomon split into n = k+m shards (verified GF(256) core). Every shard
|
| 6 |
+
carries its own SHA-256, so SILENT corruption (bit-rot flipping bytes) is
|
| 7 |
+
DETECTED, marked as an erasure, and REPAIRED from the surviving shards -- as long
|
| 8 |
+
as at least k of the n shards per chunk are still intact.
|
| 9 |
+
|
| 10 |
+
The whole thing serializes to a single .pt (torch.save):
|
| 11 |
+
{ meta, manifest:[chunk_hash...], chunks:{ hash: {shards, shard_hashes, L, orig} } }
|
| 12 |
+
|
| 13 |
+
Honesty: RS adds redundancy (archive is ~(k+m)/k x the deduped image). It repairs
|
| 14 |
+
up to m damaged shards per chunk; beyond that a chunk is flagged as lost, never
|
| 15 |
+
silently wrong. No entropy is beaten.
|
| 16 |
+
"""
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
import hashlib
|
| 19 |
+
import time
|
| 20 |
+
import torch
|
| 21 |
+
from . import rs
|
| 22 |
+
|
| 23 |
+
CHUNK = 4096
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _h(b: bytes) -> str:
|
| 27 |
+
return hashlib.sha256(b).hexdigest()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def build(data: bytes, label="disc", k=4, m=2, chunk=CHUNK, bad_sectors=None) -> dict:
|
| 31 |
+
chunks = {}
|
| 32 |
+
manifest = []
|
| 33 |
+
for off in range(0, len(data), chunk):
|
| 34 |
+
ch = data[off:off + chunk]
|
| 35 |
+
h = _h(ch)
|
| 36 |
+
manifest.append(h)
|
| 37 |
+
if h in chunks:
|
| 38 |
+
continue
|
| 39 |
+
shards, L, orig = rs.encode(ch, k, m)
|
| 40 |
+
shards = [bytes(s) for s in shards]
|
| 41 |
+
chunks[h] = {"shards": shards, "shard_hashes": [_h(s) for s in shards],
|
| 42 |
+
"L": L, "orig": orig, "k": k, "m": m}
|
| 43 |
+
return {
|
| 44 |
+
"meta": {"label": label, "size": len(data), "chunk": chunk, "k": k, "m": m,
|
| 45 |
+
"bad_sectors": list(bad_sectors or []), "created": time.time(),
|
| 46 |
+
"format": "neural-cd-preserve/1"},
|
| 47 |
+
"manifest": manifest, "chunks": chunks,
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def save(arc: dict, path: str):
|
| 52 |
+
torch.save(arc, path)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def load(path: str) -> dict:
|
| 56 |
+
return torch.load(path, map_location="cpu", weights_only=False)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _good_shards(rec: dict) -> dict:
|
| 60 |
+
"""{index: bytes} for shards whose hash still matches (uncorrupted)."""
|
| 61 |
+
out = {}
|
| 62 |
+
for i, s in enumerate(rec["shards"]):
|
| 63 |
+
if _h(s) == rec["shard_hashes"][i]:
|
| 64 |
+
out[i] = s
|
| 65 |
+
return out
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def verify(arc: dict) -> dict:
|
| 69 |
+
"""Report health without modifying. -> {chunks, corrupted_shards, repairable,
|
| 70 |
+
lost} where lost chunks have < k intact shards (unrecoverable)."""
|
| 71 |
+
corrupted = repairable = lost = 0
|
| 72 |
+
for h, rec in arc["chunks"].items():
|
| 73 |
+
good = _good_shards(rec)
|
| 74 |
+
nbad = len(rec["shards"]) - len(good)
|
| 75 |
+
corrupted += nbad
|
| 76 |
+
if nbad and len(good) >= rec["k"]:
|
| 77 |
+
repairable += 1
|
| 78 |
+
if len(good) < rec["k"]:
|
| 79 |
+
lost += 1
|
| 80 |
+
return {"chunks": len(arc["chunks"]), "corrupted_shards": corrupted,
|
| 81 |
+
"repairable_chunks": repairable, "lost_chunks": lost,
|
| 82 |
+
"healthy": corrupted == 0 and lost == 0}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def heal(arc: dict) -> int:
|
| 86 |
+
"""Detect corrupted shards and regenerate them from survivors. Returns the
|
| 87 |
+
number of shards repaired. Chunks with < k intact shards cannot be healed."""
|
| 88 |
+
repaired = 0
|
| 89 |
+
for h, rec in arc["chunks"].items():
|
| 90 |
+
good = _good_shards(rec)
|
| 91 |
+
if len(good) == len(rec["shards"]):
|
| 92 |
+
continue
|
| 93 |
+
if len(good) < rec["k"]:
|
| 94 |
+
continue # unrecoverable, leave as-is
|
| 95 |
+
ch = rs.decode(good, rec["k"], rec["m"], rec["L"], rec["orig"])
|
| 96 |
+
if _h(ch) != h:
|
| 97 |
+
continue # decoded wrong -> don't trust
|
| 98 |
+
fresh, _, _ = rs.encode(ch, rec["k"], rec["m"])
|
| 99 |
+
for i in range(len(rec["shards"])):
|
| 100 |
+
if i not in good:
|
| 101 |
+
rec["shards"][i] = bytes(fresh[i])
|
| 102 |
+
rec["shard_hashes"][i] = _h(rec["shards"][i])
|
| 103 |
+
repaired += 1
|
| 104 |
+
return repaired
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def restore(arc: dict) -> bytes:
|
| 108 |
+
"""Reconstruct the disc image, RS-correcting any detected corruption."""
|
| 109 |
+
out = bytearray()
|
| 110 |
+
for h in arc["manifest"]:
|
| 111 |
+
rec = arc["chunks"][h]
|
| 112 |
+
good = _good_shards(rec)
|
| 113 |
+
if len(good) < rec["k"]:
|
| 114 |
+
raise IOError(f"chunk {h[:8]}: only {len(good)} intact shards, "
|
| 115 |
+
f"need {rec['k']} -- unrecoverable")
|
| 116 |
+
ch = rs.decode(good, rec["k"], rec["m"], rec["L"], rec["orig"])
|
| 117 |
+
if _h(ch) != h:
|
| 118 |
+
raise IOError(f"chunk {h[:8]}: hash mismatch after decode")
|
| 119 |
+
out += ch
|
| 120 |
+
return bytes(out[:arc["meta"]["size"]])
|
cdpreserve/common.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared helpers (same as the neural-DDR project): bit<->int, MLP, verify, train."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
|
| 6 |
+
DEV = "cuda" if torch.cuda.is_available() else "cpu"
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def bits_of(v: int, n: int) -> list[int]:
|
| 10 |
+
return [(v >> k) & 1 for k in range(n)]
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def int_of(bits) -> int:
|
| 14 |
+
return sum((1 << k) for k, b in enumerate(bits) if b > 0)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def pm(bits) -> torch.Tensor:
|
| 18 |
+
return torch.tensor([1.0 if b else -1.0 for b in bits], dtype=torch.float32)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def mlp(inp: int, out: int, h: int = 256, layers: int = 2) -> nn.Sequential:
|
| 22 |
+
mods = [nn.Linear(inp, h), nn.GELU()]
|
| 23 |
+
for _ in range(layers - 1):
|
| 24 |
+
mods += [nn.Linear(h, h), nn.GELU()]
|
| 25 |
+
mods += [nn.Linear(h, out)]
|
| 26 |
+
return nn.Sequential(*mods)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@torch.no_grad()
|
| 30 |
+
def verify(net, X, Ybits) -> tuple[int, int]:
|
| 31 |
+
net = net.to("cpu")
|
| 32 |
+
pred = (net(X) > 0).int()
|
| 33 |
+
ok = (pred == Ybits.int()).all(dim=1).sum().item()
|
| 34 |
+
return ok, X.shape[0]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def train(net, X, Ybits, steps=8000, lr=2e-3, tag="", report=2000):
|
| 38 |
+
net = net.to(DEV)
|
| 39 |
+
Xd, Yd = X.to(DEV), Ybits.to(DEV)
|
| 40 |
+
opt = torch.optim.Adam(net.parameters(), lr=lr)
|
| 41 |
+
sch = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=steps)
|
| 42 |
+
lossfn = nn.BCEWithLogitsLoss()
|
| 43 |
+
for e in range(steps):
|
| 44 |
+
opt.zero_grad()
|
| 45 |
+
loss = lossfn(net(Xd), Yd)
|
| 46 |
+
loss.backward(); opt.step(); sch.step()
|
| 47 |
+
if e % report == 0 or e == steps - 1:
|
| 48 |
+
ok, tot = verify(net, X, Ybits); net.to(DEV)
|
| 49 |
+
print(f" [{tag}] epoch {e:5d} loss {loss.item():.2e} verified {ok}/{tot}")
|
| 50 |
+
if ok == tot:
|
| 51 |
+
print(f" [{tag}] -> N/N"); break
|
| 52 |
+
return net.to("cpu")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def run8(net, v: int) -> int:
|
| 56 |
+
return int_of((net(pm(bits_of(v, 8)).unsqueeze(0))[0] > 0).int().tolist())
|
cdpreserve/gf256.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Step 1 keystone: verified GF(2^8) multiply -- the arithmetic core of Reed-Solomon
|
| 3 |
+
erasure coding (Step 2).
|
| 4 |
+
|
| 5 |
+
A direct 16-bit -> 8-bit neural multiply is hard to drive to N/N. So we decompose
|
| 6 |
+
it the way real RS implementations do -- via log/exp tables:
|
| 7 |
+
|
| 8 |
+
a * b = EXP[ (LOG[a] + LOG[b]) mod 255 ] (and 0 if a==0 or b==0)
|
| 9 |
+
|
| 10 |
+
LOG and EXP are 256-entry lookups -> trivially N/N verifiable neural units. We
|
| 11 |
+
train both, compose them into a full multiply, and verify the COMPOSED multiply
|
| 12 |
+
against the golden field over ALL 65,536 (a,b) pairs.
|
| 13 |
+
|
| 14 |
+
Field: GF(2^8), reduction polynomial 0x11D, generator 2 (standard for RS).
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
import torch
|
| 18 |
+
from .common import mlp, train, verify, pm, bits_of, int_of, run8
|
| 19 |
+
|
| 20 |
+
POLY = 0x11D
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def build_tables():
|
| 24 |
+
exp = [0] * 512
|
| 25 |
+
log = [0] * 256
|
| 26 |
+
x = 1
|
| 27 |
+
for i in range(255):
|
| 28 |
+
exp[i] = x
|
| 29 |
+
log[x] = i
|
| 30 |
+
x <<= 1
|
| 31 |
+
if x & 0x100:
|
| 32 |
+
x ^= POLY
|
| 33 |
+
for i in range(255, 512):
|
| 34 |
+
exp[i] = exp[i - 255]
|
| 35 |
+
return exp, log
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
EXP, LOG = build_tables()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def gf_mul(a: int, b: int) -> int:
|
| 42 |
+
if a == 0 or b == 0:
|
| 43 |
+
return 0
|
| 44 |
+
return EXP[LOG[a] + LOG[b]]
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# ---- neural LOG (inputs 1..255) and EXP (indices 0..254) ----
|
| 48 |
+
def log_domain():
|
| 49 |
+
xs = [pm(bits_of(a, 8)) for a in range(1, 256)]
|
| 50 |
+
ys = [[float(b) for b in bits_of(LOG[a], 8)] for a in range(1, 256)]
|
| 51 |
+
return torch.stack(xs), torch.tensor(ys)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def exp_domain():
|
| 55 |
+
xs = [pm(bits_of(i, 8)) for i in range(255)]
|
| 56 |
+
ys = [[float(b) for b in bits_of(EXP[i], 8)] for i in range(255)]
|
| 57 |
+
return torch.stack(xs), torch.tensor(ys)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def gf_mul_neural(a: int, b: int, net_log, net_exp) -> int:
|
| 61 |
+
if a == 0 or b == 0:
|
| 62 |
+
return 0
|
| 63 |
+
s = (run8(net_log, a) + run8(net_log, b)) % 255
|
| 64 |
+
return run8(net_exp, s)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def train_units():
|
| 68 |
+
print(" training LOG (8->8):")
|
| 69 |
+
net_log = train(mlp(8, 8, 256, 2), *log_domain(), steps=8000, tag="log")
|
| 70 |
+
print(" training EXP (8->8):")
|
| 71 |
+
net_exp = train(mlp(8, 8, 256, 2), *exp_domain(), steps=8000, tag="exp")
|
| 72 |
+
return net_log, net_exp
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def verify_mul(net_log, net_exp) -> tuple[int, int]:
|
| 76 |
+
ok = 0
|
| 77 |
+
for a in range(256):
|
| 78 |
+
for b in range(256):
|
| 79 |
+
ok += (gf_mul_neural(a, b, net_log, net_exp) == gf_mul(a, b))
|
| 80 |
+
return ok, 256 * 256
|
cdpreserve/rs.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Step 2: Reed-Solomon erasure coding over the verified GF(256) field.
|
| 3 |
+
|
| 4 |
+
Split data into k systematic shards + m parity shards (n = k+m). ANY k of the n
|
| 5 |
+
shards reconstruct the original -- so up to m shards can be lost or corrupted and
|
| 6 |
+
the data still survives. This is the honest sense of "small pieces that together
|
| 7 |
+
rebuild the whole": redundancy, not shrinkage (n shards total n/k x the data).
|
| 8 |
+
|
| 9 |
+
All coefficients come from a Cauchy matrix over GF(256) (every square submatrix
|
| 10 |
+
invertible => any-k-of-n recovery). Every multiply is the GF(256) multiply that
|
| 11 |
+
the neural LOG/EXP units were verified bit-exact against (65536/65536), so the
|
| 12 |
+
fast table path here is the proven-equivalent of the neural coding core.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
from .gf256 import EXP, LOG, gf_mul
|
| 16 |
+
|
| 17 |
+
MUL = [[gf_mul(a, b) for b in range(256)] for a in range(256)] # full product table
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def gf_inv(a: int) -> int:
|
| 21 |
+
return EXP[255 - LOG[a]] if a else 0
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def cauchy(k: int, m: int):
|
| 25 |
+
return [[gf_inv((k + i) ^ j) for j in range(k)] for i in range(m)]
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _gen_row(r: int, k: int, C):
|
| 29 |
+
if r < k:
|
| 30 |
+
row = [0] * k
|
| 31 |
+
row[r] = 1
|
| 32 |
+
return row
|
| 33 |
+
return C[r - k]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def encode(data: bytes, k: int, m: int):
|
| 37 |
+
"""-> (shards[n], shard_len, orig_len). shards[:k] are systematic data."""
|
| 38 |
+
orig = len(data)
|
| 39 |
+
L = (orig + k - 1) // k
|
| 40 |
+
d = data + bytes(L * k - orig)
|
| 41 |
+
shards = [bytearray(d[j * L:(j + 1) * L]) for j in range(k)]
|
| 42 |
+
C = cauchy(k, m)
|
| 43 |
+
for i in range(m):
|
| 44 |
+
P = bytearray(L)
|
| 45 |
+
Ci = C[i]
|
| 46 |
+
for j in range(k):
|
| 47 |
+
MC = MUL[Ci[j]]
|
| 48 |
+
Dj = shards[j]
|
| 49 |
+
for b in range(L):
|
| 50 |
+
P[b] ^= MC[Dj[b]]
|
| 51 |
+
shards.append(P)
|
| 52 |
+
return shards, L, orig
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _invert(M, k):
|
| 56 |
+
A = [list(M[i]) + [1 if j == i else 0 for j in range(k)] for i in range(k)]
|
| 57 |
+
for col in range(k):
|
| 58 |
+
piv = col
|
| 59 |
+
while A[piv][col] == 0:
|
| 60 |
+
piv += 1
|
| 61 |
+
A[col], A[piv] = A[piv], A[col]
|
| 62 |
+
inv = gf_inv(A[col][col])
|
| 63 |
+
A[col] = [MUL[inv][x] for x in A[col]]
|
| 64 |
+
for r in range(k):
|
| 65 |
+
if r != col and A[r][col]:
|
| 66 |
+
f = A[r][col]
|
| 67 |
+
A[r] = [A[r][x] ^ MUL[f][A[col][x]] for x in range(2 * k)]
|
| 68 |
+
return [A[i][k:] for i in range(k)]
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def decode(present: dict, k: int, m: int, L: int, orig: int) -> bytes:
|
| 72 |
+
"""present: {shard_index: bytes}; needs >= k entries."""
|
| 73 |
+
idxs = sorted(present.keys())[:k]
|
| 74 |
+
C = cauchy(k, m)
|
| 75 |
+
Minv = _invert([_gen_row(r, k, C) for r in idxs], k)
|
| 76 |
+
data_shards = [bytearray(L) for _ in range(k)]
|
| 77 |
+
for j in range(k):
|
| 78 |
+
row = Minv[j]
|
| 79 |
+
Dj = data_shards[j]
|
| 80 |
+
for t, r in enumerate(idxs):
|
| 81 |
+
c = row[t]
|
| 82 |
+
if not c:
|
| 83 |
+
continue
|
| 84 |
+
MC = MUL[c]
|
| 85 |
+
src = present[r]
|
| 86 |
+
for b in range(L):
|
| 87 |
+
Dj[b] ^= MC[src[b]]
|
| 88 |
+
return b"".join(bytes(s) for s in data_shards)[:orig]
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def neural_parity_equiv(net_log, net_exp, k=4, m=2, L=16) -> bool:
|
| 92 |
+
"""Compute one parity shard with the NEURAL gf_mul and confirm it matches the
|
| 93 |
+
golden coding -- i.e. the verified units drive the erasure code bit-exactly."""
|
| 94 |
+
import os
|
| 95 |
+
from .gf256 import gf_mul_neural
|
| 96 |
+
data = os.urandom(L * k)
|
| 97 |
+
shards, _, _ = encode(data, k, m)
|
| 98 |
+
C = cauchy(k, m)
|
| 99 |
+
Dsh = [data[j * L:(j + 1) * L] for j in range(k)]
|
| 100 |
+
P = bytearray(L)
|
| 101 |
+
for j in range(k):
|
| 102 |
+
for b in range(L):
|
| 103 |
+
P[b] ^= gf_mul_neural(C[0][j], Dsh[j][b], net_log, net_exp)
|
| 104 |
+
return bytes(P) == bytes(shards[k]) # neural parity == golden parity
|
cdpreserve/scan.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Disc scanner.
|
| 3 |
+
|
| 4 |
+
Reads a source sector by sector into bytes. The source can be a raw optical
|
| 5 |
+
drive (Windows: r"\\\\.\\D:", may need admin) or an .iso / image file (for
|
| 6 |
+
testing). Unreadable sectors (scratches / disc rot) are recorded as BAD and
|
| 7 |
+
zero-filled -- ddrescue-style -- rather than aborting the scan.
|
| 8 |
+
|
| 9 |
+
Important: bad sectors are data you never read. RS self-healing (archive.py)
|
| 10 |
+
protects the STORED copy against future corruption; it cannot invent sectors the
|
| 11 |
+
drive could not read. To recover those, re-scan the disc and merge good sectors
|
| 12 |
+
across attempts (`merge_scans`).
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
SECTOR = 2048 # CD-ROM Mode-1 user data sector size
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def scan_image(path: str, sector: int = SECTOR):
|
| 20 |
+
"""-> (data: bytes, bad_sectors: list[int]). Robust to unreadable sectors."""
|
| 21 |
+
data = bytearray()
|
| 22 |
+
bad = []
|
| 23 |
+
with open(path, "rb", buffering=0) as f:
|
| 24 |
+
idx = 0
|
| 25 |
+
while True:
|
| 26 |
+
try:
|
| 27 |
+
blk = f.read(sector)
|
| 28 |
+
except OSError:
|
| 29 |
+
bad.append(idx)
|
| 30 |
+
data += bytes(sector)
|
| 31 |
+
try:
|
| 32 |
+
f.seek((idx + 1) * sector)
|
| 33 |
+
except OSError:
|
| 34 |
+
break
|
| 35 |
+
idx += 1
|
| 36 |
+
continue
|
| 37 |
+
if not blk:
|
| 38 |
+
break
|
| 39 |
+
data += blk # keep true length (last
|
| 40 |
+
idx += 1 # sector may be partial)
|
| 41 |
+
return bytes(data), bad
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def merge_scans(scans, sector: int = SECTOR):
|
| 45 |
+
"""Combine multiple (data, bad) scans of the same failing disc: a sector is
|
| 46 |
+
good if ANY scan read it. Returns (merged_data, still_bad_sectors)."""
|
| 47 |
+
n = max(len(d) // sector for d, _ in scans)
|
| 48 |
+
merged = bytearray(n * sector)
|
| 49 |
+
known = [False] * n
|
| 50 |
+
for data, bad in scans:
|
| 51 |
+
badset = set(bad)
|
| 52 |
+
for s in range(len(data) // sector):
|
| 53 |
+
if s not in badset and not known[s]:
|
| 54 |
+
merged[s * sector:(s + 1) * sector] = data[s * sector:(s + 1) * sector]
|
| 55 |
+
known[s] = True
|
| 56 |
+
still_bad = [s for s in range(n) if not known[s]]
|
| 57 |
+
return bytes(merged), still_bad
|
cli.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
neural-cd-preserve CLI -- scan a disc into a self-healing .pt archive.
|
| 3 |
+
|
| 4 |
+
archive <source> <out.pt> [--label L --k K --m M --sector S]
|
| 5 |
+
source = optical drive (e.g. \\\\.\\D:) or an .iso/image file
|
| 6 |
+
info <archive.pt> show metadata
|
| 7 |
+
verify <archive.pt> check health (corruption / recoverability)
|
| 8 |
+
heal <archive.pt> [--out O] repair detected corruption (RS)
|
| 9 |
+
restore <archive.pt> <out.iso> reconstruct the disc image (RS-corrected)
|
| 10 |
+
"""
|
| 11 |
+
import argparse
|
| 12 |
+
from cdpreserve import scan, archive
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def main():
|
| 16 |
+
ap = argparse.ArgumentParser(prog="neural-cd-preserve")
|
| 17 |
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
| 18 |
+
a = sub.add_parser("archive"); a.add_argument("source"); a.add_argument("out")
|
| 19 |
+
a.add_argument("--label", default="disc"); a.add_argument("--k", type=int, default=4)
|
| 20 |
+
a.add_argument("--m", type=int, default=2); a.add_argument("--sector", type=int, default=scan.SECTOR)
|
| 21 |
+
for name in ("info", "verify"):
|
| 22 |
+
q = sub.add_parser(name); q.add_argument("archive")
|
| 23 |
+
h = sub.add_parser("heal"); h.add_argument("archive"); h.add_argument("--out")
|
| 24 |
+
r = sub.add_parser("restore"); r.add_argument("archive"); r.add_argument("out")
|
| 25 |
+
args = ap.parse_args()
|
| 26 |
+
|
| 27 |
+
if args.cmd == "archive":
|
| 28 |
+
data, bad = scan.scan_image(args.source, args.sector)
|
| 29 |
+
arc = archive.build(data, label=args.label, k=args.k, m=args.m, bad_sectors=bad)
|
| 30 |
+
archive.save(arc, args.out)
|
| 31 |
+
red = args.k + args.m
|
| 32 |
+
print(f"archived '{args.label}': {len(data)//1024} KB, {len(arc['chunks'])} unique chunks, "
|
| 33 |
+
f"RS {args.k}+{args.m} (~{red/args.k:.2f}x). bad sectors at scan: {len(bad)}")
|
| 34 |
+
if bad:
|
| 35 |
+
print(f" WARNING: {len(bad)} sectors were unreadable at scan time (zero-filled). "
|
| 36 |
+
f"Re-scan and merge to recover them.")
|
| 37 |
+
elif args.cmd == "info":
|
| 38 |
+
meta = archive.load(args.archive)["meta"]
|
| 39 |
+
for k, v in meta.items():
|
| 40 |
+
print(f" {k}: {v if k != 'bad_sectors' else f'{len(v)} sectors'}")
|
| 41 |
+
elif args.cmd == "verify":
|
| 42 |
+
h = archive.verify(archive.load(args.archive))
|
| 43 |
+
print(f"health: {'HEALTHY' if h['healthy'] else 'DAMAGED'} "
|
| 44 |
+
f"chunks={h['chunks']} corrupted_shards={h['corrupted_shards']} "
|
| 45 |
+
f"repairable={h['repairable_chunks']} lost={h['lost_chunks']}")
|
| 46 |
+
elif args.cmd == "heal":
|
| 47 |
+
arc = archive.load(args.archive)
|
| 48 |
+
n = archive.heal(arc)
|
| 49 |
+
archive.save(arc, args.out or args.archive)
|
| 50 |
+
print(f"repaired {n} shard(s) -> {args.out or args.archive}")
|
| 51 |
+
elif args.cmd == "restore":
|
| 52 |
+
data = archive.restore(archive.load(args.archive))
|
| 53 |
+
open(args.out, "wb").write(data)
|
| 54 |
+
print(f"restored {len(data)//1024} KB -> {args.out}")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
if __name__ == "__main__":
|
| 58 |
+
main()
|
requirements.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
torch
|
step_cd.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
CD preservation demo: scan -> self-healing .pt archive -> corrupt -> detect ->
|
| 3 |
+
heal -> restore bit-exact. Uses a synthetic disc image (no drive needed here).
|
| 4 |
+
|
| 5 |
+
(a) scan an image (bad-sector-tolerant)
|
| 6 |
+
(b) build .pt archive + restore bit-exact
|
| 7 |
+
(c) inject silent bit-rot (<= m shards/chunk) -> verify DETECTS it
|
| 8 |
+
(d) heal() repairs from survivors -> healthy again -> restore bit-exact
|
| 9 |
+
(e) honesty: corrupt > m shards in a chunk -> flagged LOST, never silently wrong
|
| 10 |
+
"""
|
| 11 |
+
import os
|
| 12 |
+
import copy
|
| 13 |
+
import random
|
| 14 |
+
import tempfile
|
| 15 |
+
from cdpreserve import scan, archive
|
| 16 |
+
|
| 17 |
+
random.seed(0)
|
| 18 |
+
|
| 19 |
+
print("=" * 62)
|
| 20 |
+
print("neural-cd-preserve -- self-healing .pt disc archive")
|
| 21 |
+
print("=" * 62)
|
| 22 |
+
|
| 23 |
+
# a synthetic 'disc': structured header + repeated table + random payload
|
| 24 |
+
disc = (b"CD-IMAGE-V1" + bytes(2037) # a 'sector'
|
| 25 |
+
+ (b"TABLE" * 400) * 3 # repeated -> dedup
|
| 26 |
+
+ os.urandom(400 * 1024))
|
| 27 |
+
|
| 28 |
+
# (a) scan (via a temp file standing in for the drive)
|
| 29 |
+
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".iso")
|
| 30 |
+
tmp.write(disc); tmp.close()
|
| 31 |
+
data, bad = scan.scan_image(tmp.name)
|
| 32 |
+
print(f"(a) scan: {len(data)//1024} KB, bad sectors: {len(bad)} "
|
| 33 |
+
f"-> {'PASS' if data == disc else 'FAIL'}")
|
| 34 |
+
|
| 35 |
+
# (b) build archive + restore
|
| 36 |
+
arc = archive.build(data, label="DEMO_DISC", k=4, m=2)
|
| 37 |
+
red = 6 / 4
|
| 38 |
+
print(f"(b) archive: {len(arc['chunks'])} unique chunks, RS 4+2 (~{red:.2f}x); "
|
| 39 |
+
f"restore bit-exact: {'PASS' if archive.restore(arc) == data else 'FAIL'}")
|
| 40 |
+
|
| 41 |
+
# (c) inject silent bit-rot: flip bytes in 1..2 shards per chunk
|
| 42 |
+
def corrupt(rec, nshards):
|
| 43 |
+
idxs = random.sample(range(len(rec["shards"])), nshards)
|
| 44 |
+
for i in idxs:
|
| 45 |
+
s = bytearray(rec["shards"][i]); s[0] ^= 0xFF; s[-1] ^= 0xFF
|
| 46 |
+
rec["shards"][i] = bytes(s) # hash now mismatches
|
| 47 |
+
|
| 48 |
+
damaged = 0
|
| 49 |
+
for rec in arc["chunks"].values():
|
| 50 |
+
n = random.randint(1, 2) # <= m, recoverable
|
| 51 |
+
corrupt(rec, n); damaged += n
|
| 52 |
+
h1 = archive.verify(arc)
|
| 53 |
+
print(f"(c) injected bit-rot in {damaged} shards; verify DETECTS: "
|
| 54 |
+
f"corrupted={h1['corrupted_shards']} repairable={h1['repairable_chunks']} "
|
| 55 |
+
f"lost={h1['lost_chunks']} -> {'PASS' if h1['corrupted_shards'] == damaged and h1['lost_chunks'] == 0 else 'FAIL'}")
|
| 56 |
+
|
| 57 |
+
# (d) heal + restore
|
| 58 |
+
repaired = archive.heal(arc)
|
| 59 |
+
h2 = archive.verify(arc)
|
| 60 |
+
restore_ok = archive.restore(arc) == data
|
| 61 |
+
print(f"(d) heal repaired {repaired} shards; now {'HEALTHY' if h2['healthy'] else 'DAMAGED'}; "
|
| 62 |
+
f"restore bit-exact: {'PASS' if restore_ok and h2['healthy'] else 'FAIL'}")
|
| 63 |
+
|
| 64 |
+
# (e) honesty: beyond RS capacity -> LOST, not silently wrong
|
| 65 |
+
arc2 = copy.deepcopy(arc)
|
| 66 |
+
victim = next(iter(arc2["chunks"].values()))
|
| 67 |
+
corrupt(victim, 3) # > m=2 -> unrecoverable
|
| 68 |
+
h3 = archive.verify(arc2)
|
| 69 |
+
try:
|
| 70 |
+
archive.restore(arc2); raised = False
|
| 71 |
+
except IOError:
|
| 72 |
+
raised = True
|
| 73 |
+
print(f"(e) corrupt 3 shards (> m) in one chunk -> lost={h3['lost_chunks']}, "
|
| 74 |
+
f"restore refuses: {'PASS (honest failure)' if h3['lost_chunks'] == 1 and raised else 'FAIL'}")
|
| 75 |
+
|
| 76 |
+
os.unlink(tmp.name)
|
| 77 |
+
allpass = (archive.restore(arc) == data and h1['lost_chunks'] == 0 and h2['healthy']
|
| 78 |
+
and restore_ok and h3['lost_chunks'] == 1 and raised)
|
| 79 |
+
print("=" * 62)
|
| 80 |
+
print(f"OVERALL: {'ALL PASS' if allpass else 'some checks failed'}")
|