File size: 2,506 Bytes
6c223ff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | #!/usr/bin/env python
"""Convert Princeton WordNet (public domain) into a dictionary-style JSONL.
One natural-language entry per synset: "<lemma> (<pos>): <definition>.
Example: '<example>'. Synonyms: <lemmas>." This gives the LM real
lexical knowledge — definitions, word senses, examples, synonymy.
Usage:
python scripts/wordnet_to_jsonl.py
"""
import os, sys, json
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import nltk
HF_REPO = "thefinalboss/fractus-datasets"
POS_NAME = {"n": "noun", "v": "verb", "a": "adjective", "s": "adjective", "r": "adverb"}
def main():
out = "data/_wordnet/wordnet.jsonl"
os.makedirs(os.path.dirname(out), exist_ok=True)
# Ensure the corpus is present (small, one-time).
try:
from nltk.corpus import wordnet as wn
list(wn.synsets("test"))
except LookupError:
print("downloading wordnet corpus…", flush=True)
nltk.download("wordnet", quiet=True)
from nltk.corpus import wordnet as wn
n = 0
total_chars = 0
with open(out, "w", encoding="utf-8") as f:
for syn in wn.all_synsets():
lemmas = sorted(set(l.name().replace("_", " ") for l in syn.lemmas()))
if not lemmas:
continue
head = lemmas[0]
pos = POS_NAME.get(syn.pos(), syn.pos())
definition = (syn.definition() or "").strip()
if not definition:
continue
examples = syn.examples()
parts = [f"{head} ({pos}): {definition}."]
if examples:
parts.append("Example: " + " ".join(examples))
if len(lemmas) > 1:
parts.append("Synonyms: " + ", ".join(lemmas[1:8]))
text = " ".join(parts)
f.write(json.dumps({
"text": text,
"word": head,
"pos": pos,
"source": f"wordnet/{syn.name()}",
}, ensure_ascii=False) + "\n")
n += 1
total_chars += len(text)
print(f"{n} synset entries, {total_chars:,} chars (~{total_chars//4:,} tokens)",
flush=True)
from huggingface_hub import HfApi
api = HfApi()
api.upload_file(path_or_fileobj=out,
path_in_repo="dictionaries/wordnet.jsonl",
repo_id=HF_REPO, repo_type="dataset")
print(f" → uploaded to {HF_REPO}/dictionaries/wordnet.jsonl", flush=True)
if __name__ == "__main__":
main()
|