app.py

import gradio as gr

# Africa Environmental Law Claim Verifier — Eval Space
# AutoScientist Challenge 2026 | Standalone honest-finding analysis
# Author: Hussein Adeiza (mabera) — Licensed Environmental Health Officer, Abuja Nigeria

CLAIMS = [
    {"claim": "Nigeria's EIA process is governed by the EIA Act No. 86 of 1992.",
     "country": "Nigeria", "actual": "TRUE", "predicted": "TRUE", "correct": True,
     "note": "Correct. Direct restatement of source text, high keyword overlap correctly read as TRUE."},
    {"claim": "Ghana requires EIA approval before mining operations can begin.",
     "country": "Ghana", "actual": "TRUE", "predicted": "TRUE", "correct": True,
     "note": "Correct. Matches Ghana Minerals and Mining Act 703 of 2006 source content."},
    {"claim": "Kenya's EIA process is regulated by the Environmental Management and Coordination Act (EMCA) 1999.",
     "country": "Kenya", "actual": "TRUE", "predicted": "TRUE", "correct": True,
     "note": "Correct. Matches Kenya EMCA 1999 source content."},
    {"claim": "Rwanda has banned single-use plastics since 2008.",
     "country": "Rwanda", "actual": "TRUE", "predicted": "TRUE", "correct": True,
     "note": "Correct. Matches Rwanda REMA source content."},
    {"claim": "The African Union's Maputo Convention addresses conservation of nature and natural resources.",
     "country": "Pan-Africa", "actual": "TRUE", "predicted": "TRUE", "correct": True,
     "note": "Correct. Matches AU Maputo Convention 2003 source content."},
    {"claim": "Rwanda has no EIA requirements for development projects.",
     "country": "Rwanda", "actual": "FALSE", "predicted": "TRUE", "correct": False,
     "note": "WRONG. This is the negation-blindness failure. The claim shares nearly all vocabulary with the true statement (Rwanda, EIA, requirements, development projects) but the word 'no' inverts the meaning entirely. Keyword overlap cannot detect this."},
    {"claim": "Nigeria allows unrestricted importation of hazardous waste as long as it is for industrial reuse.",
     "country": "Nigeria", "actual": "FALSE", "predicted": "UNVERIFIABLE", "correct": False,
     "note": "WRONG. Source text uses different phrasing (prohibits importation, transit, deposit) so overlap was low, landing in UNVERIFIABLE territory by accident rather than correctly flagging this as a direct contradiction."},
    {"claim": "South Africa's EIA Basic Assessment process has no fixed legal timeframe for a decision.",
     "country": "South Africa", "actual": "FALSE", "predicted": "TRUE", "correct": False,
     "note": "WRONG. Same negation-blindness pattern. Shares vocabulary with the true 107-day timeframe statement, but 'no fixed timeframe' is the opposite claim."},
    {"claim": "Senegal's environmental assessment process has no public participation requirement.",
     "country": "Senegal", "actual": "FALSE", "predicted": "TRUE", "correct": False,
     "note": "WRONG. Again, negation inverts a true statement (public participation IS mandatory for Category 1) into a false one, undetected by keyword overlap."},
    {"claim": "ECOWAS has no shared environmental policy and leaves all environmental regulation entirely to individual member states.",
     "country": "ECOWAS Region", "actual": "FALSE", "predicted": "TRUE", "correct": False,
     "note": "WRONG. Same pattern. High vocabulary overlap with the true ECOWAS Environmental Policy 2008 statement, but negated into a false claim."},
    {"claim": "Nigeria's NESREA has approved over 5,000 EIA certificates in the last year.",
     "country": "Nigeria", "actual": "UNVERIFIABLE", "predicted": "TRUE", "correct": False,
     "note": "WRONG. A specific operational statistic not in the source dataset. Shares enough vocabulary (NESREA, EIA, certificates) with general true statements about NESREA's role to be wrongly read as TRUE."},
    {"claim": "Ethiopia is planning to repeal its EIA Proclamation 299/2002 next year.",
     "country": "Ethiopia", "actual": "UNVERIFIABLE", "predicted": "TRUE", "correct": False,
     "note": "WRONG. A forward-looking legislative prediction, not covered by a dataset documenting the current framework. High overlap on the law's name wrongly suggests confirmation."},
    {"claim": "Ghana's EPA processes EIA applications faster than South Africa's DFFE.",
     "country": "Ghana / South Africa", "actual": "UNVERIFIABLE", "predicted": "UNVERIFIABLE", "correct": True,
     "note": "Correct. A cross-country comparative speed claim with no shared source content at all, correctly flagged as unverifiable."},
]

def get_summary():
    total = len(CLAIMS)
    correct = sum(1 for c in CLAIMS if c["correct"])
    true_claims = [c for c in CLAIMS if c["actual"] == "TRUE"]
    false_claims = [c for c in CLAIMS if c["actual"] == "FALSE"]
    unverif_claims = [c for c in CLAIMS if c["actual"] == "UNVERIFIABLE"]

    true_correct = sum(1 for c in true_claims if c["correct"])
    false_correct = sum(1 for c in false_claims if c["correct"])
    unverif_correct = sum(1 for c in unverif_claims if c["correct"])

    return f"""
## 📊 Overall Result: {correct}/{total} correct ({correct/total*100:.1f}%)

| Claim Type | Accuracy |
|------------|----------|
| TRUE claims | {true_correct}/{len(true_claims)} ({true_correct/len(true_claims)*100:.0f}%) |
| FALSE claims | {false_correct}/{len(false_claims)} ({false_correct/len(false_claims)*100:.0f}%) |
| UNVERIFIABLE claims | {unverif_correct}/{len(unverif_claims)} ({unverif_correct/len(unverif_claims)*100:.0f}%) |

### 🔍 The Headline Finding: Negation Blindness

A simple keyword-overlap verifier got **every single TRUE claim right** but **every single FALSE claim wrong**, always defaulting to TRUE.

The reason is structural, not random. Claims like *"Rwanda has no EIA requirements"* share almost all the same vocabulary as the true statement *"Rwanda has EIA requirements"*, same country, same law, same nouns, just inverted by one negation word. Surface-level keyword matching cannot detect that inversion.

This means a verification layer built only on lexical overlap is **systematically blind to the most dangerous category of misinformation**, a confidently worded false claim that uses all the right vocabulary.
"""

def show_claim(index):
    c = CLAIMS[int(index)]
    status_emoji = "✅" if c["correct"] else "❌"
    return f"""
### {status_emoji} Claim {int(index)+1} of {len(CLAIMS)}

**Claim:** {c['claim']}

**Country:** {c['country']}

**Actual verdict:** {c['actual']}
**Predicted verdict:** {c['predicted']}
**Verifier was:** {"Correct ✅" if c['correct'] else "Wrong ❌"}

**Analysis:** {c['note']}
"""

with gr.Blocks(title="Africa Law Claim Verifier Eval", theme=gr.themes.Soft()) as demo:
    gr.Markdown("""
    # ⚖️ Africa Environmental Law Claim Verifier
    ## Standalone Eval — AutoScientist Challenge 2026

    **Author:** Hussein Adeiza (mabera) — Licensed Environmental Health Officer, Abuja Nigeria
    **Built on:** Africa Environmental Law Model (Legal Category submission)

    This is a complementary analysis, not the official challenge metric. It stress-tests
    a claim verification approach against 13 real claims about African environmental law,
    5 true, 5 false, 3 genuinely unverifiable, all grounded in the structured legal dataset
    from this challenge's Legal category submission.

    Inspired by the community's honest-finding eval Space pattern. Reporting the result as
    it actually came out, including where it failed.
    """)

    gr.Markdown(get_summary())

    gr.Markdown("---\n### Browse individual claims")
    with gr.Row():
        with gr.Column():
            claim_slider = gr.Slider(0, len(CLAIMS)-1, value=0, step=1, label="Claim Index")
        with gr.Column():
            claim_output = gr.Markdown()
    claim_slider.change(show_claim, inputs=claim_slider, outputs=claim_output)
    demo.load(lambda: show_claim(0), outputs=claim_output)

    gr.Markdown("""
    ---
    ### Why This Matters
    NGOs, investors and businesses increasingly rely on AI for quick answers about African
    regulatory environments. A model that can fluently restate true facts but cannot detect
    a negated false claim is a real risk, it sounds equally confident either way. This finding
    suggests verification layers need negation-aware reasoning, not just topical relevance
    matching, before being trusted for compliance-adjacent use cases.

    🤗 [Legal Model](https://huggingface.co/mabera/africa-environmental-law-model) |
    📊 [Source Dataset](https://huggingface.co/datasets/mabera/africa-environmental-law-dataset) |
    🚀 [Main Demo](https://huggingface.co/spaces/mabera/nigeria-health-ai-demo)

    Powered by Adaptive Data — Adaption Labs
    """)

demo.launch()