Add year-over-year trends + expanded ratios (EBITDA, turnover, FCF conversion)

README.md

@@ -28,12 +28,14 @@ citations** — instead of making things up.
 
 - **Grounded answers with citations** — every response is backed by excerpts
   from real 10-K filings, shown in an expandable *Sources* panel.
-- **Answers financial figures (hybrid RAG)** — plain text RAG can't read numbers
-  out of financial-statement tables. FinChat also extracts each filing's
-  **XBRL** structured financials (revenue, net income, assets, cash flow…) and
-  a hybrid retriever *guarantees* those facts are in context for numeric
-  questions — so *"What was Apple's FY2023 revenue?"* returns **$383.29 billion**,
-  not a shrug.
+- **Answers financial figures, ratios & trends (hybrid RAG)** — plain text RAG
+  can't read numbers out of financial-statement tables. FinChat extracts each
+  filing's **XBRL** structured financials, **computes standard ratios**
+  (margins, liquidity, returns, EBITDA, turnover, free cash flow)
+  deterministically in Python, and builds **year-over-year trend** facts — then
+  a hybrid retriever *guarantees* these are in context for numeric questions.
+  So *"Apple's FY2023 revenue?"* → **$383.29B**, *"quick ratio?"* → **0.94**,
+  *"did its margin improve YoY?"* → answered straight from the data.
 - **Query routing ("knows where to look")** — FinChat detects which company a
   question is about and searches *only* that company's filings via metadata
   filtering, with graceful semantic fallback when the company is ambiguous.

src/financials.py

@@ -1,15 +1,14 @@
-"""Extract structured financial figures + computed ratios from a 10-K's XBRL data.
-
-SEC filings carry machine-readable XBRL financials. We pull the income
-statement, balance sheet, and cash-flow statement as clean "label: value"
-facts, AND compute standard ratios (margins, liquidity, returns, leverage,
-free cash flow) deterministically in Python, then index them alongside the
-filing text.
-
-This lets FinChat answer numeric questions -- both direct figures ("revenue")
-and computed metrics ("quick ratio", "operating margin") -- that plain text RAG
-cannot, because the filing's tables collapse into unusable "number soup" and
-LLMs are unreliable at arithmetic.
+"""Extract structured figures, computed ratios, and YoY trends from XBRL data.
+
+SEC filings carry machine-readable XBRL financials. For each filing we build:
+  * one "label: value" chunk per statement (income / balance / cash flow),
+  * a "Key Ratios" chunk (margins, liquidity, returns, leverage, turnover,
+    EBITDA, free cash flow) computed deterministically in Python, and
+  * a "Financial Trends (YoY)" chunk comparing this year to the prior year.
+
+This lets FinChat answer numeric questions -- direct figures, computed metrics,
+and year-over-year comparisons -- that plain text RAG cannot (the filing's
+tables collapse into unusable "number soup", and LLMs are unreliable at math).
 """
 from __future__ import annotations
 
@@ -26,9 +25,25 @@ _STATEMENTS = [
 ]
 _KEYWORDS = ("revenue, sales, income, earnings, profit, margin, assets, "
             "liabilities, equity, cash flow, expenses, EPS")
-_RATIO_KEYWORDS = ("margin, ratio, return, ROE, ROA, liquidity, leverage, "
-                  "profitability, quick ratio, current ratio, debt, free cash flow")
+_RATIO_KEYWORDS = ("margin, ratio, return, ROE, ROA, EBITDA, liquidity, leverage, "
+                  "turnover, profitability, quick ratio, current ratio, debt, "
+                  "free cash flow")
+_TREND_KEYWORDS = ("year-over-year, YoY, trend, change, improved, declined, grew, "
+                  "growth, increase, decrease, historical, compared to prior year")
+
+# Figures to show in the year-over-year trends chunk.
+_TREND_FIGURES = [
+    ("Revenue", "Revenue"),
+    ("Gross profit", "GrossProfit"),
+    ("Operating income", "OperatingIncomeLoss"),
+    ("Net income", "NetIncome"),
+    ("Total assets", "LiabilitiesAndEquity"),
+    ("Property, plant & equipment (net)", "PlantPropertyEquipmentNet"),
+    ("Inventory", "Inventories"),
+]
+
 
+# --- formatting -------------------------------------------------------------
 
 def _fmt(value, label: str = "") -> str | None:
     try:
@@ -44,13 +59,7 @@ def _fmt(value, label: str = "") -> str | None:
         if a >= 1e6:
             return f"{v / 1e6:,.1f} million shares"
         return f"{v:,.0f} shares"
-    if a >= 1e9:
-        return f"${v / 1e9:,.2f} billion"
-    if a >= 1e6:
-        return f"${v / 1e6:,.1f} million"
-    if a >= 1000:
-        return f"${v:,.0f}"
-    return f"{v:,.2f}"
+    return _money(v) if a >= 1000 else f"{v:,.2f}"
 
 
 def _money(v: float) -> str:
@@ -62,20 +71,34 @@ def _money(v: float) -> str:
     return f"${v:,.0f}"
 
 
-def _value_column(df: pd.DataFrame, fiscal_year: int):
+def _fmt_metric(value: float, kind: str) -> str:
+    if kind == "pct":
+        return f"{value * 100:.1f}%"
+    if kind == "x":
+        return f"{value:.2f}"
+    return _money(value)
+
+
+# --- extraction -------------------------------------------------------------
+
+def _value_column(df: pd.DataFrame, year: int):
     date_cols = [c for c in df.columns if re.match(r"\d{4}-\d{2}-\d{2}", str(c))]
     for c in date_cols:
-        if str(c).startswith(str(fiscal_year)):
+        if str(c).startswith(str(year)):
             return c
-    return date_cols[0] if date_cols else None
+    return None
 
 
-def _statement_lines(stmt, fiscal_year: int) -> list[str]:
+def _statement_lines(stmt, year: int) -> list[str]:
     try:
         df = stmt.to_dataframe()
     except Exception:
         return []
-    col = _value_column(df, fiscal_year)
+    col = _value_column(df, year)
+    if col is None:
+        # primary statement may label the column with the filing year only
+        col = next((c for c in df.columns if re.match(r"\d{4}-\d{2}-\d{2}", str(c))),
+                   None)
     if col is None:
         return []
     lines = []
@@ -92,10 +115,8 @@ def _statement_lines(stmt, fiscal_year: int) -> list[str]:
     return lines
 
 
-# --- computed ratios --------------------------------------------------------
-
-def _collect_figures(tenk, fiscal_year: int) -> dict[str, list]:
-    """Map each XBRL standard_concept -> list of (label, value) for the year."""
+def _collect_figures(tenk, year: int) -> dict[str, list]:
+    """{standard_concept: [(label, value), ...]} for a given fiscal year."""
     figs: dict[str, list] = defaultdict(list)
     for attr, _name in _STATEMENTS:
         stmt = getattr(tenk, attr, None)
@@ -105,7 +126,7 @@ def _collect_figures(tenk, fiscal_year: int) -> dict[str, list]:
             df = stmt.to_dataframe()
         except Exception:
             continue
-        col = _value_column(df, fiscal_year)
+        col = _value_column(df, year)
         if col is None:
             continue
         for _, row in df.iterrows():
@@ -128,69 +149,103 @@ def _first(figs, concept):
 
 
 def _operating_cash_flow(figs):
-    rows = figs.get("NetCashFromOperatingActivities", [])
-    for label, v in rows:
-        if "operating activ" in label.lower():   # the total line
+    for label, v in figs.get("NetCashFromOperatingActivities", []):
+        if "operating activ" in label.lower():
             return v
     return None
 
 
-def _compute_ratios(figs) -> dict[str, str]:
-    """Standard ratios, computed only where inputs are reliable for the sector."""
-    rev = _first(figs, "Revenue")
-    ni = _first(figs, "NetIncome") or _first(figs, "ProfitLoss")
-    total_assets = _first(figs, "LiabilitiesAndEquity")   # == total assets
-    equity = _first(figs, "AllEquityBalance")
-    ca = _first(figs, "CurrentAssetsTotal")
-    cl = _first(figs, "CurrentLiabilitiesTotal")
-    inv = _first(figs, "Inventories") or 0
-    ltd = _first(figs, "LongTermDebt") or 0
-    std = _first(figs, "ShortTermDebt") or 0
-    capex = _first(figs, "CapitalExpenses")
-    ocf = _operating_cash_flow(figs)
-
-    # Banks / financials have no classified balance sheet (no current assets),
-    # and their revenue/COGS concepts are unreliable -> only compute ROA & ROE.
-    is_financial = ca is None
+# --- ratios -----------------------------------------------------------------
 
-    pct = lambda x: f"{x * 100:.1f}%"
-    rat = lambda x: f"{x:.2f}"
-    m: dict[str, str] = {}
+def _compute_ratios(figs) -> dict[str, tuple[float, str]]:
+    """{name: (value, kind)} -- computed only where inputs are reliable."""
+    g = lambda c: _first(figs, c)
+    rev = g("Revenue")
+    ni = g("NetIncome") or g("ProfitLoss")
+    ta = g("LiabilitiesAndEquity")            # == total assets (identity)
+    eq = g("AllEquityBalance")
+    ca, cl = g("CurrentAssetsTotal"), g("CurrentLiabilitiesTotal")
+    inv = g("Inventories")
+    ltd, std = g("LongTermDebt") or 0, g("ShortTermDebt") or 0
+    capex, ocf = g("CapitalExpenses"), _operating_cash_flow(figs)
+    da, ppe, rec = g("DepreciationExpense"), g("PlantPropertyEquipmentNet"), g("TradeReceivables")
+    oi, cogs = g("OperatingIncomeLoss"), g("CostOfGoodsAndServicesSold")
+
+    # Banks / financials: no classified balance sheet, unreliable revenue/COGS.
+    is_financial = ca is None
+    m: dict[str, tuple[float, str]] = {}
 
     if not is_financial and rev:
-        gp = _first(figs, "GrossProfit")
-        cogs = _first(figs, "CostOfGoodsAndServicesSold")
+        gp = g("GrossProfit")
         if gp is None and cogs is not None:
             gp = rev - cogs
-        oi = _first(figs, "OperatingIncomeLoss")
         for name, num in (("Gross margin", gp), ("Operating margin", oi),
                           ("Net profit margin", ni)):
-            if num is not None and abs(num / rev) <= 1.5:   # guard mis-maps
-                m[name] = pct(num / rev)
+            if num is not None and abs(num / rev) <= 1.5:
+                m[name] = (num / rev, "pct")
         if capex is not None:
-            m["Capital expenditure as % of revenue"] = pct(abs(capex) / rev)
-        if total_assets:
-            m["Asset turnover"] = rat(rev / total_assets)
-
-    if ni is not None and total_assets:
-        m["Return on assets (ROA)"] = pct(ni / total_assets)
-    if ni is not None and equity:
-        m["Return on equity (ROE)"] = pct(ni / equity)
-
+            m["Capital expenditure as % of revenue"] = (abs(capex) / rev, "pct")
+        if ta:
+            m["Asset turnover"] = (rev / ta, "x")
+        if cogs is not None and inv:
+            m["Inventory turnover"] = (cogs / inv, "x")
+        if rec:
+            m["Receivables turnover"] = (rev / rec, "x")
+        if oi is not None and da is not None:
+            ebitda = oi + da
+            m["EBITDA"] = (ebitda, "money")
+            if capex is not None:
+                m["EBITDA less capex"] = (ebitda - abs(capex), "money")
+
+    if ni is not None and ta:
+        m["Return on assets (ROA)"] = (ni / ta, "pct")
+    if ni is not None and eq:
+        m["Return on equity (ROE)"] = (ni / eq, "pct")
     if not is_financial and ca and cl and cl > 0:
-        m["Current ratio"] = rat(ca / cl)
-        m["Quick ratio"] = rat((ca - inv) / cl)
-    if not is_financial and equity and (ltd or std):
-        m["Debt-to-equity ratio"] = rat((ltd + std) / equity)
+        m["Current ratio"] = (ca / cl, "x")
+        m["Quick ratio"] = ((ca - (inv or 0)) / cl, "x")
+    if not is_financial and eq and (ltd or std):
+        m["Debt-to-equity ratio"] = ((ltd + std) / eq, "x")
+    if not is_financial and ta and ppe:
+        m["Property, plant & equipment as % of assets"] = (ppe / ta, "pct")
     if not is_financial and ocf is not None and capex is not None:
-        m["Free cash flow (operating cash flow - capex)"] = _money(ocf - abs(capex))
-
+        fcf = ocf - abs(capex)
+        m["Free cash flow"] = (fcf, "money")
+        if ni:
+            m["Free cash flow conversion (FCF / net income)"] = (fcf / ni, "pct")
     return m
 
 
+def _trend_line(name, cur, prev, kind, cy, py) -> str:
+    cs, ps = _fmt_metric(cur, kind), _fmt_metric(prev, kind)
+    if kind == "pct":
+        d = (cur - prev) * 100
+        w = "increased" if d > 0.05 else "decreased" if d < -0.05 else "roughly unchanged"
+        return f"{name}: FY{cy} {cs} vs FY{py} {ps} ({w} {abs(d):.1f} pp)"
+    if prev == 0:
+        return f"{name}: FY{cy} {cs} vs FY{py} {ps}"
+    d = (cur - prev) / abs(prev) * 100
+    w = "up" if d > 0.5 else "down" if d < -0.5 else "roughly flat"
+    return f"{name}: FY{cy} {cs} vs FY{py} {ps} ({w} {abs(d):.1f}%)"
+
+
+def _trend_lines(figs_cur, figs_prev, cy, py) -> list[str]:
+    lines = []
+    for name, concept in _TREND_FIGURES:
+        cur, prev = _first(figs_cur, concept), _first(figs_prev, concept)
+        if cur is not None and prev is not None:
+            lines.append(_trend_line(name, cur, prev, "money", cy, py))
+    rc, rp = _compute_ratios(figs_cur), _compute_ratios(figs_prev)
+    for name, (cur, kind) in rc.items():
+        if name in rp:
+            lines.append(_trend_line(name, cur, rp[name][0], kind, cy, py))
+    return lines
+
+
+# --- documents --------------------------------------------------------------
+
 def financial_documents(filing, ticker: str, company: str,
                         fiscal_year: int) -> list[Document]:
-    """Documents: one per financial statement + one of computed key ratios."""
     try:
         tenk = filing.obj()
     except Exception:
@@ -200,9 +255,7 @@ def financial_documents(filing, ticker: str, company: str,
         return Document(
             page_content=content,
             metadata={
-                "ticker": ticker,
-                "company": company,
-                "year": str(fiscal_year),
+                "ticker": ticker, "company": company, "year": str(fiscal_year),
                 "accession": filing.accession_no,
                 "source": f"{company} 10-K (FY{fiscal_year}) - {name} (XBRL)",
                 "type": "financials",
@@ -215,21 +268,25 @@ def financial_documents(filing, ticker: str, company: str,
         if stmt is None:
             continue
         lines = _statement_lines(stmt, fiscal_year)
-        if not lines:
-            continue
-        header = (
-            f"{company} ({ticker}) FY{fiscal_year} {name} "
-            f"(financial figures: {_KEYWORDS}; from SEC XBRL data):"
-        )
-        docs.append(_doc(name, header + "\n" + "\n".join(lines)))
+        if lines:
+            header = (f"{company} ({ticker}) FY{fiscal_year} {name} "
+                     f"(financial figures: {_KEYWORDS}; from SEC XBRL data):")
+            docs.append(_doc(name, header + "\n" + "\n".join(lines)))
 
-    ratios = _compute_ratios(_collect_figures(tenk, fiscal_year))
+    figs_cur = _collect_figures(tenk, fiscal_year)
+    ratios = _compute_ratios(figs_cur)
     if ratios:
-        header = (
-            f"{company} ({ticker}) FY{fiscal_year} Key Financial Ratios "
-            f"(computed from SEC XBRL data - {_RATIO_KEYWORDS}):"
-        )
-        body = "\n".join(f"{k}: {v}" for k, v in ratios.items())
+        header = (f"{company} ({ticker}) FY{fiscal_year} Key Financial Ratios "
+                 f"(computed from SEC XBRL data - {_RATIO_KEYWORDS}):")
+        body = "\n".join(f"{k}: {_fmt_metric(*v)}" for k, v in ratios.items())
         docs.append(_doc("Key Ratios", header + "\n" + body))
 
+    figs_prev = _collect_figures(tenk, fiscal_year - 1)
+    trends = _trend_lines(figs_cur, figs_prev, fiscal_year, fiscal_year - 1)
+    if trends:
+        header = (f"{company} ({ticker}) Financial Trends "
+                 f"FY{fiscal_year} vs FY{fiscal_year - 1} "
+                 f"({_TREND_KEYWORDS}; from SEC XBRL data):")
+        docs.append(_doc("Financial Trends", header + "\n" + "\n".join(trends)))
+
     return docs

src/rag.py

@@ -175,7 +175,7 @@ def retrieve(question: str, ticker: str | None):
     if ticker and _is_financial_query(question):
         fin = store.as_retriever(
             search_kwargs={
-                "k": 3,
+                "k": 4,
                 "filter": {"$and": [{"ticker": ticker}, {"type": "financials"}]},
             }
         ).invoke(question)

vectorstore/{1b87b035-cb38-4524-82b2-6713a29f0208 → 65beb4f5-d997-4646-b2fc-3ac731fba06f}/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0d7fe0d52b2a73ee4d97d60ad3aa744a87b5c8cd1a4702421d432763083a440
-size 35708856
+oid sha256:b9fd1b96c87c7aeb6ab9781a5ad48c5db0d242fcd91469621fe679bc98f671f8
+size 35750756

vectorstore/{1b87b035-cb38-4524-82b2-6713a29f0208 → 65beb4f5-d997-4646-b2fc-3ac731fba06f}/header.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b0b0fbfa819fce87b1b59910538fcbad67c65d698d4b9b8721b4dfd0a63ffcf
+oid sha256:bafd2c0ed962055277d16427a6588bbeec9442666e126d7b26de71c3a521bbab
 size 100

vectorstore/{1b87b035-cb38-4524-82b2-6713a29f0208 → 65beb4f5-d997-4646-b2fc-3ac731fba06f}/index_metadata.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d1db3fe7fb573f958743a8270cd28a603cdcb24a05cbef8692b4aba71c14cc6
-size 1960364
+oid sha256:7bc200edd42cccc33aaf70bedd3a0aceadde01eec160b6d8467cb87d7f5f7ad0
+size 1962664

vectorstore/{1b87b035-cb38-4524-82b2-6713a29f0208 → 65beb4f5-d997-4646-b2fc-3ac731fba06f}/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa4432349f4213ed508590fd86b5166b035d04f8c3fdd2cc0486ab84583111bd
-size 85224
+oid sha256:9c5a3b3cf7e71f0098ce2ffe4f11463733d3b7c5beb72ace5f369e4936446693
+size 85324

vectorstore/{1b87b035-cb38-4524-82b2-6713a29f0208 → 65beb4f5-d997-4646-b2fc-3ac731fba06f}/link_lists.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37729da79d7646776efd5cd0bab55cdc5c9ecc1dcc3613ee86719f7e329dfe6c
-size 182940
+oid sha256:a79fa52068d7fd8b28f2ac047b1e0c17eeee1b0eb90b0a30616d7cfc3a195d20
+size 183108

vectorstore/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fee2efebe0e59a292b3703ca9bf3579462ae27397cd13418bf03f8e103ff88d8
-size 124620800
+oid sha256:3c5479c43914cb268b20ca1b1d97b897ac42ea7363ed8725598c4149cab3758d
+size 124878848