feat(lme): full multi-year diagram engine for all symbols with entire data timeline

2026-08-26 10:54:18 +03:30 · 2026-08-26 10:54:18 +03:30 · 380605daa4
parent a857102b98
commit 380605daa4
3 changed files with 690 additions and 558 deletions
--- a/backend/main.py
+++ b/backend/main.py
@ -250,6 +250,14 @@ def get_crypto(_: object = Depends(require_session)):



+
+@app.get("/api/lme/diagram")
+def get_lme_diagram(field: str = Query("WM_Cu_low"), _: object = Depends(require_session)):
+    try:
+        return scrape_lme.get_entire_diagram_data(field)
+    except Exception as e:
+        raise HTTPException(status_code=502, detail=f"Failed to fetch diagram data: {e}")
+
@app.get("/api/lme/history")
 def get_lme_history_route(metal: str = Query("Copper"), _: object = Depends(require_session)):
    try:
@ -439,6 +447,10 @@ def v1_world_economy(): return get_world_economy(None)



+
+@data_api.get("/lme/diagram")
+def v1_lme_diagram(field: str = "WM_Cu_low"): return scrape_lme.get_entire_diagram_data(field)
+
@data_api.get("/lme/history")
 def v1_lme_history(metal: str = "Copper"): return scrape_lme.get_metal_history(metal)

--- a/backend/scrape_lme.py
+++ b/backend/scrape_lme.py
@ -1,4 +1,5 @@
 import urllib.request
+import xml.etree.ElementTree as ET
 import re
 from datetime import datetime
 import threading
@ -14,7 +15,7 @@ _cache = {
    "data": None,
    "last_fetched": 0
 }
-_history_cache = {}
+_diagram_cache = {}
 _lock = threading.Lock()

 FIELD_MAP = {
@ -40,12 +41,20 @@ FA_NAMES = {
    "Zinc": "روی (Zinc)",
    "Aluminium": "آلومینیوم (Aluminium)",
    "Nickel": "نیکل (Nickel)",
-}
-
-MONTHS_MAP = {
-    "january": "01", "february": "02", "march": "03", "april": "04",
-    "may": "05", "june": "06", "july": "07", "august": "08",
-    "september": "09", "october": "10", "november": "11", "december": "12"
+    "WM_Cu_low": "مس کاتد آلمان (WM-Notiz کف)",
+    "WM_Cu_high": "مس کاتد آلمان (WM-Notiz سقف)",
+    "WI_Cu": "مس ویلند (Wieland Copper)",
+    "ACI": "شاخص مس پیشرفته (ACI)",
+    "MB_bronze_94_6": "برنز 94/6",
+    "MB_MS_58_1": "برنج MS 58 (مرحله ۱)",
+    "MB_MS_58_2": "برنج MS 58 (مرحله ۲)",
+    "MB_MS_63_37": "برنج MS 63/37",
+    "MB_MS_63_wire": "مفتول برنجی MS 63",
+    "USD_ozt_London": "انس طلای لندن (Fixing)",
+    "Au": "طلای شمش در اروپا",
+    "Au_processed": "طلای ساخته‌شده در اروپا",
+    "Ag": "نقره خالص در اروپا",
+    "Ag_processed": "نقره ساخته‌شده در اروپا",
 }

 def clean_num(val: str) -> float:
@ -57,19 +66,6 @@ def clean_num(val: str) -> float:
    except:
        return 0.0

-def parse_date_iso(raw: str) -> str:
-    # Example: "25. August 2026"
-    try:
-        parts = raw.replace('.', '').split()
-        if len(parts) >= 3:
-            day = parts[0].zfill(2)
-            month = MONTHS_MAP.get(parts[1].lower(), "01")
-            year = parts[2]
-            return f"{year}-{month}-{day}"
-    except:
-        pass
-    return raw
-
 def fetch_raw():
    url = 'https://www.westmetall.com/en/markdaten.php'
    req = urllib.request.Request(url, headers=HEADERS)
@ -91,10 +87,13 @@ def fetch_raw():

        for r in t0_rows[2:]:
            cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else f"LME_{cells[0][:2]}_cash"
            if len(cells) >= 3 and cells[0]:
                prices.append({
                    "metal": cells[0],
                    "name_fa": FA_NAMES.get(cells[0], cells[0]),
+                    "field": field,
                    "cash": clean_num(cells[1]),
                    "cash_str": cells[1],
                    "three_months": clean_num(cells[2]),
@ -108,10 +107,13 @@ def fetch_raw():
    if len(t1_rows) > 2:
        for r in t1_rows[2:]:
            cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else ""
            if len(cells) >= 3 and cells[0]:
                stocks.append({
                    "metal": cells[0],
                    "name_fa": FA_NAMES.get(cells[0], cells[0]),
+                    "field": field,
                    "stocks": clean_num(cells[1]),
                    "stocks_str": cells[1],
                    "change": clean_num(cells[2]),
@ -129,6 +131,7 @@ def fetch_raw():
        combined_metals.append({
            "metal": m_name,
            "name_fa": p["name_fa"],
+            "field": p["field"],
            "cash_settlement": p["cash"],
            "cash_str": p["cash_str"],
            "three_months": p["three_months"],
@ -154,15 +157,37 @@ def fetch_raw():
                    "rate_str": cells[1]
                })

+    # Table 3: German Metal Quotations
+    german_symbols = []
+    if len(tables) > 3:
+        for r in re.findall(r'<tr[^>]*>([\s\S]*?)</tr>', tables[3])[2:]:
+            cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else ""
+            if len(cells) >= 3 and cells[0]:
+                german_symbols.append({
+                    "name": cells[0],
+                    "name_fa": FA_NAMES.get(field, cells[0]),
+                    "field": field,
+                    "price_str": cells[1],
+                    "prev_str": cells[2],
+                    "unit": "EUR/100kg"
+                })
+
    # Table 4: Precious Metals
    precious = []
    if len(tables) > 4:
        for r in re.findall(r'<tr[^>]*>([\s\S]*?)</tr>', tables[4])[1:]:
            cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else ""
            if len(cells) >= 2 and cells[0]:
                precious.append({
                    "name": cells[0],
-                    "price_str": cells[1]
+                    "name_fa": FA_NAMES.get(field, cells[0]),
+                    "field": field,
+                    "price_str": cells[1],
+                    "prev_str": cells[2] if len(cells) > 2 else "-"
                })

    return {
@ -170,6 +195,7 @@ def fetch_raw():
        "updated_at": datetime.utcnow().isoformat(),
        "metals": combined_metals,
        "stocks": stocks,
+        "german_symbols": german_symbols,
        "exchange_rates": fx,
        "precious_metals": precious
    }
@ -191,65 +217,79 @@ def get_lme_data(ttl_seconds: int = 180):
            raise e
        return _cache["data"]

-def get_metal_history(metal_key: str, ttl_seconds: int = 300):
-    norm_key = metal_key.lower().strip()
-    field = FIELD_MAP.get(norm_key)
-    if not field:
-        # Fallback: check if the key already is a field
-        if norm_key.startswith("lme_"):
-            field = metal_key
-        else:
-            field = f"LME_{norm_key[:2].capitalize()}_cash"
+def get_entire_diagram_data(field: str, ttl_seconds: int = 3600):
+    """
+    Fetches the ENTIRE multi-year historical dataset for any symbol
+    from the official XML marketdata API.
+    """
+    field_clean = field.strip()
+    # Resolve aliases (e.g. "copper" -> "LME_Cu_cash")
+    field_clean = FIELD_MAP.get(field_clean.lower(), field_clean)

    now = time.time()
    with _lock:
-        cached = _history_cache.get(field)
+        cached = _diagram_cache.get(field_clean)
        if cached and (now - cached["time"]) < ttl_seconds:
            return cached["data"]

-    url = f'https://www.westmetall.com/en/markdaten.php?action=table&field={field}'
+    url = f'https://www.westmetall.com/api/marketdata/en/{field_clean}/'
    req = urllib.request.Request(url, headers=HEADERS)
-    with urllib.request.urlopen(req, timeout=12) as resp:
-        html = resp.read().decode('utf-8', errors='ignore')
+    with urllib.request.urlopen(req, timeout=15) as r:
+        xml_text = r.read().decode('utf-8')

-    tables = re.findall(r'<table[^>]*>([\s\S]*?)</table>', html)
-    if not tables:
-        return {"metal": metal_key, "field": field, "points": []}
+    root = ET.fromstring(xml_text)
+    series_elem = root.find('series')
+    if series_elem is None:
+        return {"field": field_clean, "lines": []}

-    rows = re.findall(r'<tr[^>]*>([\s\S]*?)</tr>', tables[0])
-    points = []
-    # Skip header
-    for r in rows[1:]:
-        cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
-        if len(cells) >= 4 and cells[0]:
-            c_val = clean_num(cells[1])
-            tm_val = clean_num(cells[2])
-            s_val = clean_num(cells[3])
-            date_iso = parse_date_iso(cells[0])
-            points.append({
-                "date": date_iso,
-                "date_display": cells[0],
-                "cash": c_val,
-                "cash_str": cells[1],
-                "three_months": tm_val,
-                "three_months_str": cells[2],
-                "spread": round(tm_val - c_val, 2),
-                "stocks": s_val,
-                "stocks_str": cells[3],
+    lines = []
+    for line in series_elem:
+        raw_name = line.attrib.get('name', field_clean)
+        color = line.attrib.get('lineColor', '#D02E00')
+        unit = line.attrib.get('yUnit', '')
+        precision = int(line.attrib.get('yPrecision', '2'))
+
+        pts = []
+        for val in line:
+            raw_x = val.attrib.get('x', '') # YYYY/MM/DD
+            date_iso = raw_x.replace('/', '-')
+            # Also format DD/MM/YYYY for tooltips as shown in user screenshot
+            parts = date_iso.split('-')
+            date_display = f"{parts[2]}/{parts[1]}/{parts[0]}" if len(parts) == 3 else date_iso
+
+            y_str = val.attrib.get('y', '0').replace(',', '')
+            try:
+                y_val = float(y_str)
+            except:
+                y_val = 0.0
+
+            pts.append({
+                "x": date_iso,
+                "date_display": date_display,
+                "y": y_val
            })

-    # Reverse points to chronological order (oldest to newest) for charting
-    points.reverse()
+        lines.append({
+            "name": raw_name,
+            "color": color,
+            "unit": unit,
+            "precision": precision,
+            "count": len(pts),
+            "points": pts
+        })

    result = {
-        "metal": metal_key,
-        "name_fa": FA_NAMES.get(metal_key.capitalize(), metal_key),
-        "field": field,
-        "total_points": len(points),
-        "points": points
+        "field": field_clean,
+        "name_fa": FA_NAMES.get(field_clean, field_clean),
+        "total_points": lines[0]["count"] if lines else 0,
+        "lines": lines
    }

    with _lock:
-        _history_cache[field] = {"time": now, "data": result}
+        _diagram_cache[field_clean] = {"time": now, "data": result}

    return result
+
+# Backward compatibility alias
+def get_metal_history(metal_key: str, ttl_seconds: int = 300):
+    return get_entire_diagram_data(metal_key, ttl_seconds)
--- a/frontend/src/components/LmeView.tsx
+++ b/frontend/src/components/LmeView.tsx