feat(lme): full multi-year diagram engine for all symbols with entire data timeline

backend/main.py

@@ -250,6 +250,14 @@ def get_crypto(_: object = Depends(require_session)):
 
 
 
+
+@app.get("/api/lme/diagram")
+def get_lme_diagram(field: str = Query("WM_Cu_low"), _: object = Depends(require_session)):
+    try:
+        return scrape_lme.get_entire_diagram_data(field)
+    except Exception as e:
+        raise HTTPException(status_code=502, detail=f"Failed to fetch diagram data: {e}")
+
 @app.get("/api/lme/history")
 def get_lme_history_route(metal: str = Query("Copper"), _: object = Depends(require_session)):
     try:
@@ -439,6 +447,10 @@ def v1_world_economy(): return get_world_economy(None)
 
 
 
+
+@data_api.get("/lme/diagram")
+def v1_lme_diagram(field: str = "WM_Cu_low"): return scrape_lme.get_entire_diagram_data(field)
+
 @data_api.get("/lme/history")
 def v1_lme_history(metal: str = "Copper"): return scrape_lme.get_metal_history(metal)
 

backend/scrape_lme.py

@@ -1,4 +1,5 @@
 import urllib.request
+import xml.etree.ElementTree as ET
 import re
 from datetime import datetime
 import threading
@@ -14,7 +15,7 @@ _cache = {
     "data": None,
     "last_fetched": 0
 }
-_history_cache = {}
+_diagram_cache = {}
 _lock = threading.Lock()
 
 FIELD_MAP = {
@@ -40,12 +41,20 @@ FA_NAMES = {
     "Zinc": "روی (Zinc)",
     "Aluminium": "آلومینیوم (Aluminium)",
     "Nickel": "نیکل (Nickel)",
-}
+    "WM_Cu_low": "مس کاتد آلمان (WM-Notiz کف)",
-
+    "WM_Cu_high": "مس کاتد آلمان (WM-Notiz سقف)",
-MONTHS_MAP = {
+    "WI_Cu": "مس ویلند (Wieland Copper)",
-    "january": "01", "february": "02", "march": "03", "april": "04",
+    "ACI": "شاخص مس پیشرفته (ACI)",
-    "may": "05", "june": "06", "july": "07", "august": "08",
+    "MB_bronze_94_6": "برنز 94/6",
-    "september": "09", "october": "10", "november": "11", "december": "12"
+    "MB_MS_58_1": "برنج MS 58 (مرحله ۱)",
+    "MB_MS_58_2": "برنج MS 58 (مرحله ۲)",
+    "MB_MS_63_37": "برنج MS 63/37",
+    "MB_MS_63_wire": "مفتول برنجی MS 63",
+    "USD_ozt_London": "انس طلای لندن (Fixing)",
+    "Au": "طلای شمش در اروپا",
+    "Au_processed": "طلای ساخته‌شده در اروپا",
+    "Ag": "نقره خالص در اروپا",
+    "Ag_processed": "نقره ساخته‌شده در اروپا",
 }
 
 def clean_num(val: str) -> float:
@@ -57,19 +66,6 @@ def clean_num(val: str) -> float:
     except:
         return 0.0
 
-def parse_date_iso(raw: str) -> str:
-    # Example: "25. August 2026"
-    try:
-        parts = raw.replace('.', '').split()
-        if len(parts) >= 3:
-            day = parts[0].zfill(2)
-            month = MONTHS_MAP.get(parts[1].lower(), "01")
-            year = parts[2]
-            return f"{year}-{month}-{day}"
-    except:
-        pass
-    return raw
-
 def fetch_raw():
     url = 'https://www.westmetall.com/en/markdaten.php'
     req = urllib.request.Request(url, headers=HEADERS)
@@ -91,10 +87,13 @@ def fetch_raw():
 
         for r in t0_rows[2:]:
             cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else f"LME_{cells[0][:2]}_cash"
             if len(cells) >= 3 and cells[0]:
                 prices.append({
                     "metal": cells[0],
                     "name_fa": FA_NAMES.get(cells[0], cells[0]),
+                    "field": field,
                     "cash": clean_num(cells[1]),
                     "cash_str": cells[1],
                     "three_months": clean_num(cells[2]),
@@ -108,10 +107,13 @@ def fetch_raw():
     if len(t1_rows) > 2:
         for r in t1_rows[2:]:
             cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else ""
             if len(cells) >= 3 and cells[0]:
                 stocks.append({
                     "metal": cells[0],
                     "name_fa": FA_NAMES.get(cells[0], cells[0]),
+                    "field": field,
                     "stocks": clean_num(cells[1]),
                     "stocks_str": cells[1],
                     "change": clean_num(cells[2]),
@@ -129,6 +131,7 @@ def fetch_raw():
         combined_metals.append({
             "metal": m_name,
             "name_fa": p["name_fa"],
+            "field": p["field"],
             "cash_settlement": p["cash"],
             "cash_str": p["cash_str"],
             "three_months": p["three_months"],
@@ -154,15 +157,37 @@ def fetch_raw():
                     "rate_str": cells[1]
                 })
 
+    # Table 3: German Metal Quotations
+    german_symbols = []
+    if len(tables) > 3:
+        for r in re.findall(r'<tr[^>]*>([\s\S]*?)</tr>', tables[3])[2:]:
+            cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else ""
+            if len(cells) >= 3 and cells[0]:
+                german_symbols.append({
+                    "name": cells[0],
+                    "name_fa": FA_NAMES.get(field, cells[0]),
+                    "field": field,
+                    "price_str": cells[1],
+                    "prev_str": cells[2],
+                    "unit": "EUR/100kg"
+                })
+
     # Table 4: Precious Metals
     precious = []
     if len(tables) > 4:
         for r in re.findall(r'<tr[^>]*>([\s\S]*?)</tr>', tables[4])[1:]:
             cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+            links = re.findall(r'field=([a-zA-Z0-9_]+)', r)
+            field = links[0] if links else ""
             if len(cells) >= 2 and cells[0]:
                 precious.append({
                     "name": cells[0],
-                    "price_str": cells[1]
+                    "name_fa": FA_NAMES.get(field, cells[0]),
+                    "field": field,
+                    "price_str": cells[1],
+                    "prev_str": cells[2] if len(cells) > 2 else "-"
                 })
 
     return {
@@ -170,6 +195,7 @@ def fetch_raw():
         "updated_at": datetime.utcnow().isoformat(),
         "metals": combined_metals,
         "stocks": stocks,
+        "german_symbols": german_symbols,
         "exchange_rates": fx,
         "precious_metals": precious
     }
@@ -191,65 +217,79 @@ def get_lme_data(ttl_seconds: int = 180):
             raise e
         return _cache["data"]
 
-def get_metal_history(metal_key: str, ttl_seconds: int = 300):
+def get_entire_diagram_data(field: str, ttl_seconds: int = 3600):
-    norm_key = metal_key.lower().strip()
+    """
-    field = FIELD_MAP.get(norm_key)
+    Fetches the ENTIRE multi-year historical dataset for any symbol
-    if not field:
+    from the official XML marketdata API.
-        # Fallback: check if the key already is a field
+    """
-        if norm_key.startswith("lme_"):
+    field_clean = field.strip()
-            field = metal_key
+    # Resolve aliases (e.g. "copper" -> "LME_Cu_cash")
-        else:
+    field_clean = FIELD_MAP.get(field_clean.lower(), field_clean)
-            field = f"LME_{norm_key[:2].capitalize()}_cash"
 
     now = time.time()
     with _lock:
-        cached = _history_cache.get(field)
+        cached = _diagram_cache.get(field_clean)
         if cached and (now - cached["time"]) < ttl_seconds:
             return cached["data"]
 
-    url = f'https://www.westmetall.com/en/markdaten.php?action=table&field={field}'
+    url = f'https://www.westmetall.com/api/marketdata/en/{field_clean}/'
     req = urllib.request.Request(url, headers=HEADERS)
-    with urllib.request.urlopen(req, timeout=12) as resp:
+    with urllib.request.urlopen(req, timeout=15) as r:
-        html = resp.read().decode('utf-8', errors='ignore')
+        xml_text = r.read().decode('utf-8')
 
-    tables = re.findall(r'<table[^>]*>([\s\S]*?)</table>', html)
+    root = ET.fromstring(xml_text)
-    if not tables:
+    series_elem = root.find('series')
-        return {"metal": metal_key, "field": field, "points": []}
+    if series_elem is None:
+        return {"field": field_clean, "lines": []}
 
-    rows = re.findall(r'<tr[^>]*>([\s\S]*?)</tr>', tables[0])
+    lines = []
-    points = []
+    for line in series_elem:
-    # Skip header
+        raw_name = line.attrib.get('name', field_clean)
-    for r in rows[1:]:
+        color = line.attrib.get('lineColor', '#D02E00')
-        cells = [re.sub(r'<.*?>', '', c).replace('&nbsp;', ' ').strip() for c in re.findall(r'<t[dh][^>]*>([\s\S]*?)</t[dh]>', r)]
+        unit = line.attrib.get('yUnit', '')
-        if len(cells) >= 4 and cells[0]:
+        precision = int(line.attrib.get('yPrecision', '2'))
-            c_val = clean_num(cells[1])
+
-            tm_val = clean_num(cells[2])
+        pts = []
-            s_val = clean_num(cells[3])
+        for val in line:
-            date_iso = parse_date_iso(cells[0])
+            raw_x = val.attrib.get('x', '') # YYYY/MM/DD
-            points.append({
+            date_iso = raw_x.replace('/', '-')
-                "date": date_iso,
+            # Also format DD/MM/YYYY for tooltips as shown in user screenshot
-                "date_display": cells[0],
+            parts = date_iso.split('-')
-                "cash": c_val,
+            date_display = f"{parts[2]}/{parts[1]}/{parts[0]}" if len(parts) == 3 else date_iso
-                "cash_str": cells[1],
+
-                "three_months": tm_val,
+            y_str = val.attrib.get('y', '0').replace(',', '')
-                "three_months_str": cells[2],
+            try:
-                "spread": round(tm_val - c_val, 2),
+                y_val = float(y_str)
-                "stocks": s_val,
+            except:
-                "stocks_str": cells[3],
+                y_val = 0.0
+
+            pts.append({
+                "x": date_iso,
+                "date_display": date_display,
+                "y": y_val
             })
 
-    # Reverse points to chronological order (oldest to newest) for charting
+        lines.append({
-    points.reverse()
+            "name": raw_name,
+            "color": color,
+            "unit": unit,
+            "precision": precision,
+            "count": len(pts),
+            "points": pts
+        })
 
     result = {
-        "metal": metal_key,
+        "field": field_clean,
-        "name_fa": FA_NAMES.get(metal_key.capitalize(), metal_key),
+        "name_fa": FA_NAMES.get(field_clean, field_clean),
-        "field": field,
+        "total_points": lines[0]["count"] if lines else 0,
-        "total_points": len(points),
+        "lines": lines
-        "points": points
     }
 
     with _lock:
-        _history_cache[field] = {"time": now, "data": result}
+        _diagram_cache[field_clean] = {"time": now, "data": result}
 
     return result
+
+# Backward compatibility alias
+def get_metal_history(metal_key: str, ttl_seconds: int = 300):
+    return get_entire_diagram_data(metal_key, ttl_seconds)