scripts/sync_hf.py

#!/usr/bin/env python3
"""
Hermes Agent HF Spaces Persistence — Full Directory Sync
=========================================================

Simplified persistence: upload/download the entire /opt/data directory
as-is to/from a Hugging Face Dataset repo.

- Startup:  snapshot_download  →  /opt/data
- Periodic: upload_folder      →  dataset hermes_data/
- Shutdown: final upload_folder →  dataset hermes_data/
"""

import os
import sys
import time
import json
import hashlib
import threading
import subprocess
import signal
import shutil
import secrets
import tempfile
import traceback
from pathlib import Path
from datetime import datetime
# Set timeout BEFORE importing huggingface_hub
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")
os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_VERBOSITY", "warning")

import logging as _logging
_logging.getLogger("huggingface_hub").setLevel(_logging.WARNING)
_logging.getLogger("huggingface_hub.utils").setLevel(_logging.WARNING)
_logging.getLogger("filelock").setLevel(_logging.WARNING)

from huggingface_hub import HfApi, snapshot_download
from runtime_policy import apply_optional_mcp_policy

# ── Logging helper ──────────────────────────────────────────────────────────

class TeeLogger:
    """Duplicate output to stream and file."""
    def __init__(self, filename, stream):
        self.stream = stream
        self.file = open(filename, "a", encoding="utf-8")
    def write(self, message):
        self.stream.write(message)
        self.file.write(message)
        self.flush()
    def flush(self):
        self.stream.flush()
        self.file.flush()
    def fileno(self):
        return self.stream.fileno()

# ── Configuration ───────────────────────────────────────────────────────────

HF_TOKEN      = os.environ.get("HF_TOKEN")
HERMES_DATA   = Path("/opt/data")
APP_DIR       = Path("/opt/hermes")
DATASET_PATH  = "hermes_data"

AGENT_NAME = os.environ.get("AGENT_NAME", "HermesFace")

# HF Spaces built-in env vars (auto-set by HF runtime)
SPACE_HOST = os.environ.get("SPACE_HOST", "")
SPACE_ID   = os.environ.get("SPACE_ID", "")

SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "60"))
AUTO_CREATE_DATASET = os.environ.get("AUTO_CREATE_DATASET", "true").lower() in ("true", "1", "yes")

# Dataset repo: auto-derive from SPACE_ID when not explicitly set.
# Format: {username}/{SpaceName}-data
HF_REPO_ID = os.environ.get("HERMES_DATASET_REPO", "")
if not HF_REPO_ID and SPACE_ID:
    HF_REPO_ID = f"{SPACE_ID}-data"
    print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from SPACE_ID: {HF_REPO_ID}")
elif not HF_REPO_ID and HF_TOKEN:
    try:
        _api = HfApi(token=HF_TOKEN)
        _username = _api.whoami()["name"]
        HF_REPO_ID = f"{_username}/HermesFace-data"
        print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from HF_TOKEN: {HF_REPO_ID}")
        del _api, _username
    except Exception as e:
        print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")
        HF_REPO_ID = ""

# Setup logging
log_dir = HERMES_DATA / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)
sys.stderr = sys.stdout


# ── Sync Manager ────────────────────────────────────────────────────────────

class HermesFullSync:
    """Upload/download the entire /opt/data directory to HF Dataset."""

    def __init__(self):
        self.enabled = False
        self.dataset_exists = False
        self.api = None

        if not HF_TOKEN:
            print("[SYNC] WARNING: HF_TOKEN not set. Persistence disabled.")
            return
        if not HF_REPO_ID:
            print("[SYNC] WARNING: Could not determine dataset repo (no SPACE_ID or HERMES_DATASET_REPO).")
            print("[SYNC] Persistence disabled.")
            return

        self.enabled = True
        self.api = HfApi(token=HF_TOKEN)
        self.dataset_exists = self._ensure_repo_exists()

    # ── Repo management ────────────────────────────────────────────────

    def _ensure_repo_exists(self):
        """Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true."""
        try:
            self.api.repo_info(repo_id=HF_REPO_ID, repo_type="dataset")
            print(f"[SYNC] Dataset repo found: {HF_REPO_ID}")
            return True
        except Exception:
            if not AUTO_CREATE_DATASET:
                print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID}")
                print("[SYNC]   Set AUTO_CREATE_DATASET=true to auto-create.")
                print("[SYNC] Persistence disabled (app will still run normally).")
                return False
            print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID} — creating...")
            try:
                self.api.create_repo(
                    repo_id=HF_REPO_ID,
                    repo_type="dataset",
                    private=True,
                )
                print(f"[SYNC] Dataset repo created: {HF_REPO_ID}")
                return True
            except Exception as e:
                print(f"[SYNC] Failed to create dataset repo: {e}")
                return False

    # ── Restore (startup) ─────────────────────────────────────────────

    def load_from_repo(self):
        """Download from dataset → /opt/data"""
        if not self.enabled:
            print("[SYNC] Persistence disabled - skipping restore")
            self._ensure_default_config()
            return

        if not self.dataset_exists:
            print(f"[SYNC] Dataset {HF_REPO_ID} does not exist - starting fresh")
            self._ensure_default_config()
            return

        print(f"[SYNC] Restoring /opt/data from dataset {HF_REPO_ID} ...")
        HERMES_DATA.mkdir(parents=True, exist_ok=True)

        try:
            with tempfile.TemporaryDirectory() as tmpdir:
                snapshot_download(
                    repo_id=HF_REPO_ID,
                    repo_type="dataset",
                    allow_patterns=f"{DATASET_PATH}/**",
                    ignore_patterns=[
                        f"{DATASET_PATH}/.brv-cli/**",
                        f"{DATASET_PATH}/.npm/**",
                        f"{DATASET_PATH}/.local/share/uv/**",
                        f"{DATASET_PATH}/home/.local/share/uv/**",
                        f"{DATASET_PATH}/hermes_overlay/**",
                    ],
                    local_dir=tmpdir,
                    token=HF_TOKEN,
                )
                downloaded_root = Path(tmpdir) / DATASET_PATH
                if downloaded_root.exists():
                    for item in downloaded_root.rglob("*"):
                        if item.is_file():
                            rel = item.relative_to(downloaded_root)
                            dest = HERMES_DATA / rel
                            dest.parent.mkdir(parents=True, exist_ok=True)
                            shutil.copy2(str(item), str(dest))
                    print("[SYNC] Restore completed.")
                else:
                    print("[SYNC] Downloaded snapshot but dir not found. Starting fresh.")

        except Exception as e:
            print(f"[SYNC] Restore failed: {e}")
            traceback.print_exc()

        self._ensure_default_config()
        self._debug_list_files()

    # ── Save (periodic + shutdown) ─────────────────────────────────────

    def save_to_repo(self):
        """Upload entire /opt/data directory → dataset (all files, no filtering)"""
        if not self.enabled:
            return
        if not HERMES_DATA.exists():
            print("[SYNC] /opt/data does not exist, nothing to save.")
            return

        if not self._ensure_repo_exists():
            print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")
            return

        print(f"[SYNC] Uploading /opt/data → dataset {HF_REPO_ID}/{DATASET_PATH}/ ...")

        try:
            total_size = 0
            file_count = 0
            for root, dirs, fls in os.walk(HERMES_DATA):
                for fn in fls:
                    fp = os.path.join(root, fn)
                    total_size += os.path.getsize(fp)
                    file_count += 1
            print(f"[SYNC] Uploading: {file_count} files, {total_size} bytes total")

            if file_count == 0:
                print("[SYNC] Nothing to upload.")
                return

            self.api.upload_folder(
                folder_path=str(HERMES_DATA),
                path_in_repo=DATASET_PATH,
                repo_id=HF_REPO_ID,
                repo_type="dataset",
                token=HF_TOKEN,
                commit_message=f"Sync hermes_data — {datetime.now().isoformat()}",
                ignore_patterns=[
                    "*.log",        # Log files — regenerated on boot
                    "*.lock",       # Lock files — stale after restart
                    "*.tmp",        # Temp files
                    "*.pid",        # PID files
                    "*.db-shm",     # SQLite shared-memory sidecar — transient, rotates mid-upload
                    "*.db-wal",     # SQLite write-ahead-log sidecar — transient, rotates mid-upload
                    "*.db-journal", # SQLite rollback-journal sidecar — transient
                    "__pycache__",  # Python cache
                    "scripts/*",    # HermesFace scripts — from git, not data
                    "assets/*",     # Static assets — from git, not data
                    "hermes_overlay/**", # Immutable overlay — from current Space image
                    ".brv-cli/**",  # Generated browser CLI + node_modules
                    ".npm/**",      # npm cache/update metadata
                    ".local/share/uv/**", # uv package cache
                    "home/.local/share/uv/**", # nested uv package cache
                    ".cache/**",    # uv/pip cache — HF rejects paths under .cache/
                    "**/.cache/**", # nested cache dirs anywhere under /opt/data
                ],
            )
            print(f"[SYNC] Upload completed at {datetime.now().isoformat()}")

            try:
                files = self.api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
                data_files = [f for f in files if f.startswith(f"{DATASET_PATH}/")]
                print(f"[SYNC] Dataset now has {len(data_files)} files under {DATASET_PATH}/")
            except Exception:
                pass

        except Exception as e:
            print(f"[SYNC] Upload failed: {e}")
            traceback.print_exc()

    # ── Config helpers ─────────────────────────────────────────────────

    def _ensure_default_config(self):
        """Ensure Hermes has config.yaml and .env for HF Spaces."""
        config_path = HERMES_DATA / "config.yaml"
        env_path = HERMES_DATA / ".env"
        soul_path = HERMES_DATA / "SOUL.md"

        # Bootstrap from Hermes templates if available
        if not config_path.exists():
            template = APP_DIR / "cli-config.yaml.example"
            if template.exists():
                shutil.copy2(str(template), str(config_path))
                print("[SYNC] Created config.yaml from Hermes template")
            else:
                # Minimal fallback config
                import yaml
                config = {
                    "agent": {"name": AGENT_NAME},
                    "server": {"host": "0.0.0.0", "port": 7860},
                }
                with open(config_path, "w") as f:
                    yaml.dump(config, f, default_flow_style=False)
                print(f"[SYNC] Created minimal config.yaml (agent={AGENT_NAME}, port=7860)")

        if not env_path.exists():
            template = APP_DIR / ".env.example"
            if template.exists():
                shutil.copy2(str(template), str(env_path))
                print("[SYNC] Created .env from Hermes template")
            else:
                env_lines = []
                for key in [
                    "OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",
                    "NOUS_API_KEY", "GOOGLE_API_KEY", "MISTRAL_API_KEY",
                    "TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN",
                ]:
                    val = os.environ.get(key, "")
                    if val:
                        env_lines.append(f"{key}={val}")
                if env_lines:
                    with open(env_path, "w") as f:
                        f.write("\n".join(env_lines) + "\n")
                    print(f"[SYNC] Created .env with {len(env_lines)} keys")

        if not soul_path.exists():
            template = APP_DIR / "docker" / "SOUL.md"
            if template.exists():
                shutil.copy2(str(template), str(soul_path))
                print("[SYNC] Created SOUL.md from Hermes template")
            else:
                with open(soul_path, "w") as f:
                    f.write(f"# {AGENT_NAME}\n\nI am {AGENT_NAME}, a self-improving AI assistant powered by Hermes Agent.\n")
                print("[SYNC] Created default SOUL.md")

        self._ensure_fallback_chain()

    def _debug_list_files(self):
        try:
            count = sum(1 for _, _, files in os.walk(HERMES_DATA) for _ in files)
            print(f"[SYNC] Local /opt/data: {count} files")
        except Exception as e:
            print(f"[SYNC] listing failed: {e}")

    # ── Admin auth wiring ───────────────────────────────────────────────
    def _configure_admin_auth(self):
        """Map HERMES_ADMIN_PASSWORD onto Hermes' real dashboard auth provider.

        Hermes' bundled password provider (plugins/dashboard_auth/basic) reads
        HERMES_DASHBOARD_BASIC_AUTH_USERNAME / _PASSWORD / _SECRET — nothing in
        Hermes reads HERMES_ADMIN_PASSWORD. Previously this Secret was set but
        unused, and since Hermes now requires an auth provider on any
        non-loopback bind (HF Spaces binds 0.0.0.0), the dashboard was failing
        closed (401 on every route) with no way to log in. This wires the
        existing secret into the mechanism Hermes actually checks.
        """
        admin_password = os.environ.get("HERMES_ADMIN_PASSWORD", "").strip()
        if not admin_password:
            print("[SYNC] WARNING: HERMES_ADMIN_PASSWORD not set — dashboard "
                  "login will be unavailable (non-loopback bind requires an "
                  "auth provider; the basic-auth plugin won't register without "
                  "a username+password).")
            return

        os.environ.setdefault("HERMES_DASHBOARD_BASIC_AUTH_USERNAME", "admin")
        os.environ["HERMES_DASHBOARD_BASIC_AUTH_PASSWORD"] = admin_password

        # Persist a stable HMAC signing secret across restarts so logged-in
        # sessions survive a Space restart instead of all being invalidated.
        if "HERMES_DASHBOARD_BASIC_AUTH_SECRET" not in os.environ:
            secret_path = HERMES_DATA / ".dashboard_auth_secret"
            try:
                if secret_path.exists():
                    secret = secret_path.read_text().strip()
                else:
                    secret = secrets.token_hex(32)
                    secret_path.parent.mkdir(parents=True, exist_ok=True)
                    secret_path.write_text(secret)
                    try:
                        os.chmod(secret_path, 0o600)
                    except Exception:
                        pass
                if secret:
                    os.environ["HERMES_DASHBOARD_BASIC_AUTH_SECRET"] = secret
            except Exception as e:
                print(f"[SYNC] WARNING: could not persist dashboard auth secret: {e}")

        print("[SYNC] Admin login wired: HERMES_DASHBOARD_BASIC_AUTH_USERNAME/PASSWORD "
              "set from HERMES_ADMIN_PASSWORD (username=admin unless overridden)")

    # ── Provider fallback chain ─────────────────────────────────────────
    def _ensure_fallback_chain(self):
        """Write the required model fallback order into config.yaml.

        Order: deepseek/deepseek-r1-0528:free -> deepseek/deepseek-r1-0528-qwen3-8b:free
        (both via OpenRouter) -> Google AI Studio (gemini, GOOGLE_API_KEY) ->
        openrouter/free. Only writes if fallback_providers isn't already
        configured, so it never clobbers a config the operator (or a restored
        dataset snapshot) already set up.
        """
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = {}
            if config_path.exists():
                with open(config_path) as f:
                    data = yaml.safe_load(f) or {}
            if data.get("fallback_providers"):
                print("[SYNC] fallback_providers already configured — leaving as-is")
                return
            data["fallback_providers"] = [
                {"provider": "openrouter", "model": "deepseek/deepseek-r1-0528:free"},
                {"provider": "openrouter", "model": "deepseek/deepseek-r1-0528-qwen3-8b:free"},
                {"provider": "gemini", "model": "gemini-3.5-flash"},
                {"provider": "openrouter", "model": "openrouter/free"},
            ]
            with open(config_path, "w") as f:
                yaml.dump(data, f, default_flow_style=False)
            print("[SYNC] fallback_providers written to config.yaml: "
                  "deepseek-r1 -> deepseek-r1-qwen3-8b -> gemini -> openrouter/free")
        except Exception as e:
            print(f"[SYNC] WARNING: could not write fallback_providers: {e}")

    def _park_unavailable_optional_mcp(self):
        """Keep optional local MCP servers from entering a reconnect loop.

        Unreal Engine and Linear are opt-in in a headless Space. Linear also
        requires cached OAuth credentials before it is exposed to child
        processes; otherwise one sanitized warning is emitted and the entry is
        parked for this boot. All other MCP entries remain untouched.
        """
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
            policy = apply_optional_mcp_policy(
                data,
                linear_enabled=os.environ.get("LINEAR_MCP_ENABLED", "false").strip().lower() == "true",
                linear_tokens_available=(HERMES_DATA / "mcp-tokens" / "linear.json").is_file(),
                unreal_enabled=os.environ.get("UNREAL_ENGINE_MCP_ENABLED", "false").lower() in {"1", "true", "yes", "on"},
            )
            if policy["removed"]:
                config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
            for name in policy["removed"]:
                if name.lower() == "unreal-engine":
                    print("[SYNC] Optional unreal-engine MCP parked (not enabled)")
                elif name.lower() == "linear" and not policy["linear_unconfigured"]:
                    print("[SYNC] Linear MCP disabled; OAuth discovery skipped")
            if policy["linear_unconfigured"]:
                print("[SYNC] WARNING: Linear MCP enabled but no cached OAuth credentials are available; parked for this process")
        except Exception as exc:
            print(f"[SYNC] Optional MCP parking skipped: {exc}")

    def _disable_legacy_telegram_gateway(self):
        """Make the Gateway's persisted platform map agree with webhook mode.

        Hermes loads ~/.hermes/.env after the child environment is built, so
        removing TELEGRAM_BOT_TOKEN from one env dict is insufficient. An
        explicit YAML ``enabled: false`` is the authoritative propagation
        point and leaves the dashboard webhook process independent.
        """
        enabled = os.environ.get("TELEGRAM_ENABLED", "false").strip().lower() == "true"
        mode = os.environ.get("TELEGRAM_MODE", "webhook").strip().lower()
        if enabled and mode != "webhook":
            return
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
            platforms = data.setdefault("platforms", {})
            if isinstance(platforms, dict):
                telegram = platforms.setdefault("telegram", {})
                if isinstance(telegram, dict):
                    telegram["enabled"] = False
                    config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
                    if enabled and mode == "webhook":
                        print("[SYNC] Telegram runtime: webhook-only; Gateway polling adapter disabled")
                    else:
                        print("[SYNC] Telegram runtime: disabled; no adapter or reconnect task started")
        except Exception as exc:
            print(f"[SYNC] Telegram Gateway isolation skipped: {exc}")

    # ── Background sync loop ──────────────────────────────────────────

    def background_sync_loop(self, stop_event):
        print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")
        while not stop_event.is_set():
            if stop_event.wait(timeout=SYNC_INTERVAL):
                break
            print(f"[SYNC] Periodic sync triggered at {datetime.now().isoformat()}")
            self.save_to_repo()

    # ── Application runner ─────────────────────────────────────────────

    def _patch_web_server_cors(self):
        """Patch Hermes web_server.py:
        - Allow any origin (HF Spaces iframe, custom domains)
        - Allow iframe embedding in huggingface.co + *.hf.space
        - Add /health and /ready aliases (HermesFace-specific; Hermes itself
          only exposes the equivalent public status at /api/status)
        """
        ws_path = APP_DIR / "hermes_cli" / "web_server.py"
        if not ws_path.exists():
            return
        try:
            code = ws_path.read_text()
            changed = False

            old_cors = 'allow_origin_regex=r"^https?://(localhost|127\\.0\\.0\\.1)(:\\d+)?$"'
            new_cors = 'allow_origins=["*"]'
            if old_cors in code:
                code = code.replace(old_cors, new_cors)
                changed = True
                print("[SYNC] Patched web_server.py CORS for HF Spaces")

            # Neutralise X-Frame-Options so HF Spaces can embed the dashboard.
            for pat in ('X-Frame-Options", "DENY"', 'X-Frame-Options", "SAMEORIGIN"'):
                if pat in code:
                    code = code.replace(pat, 'X-Frame-Options", "ALLOWALL"')
                    changed = True
                    print("[SYNC] Relaxed X-Frame-Options for HF Spaces")

            # Relax CSP frame-ancestors if present.
            csp_old = 'frame-ancestors \'none\''
            csp_new = "frame-ancestors 'self' https://huggingface.co https://*.hf.space"
            if csp_old in code:
                code = code.replace(csp_old, csp_new)
                changed = True
                print("[SYNC] Relaxed CSP frame-ancestors for HF Spaces")

            # Add /health and /ready — plain, non-/api/ paths, so they are
            # never gated by the dashboard auth middleware (which only
            # intercepts paths starting with /api/). Both just delegate to
            # the existing public /api/status handler.
            health_marker = '@app.get("/health")\nasync def _hermesface_health'
            old_status_route = '@app.get("/api/status")\nasync def get_status('
            if health_marker not in code and old_status_route in code:
                new_routes = (
                    '@app.get("/health")\n'
                    'async def _hermesface_health():\n'
                    '    """HermesFace: liveness alias for /api/status."""\n'
                    '    return await get_status()\n\n\n'
                    '@app.get("/ready")\n'
                    'async def _hermesface_ready():\n'
                    '    """HermesFace: readiness alias for /api/status."""\n'
                    '    return await get_status()\n\n\n'
                    + old_status_route
                )
                code = code.replace(old_status_route, new_routes, 1)
                changed = True
                print("[SYNC] Added /health and /ready aliases (delegate to /api/status)")

            if changed:
                ws_path.write_text(code)
        except Exception as e:
            print(f"[SYNC] web_server patch failed (non-fatal): {e}")

    def _install_futures_overlay(self):
        """Copy the HermesFace Futures trading overlay into /opt/hermes.

        Source lives at /opt/data/hermes_overlay (COPY'd from this repo's
        `hermes_overlay/` at build time, see Dockerfile). `tools/` is
        auto-discovered by tools.registry.discover_builtin_tools() purely by
        living in /opt/hermes/tools — no explicit import wiring needed.
        `trading/` is a plain importable package alongside `tools/`, `agent/`,
        etc. at the /opt/hermes project root.
        """
        # Keep the deployed source outside /opt/data: persistence restore may
        # contain an older hermes_overlay snapshot and must never downgrade
        # the code shipped in the current Space image.
        overlay_src = Path("/opt/hermesface_overlay")
        if not overlay_src.exists():
            overlay_src = HERMES_DATA / "hermes_overlay"
        if not overlay_src.exists():
            print("[SYNC] No hermes_overlay found at /opt/data/hermes_overlay — skipping Futures install")
            return
        try:
            trading_src = overlay_src / "trading"
            if trading_src.exists():
                shutil.copytree(trading_src, APP_DIR / "trading", dirs_exist_ok=True)
            tools_src = overlay_src / "tools"
            if tools_src.exists():
                for f in tools_src.glob("*.py"):
                    shutil.copy2(f, APP_DIR / "tools" / f.name)
            external_ai_src = overlay_src / "external_ai"
            if external_ai_src.exists():
                shutil.copytree(external_ai_src, APP_DIR / "external_ai", dirs_exist_ok=True)
            templates_src = tools_src / "templates"
            if templates_src.exists():
                shutil.copytree(templates_src, APP_DIR / "tools" / "templates", dirs_exist_ok=True)
            tracked = {
                "router": (
                    tools_src / "futures_dashboard_api.py",
                    APP_DIR / "tools" / "futures_dashboard_api.py",
                ),
                "template": (
                    templates_src / "hermes_futures_desk_luxury.html",
                    APP_DIR / "tools" / "templates" / "hermes_futures_desk_luxury.html",
                ),
            }

            def _sha256(path):
                if not path.is_file():
                    return None
                digest = hashlib.sha256()
                with path.open("rb") as handle:
                    for chunk in iter(lambda: handle.read(1024 * 1024), b""):
                        digest.update(chunk)
                return digest.hexdigest()

            manifest = {
                "installedAt": datetime.now().astimezone().isoformat(),
                "sourceRoot": str(overlay_src),
                "destinationRoot": str(APP_DIR),
                "files": {},
            }
            for name, (source_path, destination_path) in tracked.items():
                source_hash = _sha256(source_path)
                destination_hash = _sha256(destination_path)
                manifest["files"][name] = {
                    "sourcePath": str(source_path),
                    "destinationPath": str(destination_path),
                    "sourceSha256": source_hash,
                    "destinationSha256": destination_hash,
                    "matches": bool(source_hash and source_hash == destination_hash),
                }
            manifest_path = APP_DIR / ".hermes_futures_overlay_manifest.json"
            manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
            mismatch = [name for name, item in manifest["files"].items() if not item["matches"]]
            if mismatch:
                print(f"[SYNC] WARNING: Futures overlay hash mismatch: {', '.join(mismatch)}")
            else:
                print("[SYNC] Futures overlay hashes verified")
            print("[SYNC] Futures trading overlay installed into /opt/hermes")
        except Exception as e:
            print(f"[SYNC] Futures overlay install failed (non-fatal): {e}")

    def _patch_toolsets_futures_trading(self):
        """Register the HermesFace Futures toolset with Hermes' static TOOLSETS map.

        The futures tools themselves self-register into the tools.registry
        under toolset="futures_trading" just by living in /opt/hermes/tools
        (see tools/futures_trading_tool.py, discovered the same way as any
        built-in tool). But `toolsets.py`'s static TOOLSETS dict is what
        config.yaml's `toolsets:` list and the dashboard toolset picker read,
        so without this entry the toolset can't be enabled from config. Same
        patch-on-boot approach as _patch_web_server_cors -- no new plugin
        framework, just extending the one already used here.
        """
        ts_path = APP_DIR / "toolsets.py"
        if not ts_path.exists():
            return
        try:
            code = ts_path.read_text(encoding="utf-8")
            if '"futures_trading"' in code:
                return  # already patched (e.g. re-run after restart)
            marker = "TOOLSETS = {\n"
            if marker not in code:
                print("[SYNC] WARNING: toolsets.py TOOLSETS marker not found — skipping patch")
                return
            entry = (
                marker
                + '    "futures_trading": {\n'
                + '        "description": (\n'
                + '            "HermesFace dual-datasource Futures trading desk: '
                + 'get_market_context, get_alpha_signals, get_futures_account, '
                + 'get_futures_positions, calculate_futures_size, '
                + 'set_leverage_and_margin, execute_futures_position, '
                + 'close_futures_position."\n'
                + '        ),\n'
                + '        "tools": [\n'
                + '            "get_market_context", "get_alpha_signals", '
                + '"get_futures_account", "get_futures_positions", '
                + '"calculate_futures_size", "set_leverage_and_margin", '
                + '"execute_futures_position", "close_futures_position",\n'
                + '        ],\n'
                + '        "includes": [],\n'
                + '    },\n\n'
            )
            code = code.replace(marker, entry, 1)
            ts_path.write_text(code, encoding="utf-8")
            print("[SYNC] Patched toolsets.py: added 'futures_trading' toolset")
        except Exception as e:
            print(f"[SYNC] toolsets.py patch failed (non-fatal): {e}")

    def _patch_web_server_futures_dashboard(self):
        """Mount the Futures dashboard router onto the existing FastAPI `app`.

        Same server, same port, same process as the rest of HermesFace --
        this is the additive "extend the dashboard" hook, not a separate
        frontend. See tools/futures_dashboard_api.py for the actual routes
        (/api/futures/status, /api/futures/positions, /futures).
        """
        ws_path = APP_DIR / "hermes_cli" / "web_server.py"
        if not ws_path.exists():
            return
        try:
            code = ws_path.read_text(encoding="utf-8")
            if "futures_dashboard_api" in code and "telegram_bot" in code:
                return  # already patched
            marker = 'app = FastAPI(title="Hermes Agent", version=__version__, lifespan=_lifespan)\n'
            if marker not in code:
                print("[SYNC] WARNING: web_server.py FastAPI app marker not found — skipping Futures dashboard patch")
                return
            patch = (
                marker
                + "try:\n"
                + "    from tools.futures_dashboard_api import router as _futures_dashboard_router\n"
                + "    app.include_router(_futures_dashboard_router)\n"
                + "    from tools.telegram_bot import router as _telegram_router\n"
                + "    app.include_router(_telegram_router)\n"
                + "except Exception as _futures_dash_exc:  # never block boot on this\n"
                + "    print(f\"[HermesFace] Futures dashboard not mounted: {_futures_dash_exc}\")\n"
            )
            code = code.replace(marker, patch, 1)
            ws_path.write_text(code, encoding="utf-8")
            print("[SYNC] Patched web_server.py: mounted Futures and Telegram webhook routers")
        except Exception as e:
            print(f"[SYNC] web_server.py Futures dashboard patch failed (non-fatal): {e}")

    def _patch_telegram_webhook_public_path(self):
        """Let Telegram reach its own HMAC-protected webhook before cookie auth."""
        path = APP_DIR / "hermes_cli" / "dashboard_auth" / "public_paths.py"
        if not path.exists():
            return
        try:
            code = path.read_text(encoding="utf-8")
            marker = '    "/api/cron/fire",\n'
            if marker in code:
                additions = []
                if '"/api/telegram/webhook"' not in code:
                    additions.append('    "/api/telegram/webhook",  # HMAC-protected Telegram ingress\n')
                if '"/api/telegram/bootstrap/status"' not in code:
                    additions.append('    "/api/telegram/bootstrap/status",  # secret-protected activation status\n')
                if not additions:
                    return
                code = code.replace(marker, marker + "".join(additions), 1)
                path.write_text(code, encoding="utf-8")
                print("[SYNC] Added HMAC-protected Telegram webhook to public ingress paths")
        except Exception as exc:
            print(f"[SYNC] Telegram webhook auth-path patch skipped: {exc}")

    def _start_process(self, cmd, label, env, log_path):
        """Helper to start a subprocess with output logging."""
        log_fh = open(log_path, "a")
        try:
            process = subprocess.Popen(
                cmd,
                cwd=str(APP_DIR),
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                text=True,
                bufsize=1,
                env=env,
            )

            def copy_output():
                try:
                    for line in process.stdout:
                        log_fh.write(line)
                        log_fh.flush()
                        stripped = line.strip()
                        if not stripped:
                            continue
                        if any(skip in stripped for skip in [
                            'Downloading', 'Fetching', '%|', '━', '───',
                            'Already cached', 'Using cache', 'tokenizer',
                            '.safetensors', 'model-', 'shard',
                        ]):
                            continue
                        print(line, end='')
                except Exception as e:
                    print(f"[SYNC] {label} output error: {e}")
                finally:
                    log_fh.close()

            threading.Thread(target=copy_output, daemon=True).start()
            print(f"[SYNC] {label} started (PID {process.pid})")
            return process
        except Exception as e:
            log_fh.close()
            print(f"[SYNC] ERROR starting {label}: {e}")
            traceback.print_exc()
            return None

    def run_hermes(self):
        """Start Hermes: web dashboard on port 7860, gateway in background if messaging tokens configured."""
        log_dir = HERMES_DATA / "logs"
        log_dir.mkdir(parents=True, exist_ok=True)

        if not APP_DIR.exists():
            print(f"[SYNC] ERROR: App directory does not exist: {APP_DIR}")
            return None

        hermes_bin = shutil.which("hermes") or str(APP_DIR / ".venv" / "bin" / "hermes")
        if not Path(hermes_bin).exists():
            print("[SYNC] ERROR: hermes CLI not found")
            return None

        self._configure_admin_auth()
        env = os.environ.copy()
        env["HERMES_HOME"] = str(HERMES_DATA)
        env["GATEWAY_ALLOW_ALL_USERS"] = "true"
        # Prevent gateway from grabbing port 7860
        env.pop("API_SERVER_ENABLED", None)
        env.pop("API_SERVER_PORT", None)
        # Telegram is an isolated optional platform. It is opt-in for the
        # Space so regional egress failures cannot create reconnect storms.
        if env.get("TELEGRAM_ENABLED", "false").lower() not in {"1", "true", "yes", "on"}:
            env.pop("TELEGRAM_BOT_TOKEN", None)
            print("[SYNC] Telegram platform disabled (set TELEGRAM_ENABLED=true to enable)")
        self._park_unavailable_optional_mcp()
        self._disable_legacy_telegram_gateway()

        # ── 1. Install the HermesFace Futures overlay + patch dashboard ──
        self._install_futures_overlay()
        self._patch_web_server_cors()
        self._patch_toolsets_futures_trading()
        self._patch_telegram_webhook_public_path()
        self._patch_web_server_futures_dashboard()

        # ── 2. Start web dashboard on port 7860 (HF Spaces frontend) ─
        # --insecure: required to bind 0.0.0.0; HF Spaces already sandboxes the
        # container and Repository Secrets are never exposed to the browser.
        dashboard_cmd = [hermes_bin, "dashboard", "--host", "0.0.0.0", "--port", "7860",
                         "--no-open", "--insecure"]
        print("[SYNC] Starting web dashboard on port 7860...")
        dashboard_proc = self._start_process(
            dashboard_cmd, "Dashboard", env, log_dir / "dashboard.log"
        )

        # ── 3. Start gateway in background (messaging platforms + cron) ─
        time.sleep(2)  # Let dashboard bind 7860 first
        gateway_env = env.copy()
        gateway_env["GATEWAY_ALLOW_ALL_USERS"] = "true"
        # Webhook mode owns Telegram updates; never start Hermes' polling
        # adapter alongside it. The dashboard process still retains the token
        # for optional direct replies/relay delivery.
        if gateway_env.get("TELEGRAM_MODE", "webhook").lower() == "webhook":
            gateway_env["TELEGRAM_ENABLED"] = "false"
            gateway_env.pop("TELEGRAM_BOT_TOKEN", None)
        gateway_cmd = [hermes_bin, "gateway"]
        print("[SYNC] Starting gateway (messaging platforms)...")
        self.gateway_proc = self._start_process(
            gateway_cmd, "Gateway", gateway_env, log_dir / "gateway.log"
        )

        return dashboard_proc


# ── Main ────────────────────────────────────────────────────────────────────

def main():
    try:
        t_main_start = time.time()

        t0 = time.time()
        sync = HermesFullSync()
        print(f"[TIMER] sync_hf init: {time.time() - t0:.1f}s")

        # 1. Restore
        t0 = time.time()
        sync.load_from_repo()
        print(f"[TIMER] load_from_repo (restore): {time.time() - t0:.1f}s")

        # 2. Background sync
        stop_event = threading.Event()
        t = threading.Thread(target=sync.background_sync_loop, args=(stop_event,), daemon=True)
        t.start()

        # 3. Start application (Hermes API server will bind port 7860)
        t0 = time.time()
        process = sync.run_hermes()
        print(f"[TIMER] run_hermes launch: {time.time() - t0:.1f}s")
        print(f"[TIMER] Total startup (init → app launched): {time.time() - t_main_start:.1f}s")

        # Signal handler
        def handle_signal(sig, frame):
            print(f"\n[SYNC] Signal {sig} received. Shutting down...")
            stop_event.set()
            t.join(timeout=10)
            # Stop gateway
            if hasattr(sync, 'gateway_proc') and sync.gateway_proc:
                sync.gateway_proc.terminate()
                try:
                    sync.gateway_proc.wait(timeout=5)
                except subprocess.TimeoutExpired:
                    sync.gateway_proc.kill()
            # Stop dashboard
            if process:
                process.terminate()
                try:
                    process.wait(timeout=5)
                except subprocess.TimeoutExpired:
                    process.kill()
            print("[SYNC] Final sync...")
            sync.save_to_repo()
            sys.exit(0)

        signal.signal(signal.SIGINT, handle_signal)
        signal.signal(signal.SIGTERM, handle_signal)

        # Wait
        if process is None:
            print("[SYNC] ERROR: Failed to start Hermes process. Exiting.")
            stop_event.set()
            t.join(timeout=5)
            sys.exit(1)

        exit_code = process.wait()
        print(f"[SYNC] Hermes exited with code {exit_code}")
        stop_event.set()
        t.join(timeout=10)
        print("[SYNC] Final sync...")
        sync.save_to_repo()
        sys.exit(exit_code)

    except Exception as e:
        print(f"[SYNC] FATAL ERROR in main: {e}")
        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()