module registry

app/modules/__init__.py

@@ -0,0 +1,25 @@
+"""
+Processing Modules Package
+Module registry and base interfaces
+"""
+
+from app.modules.base import BaseModule, ModuleCapability, ModuleResult
+from app.modules.registry import ModuleRegistry, ModuleSelector
+from app.modules.capabilities import (
+    ScrapingCapability,
+    ProcessingCapability,
+    VisualizationCapability,
+    OutputCapability
+)
+
+__all__ = [
+    "BaseModule",
+    "ModuleCapability",
+    "ModuleResult",
+    "ModuleRegistry",
+    "ModuleSelector",
+    "ScrapingCapability",
+    "ProcessingCapability",
+    "VisualizationCapability",
+    "OutputCapability"
+]

app/modules/base.py

@@ -0,0 +1,174 @@
+"""
+Base Module Interface
+All processing modules inherit from this
+"""
+
+from typing import Dict, Any, List, Optional, Set
+from abc import ABC, abstractmethod
+from enum import Enum
+from pydantic import BaseModel, Field
+
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class ModuleType(str, Enum):
+    """Types of processing modules"""
+    SCRAPER = "scraper"
+    PROCESSOR = "processor"
+    ANALYZER = "analyzer"
+    VISUALIZER = "visualizer"
+    EXPORTER = "exporter"
+    API_CLIENT = "api_client"
+
+
+class ModuleCapability(BaseModel):
+    """Capability definition for a module"""
+    
+    # What can this module do?
+    can_scrape_static: bool = False
+    can_scrape_dynamic: bool = False
+    can_handle_javascript: bool = False
+    can_authenticate: bool = False
+    can_handle_api: bool = False
+    can_process_data: bool = False
+    can_clean_data: bool = False
+    can_transform_data: bool = False
+    can_aggregate: bool = False
+    can_filter: bool = False
+    can_sort: bool = False
+    can_visualize: bool = False
+    can_create_charts: bool = False
+    can_create_maps: bool = False
+    can_export_csv: bool = False
+    can_export_json: bool = False
+    can_export_excel: bool = False
+    can_export_pdf: bool = False
+    
+    # What data formats can it handle?
+    supported_input_formats: Set[str] = Field(default_factory=set)
+    supported_output_formats: Set[str] = Field(default_factory=set)
+    
+    # Performance characteristics
+    max_concurrent_requests: int = 1
+    estimated_speed: str = "medium"  # fast, medium, slow
+    memory_usage: str = "medium"  # low, medium, high
+    
+    # Requirements
+    requires_browser: bool = False
+    requires_api_key: bool = False
+    requires_database: bool = False
+
+
+class ModuleResult(BaseModel):
+    """Result from module execution"""
+    
+    success: bool
+    data: Any = None
+    error: Optional[str] = None
+    warnings: List[str] = Field(default_factory=list)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+    execution_time: float = 0.0
+    
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class BaseModule(ABC):
+    """
+    Base class for all processing modules
+    All modules must inherit from this and implement required methods
+    """
+    
+    def __init__(self, name: str, module_type: ModuleType):
+        """
+        Initialize base module
+        
+        Args:
+            name: Module name
+            module_type: Type of module
+        """
+        self.name = name
+        self.module_type = module_type
+        self.logger = get_logger(f"module.{name}")
+        self._is_initialized = False
+    
+    @abstractmethod
+    def get_capabilities(self) -> ModuleCapability:
+        """
+        Return module capabilities
+        
+        Returns:
+            ModuleCapability: What this module can do
+        """
+        pass
+    
+    @abstractmethod
+    async def execute(
+        self,
+        parameters: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ModuleResult:
+        """
+        Execute module with given parameters
+        
+        Args:
+            parameters: Execution parameters
+            context: Optional execution context
+            
+        Returns:
+            ModuleResult: Execution result
+        """
+        pass
+    
+    async def initialize(self) -> bool:
+        """
+        Initialize module (load models, connect to services, etc.)
+        Override this if your module needs initialization
+        
+        Returns:
+            bool: True if initialization successful
+        """
+        self._is_initialized = True
+        return True
+    
+    async def cleanup(self):
+        """
+        Clean up module resources
+        Override this if your module needs cleanup
+        """
+        self._is_initialized = False
+    
+    def is_initialized(self) -> bool:
+        """Check if module is initialized"""
+        return self._is_initialized
+    
+    def can_handle(self, parameters: Dict[str, Any]) -> bool:
+        """
+        Check if this module can handle given parameters
+        Override this for custom logic
+        
+        Args:
+            parameters: Parameters to check
+            
+        Returns:
+            bool: True if module can handle these parameters
+        """
+        return True
+    
+    def estimate_cost(self, parameters: Dict[str, Any]) -> float:
+        """
+        Estimate execution cost/time for given parameters
+        Used for module selection
+        
+        Args:
+            parameters: Parameters to estimate for
+            
+        Returns:
+            float: Estimated cost (lower is better)
+        """
+        return 1.0
+    
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}(name='{self.name}', type='{self.module_type}')>"

app/modules/capabilities.py

@@ -0,0 +1,128 @@
+"""
+Module Capability Definitions
+Pre-defined capability sets for different module types
+"""
+
+from app.modules.base import ModuleCapability
+
+
+class ScrapingCapability:
+    """Capability definitions for scraping modules"""
+    
+    STATIC = ModuleCapability(
+        can_scrape_static=True,
+        can_scrape_dynamic=False,
+        can_handle_javascript=False,
+        can_authenticate=False,
+        supported_input_formats={'html', 'xml'},
+        supported_output_formats={'json', 'csv', 'dict'},
+        max_concurrent_requests=5,
+        estimated_speed="fast",
+        memory_usage="low",
+        requires_browser=False
+    )
+    
+    DYNAMIC = ModuleCapability(
+        can_scrape_static=True,
+        can_scrape_dynamic=True,
+        can_handle_javascript=True,
+        can_authenticate=True,
+        supported_input_formats={'html', 'javascript'},
+        supported_output_formats={'json', 'csv', 'dict'},
+        max_concurrent_requests=2,
+        estimated_speed="medium",
+        memory_usage="high",
+        requires_browser=True
+    )
+    
+    API_CLIENT = ModuleCapability(
+        can_handle_api=True,
+        supported_input_formats={'api', 'rest', 'graphql'},
+        supported_output_formats={'json', 'dict'},
+        max_concurrent_requests=10,
+        estimated_speed="fast",
+        memory_usage="low",
+        requires_api_key=True
+    )
+
+
+class ProcessingCapability:
+    """Capability definitions for data processing modules"""
+    
+    DATA_CLEANER = ModuleCapability(
+        can_process_data=True,
+        can_clean_data=True,
+        supported_input_formats={'csv', 'json', 'dict', 'dataframe'},
+        supported_output_formats={'csv', 'json', 'dict', 'dataframe'},
+        max_concurrent_requests=1,
+        estimated_speed="fast",
+        memory_usage="medium"
+    )
+    
+    DATA_TRANSFORMER = ModuleCapability(
+        can_process_data=True,
+        can_transform_data=True,
+        can_aggregate=True,
+        can_filter=True,
+        can_sort=True,
+        supported_input_formats={'csv', 'json', 'dict', 'dataframe'},
+        supported_output_formats={'csv', 'json', 'dict', 'dataframe'},
+        max_concurrent_requests=1,
+        estimated_speed="medium",
+        memory_usage="medium"
+    )
+
+
+class VisualizationCapability:
+    """Capability definitions for visualization modules"""
+    
+    CHART_CREATOR = ModuleCapability(
+        can_visualize=True,
+        can_create_charts=True,
+        supported_input_formats={'csv', 'json', 'dict', 'dataframe'},
+        supported_output_formats={'png', 'jpg', 'svg', 'html'},
+        max_concurrent_requests=1,
+        estimated_speed="medium",
+        memory_usage="medium"
+    )
+    
+    MAP_CREATOR = ModuleCapability(
+        can_visualize=True,
+        can_create_maps=True,
+        supported_input_formats={'csv', 'json', 'dict', 'geojson'},
+        supported_output_formats={'html', 'png'},
+        max_concurrent_requests=1,
+        estimated_speed="slow",
+        memory_usage="high"
+    )
+
+
+class OutputCapability:
+    """Capability definitions for output/export modules"""
+    
+    CSV_EXPORTER = ModuleCapability(
+        can_export_csv=True,
+        supported_input_formats={'dict', 'list', 'dataframe'},
+        supported_output_formats={'csv'},
+        max_concurrent_requests=1,
+        estimated_speed="fast",
+        memory_usage="low"
+    )
+    
+    EXCEL_EXPORTER = ModuleCapability(
+        can_export_excel=True,
+        supported_input_formats={'dict', 'list', 'dataframe'},
+        supported_output_formats={'xlsx', 'xls'},
+        max_concurrent_requests=1,
+        estimated_speed="medium",
+        memory_usage="medium"
+    )
+    
+    JSON_EXPORTER = ModuleCapability(
+        can_export_json=True,
+        supported_input_formats={'dict', 'list', 'dataframe'},
+        supported_output_formats={'json'},
+        max_concurrent_requests=1,
+        estimated_speed="fast",
+        memory_usage="low"
+    )

app/modules/mock_modules.py

@@ -0,0 +1,273 @@
+"""
+Mock Modules for Testing
+These simulate real modules until Phase 2 modules are built
+"""
+
+from typing import Dict, Any, Optional
+import time
+
+from app.modules.base import (
+    BaseModule,
+    ModuleType,
+    ModuleCapability,
+    ModuleResult
+)
+from app.modules.capabilities import (
+    ScrapingCapability,
+    ProcessingCapability,
+    VisualizationCapability,
+    OutputCapability
+)
+
+from app.core.logging import get_logger
+from app.modules.registry import ModuleRegistry
+
+logger = get_logger(__name__)
+
+
+class MockStaticScraper(BaseModule):
+    """Mock static web scraper"""
+    
+    def __init__(self):
+        super().__init__(name="static_scraper", module_type=ModuleType.SCRAPER)
+    
+    def get_capabilities(self) -> ModuleCapability:
+        return ScrapingCapability.STATIC
+    
+    async def execute(
+        self,
+        parameters: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ModuleResult:
+        """Mock scraping execution"""
+        self.logger.info(f"[MOCK] Scraping static HTML from: {parameters.get('url')}")
+        
+        # Simulate work
+        await self._simulate_work(1.0)
+        
+        # Mock data
+        mock_data = [
+            {'name': 'Product 1', 'price': '$10.99'},
+            {'name': 'Product 2', 'price': '$24.99'},
+            {'name': 'Product 3', 'price': '$15.49'}
+        ]
+        
+        return ModuleResult(
+            success=True,
+            data=mock_data,
+            metadata={'rows': len(mock_data), 'source': 'mock'},
+            execution_time=1.0
+        )
+    
+    async def _simulate_work(self, seconds: float):
+        """Simulate async work"""
+        import asyncio
+        await asyncio.sleep(seconds)
+
+
+class MockDynamicScraper(BaseModule):
+    """Mock dynamic web scraper (Playwright)"""
+    
+    def __init__(self):
+        super().__init__(name="dynamic_scraper", module_type=ModuleType.SCRAPER)
+    
+    def get_capabilities(self) -> ModuleCapability:
+        return ScrapingCapability.DYNAMIC
+    
+    async def execute(
+        self,
+        parameters: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ModuleResult:
+        """Mock dynamic scraping"""
+        self.logger.info(f"[MOCK] Scraping with JavaScript from: {parameters.get('url')}")
+        
+        await self._simulate_work(2.0)
+        
+        mock_data = [
+            {'name': 'Dynamic Product 1', 'price': '$29.99'},
+            {'name': 'Dynamic Product 2', 'price': '$49.99'}
+        ]
+        
+        return ModuleResult(
+            success=True,
+            data=mock_data,
+            metadata={'rows': len(mock_data), 'method': 'playwright'},
+            execution_time=2.0
+        )
+    
+    async def _simulate_work(self, seconds: float):
+        import asyncio
+        await asyncio.sleep(seconds)
+
+
+class MockDataProcessor(BaseModule):
+    """Mock data processor"""
+    
+    def __init__(self):
+        super().__init__(name="data_processor", module_type=ModuleType.PROCESSOR)
+    
+    def get_capabilities(self) -> ModuleCapability:
+        return ProcessingCapability.DATA_TRANSFORMER
+    
+    async def execute(
+        self,
+        parameters: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ModuleResult:
+        """Mock data processing"""
+        self.logger.info("[MOCK] Processing data with filters and aggregations")
+        
+        # Get input data from context
+        input_data = context.get('data', []) if context else []
+        
+        # Mock filtering
+        if parameters.get('filters'):
+            self.logger.info(f"[MOCK] Applying {len(parameters['filters'])} filters")
+        
+        # Mock aggregation
+        if parameters.get('aggregations'):
+            self.logger.info(f"[MOCK] Applying {len(parameters['aggregations'])} aggregations")
+        
+        mock_result = {
+            'filtered_rows': len(input_data),
+            'aggregated': True,
+            'data': input_data
+        }
+        
+        return ModuleResult(
+            success=True,
+            data=mock_result,
+            metadata={'processed': True},
+            execution_time=0.5
+        )
+
+
+class MockChartCreator(BaseModule):
+    """Mock chart/visualization creator"""
+    
+    def __init__(self):
+        super().__init__(name="chart_creator", module_type=ModuleType.VISUALIZER)
+    
+    def get_capabilities(self) -> ModuleCapability:
+        return VisualizationCapability.CHART_CREATOR
+    
+    async def execute(
+        self,
+        parameters: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ModuleResult:
+        """Mock chart creation"""
+        chart_type = parameters.get('chart_type', 'bar')
+        self.logger.info(f"[MOCK] Creating {chart_type} chart")
+        
+        mock_chart = {
+            'type': chart_type,
+            'file': f'/tmp/chart_{chart_type}.png',
+            'created': True
+        }
+        
+        return ModuleResult(
+            success=True,
+            data=mock_chart,
+            metadata={'chart_type': chart_type},
+            execution_time=0.8
+        )
+
+
+class MockCSVExporter(BaseModule):
+    """Mock CSV exporter"""
+    
+    def __init__(self):
+        super().__init__(name="csv_exporter", module_type=ModuleType.EXPORTER)
+    
+    def get_capabilities(self) -> ModuleCapability:
+        return OutputCapability.CSV_EXPORTER
+    
+    async def execute(
+        self,
+        parameters: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ModuleResult:
+        """Mock CSV export"""
+        filename = parameters.get('filename', 'output.csv')
+        self.logger.info(f"[MOCK] Exporting to CSV: {filename}")
+        
+        mock_export = {
+            'filename': filename,
+            'path': f'/tmp/{filename}',
+            'rows': 10,
+            'exported': True
+        }
+        
+        return ModuleResult(
+            success=True,
+            data=mock_export,
+            metadata={'format': 'csv'},
+            execution_time=0.3
+        )
+
+
+class MockAPIClient(BaseModule):
+    """Mock API client"""
+    
+    def __init__(self):
+        super().__init__(name="api_client", module_type=ModuleType.API_CLIENT)
+    
+    def get_capabilities(self) -> ModuleCapability:
+        return ScrapingCapability.API_CLIENT
+    
+    async def execute(
+        self,
+        parameters: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ModuleResult:
+        """Mock API call"""
+        api_url = parameters.get('url', 'https://api.example.com')
+        self.logger.info(f"[MOCK] Calling API: {api_url}")
+        
+        await self._simulate_work(0.5)
+        
+        mock_data = {
+            'status': 'success',
+            'data': [
+                {'id': 1, 'name': 'Item 1'},
+                {'id': 2, 'name': 'Item 2'}
+            ]
+        }
+        
+        return ModuleResult(
+            success=True,
+            data=mock_data,
+            metadata={'api_url': api_url},
+            execution_time=0.5
+        )
+    
+    async def _simulate_work(self, seconds: float):
+        import asyncio
+        await asyncio.sleep(seconds)
+
+
+def register_mock_modules(registry: Optional['ModuleRegistry'] = None):
+    """
+    Register all mock modules for testing
+    
+    Args:
+        registry: Registry to register to (creates new if None)
+    """
+    from app.modules.registry import ModuleRegistry
+    
+    if registry is None:
+        registry = ModuleRegistry()
+    
+    # Register all mock modules
+    registry.register(MockStaticScraper())
+    registry.register(MockDynamicScraper())
+    registry.register(MockDataProcessor())
+    registry.register(MockChartCreator())
+    registry.register(MockCSVExporter())
+    registry.register(MockAPIClient())
+    
+    logger.info("✅ Registered 6 mock modules for testing")
+    
+    return registry

app/modules/registry.py

@@ -0,0 +1,387 @@
+"""
+Module Registry
+Central registry for all processing modules
+Handles module registration, discovery, and selection
+"""
+
+from typing import Dict, List, Optional, Type, Set
+from collections import defaultdict
+
+from app.modules.base import BaseModule, ModuleType, ModuleCapability
+from app.orchestrator.parameter_models import ExtractedParameters
+from app.orchestrator.models import TaskClassification
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class ModuleRegistry:
+    """
+    Central registry for all processing modules
+    Singleton pattern - only one registry exists
+    """
+    
+    _instance = None
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+    
+    def __init__(self):
+        """Initialize module registry"""
+        if self._initialized:
+            return
+        
+        self.modules: Dict[str, BaseModule] = {}
+        self.modules_by_type: Dict[ModuleType, List[BaseModule]] = defaultdict(list)
+        self._initialized = True
+        
+        logger.info("ModuleRegistry initialized")
+    
+    def register(self, module: BaseModule):
+        """
+        Register a module
+        
+        Args:
+            module: Module to register
+        """
+        if module.name in self.modules:
+            logger.warning(f"Module '{module.name}' already registered, replacing")
+        
+        self.modules[module.name] = module
+        self.modules_by_type[module.module_type].append(module)
+        
+        logger.info(
+            f"✓ Registered module: {module.name} "
+            f"(type: {module.module_type.value})"
+        )
+    
+    def unregister(self, module_name: str) -> bool:
+        """
+        Unregister a module
+        
+        Args:
+            module_name: Name of module to unregister
+            
+        Returns:
+            bool: True if unregistered
+        """
+        if module_name not in self.modules:
+            return False
+        
+        module = self.modules[module_name]
+        del self.modules[module_name]
+        self.modules_by_type[module.module_type].remove(module)
+        
+        logger.info(f"Unregistered module: {module_name}")
+        return True
+    
+    def get_module(self, name: str) -> Optional[BaseModule]:
+        """Get module by name"""
+        return self.modules.get(name)
+    
+    def get_modules_by_type(self, module_type: ModuleType) -> List[BaseModule]:
+        """Get all modules of a specific type"""
+        return self.modules_by_type.get(module_type, [])
+    
+    def get_all_modules(self) -> List[BaseModule]:
+        """Get all registered modules"""
+        return list(self.modules.values())
+    
+    def list_modules(self) -> Dict[str, Dict]:
+        """
+        List all registered modules with their info
+        
+        Returns:
+            Dict: Module information
+        """
+        result = {}
+        
+        for name, module in self.modules.items():
+            capabilities = module.get_capabilities()
+            
+            result[name] = {
+                'type': module.module_type.value,
+                'initialized': module.is_initialized(),
+                'capabilities': capabilities.dict()
+            }
+        
+        return result
+    
+    def clear(self):
+        """Clear all registered modules (for testing)"""
+        self.modules.clear()
+        self.modules_by_type.clear()
+        logger.info("Registry cleared")
+
+
+class ModuleSelector:
+    """
+    Selects appropriate modules based on task requirements
+    Uses classification and parameters to find best modules
+    """
+    
+    def __init__(self, registry: Optional[ModuleRegistry] = None):
+        """
+        Initialize module selector
+        
+        Args:
+            registry: Module registry to use (creates new if None)
+        """
+        self.registry = registry or ModuleRegistry()
+        logger.debug("ModuleSelector initialized")
+    
+    def select_modules(
+        self,
+        classification: TaskClassification,
+        parameters: ExtractedParameters
+    ) -> List[BaseModule]:
+        """
+        Select appropriate modules for task
+        
+        Args:
+            classification: Task classification
+            parameters: Extracted parameters
+            
+        Returns:
+            List[BaseModule]: Selected modules in execution order
+        """
+        logger.info("🔍 Selecting modules for task")
+        logger.debug(
+            f"Task type: {classification.primary_task.value}, "
+            f"Complexity: {classification.complexity.value}"
+        )
+        
+        selected = []
+        
+        # Step 1: Select data sourcing module
+        sourcing_module = self._select_sourcing_module(classification, parameters)
+        if sourcing_module:
+            selected.append(sourcing_module)
+        
+        # Step 2: Select processing modules
+        processing_modules = self._select_processing_modules(classification, parameters)
+        selected.extend(processing_modules)
+        
+        # Step 3: Select visualization module (if needed)
+        viz_module = self._select_visualization_module(classification, parameters)
+        if viz_module:
+            selected.append(viz_module)
+        
+        # Step 4: Select output/export module
+        output_module = self._select_output_module(classification, parameters)
+        if output_module:
+            selected.append(output_module)
+        
+        logger.info(
+            f"✅ Selected {len(selected)} modules: "
+            f"{[m.name for m in selected]}"
+        )
+        
+        return selected
+    
+    def _select_sourcing_module(
+    self,
+    classification: TaskClassification,
+    parameters: ExtractedParameters
+) -> Optional[BaseModule]:
+        """Select data sourcing module (scraper, API client, etc.)"""
+        
+        # No data sources, no module needed
+        if not parameters.data_sources:
+            logger.debug("No data sources, skipping sourcing module")
+            return None
+        
+        data_source = parameters.data_sources[0]  # Use first source
+        
+        # API data source
+        if data_source.type == 'api':
+            logger.debug("Selecting API client module")
+            candidates = self.registry.get_modules_by_type(ModuleType.API_CLIENT)
+            
+            for module in candidates:
+                if module.get_capabilities().can_handle_api:
+                    logger.info(f"✓ Selected API client: {module.name}")
+                    return module
+        
+        # URL/web scraping
+        if data_source.type == 'url':
+            # Check if JavaScript needed from CLASSIFICATION (primary source)
+            needs_javascript = classification.requires_javascript  # ← Fixed: Check classification first
+            
+            # Also check parameters.urls if present
+            if parameters.urls:
+                needs_javascript = needs_javascript or any(
+                    u.requires_javascript for u in parameters.urls
+                )
+            
+            if needs_javascript:
+                logger.debug("JavaScript required, selecting dynamic scraper")
+                candidates = self.registry.get_modules_by_type(ModuleType.SCRAPER)
+                
+                for module in candidates:
+                    if module.get_capabilities().can_scrape_dynamic:
+                        logger.info(f"✓ Selected dynamic scraper: {module.name}")
+                        return module
+                
+                logger.warning("JavaScript needed but no dynamic scraper available")
+            
+            # Static HTML scraping
+            logger.debug("Selecting static scraper")
+            candidates = self.registry.get_modules_by_type(ModuleType.SCRAPER)
+            
+            for module in candidates:
+                if module.get_capabilities().can_scrape_static:
+                    logger.info(f"✓ Selected static scraper: {module.name}")
+                    return module
+        
+        logger.warning("No suitable sourcing module found")
+        return None
+    
+    def _select_processing_modules(
+        self,
+        classification: TaskClassification,
+        parameters: ExtractedParameters
+    ) -> List[BaseModule]:
+        """Select data processing modules"""
+        
+        selected = []
+        
+        # Data cleaning (if filters present)
+        if parameters.filters:
+            logger.debug("Filters detected, need data processor")
+            candidates = self.registry.get_modules_by_type(ModuleType.PROCESSOR)
+            
+            for module in candidates:
+                caps = module.get_capabilities()
+                if caps.can_filter or caps.can_transform_data:
+                    logger.info(f"✓ Selected processor: {module.name}")
+                    selected.append(module)
+                    break
+        
+        # Aggregation (if aggregations present)
+        if parameters.aggregations:
+            logger.debug("Aggregations detected")
+            candidates = self.registry.get_modules_by_type(ModuleType.PROCESSOR)
+            
+            for module in candidates:
+                if module.get_capabilities().can_aggregate:
+                    if module not in selected:
+                        logger.info(f"✓ Selected aggregator: {module.name}")
+                        selected.append(module)
+                    break
+        
+        return selected
+    
+    def _select_visualization_module(
+        self,
+        classification: TaskClassification,
+        parameters: ExtractedParameters
+    ) -> Optional[BaseModule]:
+        """Select visualization module"""
+        
+        if not parameters.visualizations:
+            return None
+        
+        viz_req = parameters.visualizations[0]  # Use first visualization
+        
+        # Map visualization
+        if viz_req.type == 'map':
+            logger.debug("Map visualization needed")
+            candidates = self.registry.get_modules_by_type(ModuleType.VISUALIZER)
+            
+            for module in candidates:
+                if module.get_capabilities().can_create_maps:
+                    logger.info(f"✓ Selected map creator: {module.name}")
+                    return module
+        
+        # Chart visualization
+        if viz_req.type == 'chart':
+            logger.debug("Chart visualization needed")
+            candidates = self.registry.get_modules_by_type(ModuleType.VISUALIZER)
+            
+            for module in candidates:
+                if module.get_capabilities().can_create_charts:
+                    logger.info(f"✓ Selected chart creator: {module.name}")
+                    return module
+        
+        return None
+    
+    def _select_output_module(
+        self,
+        classification: TaskClassification,
+        parameters: ExtractedParameters
+    ) -> Optional[BaseModule]:
+        """Select output/export module"""
+        
+        if not parameters.output:
+            logger.debug("No output format specified, using default CSV")
+            output_format = 'csv'
+        else:
+            output_format = parameters.output.format
+        
+        candidates = self.registry.get_modules_by_type(ModuleType.EXPORTER)
+        
+        # Match by format
+        for module in candidates:
+            caps = module.get_capabilities()
+            
+            if output_format == 'csv' and caps.can_export_csv:
+                logger.info(f"✓ Selected CSV exporter: {module.name}")
+                return module
+            
+            elif output_format == 'excel' and caps.can_export_excel:
+                logger.info(f"✓ Selected Excel exporter: {module.name}")
+                return module
+            
+            elif output_format == 'json' and caps.can_export_json:
+                logger.info(f"✓ Selected JSON exporter: {module.name}")
+                return module
+        
+        logger.warning(f"No exporter for format: {output_format}")
+        return None
+    
+    def can_execute_task(
+        self,
+        classification: TaskClassification,
+        parameters: ExtractedParameters
+    ) -> bool:
+        """
+        Check if task can be executed with available modules
+        
+        Args:
+            classification: Task classification
+            parameters: Extracted parameters
+            
+        Returns:
+            bool: True if task can be executed
+        """
+        selected = self.select_modules(classification, parameters)
+        
+        # Need at least one module to execute
+        if not selected:
+            logger.warning("No modules selected, cannot execute task")
+            return False
+        
+        # Check if we have sourcing module (if data sources present)
+        if parameters.data_sources and not any(
+            m.module_type in [ModuleType.SCRAPER, ModuleType.API_CLIENT] 
+            for m in selected
+        ):
+            logger.warning("Data sources present but no sourcing module")
+            return False
+        
+        return True
+
+
+# Convenience function
+def get_module_registry() -> ModuleRegistry:
+    """Get global module registry instance"""
+    return ModuleRegistry()
+
+
+def get_module_selector() -> ModuleSelector:
+    """Get module selector instance"""
+    return ModuleSelector()

app/orchestrator/models.py

@@ -41,6 +41,7 @@ class OutputFormat(str, Enum):
     CSV = "csv"
     IMAGE = "image"
     CHART = "chart"
+    EXCEL = "excel"
     HTML = "html"
     PDF = "pdf"
     UNKNOWN = "unknown"

test/test_module_registry.py

@@ -0,0 +1,398 @@
+"""
+Test Module Registry
+Comprehensive tests for module registration and selection
+"""
+
+import asyncio
+import sys
+import os
+
+# Add project root to Python path
+project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.insert(0, project_root)
+from app.modules.registry import ModuleRegistry, ModuleSelector
+from app.modules.mock_modules import register_mock_modules
+from app.orchestrator.models import TaskClassification, TaskType, ComplexityLevel, OutputFormat
+from app.orchestrator.parameter_models import (
+    ExtractedParameters,
+    DataSource,
+    FilterCondition,
+    VisualizationRequirement,
+    OutputRequirement
+)
+from app.core.logging import setup_logging, get_logger
+
+setup_logging()
+logger = get_logger(__name__)
+
+
+def test_registry_registration():
+    """Test module registration"""
+    
+    print("\n" + "=" * 60)
+    print("Test 1: Module Registration")
+    print("=" * 60)
+    
+    # Clear registry
+    registry = ModuleRegistry()
+    registry.clear()
+    
+    # Register mock modules
+    register_mock_modules(registry)
+    
+    # Check registration
+    all_modules = registry.get_all_modules()
+    
+    print(f"\n✓ Registered {len(all_modules)} modules:")
+    for module in all_modules:
+        print(f"  - {module.name} ({module.module_type.value})")
+    
+    # List with details
+    module_info = registry.list_modules()
+    
+    print(f"\n📊 Module Details:")
+    for name, info in module_info.items():
+        print(f"\n  {name}:")
+        print(f"    Type: {info['type']}")
+        print(f"    Initialized: {info['initialized']}")
+        
+        caps = info['capabilities']
+        cap_list = [k for k, v in caps.items() if v and k.startswith('can_')]
+        if cap_list:
+            print(f"    Capabilities: {', '.join(cap_list[:3])}...")
+
+
+def test_simple_scraping_selection():
+    """Test module selection for simple scraping task"""
+    
+    print("\n" + "=" * 60)
+    print("Test 2: Simple Scraping Task")
+    print("=" * 60)
+    
+    # Setup
+    registry = ModuleRegistry()
+    registry.clear()
+    register_mock_modules(registry)
+    
+    selector = ModuleSelector(registry)
+    
+    # Create simple scraping task
+    classification = TaskClassification(
+        primary_task=TaskType.WEB_SCRAPING,
+        secondary_tasks=[],
+        complexity=ComplexityLevel.SIMPLE,
+        estimated_steps=2,
+        requires_javascript=False,
+        requires_authentication=False,
+        output_format=OutputFormat.CSV,
+        confidence=0.9,  # ← Added
+        reasoning="Simple static web scraping task"  # ← Added
+    )
+    
+    parameters = ExtractedParameters(
+        data_sources=[
+            DataSource(
+                type='url',
+                location='https://example.com/products',
+                format='html',
+                description='Product listing page'
+            )
+        ],
+        output=OutputRequirement(
+            format='csv',
+            description='Export as CSV'
+        )
+    )
+    
+    print("\n📋 Task:")
+    print("  Type: Simple web scraping")
+    print("  JavaScript: No")
+    print("  Output: CSV")
+    print("-" * 60)
+    
+    # Select modules
+    selected = selector.select_modules(classification, parameters)
+    
+    print(f"\n✅ Selected {len(selected)} modules:")
+    for i, module in enumerate(selected, 1):
+        print(f"  {i}. {module.name} ({module.module_type.value})")
+    
+    # Verify
+    assert len(selected) >= 2, "Should select at least scraper + exporter"
+    assert any(m.name == 'static_scraper' for m in selected), "Should use static scraper"
+    assert any(m.name == 'csv_exporter' for m in selected), "Should use CSV exporter"
+    
+    print("\n✓ Correct modules selected!")
+
+
+def test_dynamic_scraping_selection():
+    """Test module selection for dynamic scraping (JavaScript)"""
+    
+    print("\n" + "=" * 60)
+    print("Test 3: Dynamic Scraping Task (JavaScript)")
+    print("=" * 60)
+    
+    registry = ModuleRegistry()
+    registry.clear()
+    register_mock_modules(registry)
+    
+    selector = ModuleSelector(registry)
+    
+    # Create dynamic scraping task
+    classification = TaskClassification(
+        primary_task=TaskType.WEB_SCRAPING,
+        secondary_tasks=[],
+        complexity=ComplexityLevel.MEDIUM,
+        estimated_steps=3,
+        requires_javascript=True,
+        requires_authentication=False,
+        output_format=OutputFormat.JSON,
+        confidence=0.85,  # ← Added
+        reasoning="Dynamic web scraping with JavaScript"  # ← Added
+    )
+    
+    parameters = ExtractedParameters(
+        data_sources=[
+            DataSource(
+                type='url',
+                location='https://example.com/dynamic-products',
+                format='html',
+                description='Dynamic product listing (JavaScript)'
+            )
+        ]
+    )
+    
+    print("\n📋 Task:")
+    print("  Type: Dynamic web scraping")
+    print("  JavaScript: Yes")
+    print("  Output: JSON")
+    print("-" * 60)
+    
+    # Select modules
+    selected = selector.select_modules(classification, parameters)
+    
+    print(f"\n✅ Selected {len(selected)} modules:")
+    for i, module in enumerate(selected, 1):
+        print(f"  {i}. {module.name} ({module.module_type.value})")
+    
+    # Verify dynamic scraper selected
+    assert any(m.name == 'dynamic_scraper' for m in selected), \
+        "Should use dynamic scraper for JavaScript"
+    
+    print("\n✓ Dynamic scraper selected for JavaScript task!")
+
+
+def test_complex_analysis_selection():
+    """Test module selection for complex data analysis"""
+    
+    print("\n" + "=" * 60)
+    print("Test 4: Complex Data Analysis Task")
+    print("=" * 60)
+    
+    registry = ModuleRegistry()
+    registry.clear()
+    register_mock_modules(registry)
+    
+    selector = ModuleSelector(registry)
+    
+    # Create complex analysis task
+    classification = TaskClassification(
+        primary_task=TaskType.ML_ANALYSIS,
+        secondary_tasks=[TaskType.VISUALIZATION],
+        complexity=ComplexityLevel.COMPLEX,
+        estimated_steps=5,
+        requires_javascript=False,
+        output_format=OutputFormat.EXCEL,
+        confidence=0.88,  # ← Added
+        reasoning="Complex data analysis with filtering and visualization"  # ← Added
+    )
+    
+    parameters = ExtractedParameters(
+        data_sources=[
+            DataSource(
+                type='url',
+                location='https://example.com/sales.csv',
+                format='csv',
+                description='Sales data'
+            )
+        ],
+        filters=[
+            FilterCondition(
+                field='region',
+                operator='equals',
+                value='North',
+                description='Filter for North region'
+            )
+        ],
+        visualizations=[
+            VisualizationRequirement(
+                type='chart',
+                chart_type='bar',
+                description='Bar chart of sales by category'
+            )
+        ]
+    )
+    
+    print("\n📋 Task:")
+    print("  Type: Data analysis + visualization")
+    print("  Has filters: Yes")
+    print("  Has visualizations: Yes")
+    print("  Output: Excel")
+    print("-" * 60)
+    
+    # Select modules
+    selected = selector.select_modules(classification, parameters)
+    
+    print(f"\n✅ Selected {len(selected)} modules:")
+    for i, module in enumerate(selected, 1):
+        caps = module.get_capabilities()
+        cap_names = [k for k, v in caps.dict().items() if v and k.startswith('can_')]
+        print(f"  {i}. {module.name} ({module.module_type.value})")
+        print(f"      Capabilities: {', '.join(cap_names[:2])}...")
+    
+    # Verify correct module types
+    module_types = [m.module_type.value for m in selected]
+    
+    print(f"\n📊 Module Pipeline:")
+    print(f"  Scraper: {'✓' if 'scraper' in module_types else '✗'}")
+    print(f"  Processor: {'✓' if 'processor' in module_types else '✗'}")
+    print(f"  Visualizer: {'✓' if 'visualizer' in module_types else '✗'}")
+    print(f"  Exporter: {'✓' if 'exporter' in module_types else '✗'}")
+
+
+def test_api_task_selection():
+    """Test module selection for API-based task"""
+    
+    print("\n" + "=" * 60)
+    print("Test 5: API Data Fetching Task")
+    print("=" * 60)
+    
+    registry = ModuleRegistry()
+    registry.clear()
+    register_mock_modules(registry)
+    
+    selector = ModuleSelector(registry)
+    
+    # Create API task
+    classification = TaskClassification(
+        primary_task=TaskType.WEB_SCRAPING,
+        secondary_tasks=[],
+        complexity=ComplexityLevel.SIMPLE,
+        estimated_steps=2,
+        output_format=OutputFormat.JSON,
+        confidence=0.92,  # ← Added
+        reasoning="API data fetching task"  # ← Added
+    )
+    
+    parameters = ExtractedParameters(
+        data_sources=[
+            DataSource(
+                type='api',
+                location='https://api.example.com/users',
+                format='json',
+                description='User data API'
+            )
+        ]
+    )
+    
+    print("\n📋 Task:")
+    print("  Type: API data fetching")
+    print("  Source: REST API")
+    print("  Output: JSON")
+    print("-" * 60)
+    
+    # Select modules
+    selected = selector.select_modules(classification, parameters)
+    
+    print(f"\n✅ Selected {len(selected)} modules:")
+    for module in selected:
+        print(f"  - {module.name} ({module.module_type.value})")
+    
+    # Verify API client selected
+    assert any(m.name == 'api_client' for m in selected), \
+        "Should use API client for API data source"
+    
+    print("\n✓ API client selected for API task!")
+
+
+async def test_module_execution():
+    """Test actually executing a selected module"""
+    
+    print("\n" + "=" * 60)
+    print("Test 6: Module Execution")
+    print("=" * 60)
+    
+    registry = ModuleRegistry()
+    registry.clear()
+    register_mock_modules(registry)
+    
+    # Get a module
+    scraper = registry.get_module('static_scraper')
+    
+    print(f"\n🔧 Testing module: {scraper.name}")
+    print("-" * 60)
+    
+    # Execute
+    parameters = {
+        'url': 'https://example.com/test',
+        'columns': ['name', 'price']
+    }
+    
+    print(f"\nExecuting with parameters:")
+    print(f"  URL: {parameters['url']}")
+    print(f"  Columns: {parameters['columns']}")
+    
+    result = await scraper.execute(parameters)
+    
+    print(f"\n✅ Execution Result:")
+    print(f"  Success: {result.success}")
+    print(f"  Execution Time: {result.execution_time}s")
+    print(f"  Data rows: {len(result.data) if result.data else 0}")
+    
+    if result.data:
+        print(f"\n📊 Sample Data:")
+        for item in result.data[:3]:
+            print(f"  - {item}")
+
+
+def run_all_tests():
+    """Run all registry tests"""
+    
+    print("\n" + "=" * 80)
+    print(" " * 20 + "MODULE REGISTRY TEST SUITE")
+    print("=" * 80)
+    
+    try:
+        # Synchronous tests
+        test_registry_registration()
+        test_simple_scraping_selection()
+        test_dynamic_scraping_selection()
+        test_complex_analysis_selection()
+        test_api_task_selection()
+        
+        # Async test
+        asyncio.run(test_module_execution())
+        
+        print("\n" + "=" * 80)
+        print(" " * 30 + "ALL TESTS PASSED")
+        print("=" * 80)
+        print("\n✅ Module registry tests complete!")
+        print("\n📊 Summary:")
+        print("  ✓ Module registration working")
+        print("  ✓ Module selection logic working")
+        print("  ✓ Different task types handled correctly")
+        print("  ✓ Module execution working")
+        
+    except AssertionError as e:
+        print(f"\n❌ Assertion failed: {e}")
+        logger.error("Test assertion failed", exc_info=True)
+        raise
+    
+    except Exception as e:
+        print(f"\n❌ Test failed: {e}")
+        logger.error("Test suite failed", exc_info=True)
+        raise
+
+
+if __name__ == "__main__":
+    run_all_tests()