""" Integration tests for Process Orchestration functionality. These tests start PyServe with process orchestration extension and verify that requests are correctly routed to isolated worker processes. """ import asyncio import os import sys import tempfile import pytest import httpx import socket from pathlib import Path from typing import Dict, Any import uvicorn from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import JSONResponse, PlainTextResponse from starlette.routing import Route from pyserve.config import Config, ServerConfig, HttpConfig, LoggingConfig, ExtensionConfig from pyserve.server import PyServeServer from pyserve.process_manager import ( ProcessConfig, ProcessInfo, ProcessManager, ProcessState, PortAllocator, ) def get_free_port() -> int: """Get an available port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(('', 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) return s.getsockname()[1] # ============== Test Applications (saved as temp files) ============== FASTAPI_APP_CODE = ''' from fastapi import FastAPI import os app = FastAPI() @app.get("/") def root(): return {"app": "fastapi-worker", "pid": os.getpid()} @app.get("/health") def health(): return {"status": "healthy", "pid": os.getpid()} @app.get("/users") def users(): return {"users": [{"id": 1, "name": "Alice"}], "pid": os.getpid()} @app.post("/users") def create_user(data: dict): return {"created": data, "pid": os.getpid()} ''' STARLETTE_APP_CODE = ''' from starlette.applications import Starlette from starlette.responses import JSONResponse from starlette.routing import Route import os async def root(request): return JSONResponse({"app": "starlette-worker", "pid": os.getpid()}) async def health(request): return JSONResponse({"status": "healthy", "pid": os.getpid()}) async def info(request): return JSONResponse({"info": "test", "pid": os.getpid()}) app = Starlette(routes=[ Route("/", root), Route("/health", health), Route("/info", info), ]) ''' SIMPLE_ASGI_APP_CODE = ''' import os async def app(scope, receive, send): if scope["type"] == "http": path = scope.get("path", "/") if path == "/health": body = b'{"status": "healthy", "pid": ' + str(os.getpid()).encode() + b'}' else: body = b'{"app": "simple-asgi", "path": "' + path.encode() + b'", "pid": ' + str(os.getpid()).encode() + b'}' await send({ "type": "http.response.start", "status": 200, "headers": [[b"content-type", b"application/json"]], }) await send({ "type": "http.response.body", "body": body, }) ''' # ============== Unit Tests for ProcessConfig ============== class TestProcessConfig: def test_default_values(self): config = ProcessConfig(name="test", app_path="myapp:app") assert config.name == "test" assert config.app_path == "myapp:app" assert config.app_type == "asgi" assert config.host == "127.0.0.1" assert config.port == 0 assert config.workers == 1 assert config.health_check_enabled is True assert config.health_check_path == "/health" assert config.max_restart_count == 5 def test_custom_values(self): config = ProcessConfig( name="api", app_path="myapp.api:create_app", workers=4, factory=True, health_check_path="/api/health", max_memory_mb=512, ) assert config.workers == 4 assert config.factory is True assert config.health_check_path == "/api/health" assert config.max_memory_mb == 512 class TestProcessInfo: def test_initial_state(self): config = ProcessConfig(name="test", app_path="myapp:app") info = ProcessInfo(config=config) assert info.state == ProcessState.PENDING assert info.pid is None assert info.port == 0 assert info.restart_count == 0 assert info.is_running is False def test_uptime(self): import time config = ProcessConfig(name="test", app_path="myapp:app") info = ProcessInfo(config=config) assert info.uptime == 0.0 info.start_time = time.time() - 10 assert 9 < info.uptime < 11 def test_to_dict(self): config = ProcessConfig(name="test", app_path="myapp:app", workers=2) info = ProcessInfo(config=config, state=ProcessState.RUNNING, pid=12345, port=9001) result = info.to_dict() assert result["name"] == "test" assert result["state"] == "running" assert result["pid"] == 12345 assert result["port"] == 9001 assert result["workers"] == 2 class TestPortAllocator: @pytest.mark.asyncio async def test_allocate_port(self): allocator = PortAllocator(start_port=19000, end_port=19010) port = await allocator.allocate() assert 19000 <= port <= 19010 assert port in allocator._allocated @pytest.mark.asyncio async def test_release_port(self): allocator = PortAllocator(start_port=19000, end_port=19010) port = await allocator.allocate() assert port in allocator._allocated await allocator.release(port) assert port not in allocator._allocated @pytest.mark.asyncio async def test_no_available_ports(self): allocator = PortAllocator(start_port=19000, end_port=19001) # Allocate all ports port1 = await allocator.allocate() port2 = await allocator.allocate() # Should raise error with pytest.raises(RuntimeError, match="No available ports"): await allocator.allocate() # Release one and try again await allocator.release(port1) port3 = await allocator.allocate() assert port3 == port1 class TestProcessManager: """Tests for ProcessManager.""" @pytest.mark.asyncio async def test_register_process(self): manager = ProcessManager(port_range=(19000, 19999)) config = ProcessConfig(name="test", app_path="myapp:app") info = await manager.register(config) assert info.config.name == "test" assert "test" in manager._processes @pytest.mark.asyncio async def test_register_duplicate(self): manager = ProcessManager(port_range=(19000, 19999)) config = ProcessConfig(name="test", app_path="myapp:app") await manager.register(config) with pytest.raises(ValueError, match="already registered"): await manager.register(config) @pytest.mark.asyncio async def test_unregister_process(self): manager = ProcessManager(port_range=(19000, 19999)) config = ProcessConfig(name="test", app_path="myapp:app") await manager.register(config) await manager.unregister("test") assert "test" not in manager._processes @pytest.mark.asyncio async def test_get_process(self): manager = ProcessManager(port_range=(19000, 19999)) config = ProcessConfig(name="test", app_path="myapp:app") await manager.register(config) info = manager.get_process("test") assert info is not None assert info.config.name == "test" info = manager.get_process("nonexistent") assert info is None @pytest.mark.asyncio async def test_build_command(self): manager = ProcessManager() config = ProcessConfig( name="test", app_path="myapp:app", workers=4, ) cmd = manager._build_command(config, 9001) # Command format: [python, -m, uvicorn, app_path, ...] assert "-m" in cmd assert "uvicorn" in cmd assert "myapp:app" in cmd assert "--port" in cmd assert "9001" in cmd assert "--workers" in cmd assert "4" in cmd @pytest.mark.asyncio async def test_build_command_with_factory(self): manager = ProcessManager() config = ProcessConfig( name="test", app_path="myapp:create_app", factory=True, ) cmd = manager._build_command(config, 9001) assert "--factory" in cmd @pytest.mark.asyncio async def test_get_metrics(self): manager = ProcessManager(port_range=(19000, 19999)) config1 = ProcessConfig(name="app1", app_path="myapp1:app") config2 = ProcessConfig(name="app2", app_path="myapp2:app") await manager.register(config1) await manager.register(config2) metrics = manager.get_metrics() assert metrics["managed_processes"] == 2 assert metrics["running_processes"] == 0 assert "app1" in metrics["processes"] assert "app2" in metrics["processes"] @pytest.mark.asyncio async def test_start_stop_lifecycle(self): manager = ProcessManager(port_range=(19000, 19999)) await manager.start() assert manager._started is True await manager.stop() assert manager._started is False class TestProcessState: """Tests for ProcessState enum.""" def test_all_states(self): assert ProcessState.PENDING.value == "pending" assert ProcessState.STARTING.value == "starting" assert ProcessState.RUNNING.value == "running" assert ProcessState.STOPPING.value == "stopping" assert ProcessState.STOPPED.value == "stopped" assert ProcessState.FAILED.value == "failed" assert ProcessState.RESTARTING.value == "restarting" # ============== Integration Tests ============== class TestProcessManagerIntegration: """Integration tests for ProcessManager with real processes.""" @pytest.fixture def temp_app_dir(self): """Create temp directory with test apps.""" with tempfile.TemporaryDirectory() as tmpdir: # Write starlette app (no extra dependencies needed) app_file = Path(tmpdir) / "test_app.py" app_file.write_text(STARLETTE_APP_CODE) # Write simple ASGI app simple_file = Path(tmpdir) / "simple_app.py" simple_file.write_text(SIMPLE_ASGI_APP_CODE) yield tmpdir @pytest.mark.asyncio async def test_start_real_process(self, temp_app_dir): """Test starting a real uvicorn process.""" manager = ProcessManager(port_range=(19100, 19199), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="test_starlette", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", health_check_enabled=True, ) await manager.register(config) success = await manager.start_process("test_starlette") assert success is True info = manager.get_process("test_starlette") assert info is not None assert info.state == ProcessState.RUNNING assert info.port > 0 assert info.pid is not None # Make request to the running process async with httpx.AsyncClient() as client: resp = await client.get(f"http://127.0.0.1:{info.port}/") assert resp.status_code == 200 data = resp.json() assert data["app"] == "starlette-worker" assert data["pid"] == info.pid # Health check resp = await client.get(f"http://127.0.0.1:{info.port}/health") assert resp.status_code == 200 assert resp.json()["status"] == "healthy" finally: await manager.stop() @pytest.mark.asyncio async def test_start_multiple_processes(self, temp_app_dir): """Test starting multiple processes on different ports.""" manager = ProcessManager(port_range=(19200, 19299), health_check_enabled=False) await manager.start() try: # Register two apps config1 = ProcessConfig( name="app1", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) config2 = ProcessConfig( name="app2", app_path="simple_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config1) await manager.register(config2) results = await manager.start_all() assert results["app1"] is True assert results["app2"] is True info1 = manager.get_process("app1") info2 = manager.get_process("app2") assert info1.port != info2.port assert info1.pid != info2.pid # Both should respond async with httpx.AsyncClient() as client: resp1 = await client.get(f"http://127.0.0.1:{info1.port}/") resp2 = await client.get(f"http://127.0.0.1:{info2.port}/") assert resp1.status_code == 200 assert resp2.status_code == 200 assert resp1.json()["app"] == "starlette-worker" assert resp2.json()["app"] == "simple-asgi" finally: await manager.stop() @pytest.mark.asyncio async def test_stop_process(self, temp_app_dir): """Test stopping a running process.""" manager = ProcessManager(port_range=(19300, 19399), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="test_app", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config) await manager.start_process("test_app") info = manager.get_process("test_app") port = info.port # Process should be running async with httpx.AsyncClient() as client: resp = await client.get(f"http://127.0.0.1:{port}/health") assert resp.status_code == 200 # Stop the process await manager.stop_process("test_app") assert info.state == ProcessState.STOPPED assert info.process is None # Process should no longer respond async with httpx.AsyncClient() as client: with pytest.raises(httpx.ConnectError): await client.get(f"http://127.0.0.1:{port}/health", timeout=1.0) finally: await manager.stop() @pytest.mark.asyncio async def test_restart_process(self, temp_app_dir): """Test restarting a process gets a new PID.""" manager = ProcessManager(port_range=(19400, 19499), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="test_app", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", restart_delay=0.1, ) await manager.register(config) await manager.start_process("test_app") info = manager.get_process("test_app") old_pid = info.pid # Restart await manager.restart_process("test_app") assert info.state == ProcessState.RUNNING assert info.pid != old_pid # New PID assert info.restart_count == 0 # Manual restart doesn't increment # Should still respond async with httpx.AsyncClient() as client: resp = await client.get(f"http://127.0.0.1:{info.port}/health") assert resp.status_code == 200 finally: await manager.stop() @pytest.mark.asyncio async def test_get_upstream_url(self, temp_app_dir): """Test getting upstream URL for proxy routing.""" manager = ProcessManager(port_range=(19500, 19599), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="api", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config) # Not running yet assert manager.get_upstream_url("api") is None await manager.start_process("api") info = manager.get_process("api") url = manager.get_upstream_url("api") assert url == f"http://127.0.0.1:{info.port}" # Verify the URL works async with httpx.AsyncClient() as client: resp = await client.get(f"{url}/health") assert resp.status_code == 200 finally: await manager.stop() @pytest.mark.asyncio async def test_process_isolation(self, temp_app_dir): """Test that processes are truly isolated (different PIDs).""" manager = ProcessManager(port_range=(19600, 19699), health_check_enabled=False) await manager.start() try: # Start same app twice with different names config1 = ProcessConfig( name="worker1", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) config2 = ProcessConfig( name="worker2", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config1) await manager.register(config2) await manager.start_all() info1 = manager.get_process("worker1") info2 = manager.get_process("worker2") # Get PIDs from the apps themselves async with httpx.AsyncClient() as client: resp1 = await client.get(f"http://127.0.0.1:{info1.port}/") resp2 = await client.get(f"http://127.0.0.1:{info2.port}/") pid1 = resp1.json()["pid"] pid2 = resp2.json()["pid"] # PIDs should be different - true process isolation assert pid1 != pid2 assert pid1 == info1.pid assert pid2 == info2.pid finally: await manager.stop() @pytest.mark.asyncio async def test_metrics_with_running_processes(self, temp_app_dir): """Test metrics reflect actual process state.""" manager = ProcessManager(port_range=(19700, 19799), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="monitored_app", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config) # Before start metrics = manager.get_metrics() assert metrics["managed_processes"] == 1 assert metrics["running_processes"] == 0 assert metrics["processes"]["monitored_app"]["state"] == "pending" # After start await manager.start_process("monitored_app") metrics = manager.get_metrics() assert metrics["running_processes"] == 1 assert metrics["processes"]["monitored_app"]["state"] == "running" assert metrics["processes"]["monitored_app"]["pid"] is not None assert metrics["processes"]["monitored_app"]["port"] > 0 # After stop await manager.stop_process("monitored_app") metrics = manager.get_metrics() assert metrics["running_processes"] == 0 assert metrics["processes"]["monitored_app"]["state"] == "stopped" finally: await manager.stop() # ============== Error Handling Tests ============== class TestErrorHandling: """Tests for error scenarios and edge cases.""" @pytest.fixture def temp_app_dir(self): """Create temp directory with test apps.""" with tempfile.TemporaryDirectory() as tmpdir: # Write starlette app (no extra dependencies needed) app_file = Path(tmpdir) / "test_app.py" app_file.write_text(STARLETTE_APP_CODE) yield tmpdir @pytest.mark.asyncio async def test_process_crash_detection(self, temp_app_dir): """Process dies unexpectedly - state should reflect failure.""" manager = ProcessManager(port_range=(19800, 19899), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="crash_test", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config) await manager.start_process("crash_test") info = manager.get_process("crash_test") assert info.state == ProcessState.RUNNING old_pid = info.pid # Kill the process externally import os import signal os.kill(old_pid, signal.SIGKILL) # Wait a bit for process to die await asyncio.sleep(0.5) # Process should have exited - check via poll assert info.process.poll() is not None finally: await manager.stop() @pytest.mark.asyncio async def test_startup_timeout_with_bad_app(self): """App that fails to start should timeout gracefully.""" manager = ProcessManager(port_range=(19900, 19999), health_check_enabled=False) await manager.start() try: # Create a temp dir with a broken app with tempfile.TemporaryDirectory() as tmpdir: broken_app = Path(tmpdir) / "broken_app.py" broken_app.write_text(''' # App that crashes on import raise RuntimeError("Intentional crash") ''') config = ProcessConfig( name="broken_app", app_path="broken_app:app", module_path=tmpdir, workers=1, health_check_path="/health", ) await manager.register(config) # Should fail to start success = await manager.start_process("broken_app") assert success is False info = manager.get_process("broken_app") assert info.state == ProcessState.FAILED finally: await manager.stop() @pytest.mark.asyncio async def test_malformed_app_path(self): """Invalid app_path should fail gracefully.""" manager = ProcessManager(port_range=(20000, 20099), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="bad_path", app_path="nonexistent_module:app", workers=1, ) await manager.register(config) # Should fail to start success = await manager.start_process("bad_path") assert success is False info = manager.get_process("bad_path") assert info.state == ProcessState.FAILED finally: await manager.stop() @pytest.mark.asyncio async def test_process_not_found(self): """Operations on non-existent process should return False.""" manager = ProcessManager(port_range=(20100, 20199)) await manager.start() try: # Start non-existent process result = await manager.start_process("nonexistent") assert result is False # Stop non-existent process result = await manager.stop_process("nonexistent") assert result is False # Restart non-existent process result = await manager.restart_process("nonexistent") assert result is False # Get non-existent process info = manager.get_process("nonexistent") assert info is None # Get upstream URL for non-existent process url = manager.get_upstream_url("nonexistent") assert url is None finally: await manager.stop() @pytest.mark.asyncio async def test_double_start(self, temp_app_dir): """Starting already running process should succeed (idempotent).""" manager = ProcessManager(port_range=(20200, 20299), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="double_start", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config) # First start success1 = await manager.start_process("double_start") assert success1 is True info = manager.get_process("double_start") pid1 = info.pid # Second start should be idempotent success2 = await manager.start_process("double_start") assert success2 is True # Same process, same PID assert info.pid == pid1 finally: await manager.stop() @pytest.mark.asyncio async def test_double_stop(self, temp_app_dir): """Stopping already stopped process should succeed (idempotent).""" manager = ProcessManager(port_range=(20300, 20399), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="double_stop", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config) await manager.start_process("double_stop") # First stop success1 = await manager.stop_process("double_stop") assert success1 is True info = manager.get_process("double_stop") assert info.state == ProcessState.STOPPED # Second stop should be idempotent success2 = await manager.stop_process("double_stop") assert success2 is True finally: await manager.stop() # ============== Health Check Edge Case Tests ============== class TestHealthCheckEdgeCases: """Tests for health check scenarios.""" @pytest.fixture def temp_app_dir(self): """Create temp directory with test apps.""" with tempfile.TemporaryDirectory() as tmpdir: # Write starlette app (no extra dependencies needed) app_file = Path(tmpdir) / "test_app.py" app_file.write_text(STARLETTE_APP_CODE) yield tmpdir @pytest.mark.asyncio async def test_health_check_missing_endpoint(self, temp_app_dir): """App without /health endpoint - process starts but health checks fail.""" # Create an app without /health endpoint with tempfile.TemporaryDirectory() as tmpdir: no_health_app = Path(tmpdir) / "no_health_app.py" no_health_app.write_text(''' from starlette.applications import Starlette from starlette.responses import JSONResponse from starlette.routing import Route async def root(request): return JSONResponse({"app": "no-health"}) app = Starlette(routes=[ Route("/", root), ]) ''') manager = ProcessManager(port_range=(20400, 20499), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="no_health", app_path="no_health_app:app", module_path=tmpdir, workers=1, health_check_path="/health", # This endpoint doesn't exist health_check_timeout=2.0, ) await manager.register(config) # Should fail because health check endpoint doesn't exist # _wait_for_ready expects < 500 status code success = await manager.start_process("no_health") # Process may start but health check will get 404 # Our implementation accepts < 500 so 404 is OK # (This tests current behavior - may want to change later) info = manager.get_process("no_health") assert info is not None finally: await manager.stop() @pytest.mark.asyncio async def test_health_check_returns_500(self, temp_app_dir): """App health endpoint returns 500 - should fail health check.""" with tempfile.TemporaryDirectory() as tmpdir: unhealthy_app = Path(tmpdir) / "unhealthy_app.py" unhealthy_app.write_text(''' from starlette.applications import Starlette from starlette.responses import JSONResponse from starlette.routing import Route async def root(request): return JSONResponse({"app": "unhealthy"}) async def health(request): return JSONResponse({"status": "unhealthy"}, status_code=500) app = Starlette(routes=[ Route("/", root), Route("/health", health), ]) ''') manager = ProcessManager(port_range=(20500, 20599), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="unhealthy", app_path="unhealthy_app:app", module_path=tmpdir, workers=1, health_check_path="/health", health_check_timeout=2.0, ) await manager.register(config) # Should fail to start because health returns 500 success = await manager.start_process("unhealthy") assert success is False info = manager.get_process("unhealthy") assert info.state == ProcessState.FAILED finally: await manager.stop() @pytest.mark.asyncio async def test_process_info_uptime_calculation(self, temp_app_dir): """Uptime should be calculated correctly.""" manager = ProcessManager(port_range=(20600, 20699), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="uptime_test", app_path="test_app:app", module_path=temp_app_dir, workers=1, health_check_path="/health", ) await manager.register(config) await manager.start_process("uptime_test") info = manager.get_process("uptime_test") # Wait a bit await asyncio.sleep(1.0) # Uptime should be > 1 second assert info.uptime >= 1.0 # to_dict should include uptime data = info.to_dict() assert data["uptime"] >= 1.0 finally: await manager.stop() @pytest.mark.asyncio async def test_env_variables_passed_to_process(self, temp_app_dir): """Environment variables should be passed to subprocess.""" with tempfile.TemporaryDirectory() as tmpdir: env_app = Path(tmpdir) / "env_app.py" env_app.write_text(''' import os from starlette.applications import Starlette from starlette.responses import JSONResponse from starlette.routing import Route async def root(request): return JSONResponse({ "MY_VAR": os.environ.get("MY_VAR", "not set"), "ANOTHER_VAR": os.environ.get("ANOTHER_VAR", "not set"), }) async def health(request): return JSONResponse({"status": "healthy"}) app = Starlette(routes=[ Route("/", root), Route("/health", health), ]) ''') manager = ProcessManager(port_range=(20700, 20799), health_check_enabled=False) await manager.start() try: config = ProcessConfig( name="env_test", app_path="env_app:app", module_path=tmpdir, workers=1, health_check_path="/health", env={ "MY_VAR": "hello", "ANOTHER_VAR": "world", }, ) await manager.register(config) await manager.start_process("env_test") info = manager.get_process("env_test") # Check that env vars were passed async with httpx.AsyncClient() as client: resp = await client.get(f"http://127.0.0.1:{info.port}/") data = resp.json() assert data["MY_VAR"] == "hello" assert data["ANOTHER_VAR"] == "world" finally: await manager.stop()