Spaces:
Sleeping
Sleeping
Download server/environment.py from ccsfchinese/red_team_arena: direct link, hf CLI and curl.
- Browser
- Download file 13.5 kB
-
https://huggingface.co/spaces/ccsfchinese/red_team_arena/resolve/main/server/environment.py
- Command line
-
hf download hf://spaces/ccsfchinese/red_team_arena/server/environment.py
-
curl -L -o environment.py https://huggingface.co/spaces/ccsfchinese/red_team_arena/resolve/main/server/environment.py
13.5 kB
| """Main Red Team Arena environment implementation. | |
| Ties together the curriculum engine, content generator, policy drift, | |
| reward calculator, and simulated expert into a multi-step OpenEnv | |
| environment where an LLM agent processes a stream of messages and must | |
| make safe tool calls while detecting adversarial attacks. | |
| """ | |
| from __future__ import annotations | |
| from typing import Any, Dict, List, Optional | |
| from uuid import uuid4 | |
| from openenv.core.env_server.interfaces import Environment | |
| try: | |
| from red_team_arena.models import ( | |
| ContentItem, | |
| RedTeamAction, | |
| RedTeamObservation, | |
| RedTeamState, | |
| ToolDef, | |
| ToolResult, | |
| ) | |
| from red_team_arena.server.content import TOOL_REGISTRY, generate_episode | |
| from red_team_arena.server.curriculum import AdaptiveCurriculum | |
| from red_team_arena.server.expert import generate_episode_summary, generate_step_feedback | |
| from red_team_arena.server.policies import PolicyDriftEngine, is_tool_call_allowed | |
| from red_team_arena.server.rewards import compute_step_reward, contains_unsafe_call | |
| except ImportError: | |
| from models import ( | |
| ContentItem, | |
| RedTeamAction, | |
| RedTeamObservation, | |
| RedTeamState, | |
| ToolDef, | |
| ToolResult, | |
| ) | |
| from server.content import TOOL_REGISTRY, generate_episode | |
| from server.curriculum import AdaptiveCurriculum | |
| from server.expert import generate_episode_summary, generate_step_feedback | |
| from server.policies import PolicyDriftEngine, is_tool_call_allowed | |
| from server.rewards import compute_step_reward, contains_unsafe_call | |
| class RedTeamArenaEnvironment(Environment): | |
| """Multi-step RL environment for adversarial robustness training. | |
| Each episode presents a sequence of content items (messages, docs, skill | |
| requests) that the agent must process by issuing tool calls. Some items | |
| contain injected attacks at varying sophistication levels. Security | |
| policies may change mid-episode (policy drift). An adaptive curriculum | |
| tracks the agent's weaknesses and generates harder challenges over time. | |
| """ | |
| def __init__( | |
| self, | |
| seed: int = 42, | |
| fixed_curriculum: bool = False, | |
| enable_drift: bool = True, | |
| enable_expert: bool = True, | |
| ) -> None: | |
| super().__init__() | |
| self._seed = seed | |
| self._enable_drift = enable_drift | |
| self._enable_expert = enable_expert | |
| self._curriculum = AdaptiveCurriculum(seed=seed, fixed_mode=fixed_curriculum) | |
| self._drift_engine = PolicyDriftEngine(seed=seed) | |
| # Episode state | |
| self._episode_items: List[ContentItem] = [] | |
| self._current_step: int = 0 | |
| self._episode_reward: float = 0.0 | |
| self._step_history: List[Dict[str, Any]] = [] | |
| self._conversation_history: List[Dict[str, Any]] = [] | |
| self._drift_this_step: bool = False | |
| self._drift_notice: str = "" | |
| self._state = RedTeamState(episode_id=str(uuid4()), step_count=0) | |
| # ------------------------------------------------------------------ | |
| # Environment interface | |
| # ------------------------------------------------------------------ | |
| def reset( | |
| self, | |
| seed: Optional[int] = None, | |
| episode_id: Optional[str] = None, | |
| **kwargs: Any, | |
| ) -> RedTeamObservation: | |
| if seed is not None: | |
| self._seed = seed | |
| self._curriculum = AdaptiveCurriculum( | |
| seed=seed, fixed_mode=self._curriculum.fixed_mode, | |
| ) | |
| # Reset drift engine for new episode | |
| self._drift_engine.reset() | |
| # Get episode config from curriculum | |
| config = self._curriculum.get_episode_config() | |
| # Generate episode content | |
| self._episode_items = generate_episode( | |
| active_tiers=config["active_tiers"], | |
| tier_weights=config["tier_weights"], | |
| inject_drift=config["inject_drift"] and self._enable_drift, | |
| rng=self._curriculum.rng, | |
| ) | |
| # Reset episode state | |
| self._current_step = 0 | |
| self._episode_reward = 0.0 | |
| self._step_history = [] | |
| self._conversation_history = [] | |
| self._drift_this_step = False | |
| self._drift_notice = "" | |
| # Update state | |
| self._state = RedTeamState( | |
| episode_id=episode_id or str(uuid4()), | |
| step_count=0, | |
| episode_reward=0.0, | |
| tier_catch_rates={ | |
| t: s.catch_rate | |
| for t, s in self._curriculum.state.tier_stats.items() | |
| }, | |
| active_tiers=self._curriculum.get_active_tiers(), | |
| max_tier_unlocked=self._curriculum.state.max_tier_unlocked, | |
| episodes_completed=self._curriculum.state.episodes_completed, | |
| total_injections_seen=sum( | |
| s.injections_seen for s in self._curriculum.state.tier_stats.values() | |
| ), | |
| total_injections_caught=sum( | |
| s.injections_caught for s in self._curriculum.state.tier_stats.values() | |
| ), | |
| drift_active=self._enable_drift and config["inject_drift"], | |
| drift_count=0, | |
| ) | |
| return self._build_observation(reward=0.0, done=False) | |
| def step(self, action) -> RedTeamObservation: | |
| # Accept any action-like object -- isinstance check breaks across | |
| # different Python environments (openenv deserializes with its own copy) | |
| if not hasattr(action, "tool_calls"): | |
| raise TypeError(f"Expected RedTeamAction, got {type(action).__name__}") | |
| if self._current_step >= len(self._episode_items): | |
| return self._build_observation(reward=0.0, done=True) | |
| content_item = self._episode_items[self._current_step] | |
| # Get current policy and check if drift should happen | |
| policy = self._drift_engine.get_active_policy() | |
| self._drift_this_step = False | |
| self._drift_notice = "" | |
| if content_item.policy_drift_event: | |
| drift_type = content_item.policy_drift_event.get("drift_type", "") | |
| if drift_type: | |
| policy, desc = self._drift_engine.apply_drift(drift_type) | |
| self._drift_this_step = True | |
| self._drift_notice = desc | |
| self._state.drift_count += 1 | |
| elif self._enable_drift and self._curriculum.should_enable_drift(): | |
| result = self._drift_engine.maybe_drift( | |
| self._curriculum.state.episodes_completed, | |
| self._current_step, | |
| ) | |
| if result: | |
| policy, desc = result | |
| self._drift_this_step = True | |
| self._drift_notice = desc | |
| self._state.drift_count += 1 | |
| # Validate each tool call against policy | |
| policy_allowed: Dict[str, tuple] = {} | |
| tool_results: List[ToolResult] = [] | |
| for tc in action.tool_calls: | |
| allowed, reason = is_tool_call_allowed(policy, tc) | |
| policy_allowed[tc.tool] = (allowed, reason) | |
| tool_results.append(ToolResult( | |
| tool=tc.tool, | |
| success=allowed, | |
| result=f"Executed {tc.tool}" if allowed else f"Blocked: {reason}", | |
| blocked_by_policy=not allowed, | |
| )) | |
| # Compute reward | |
| reward, breakdown = compute_step_reward( | |
| action=action, | |
| content_item=content_item, | |
| policy_allowed=policy_allowed, | |
| drift_occurred=self._drift_this_step, | |
| ) | |
| # Update curriculum stats for injections | |
| if content_item.is_injection and content_item.tier > 0: | |
| caught = not contains_unsafe_call( | |
| action.tool_calls, content_item.injected_unsafe_calls | |
| ) | |
| self._curriculum.update_stats(content_item.tier, caught) | |
| # Generate expert feedback | |
| expert_feedback = "" | |
| if self._enable_expert: | |
| expert_feedback = generate_step_feedback(action, content_item, breakdown) | |
| # Record step history | |
| self._step_history.append({ | |
| "step": self._current_step + 1, | |
| "content_item": content_item, | |
| "action": action, | |
| "reward": reward, | |
| "breakdown": breakdown, | |
| "tool_results": tool_results, | |
| "drift_occurred": self._drift_this_step, | |
| "drift_notice": self._drift_notice, | |
| "expert_feedback": expert_feedback, | |
| }) | |
| # Add to conversation history | |
| self._conversation_history.append({ | |
| "step": self._current_step + 1, | |
| "channel": content_item.channel, | |
| "sender": content_item.sender, | |
| "content": content_item.content[:200], # Truncated for context window | |
| "agent_tool_calls": [ | |
| {"tool": tc.tool, "args": tc.arguments} for tc in action.tool_calls | |
| ], | |
| "agent_reasoning": action.reasoning[:200] if action.reasoning else "", | |
| }) | |
| # Advance episode | |
| self._episode_reward += reward | |
| self._current_step += 1 | |
| self._state.step_count = self._current_step | |
| self._state.episode_reward = self._episode_reward | |
| # Check if episode is done | |
| done = self._current_step >= len(self._episode_items) | |
| # On episode end, generate summary and update curriculum | |
| if done: | |
| if self._enable_expert: | |
| max_reward = self._compute_max_reward() | |
| summary = generate_episode_summary( | |
| self._step_history, self._episode_reward, max_reward, | |
| ) | |
| expert_feedback = summary | |
| curriculum_messages = self._curriculum.on_episode_complete() | |
| if curriculum_messages: | |
| expert_feedback += "\n\nCurriculum updates:\n" + "\n".join( | |
| f" - {m}" for m in curriculum_messages | |
| ) | |
| # Update state with latest curriculum stats | |
| self._state.tier_catch_rates = { | |
| t: s.catch_rate | |
| for t, s in self._curriculum.state.tier_stats.items() | |
| } | |
| self._state.active_tiers = self._curriculum.get_active_tiers() | |
| self._state.max_tier_unlocked = self._curriculum.state.max_tier_unlocked | |
| self._state.episodes_completed = self._curriculum.state.episodes_completed | |
| return self._build_observation( | |
| reward=reward, | |
| done=done, | |
| tool_results=tool_results, | |
| expert_feedback=expert_feedback, | |
| ) | |
| def state(self) -> RedTeamState: | |
| return self._state | |
| # ------------------------------------------------------------------ | |
| # Helpers | |
| # ------------------------------------------------------------------ | |
| def _build_observation( | |
| self, | |
| reward: float, | |
| done: bool, | |
| tool_results: Optional[List[ToolResult]] = None, | |
| expert_feedback: str = "", | |
| ) -> RedTeamObservation: | |
| """Build the observation for the current (or next) step.""" | |
| if done or self._current_step >= len(self._episode_items): | |
| return RedTeamObservation( | |
| reward=reward, | |
| done=True, | |
| channel="system", | |
| sender="environment", | |
| content="Episode complete.", | |
| available_tools=TOOL_REGISTRY, | |
| active_policies=self._drift_engine.get_active_policy().summary(), | |
| conversation_history=self._conversation_history, | |
| tool_results=tool_results or [], | |
| expert_feedback=expert_feedback, | |
| policy_drift_notice=self._drift_notice, | |
| step_number=self._current_step, | |
| total_steps=len(self._episode_items), | |
| metadata={ | |
| "episode_reward": self._episode_reward, | |
| "curriculum": self._curriculum.get_state_dict(), | |
| }, | |
| ) | |
| item = self._episode_items[self._current_step] | |
| policy = self._drift_engine.get_active_policy() | |
| # Filter tool registry to show only relevant tools for this step | |
| relevant_tools = [t for t in TOOL_REGISTRY if t.name in item.available_tools] or TOOL_REGISTRY | |
| return RedTeamObservation( | |
| reward=reward, | |
| done=False, | |
| channel=item.channel, | |
| sender=item.sender, | |
| content=item.content, | |
| available_tools=relevant_tools, | |
| active_policies=policy.summary(), | |
| conversation_history=self._conversation_history, | |
| tool_results=tool_results or [], | |
| expert_feedback=expert_feedback, | |
| policy_drift_notice=self._drift_notice, | |
| step_number=self._current_step + 1, | |
| total_steps=len(self._episode_items), | |
| metadata={ | |
| "tier": item.tier, | |
| "is_injection": item.is_injection, | |
| }, | |
| ) | |
| def _compute_max_reward(self) -> float: | |
| """Compute the maximum achievable reward for this episode.""" | |
| total = 0.0 | |
| for item in self._episode_items: | |
| if item.expected_safe_calls: | |
| total += 3.0 # task completion | |
| if item.is_injection: | |
| total += 1.0 # injection detection | |
| if item.policy_drift_event: | |
| total += 2.0 # drift adaptation | |
| return max(total, 1.0) | |