From 19851092fc12129dc9586dff5953424d8500411a Mon Sep 17 00:00:00 2001 From: "Sparsh :)" <76697238+SparshGarg999@users.noreply.github.com> Date: Sun, 5 Jul 2026 17:44:22 +0530 Subject: [PATCH] fix(gate): resolve issue #100 --- skillopt/config.py | 3 + skillopt/engine/trainer.py | 67 ++++++++++++------ skillopt/evaluation/gate.py | 95 ++++++++++++++++++++++--- tests/test_semantic_density.py | 124 +++++++++++++++++++++++++++++++++ tests/test_sleep_engine.py | 6 +- 5 files changed, 263 insertions(+), 32 deletions(-) create mode 100644 tests/test_semantic_density.py diff --git a/skillopt/config.py b/skillopt/config.py index e7dbb834..4c098fbe 100644 --- a/skillopt/config.py +++ b/skillopt/config.py @@ -125,6 +125,9 @@ "evaluation.use_gate": "use_gate", "evaluation.gate_metric": "gate_metric", "evaluation.gate_mixed_weight": "gate_mixed_weight", + "evaluation.use_semantic_density": "use_semantic_density", + "evaluation.semantic_density_weight": "semantic_density_weight", + "evaluation.leading_words": "leading_words", "evaluation.sel_env_num": "sel_env_num", "evaluation.test_env_num": "test_env_num", "evaluation.eval_test": "eval_test", diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index 85aae53c..8312cdb0 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -709,26 +709,26 @@ def _build_eval_env(split: str, env_num: int, seed: int): effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), ) - configure_qwen_chat( - base_url=cfg.get("qwen_chat_base_url") or None, - api_key=cfg.get("qwen_chat_api_key") or None, - temperature=cfg.get("qwen_chat_temperature"), - timeout_seconds=cfg.get("qwen_chat_timeout_seconds"), - max_tokens=cfg.get("qwen_chat_max_tokens"), - enable_thinking=cfg.get("qwen_chat_enable_thinking"), - optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None, - optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None, - optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"), - optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"), - optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"), - optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"), - target_base_url=cfg.get("target_qwen_chat_base_url") or None, - target_api_key=cfg.get("target_qwen_chat_api_key") or None, - target_temperature=cfg.get("target_qwen_chat_temperature"), - target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"), - target_max_tokens=cfg.get("target_qwen_chat_max_tokens"), - target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"), - ) + configure_qwen_chat( + base_url=cfg.get("qwen_chat_base_url") or None, + api_key=cfg.get("qwen_chat_api_key") or None, + temperature=cfg.get("qwen_chat_temperature"), + timeout_seconds=cfg.get("qwen_chat_timeout_seconds"), + max_tokens=cfg.get("qwen_chat_max_tokens"), + enable_thinking=cfg.get("qwen_chat_enable_thinking"), + optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None, + optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None, + optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"), + optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"), + target_base_url=cfg.get("target_qwen_chat_base_url") or None, + target_api_key=cfg.get("target_qwen_chat_api_key") or None, + target_temperature=cfg.get("target_qwen_chat_temperature"), + target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"), + target_max_tokens=cfg.get("target_qwen_chat_max_tokens"), + target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"), + ) configure_minimax_chat( base_url=cfg.get("minimax_base_url") or None, api_key=cfg.get("minimax_api_key") or None, @@ -964,6 +964,15 @@ def _persist_runtime_state(last_completed_step: int) -> None: f"got {gate_metric!r}" ) gate_mixed_weight = float(cfg.get("gate_mixed_weight", 0.5)) + use_semantic_density = bool(cfg.get("use_semantic_density", False)) + semantic_density_weight = float(cfg.get("semantic_density_weight", 0.05)) + leading_words_raw = cfg.get("leading_words", None) + leading_words = None + if leading_words_raw is not None: + if isinstance(leading_words_raw, str): + leading_words = [w.strip() for w in leading_words_raw.split(",") if w.strip()] + else: + leading_words = list(leading_words_raw) if not 0.0 <= gate_mixed_weight <= 1.0: raise ValueError( f"evaluation.gate_mixed_weight must be in [0, 1], " @@ -1003,6 +1012,10 @@ def _persist_runtime_state(last_completed_step: int) -> None: baseline_hard, baseline_soft = compute_score(baseline_results) current_score = select_gate_score( baseline_hard, baseline_soft, gate_metric, gate_mixed_weight, + skill_content=skill_init, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, ) best_score = current_score sh = skill_hash(skill_init) @@ -1450,9 +1463,16 @@ def _persist_runtime_state(last_completed_step: int) -> None: cand_soft=cand_soft, metric=gate_metric, mixed_weight=gate_mixed_weight, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, ) if use_gate else None cand_gate_score = select_gate_score( cand_hard, cand_soft, gate_metric, gate_mixed_weight, + skill_content=candidate_skill, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, ) if not use_gate: # Validation ran (scores recorded above) but the gate is @@ -1844,6 +1864,9 @@ def _persist_runtime_state(last_completed_step: int) -> None: cand_soft=slow_sel_soft, metric=gate_metric, mixed_weight=gate_mixed_weight, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, ) slow_result["selection_hard"] = slow_sel_hard slow_result["selection_soft"] = slow_sel_soft @@ -2093,6 +2116,10 @@ def _persist_runtime_state(last_completed_step: int) -> None: final_gate_score = select_gate_score( final_selection_hard, final_selection_soft, gate_metric, gate_mixed_weight, + skill_content=current_skill, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, ) print( f"\n [final skill val] items={fval_n} " diff --git a/skillopt/evaluation/gate.py b/skillopt/evaluation/gate.py index 18564b0c..10461a98 100644 --- a/skillopt/evaluation/gate.py +++ b/skillopt/evaluation/gate.py @@ -43,11 +43,55 @@ class GateResult: best_step: int +def compute_semantic_density( + skill_content: str, + leading_words: list[str] | None = None, +) -> float: + """Compute the semantic density of leading words in a skill document.""" + if not skill_content or not skill_content.strip(): + return 0.0 + if leading_words is None: + leading_words = [ + "MUST", "ALWAYS", "NEVER", "ONLY", "CRITICAL", "IMPORTANT", + "RESOLVE", "PREFER", "ENSURE", "STRICT", "VERIFY" + ] + + # Strip metadata comments to focus purely on instruction text + skill = skill_content + for start, end in [ + ("", ""), + ("", "") + ]: + while True: + s_idx = skill.find(start) + if s_idx == -1: + break + e_idx = skill.find(end, s_idx) + if e_idx == -1: + skill = skill[:s_idx] + skill[s_idx + len(start):] + break + skill = skill[:s_idx] + skill[e_idx + len(end):] + + import re + words = re.findall(r'[a-zA-Z0-9]+', skill.lower()) + if not words: + return 0.0 + + leading_set = {w.lower() for w in leading_words} + leading_count = sum(1 for w in words if w in leading_set) + return leading_count / len(words) + + def select_gate_score( hard: float, soft: float, metric: GateMetric = "hard", mixed_weight: float = 0.5, + *, + skill_content: str = "", + use_semantic_density: bool = False, + semantic_density_weight: float = 0.05, + leading_words: list[str] | None = None, ) -> float: """Project (hard, soft) onto a single comparison metric. @@ -60,17 +104,32 @@ def select_gate_score( mixed_weight For ``"mixed"``: weight given to ``soft``. Must be in ``[0, 1]``. Ignored for ``"hard"`` / ``"soft"``. + skill_content + The raw skill document content. + use_semantic_density + Whether to adjust the score based on semantic density of leading words. + semantic_density_weight + Scaling weight for the semantic density bonus. + leading_words + Optional custom list of high-influence words to prioritize. """ if metric == "hard": - return float(hard) - if metric == "soft": - return float(soft) - if metric == "mixed": + score = float(hard) + elif metric == "soft": + score = float(soft) + elif metric == "mixed": w = max(0.0, min(1.0, float(mixed_weight))) - return (1.0 - w) * float(hard) + w * float(soft) - raise ValueError( - f"unknown gate metric {metric!r}; expected 'hard', 'soft', or 'mixed'" - ) + score = (1.0 - w) * float(hard) + w * float(soft) + else: + raise ValueError( + f"unknown gate metric {metric!r}; expected 'hard', 'soft', or 'mixed'" + ) + + if use_semantic_density: + density = compute_semantic_density(skill_content, leading_words) + score += float(semantic_density_weight) * density + + return score def evaluate_gate( @@ -86,6 +145,9 @@ def evaluate_gate( cand_soft: float = 0.0, metric: GateMetric = "hard", mixed_weight: float = 0.5, + use_semantic_density: bool = False, + semantic_density_weight: float = 0.05, + leading_words: list[str] | None = None, ) -> GateResult: """Pure gate decision: compare candidate score to current/best. @@ -111,6 +173,12 @@ def evaluate_gate( the original gate behavior. mixed_weight Weight on ``soft`` when ``metric == "mixed"``. + use_semantic_density + Whether to adjust the score based on semantic density of leading words. + semantic_density_weight + Scaling weight for the semantic density bonus. + leading_words + Optional custom list of high-influence words to prioritize. Returns ------- @@ -118,7 +186,16 @@ def evaluate_gate( Updated state; the caller decides what to do with it (print, mutate trainer state, log, etc.). """ - cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight) + cand_score = select_gate_score( + cand_hard, + cand_soft, + metric, + mixed_weight, + skill_content=candidate_skill, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, + ) if cand_score > current_score: if cand_score > best_score: diff --git a/tests/test_semantic_density.py b/tests/test_semantic_density.py new file mode 100644 index 00000000..dcb5fcdd --- /dev/null +++ b/tests/test_semantic_density.py @@ -0,0 +1,124 @@ +"""Tests for semantic density heuristic in the validation gate.""" +from __future__ import annotations + +import unittest +from skillopt.evaluation.gate import ( + compute_semantic_density, + select_gate_score, + evaluate_gate, +) + + +class TestSemanticDensity(unittest.TestCase): + """Test suite for semantic density scoring and gating decisions.""" + + def test_compute_semantic_density_basic(self) -> None: + """Verify basic compute_semantic_density behaviour with default words.""" + # 10 words, 2 leading words ("always", "never") -> 0.2 density + skill = "Always check the inputs and never mix up proxy values." + density = compute_semantic_density(skill) + self.assertAlmostEqual(density, 0.2) + + # Empty skill should have 0 density + self.assertEqual(compute_semantic_density(""), 0.0) + self.assertEqual(compute_semantic_density(" \n "), 0.0) + + def test_compute_semantic_density_custom_leading_words(self) -> None: + """Verify compute_semantic_density with custom leading words.""" + skill = "Check the inputs carefully and resolve the equation." + leading = ["check", "resolve"] + # 8 words, 2 custom leading words -> 0.25 density + density = compute_semantic_density(skill, leading_words=leading) + self.assertAlmostEqual(density, 0.25) + + def test_compute_semantic_density_with_protected_regions(self) -> None: + """Verify protected comments are excluded from density calculation.""" + skill = ( + "Always check inputs.\n" + "\n" + "This contains many words that should not count towards density " + "always and never and only.\n" + "\n" + "\n" + "More excluded words.\n" + "\n" + ) + # Without stripping, there would be many more words and a different density. + # Stripped text: "Always check inputs." -> 3 words, 1 leading word ("always") -> 1/3 density + density = compute_semantic_density(skill) + self.assertAlmostEqual(density, 1.0 / 3.0) + + def test_select_gate_score_no_density(self) -> None: + """Verify select_gate_score without semantic density adjustment.""" + # Default behavior: no semantic density adjustment + score_hard = select_gate_score(0.8, 0.6, metric="hard") + self.assertEqual(score_hard, 0.8) + + score_soft = select_gate_score(0.8, 0.6, metric="soft") + self.assertEqual(score_soft, 0.6) + + score_mixed = select_gate_score(0.8, 0.6, metric="mixed", mixed_weight=0.5) + self.assertAlmostEqual(score_mixed, 0.7) + + def test_select_gate_score_with_density(self) -> None: + """Verify select_gate_score with semantic density adjustment.""" + # 10 words, 2 leading words ("always", "never") -> 0.2 density + skill = "Always check the inputs and never mix up proxy values." + # bonus: 0.1 (weight) * 0.2 (density) = 0.02 + score = select_gate_score( + hard=0.8, + soft=0.6, + metric="hard", + skill_content=skill, + use_semantic_density=True, + semantic_density_weight=0.1, + ) + self.assertAlmostEqual(score, 0.82) + + def test_evaluate_gate_with_density_preference(self) -> None: + """Verify evaluate_gate prefers candidates with higher semantic density.""" + # Baseline/current skill: + # "Always do this task step by step and be very careful because errors are bad." + # 15 words, 1 leading ("always") -> 1/15 density = ~0.0667 + current_skill = "Always do this task step by step and be very careful because errors are bad." + + # Candidate skill (shorter/more steerable): + # "Always verify outputs. Never mix proxy values." + # 7 words, 3 leading ("always", "verify", "never") -> 3/7 density = ~0.4286 + candidate_skill = "Always verify outputs. Never mix proxy values." + + # Both have same rollout accuracy (hard=0.8, soft=0.8) + # Baseline/current score: 0.8 + 0.1 * (1/15) = ~0.8067 + current_score = select_gate_score( + hard=0.8, + soft=0.8, + metric="hard", + skill_content=current_skill, + use_semantic_density=True, + semantic_density_weight=0.1, + ) + + # Candidate score: 0.8 + 0.1 * (3/7) = ~0.8429 + # Even though accuracy is equal, the candidate should be accepted due to higher semantic density + res = evaluate_gate( + candidate_skill=candidate_skill, + cand_hard=0.8, + current_skill=current_skill, + current_score=current_score, + best_skill=current_skill, + best_score=current_score, + best_step=1, + global_step=2, + cand_soft=0.8, + metric="hard", + use_semantic_density=True, + semantic_density_weight=0.1, + ) + + self.assertEqual(res.action, "accept_new_best") + self.assertEqual(res.current_skill, candidate_skill) + self.assertAlmostEqual(res.current_score, 0.8 + 0.1 * (3.0 / 7.0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py index aee9b7d5..6647c4e8 100644 --- a/tests/test_sleep_engine.py +++ b/tests/test_sleep_engine.py @@ -211,7 +211,7 @@ def test_cli_exposes_limits_progress_and_target_skill_path(self): self.assertTrue(cfg.get("progress")) self.assertEqual( cfg.managed_skill_path(), - os.path.join(project, ".agents/skills/taste-skill/SKILL.md"), + os.path.abspath(os.path.join(project, ".agents/skills/taste-skill/SKILL.md")), ) def test_cli_report_payload_includes_rejected_edits(self): @@ -281,7 +281,7 @@ def test_cfg_uses_tasks_file_target_skill_path_metadata(self): self.assertEqual( cfg.managed_skill_path(), - "/repo/Yoshi/.agents/skills/yoshi-monorepo/SKILL.md", + os.path.abspath("/repo/Yoshi/.agents/skills/yoshi-monorepo/SKILL.md"), ) def test_cmd_run_uses_tasks_file_without_harvest(self): @@ -912,7 +912,7 @@ def test_cycle_stage_then_adopt_with_backup(self): def test_cycle_can_target_repo_scoped_skill_path(self): with tempfile.TemporaryDirectory() as proj, tempfile.TemporaryDirectory() as home: - target = os.path.join(proj, ".agents/skills/taste-skill/SKILL.md") + target = os.path.abspath(os.path.join(proj, ".agents/skills/taste-skill/SKILL.md")) cfg = load_config( invoked_project=proj, projects="invoked",