Krishna1107 Claude Opus 4.7 commited on
Commit
576dfc3
·
1 Parent(s): d681074

Drop GRPO temp to 0.3, bump max_new_tokens to 2048, add inference smoke test

Browse files
scripts/run_hf_job_7b.sh CHANGED
@@ -165,6 +165,21 @@ print("Baselines downloaded from HF Hub")
165
  PY
166
  fi
167
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
  # ==============================================================================
169
  # Phase 3: GRPO training (~4h)
170
  # ==============================================================================
@@ -173,7 +188,7 @@ python training/train_grpo.py \
173
  --steps 80 \
174
  --rollouts-per-step 3 \
175
  --base-model Qwen/Qwen2.5-Coder-7B-Instruct \
176
- --max-new-tokens 1024 \
177
  --learning-rate 5e-6 \
178
  --seed 42 \
179
  --no-unsloth \
 
165
  PY
166
  fi
167
 
168
+ # ==============================================================================
169
+ # Phase 2.5: GRPO inference smoke test (~5 min on GPU)
170
+ # Catches malformed-pytest / truncation issues with the new sampling config
171
+ # BEFORE we burn ~4h on GRPO training.
172
+ # ==============================================================================
173
+ echo "=== Phase 2.5: GRPO inference smoke test ==="
174
+ python training/smoke_grpo_inference.py \
175
+ --model Qwen/Qwen2.5-Coder-7B-Instruct \
176
+ --n-completions 5 \
177
+ --episode-seed 42 \
178
+ --temperature 0.3 \
179
+ --top-p 0.9 \
180
+ --max-new-tokens 2048 \
181
+ --device auto
182
+
183
  # ==============================================================================
184
  # Phase 3: GRPO training (~4h)
185
  # ==============================================================================
 
188
  --steps 80 \
189
  --rollouts-per-step 3 \
190
  --base-model Qwen/Qwen2.5-Coder-7B-Instruct \
191
+ --max-new-tokens 2048 \
192
  --learning-rate 5e-6 \
193
  --seed 42 \
194
  --no-unsloth \
training/smoke_grpo_inference.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pre-flight smoke test for GRPO rollouts.
2
+
3
+ Runs the SAME (model, prompt, sampling-config) combo GRPOTrainer would use
4
+ during rollouts, generates a small batch of completions, and reports how
5
+ many pass ``validate_test_code_structure``. The goal is to catch
6
+ malformed-pytest / truncation issues BEFORE paying for a full GRPO run.
7
+
8
+ Usage:
9
+
10
+ # Default: Qwen 2.5-Coder 7B, 5 completions, CPU-friendly fallback
11
+ python training/smoke_grpo_inference.py
12
+
13
+ # On a GPU box:
14
+ python training/smoke_grpo_inference.py --device cuda
15
+
16
+ Pass criterion (printed only — no exit code gate):
17
+ >= 4 / 5 completions pass validate_test_code_structure
18
+
19
+ If you see 1/5 pass, the GRPO sampling config will not give you positive
20
+ reward signal at scale — fix the prompt, the model, or the sampling knobs
21
+ before launching Phase 3.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import argparse
27
+ import sys
28
+ import time
29
+ import traceback
30
+ from pathlib import Path
31
+
32
+ ROOT = Path(__file__).resolve().parents[1]
33
+ SRC = ROOT / "src"
34
+ if str(SRC) not in sys.path:
35
+ sys.path.insert(0, str(SRC))
36
+ if str(ROOT) not in sys.path:
37
+ sys.path.insert(0, str(ROOT))
38
+
39
+ from evaluation.zero_shot_distribution import validate_test_code_structure # noqa: E402
40
+ from mutant_hunter.server.mutant_hunter_environment import MutantHunterEnvironment # noqa: E402
41
+ from training.train_grpo import _strip_markdown_fences, build_prompt # noqa: E402
42
+
43
+
44
+ def _generate_one(model, tokenizer, prompt: str, device: str,
45
+ max_new_tokens: int, temperature: float, top_p: float,
46
+ seed: int) -> str:
47
+ import torch
48
+
49
+ messages = [{"role": "user", "content": prompt}]
50
+ try:
51
+ templated = tokenizer.apply_chat_template(
52
+ messages, tokenize=False, add_generation_prompt=True
53
+ )
54
+ except Exception:
55
+ templated = prompt
56
+ inputs = tokenizer(
57
+ templated, return_tensors="pt", truncation=True, max_length=4096
58
+ )
59
+ inputs = {k: v.to(device) for k, v in inputs.items()}
60
+
61
+ gen = torch.Generator(device=device)
62
+ gen.manual_seed(seed)
63
+ with torch.no_grad():
64
+ out = model.generate(
65
+ **inputs,
66
+ max_new_tokens=max_new_tokens,
67
+ do_sample=True,
68
+ temperature=temperature,
69
+ top_p=top_p,
70
+ top_k=50,
71
+ pad_token_id=tokenizer.pad_token_id,
72
+ )
73
+ return tokenizer.decode(
74
+ out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True
75
+ )
76
+
77
+
78
+ def main() -> int:
79
+ ap = argparse.ArgumentParser(
80
+ description="Pre-flight smoke test: generate N completions with the "
81
+ "GRPO rollout sampling config and report validation pass rate."
82
+ )
83
+ ap.add_argument(
84
+ "--model",
85
+ type=str,
86
+ default="Qwen/Qwen2.5-Coder-7B-Instruct",
87
+ help="HF model id. Same as Phase 3 base model by default.",
88
+ )
89
+ ap.add_argument(
90
+ "--n-completions",
91
+ type=int,
92
+ default=5,
93
+ help="Number of completions to sample for the same prompt.",
94
+ )
95
+ ap.add_argument(
96
+ "--episode-seed",
97
+ type=int,
98
+ default=42,
99
+ help="Seed passed to env.reset() to pick the (repo, module) episode "
100
+ "the prompt is built from.",
101
+ )
102
+ ap.add_argument("--max-new-tokens", type=int, default=2048)
103
+ ap.add_argument("--temperature", type=float, default=0.3)
104
+ ap.add_argument("--top-p", type=float, default=0.9)
105
+ ap.add_argument(
106
+ "--device",
107
+ type=str,
108
+ default="auto",
109
+ help="'auto' | 'cuda' | 'cpu'",
110
+ )
111
+ args = ap.parse_args()
112
+
113
+ try:
114
+ import torch
115
+ from transformers import AutoModelForCausalLM, AutoTokenizer
116
+ except ImportError as exc:
117
+ print(f"[smoke] FAIL — transformers/torch not installed: {exc}")
118
+ return 1
119
+
120
+ if args.device == "auto":
121
+ device = "cuda" if torch.cuda.is_available() else "cpu"
122
+ else:
123
+ device = args.device
124
+ print(f"[smoke] device={device} model={args.model}", flush=True)
125
+ if device == "cpu":
126
+ print(
127
+ "[smoke] WARN — CPU inference of a 7B model with "
128
+ f"{args.max_new_tokens} new tokens will take many minutes per "
129
+ "completion. Consider --device cuda or a smaller --model.",
130
+ flush=True,
131
+ )
132
+
133
+ print("[smoke] building env + prompt ...", flush=True)
134
+ env = MutantHunterEnvironment()
135
+ obs = env.reset(seed=args.episode_seed)
136
+ prompt = build_prompt(obs)
137
+ print(
138
+ f"[smoke] episode seed={args.episode_seed} repo={obs.repo_name} "
139
+ f"module={obs.module_path} prompt_chars={len(prompt)}",
140
+ flush=True,
141
+ )
142
+
143
+ print("[smoke] loading model ...", flush=True)
144
+ t0 = time.time()
145
+ try:
146
+ tokenizer = AutoTokenizer.from_pretrained(args.model)
147
+ model = AutoModelForCausalLM.from_pretrained(args.model)
148
+ model = model.to(device)
149
+ model.eval()
150
+ except Exception as exc:
151
+ print(f"[smoke] FAIL — could not load model {args.model}: {exc}")
152
+ traceback.print_exc()
153
+ return 1
154
+ if tokenizer.pad_token is None:
155
+ tokenizer.pad_token = tokenizer.eos_token
156
+ print(f"[smoke] model loaded in {time.time() - t0:.1f}s", flush=True)
157
+
158
+ print(
159
+ f"[smoke] generating {args.n_completions} completions @ "
160
+ f"temp={args.temperature} top_p={args.top_p} "
161
+ f"max_new_tokens={args.max_new_tokens}",
162
+ flush=True,
163
+ )
164
+
165
+ n_valid = 0
166
+ sample_valid: tuple[int, str] | None = None
167
+ sample_invalid: tuple[int, str, str] | None = None
168
+ per_attempt: list[dict] = []
169
+
170
+ for i in range(args.n_completions):
171
+ t1 = time.time()
172
+ try:
173
+ full = _generate_one(
174
+ model, tokenizer, prompt, device,
175
+ max_new_tokens=args.max_new_tokens,
176
+ temperature=args.temperature,
177
+ top_p=args.top_p,
178
+ seed=args.episode_seed * 1000 + i,
179
+ )
180
+ except Exception as exc:
181
+ print(f" [{i}] generation_error: {type(exc).__name__}: {exc}",
182
+ flush=True)
183
+ per_attempt.append({"i": i, "valid": False,
184
+ "reason": f"generation_error: {exc}"})
185
+ continue
186
+ gen_s = time.time() - t1
187
+
188
+ test_code = _strip_markdown_fences(full)
189
+ err = validate_test_code_structure(test_code)
190
+ valid = err is None
191
+ if valid:
192
+ n_valid += 1
193
+ if sample_valid is None:
194
+ sample_valid = (i, test_code)
195
+ else:
196
+ if sample_invalid is None:
197
+ sample_invalid = (i, test_code, err or "unknown")
198
+ per_attempt.append({
199
+ "i": i,
200
+ "valid": valid,
201
+ "reason": err,
202
+ "raw_chars": len(full),
203
+ "stripped_chars": len(test_code),
204
+ "gen_s": round(gen_s, 1),
205
+ })
206
+ print(
207
+ f" [{i}] valid={valid} reason={err!r} raw={len(full)}c "
208
+ f"stripped={len(test_code)}c gen={gen_s:.1f}s",
209
+ flush=True,
210
+ )
211
+
212
+ print()
213
+ print(f"[smoke] RESULT: {n_valid}/{args.n_completions} completions "
214
+ f"pass validate_test_code_structure")
215
+ print()
216
+
217
+ if sample_valid is not None:
218
+ i, code = sample_valid
219
+ snippet = code[:1000]
220
+ ellipsis = "" if len(code) <= 1000 else "\n... (truncated)"
221
+ print(f"[smoke] sample VALID completion (i={i}):")
222
+ print("---8<---")
223
+ print(snippet + ellipsis)
224
+ print("---8<---")
225
+ else:
226
+ print("[smoke] no valid completion to show")
227
+
228
+ print()
229
+ if sample_invalid is not None:
230
+ i, code, reason = sample_invalid
231
+ snippet = code[:1000]
232
+ ellipsis = "" if len(code) <= 1000 else "\n... (truncated)"
233
+ print(f"[smoke] sample INVALID completion (i={i}, reason={reason!r}):")
234
+ print("---8<---")
235
+ print(snippet + ellipsis)
236
+ print("---8<---")
237
+ else:
238
+ print("[smoke] no invalid completion to show (all passed!)")
239
+
240
+ print()
241
+ pass_rate = n_valid / args.n_completions if args.n_completions else 0.0
242
+ if pass_rate >= 0.8:
243
+ print(f"[smoke] PASS — {pass_rate:.0%} valid. GRPO config looks good.")
244
+ return 0
245
+ print(
246
+ f"[smoke] WARN — only {pass_rate:.0%} valid. GRPO will likely give "
247
+ "near-zero reward signal at this rate. Re-tune sampling or prompt "
248
+ "before launching Phase 3."
249
+ )
250
+ return 0
251
+
252
+
253
+ if __name__ == "__main__":
254
+ raise SystemExit(main())
training/train_grpo.py CHANGED
@@ -380,11 +380,11 @@ def run(cfg: TrainingConfig) -> int:
380
  learning_rate=cfg.learning_rate,
381
  max_steps=cfg.steps,
382
  max_completion_length=cfg.max_new_tokens,
383
- # Rollout sampling: keep generations close to what the zero-shot eval
384
- # used. The TRL default (~1.0) produced too much malformed/truncated
385
- # pytest, pinning rewards at 0 even after the seed-routing fix.
386
- temperature=0.7,
387
- top_p=0.95,
388
  top_k=50,
389
  bf16=False,
390
  fp16=False,
@@ -422,7 +422,7 @@ def main() -> int:
422
  ap.add_argument("--steps", type=int, default=10)
423
  ap.add_argument("--rollouts-per-step", type=int, default=2)
424
  ap.add_argument("--learning-rate", type=float, default=5e-6)
425
- ap.add_argument("--max-new-tokens", type=int, default=512)
426
  ap.add_argument("--seed", type=int, default=0)
427
  ap.add_argument("--output-dir", type=str, default="./training/_runs/grpo")
428
  ap.add_argument("--wandb-project", type=str, default=None,
 
380
  learning_rate=cfg.learning_rate,
381
  max_steps=cfg.steps,
382
  max_completion_length=cfg.max_new_tokens,
383
+ # Rollout sampling: temp=0.7 still produced too much malformed pytest
384
+ # at 1024 max_new_tokens. Drop to 0.3 + bump tokens to 2048 so the
385
+ # model has both the determinism and the room to finish a coherent file.
386
+ temperature=0.3,
387
+ top_p=0.9,
388
  top_k=50,
389
  bf16=False,
390
  fp16=False,
 
422
  ap.add_argument("--steps", type=int, default=10)
423
  ap.add_argument("--rollouts-per-step", type=int, default=2)
424
  ap.add_argument("--learning-rate", type=float, default=5e-6)
425
+ ap.add_argument("--max-new-tokens", type=int, default=2048)
426
  ap.add_argument("--seed", type=int, default=0)
427
  ap.add_argument("--output-dir", type=str, default="./training/_runs/grpo")
428
  ap.add_argument("--wandb-project", type=str, default=None,