Spaces:
Sleeping
Sleeping
Commit ·
576dfc3
1
Parent(s): d681074
Drop GRPO temp to 0.3, bump max_new_tokens to 2048, add inference smoke test
Browse files- scripts/run_hf_job_7b.sh +16 -1
- training/smoke_grpo_inference.py +254 -0
- training/train_grpo.py +6 -6
scripts/run_hf_job_7b.sh
CHANGED
|
@@ -165,6 +165,21 @@ print("Baselines downloaded from HF Hub")
|
|
| 165 |
PY
|
| 166 |
fi
|
| 167 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
# ==============================================================================
|
| 169 |
# Phase 3: GRPO training (~4h)
|
| 170 |
# ==============================================================================
|
|
@@ -173,7 +188,7 @@ python training/train_grpo.py \
|
|
| 173 |
--steps 80 \
|
| 174 |
--rollouts-per-step 3 \
|
| 175 |
--base-model Qwen/Qwen2.5-Coder-7B-Instruct \
|
| 176 |
-
--max-new-tokens
|
| 177 |
--learning-rate 5e-6 \
|
| 178 |
--seed 42 \
|
| 179 |
--no-unsloth \
|
|
|
|
| 165 |
PY
|
| 166 |
fi
|
| 167 |
|
| 168 |
+
# ==============================================================================
|
| 169 |
+
# Phase 2.5: GRPO inference smoke test (~5 min on GPU)
|
| 170 |
+
# Catches malformed-pytest / truncation issues with the new sampling config
|
| 171 |
+
# BEFORE we burn ~4h on GRPO training.
|
| 172 |
+
# ==============================================================================
|
| 173 |
+
echo "=== Phase 2.5: GRPO inference smoke test ==="
|
| 174 |
+
python training/smoke_grpo_inference.py \
|
| 175 |
+
--model Qwen/Qwen2.5-Coder-7B-Instruct \
|
| 176 |
+
--n-completions 5 \
|
| 177 |
+
--episode-seed 42 \
|
| 178 |
+
--temperature 0.3 \
|
| 179 |
+
--top-p 0.9 \
|
| 180 |
+
--max-new-tokens 2048 \
|
| 181 |
+
--device auto
|
| 182 |
+
|
| 183 |
# ==============================================================================
|
| 184 |
# Phase 3: GRPO training (~4h)
|
| 185 |
# ==============================================================================
|
|
|
|
| 188 |
--steps 80 \
|
| 189 |
--rollouts-per-step 3 \
|
| 190 |
--base-model Qwen/Qwen2.5-Coder-7B-Instruct \
|
| 191 |
+
--max-new-tokens 2048 \
|
| 192 |
--learning-rate 5e-6 \
|
| 193 |
--seed 42 \
|
| 194 |
--no-unsloth \
|
training/smoke_grpo_inference.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pre-flight smoke test for GRPO rollouts.
|
| 2 |
+
|
| 3 |
+
Runs the SAME (model, prompt, sampling-config) combo GRPOTrainer would use
|
| 4 |
+
during rollouts, generates a small batch of completions, and reports how
|
| 5 |
+
many pass ``validate_test_code_structure``. The goal is to catch
|
| 6 |
+
malformed-pytest / truncation issues BEFORE paying for a full GRPO run.
|
| 7 |
+
|
| 8 |
+
Usage:
|
| 9 |
+
|
| 10 |
+
# Default: Qwen 2.5-Coder 7B, 5 completions, CPU-friendly fallback
|
| 11 |
+
python training/smoke_grpo_inference.py
|
| 12 |
+
|
| 13 |
+
# On a GPU box:
|
| 14 |
+
python training/smoke_grpo_inference.py --device cuda
|
| 15 |
+
|
| 16 |
+
Pass criterion (printed only — no exit code gate):
|
| 17 |
+
>= 4 / 5 completions pass validate_test_code_structure
|
| 18 |
+
|
| 19 |
+
If you see 1/5 pass, the GRPO sampling config will not give you positive
|
| 20 |
+
reward signal at scale — fix the prompt, the model, or the sampling knobs
|
| 21 |
+
before launching Phase 3.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import argparse
|
| 27 |
+
import sys
|
| 28 |
+
import time
|
| 29 |
+
import traceback
|
| 30 |
+
from pathlib import Path
|
| 31 |
+
|
| 32 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 33 |
+
SRC = ROOT / "src"
|
| 34 |
+
if str(SRC) not in sys.path:
|
| 35 |
+
sys.path.insert(0, str(SRC))
|
| 36 |
+
if str(ROOT) not in sys.path:
|
| 37 |
+
sys.path.insert(0, str(ROOT))
|
| 38 |
+
|
| 39 |
+
from evaluation.zero_shot_distribution import validate_test_code_structure # noqa: E402
|
| 40 |
+
from mutant_hunter.server.mutant_hunter_environment import MutantHunterEnvironment # noqa: E402
|
| 41 |
+
from training.train_grpo import _strip_markdown_fences, build_prompt # noqa: E402
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _generate_one(model, tokenizer, prompt: str, device: str,
|
| 45 |
+
max_new_tokens: int, temperature: float, top_p: float,
|
| 46 |
+
seed: int) -> str:
|
| 47 |
+
import torch
|
| 48 |
+
|
| 49 |
+
messages = [{"role": "user", "content": prompt}]
|
| 50 |
+
try:
|
| 51 |
+
templated = tokenizer.apply_chat_template(
|
| 52 |
+
messages, tokenize=False, add_generation_prompt=True
|
| 53 |
+
)
|
| 54 |
+
except Exception:
|
| 55 |
+
templated = prompt
|
| 56 |
+
inputs = tokenizer(
|
| 57 |
+
templated, return_tensors="pt", truncation=True, max_length=4096
|
| 58 |
+
)
|
| 59 |
+
inputs = {k: v.to(device) for k, v in inputs.items()}
|
| 60 |
+
|
| 61 |
+
gen = torch.Generator(device=device)
|
| 62 |
+
gen.manual_seed(seed)
|
| 63 |
+
with torch.no_grad():
|
| 64 |
+
out = model.generate(
|
| 65 |
+
**inputs,
|
| 66 |
+
max_new_tokens=max_new_tokens,
|
| 67 |
+
do_sample=True,
|
| 68 |
+
temperature=temperature,
|
| 69 |
+
top_p=top_p,
|
| 70 |
+
top_k=50,
|
| 71 |
+
pad_token_id=tokenizer.pad_token_id,
|
| 72 |
+
)
|
| 73 |
+
return tokenizer.decode(
|
| 74 |
+
out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def main() -> int:
|
| 79 |
+
ap = argparse.ArgumentParser(
|
| 80 |
+
description="Pre-flight smoke test: generate N completions with the "
|
| 81 |
+
"GRPO rollout sampling config and report validation pass rate."
|
| 82 |
+
)
|
| 83 |
+
ap.add_argument(
|
| 84 |
+
"--model",
|
| 85 |
+
type=str,
|
| 86 |
+
default="Qwen/Qwen2.5-Coder-7B-Instruct",
|
| 87 |
+
help="HF model id. Same as Phase 3 base model by default.",
|
| 88 |
+
)
|
| 89 |
+
ap.add_argument(
|
| 90 |
+
"--n-completions",
|
| 91 |
+
type=int,
|
| 92 |
+
default=5,
|
| 93 |
+
help="Number of completions to sample for the same prompt.",
|
| 94 |
+
)
|
| 95 |
+
ap.add_argument(
|
| 96 |
+
"--episode-seed",
|
| 97 |
+
type=int,
|
| 98 |
+
default=42,
|
| 99 |
+
help="Seed passed to env.reset() to pick the (repo, module) episode "
|
| 100 |
+
"the prompt is built from.",
|
| 101 |
+
)
|
| 102 |
+
ap.add_argument("--max-new-tokens", type=int, default=2048)
|
| 103 |
+
ap.add_argument("--temperature", type=float, default=0.3)
|
| 104 |
+
ap.add_argument("--top-p", type=float, default=0.9)
|
| 105 |
+
ap.add_argument(
|
| 106 |
+
"--device",
|
| 107 |
+
type=str,
|
| 108 |
+
default="auto",
|
| 109 |
+
help="'auto' | 'cuda' | 'cpu'",
|
| 110 |
+
)
|
| 111 |
+
args = ap.parse_args()
|
| 112 |
+
|
| 113 |
+
try:
|
| 114 |
+
import torch
|
| 115 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 116 |
+
except ImportError as exc:
|
| 117 |
+
print(f"[smoke] FAIL — transformers/torch not installed: {exc}")
|
| 118 |
+
return 1
|
| 119 |
+
|
| 120 |
+
if args.device == "auto":
|
| 121 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 122 |
+
else:
|
| 123 |
+
device = args.device
|
| 124 |
+
print(f"[smoke] device={device} model={args.model}", flush=True)
|
| 125 |
+
if device == "cpu":
|
| 126 |
+
print(
|
| 127 |
+
"[smoke] WARN — CPU inference of a 7B model with "
|
| 128 |
+
f"{args.max_new_tokens} new tokens will take many minutes per "
|
| 129 |
+
"completion. Consider --device cuda or a smaller --model.",
|
| 130 |
+
flush=True,
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
print("[smoke] building env + prompt ...", flush=True)
|
| 134 |
+
env = MutantHunterEnvironment()
|
| 135 |
+
obs = env.reset(seed=args.episode_seed)
|
| 136 |
+
prompt = build_prompt(obs)
|
| 137 |
+
print(
|
| 138 |
+
f"[smoke] episode seed={args.episode_seed} repo={obs.repo_name} "
|
| 139 |
+
f"module={obs.module_path} prompt_chars={len(prompt)}",
|
| 140 |
+
flush=True,
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
print("[smoke] loading model ...", flush=True)
|
| 144 |
+
t0 = time.time()
|
| 145 |
+
try:
|
| 146 |
+
tokenizer = AutoTokenizer.from_pretrained(args.model)
|
| 147 |
+
model = AutoModelForCausalLM.from_pretrained(args.model)
|
| 148 |
+
model = model.to(device)
|
| 149 |
+
model.eval()
|
| 150 |
+
except Exception as exc:
|
| 151 |
+
print(f"[smoke] FAIL — could not load model {args.model}: {exc}")
|
| 152 |
+
traceback.print_exc()
|
| 153 |
+
return 1
|
| 154 |
+
if tokenizer.pad_token is None:
|
| 155 |
+
tokenizer.pad_token = tokenizer.eos_token
|
| 156 |
+
print(f"[smoke] model loaded in {time.time() - t0:.1f}s", flush=True)
|
| 157 |
+
|
| 158 |
+
print(
|
| 159 |
+
f"[smoke] generating {args.n_completions} completions @ "
|
| 160 |
+
f"temp={args.temperature} top_p={args.top_p} "
|
| 161 |
+
f"max_new_tokens={args.max_new_tokens}",
|
| 162 |
+
flush=True,
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
n_valid = 0
|
| 166 |
+
sample_valid: tuple[int, str] | None = None
|
| 167 |
+
sample_invalid: tuple[int, str, str] | None = None
|
| 168 |
+
per_attempt: list[dict] = []
|
| 169 |
+
|
| 170 |
+
for i in range(args.n_completions):
|
| 171 |
+
t1 = time.time()
|
| 172 |
+
try:
|
| 173 |
+
full = _generate_one(
|
| 174 |
+
model, tokenizer, prompt, device,
|
| 175 |
+
max_new_tokens=args.max_new_tokens,
|
| 176 |
+
temperature=args.temperature,
|
| 177 |
+
top_p=args.top_p,
|
| 178 |
+
seed=args.episode_seed * 1000 + i,
|
| 179 |
+
)
|
| 180 |
+
except Exception as exc:
|
| 181 |
+
print(f" [{i}] generation_error: {type(exc).__name__}: {exc}",
|
| 182 |
+
flush=True)
|
| 183 |
+
per_attempt.append({"i": i, "valid": False,
|
| 184 |
+
"reason": f"generation_error: {exc}"})
|
| 185 |
+
continue
|
| 186 |
+
gen_s = time.time() - t1
|
| 187 |
+
|
| 188 |
+
test_code = _strip_markdown_fences(full)
|
| 189 |
+
err = validate_test_code_structure(test_code)
|
| 190 |
+
valid = err is None
|
| 191 |
+
if valid:
|
| 192 |
+
n_valid += 1
|
| 193 |
+
if sample_valid is None:
|
| 194 |
+
sample_valid = (i, test_code)
|
| 195 |
+
else:
|
| 196 |
+
if sample_invalid is None:
|
| 197 |
+
sample_invalid = (i, test_code, err or "unknown")
|
| 198 |
+
per_attempt.append({
|
| 199 |
+
"i": i,
|
| 200 |
+
"valid": valid,
|
| 201 |
+
"reason": err,
|
| 202 |
+
"raw_chars": len(full),
|
| 203 |
+
"stripped_chars": len(test_code),
|
| 204 |
+
"gen_s": round(gen_s, 1),
|
| 205 |
+
})
|
| 206 |
+
print(
|
| 207 |
+
f" [{i}] valid={valid} reason={err!r} raw={len(full)}c "
|
| 208 |
+
f"stripped={len(test_code)}c gen={gen_s:.1f}s",
|
| 209 |
+
flush=True,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
print()
|
| 213 |
+
print(f"[smoke] RESULT: {n_valid}/{args.n_completions} completions "
|
| 214 |
+
f"pass validate_test_code_structure")
|
| 215 |
+
print()
|
| 216 |
+
|
| 217 |
+
if sample_valid is not None:
|
| 218 |
+
i, code = sample_valid
|
| 219 |
+
snippet = code[:1000]
|
| 220 |
+
ellipsis = "" if len(code) <= 1000 else "\n... (truncated)"
|
| 221 |
+
print(f"[smoke] sample VALID completion (i={i}):")
|
| 222 |
+
print("---8<---")
|
| 223 |
+
print(snippet + ellipsis)
|
| 224 |
+
print("---8<---")
|
| 225 |
+
else:
|
| 226 |
+
print("[smoke] no valid completion to show")
|
| 227 |
+
|
| 228 |
+
print()
|
| 229 |
+
if sample_invalid is not None:
|
| 230 |
+
i, code, reason = sample_invalid
|
| 231 |
+
snippet = code[:1000]
|
| 232 |
+
ellipsis = "" if len(code) <= 1000 else "\n... (truncated)"
|
| 233 |
+
print(f"[smoke] sample INVALID completion (i={i}, reason={reason!r}):")
|
| 234 |
+
print("---8<---")
|
| 235 |
+
print(snippet + ellipsis)
|
| 236 |
+
print("---8<---")
|
| 237 |
+
else:
|
| 238 |
+
print("[smoke] no invalid completion to show (all passed!)")
|
| 239 |
+
|
| 240 |
+
print()
|
| 241 |
+
pass_rate = n_valid / args.n_completions if args.n_completions else 0.0
|
| 242 |
+
if pass_rate >= 0.8:
|
| 243 |
+
print(f"[smoke] PASS — {pass_rate:.0%} valid. GRPO config looks good.")
|
| 244 |
+
return 0
|
| 245 |
+
print(
|
| 246 |
+
f"[smoke] WARN — only {pass_rate:.0%} valid. GRPO will likely give "
|
| 247 |
+
"near-zero reward signal at this rate. Re-tune sampling or prompt "
|
| 248 |
+
"before launching Phase 3."
|
| 249 |
+
)
|
| 250 |
+
return 0
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
if __name__ == "__main__":
|
| 254 |
+
raise SystemExit(main())
|
training/train_grpo.py
CHANGED
|
@@ -380,11 +380,11 @@ def run(cfg: TrainingConfig) -> int:
|
|
| 380 |
learning_rate=cfg.learning_rate,
|
| 381 |
max_steps=cfg.steps,
|
| 382 |
max_completion_length=cfg.max_new_tokens,
|
| 383 |
-
# Rollout sampling:
|
| 384 |
-
#
|
| 385 |
-
#
|
| 386 |
-
temperature=0.
|
| 387 |
-
top_p=0.
|
| 388 |
top_k=50,
|
| 389 |
bf16=False,
|
| 390 |
fp16=False,
|
|
@@ -422,7 +422,7 @@ def main() -> int:
|
|
| 422 |
ap.add_argument("--steps", type=int, default=10)
|
| 423 |
ap.add_argument("--rollouts-per-step", type=int, default=2)
|
| 424 |
ap.add_argument("--learning-rate", type=float, default=5e-6)
|
| 425 |
-
ap.add_argument("--max-new-tokens", type=int, default=
|
| 426 |
ap.add_argument("--seed", type=int, default=0)
|
| 427 |
ap.add_argument("--output-dir", type=str, default="./training/_runs/grpo")
|
| 428 |
ap.add_argument("--wandb-project", type=str, default=None,
|
|
|
|
| 380 |
learning_rate=cfg.learning_rate,
|
| 381 |
max_steps=cfg.steps,
|
| 382 |
max_completion_length=cfg.max_new_tokens,
|
| 383 |
+
# Rollout sampling: temp=0.7 still produced too much malformed pytest
|
| 384 |
+
# at 1024 max_new_tokens. Drop to 0.3 + bump tokens to 2048 so the
|
| 385 |
+
# model has both the determinism and the room to finish a coherent file.
|
| 386 |
+
temperature=0.3,
|
| 387 |
+
top_p=0.9,
|
| 388 |
top_k=50,
|
| 389 |
bf16=False,
|
| 390 |
fp16=False,
|
|
|
|
| 422 |
ap.add_argument("--steps", type=int, default=10)
|
| 423 |
ap.add_argument("--rollouts-per-step", type=int, default=2)
|
| 424 |
ap.add_argument("--learning-rate", type=float, default=5e-6)
|
| 425 |
+
ap.add_argument("--max-new-tokens", type=int, default=2048)
|
| 426 |
ap.add_argument("--seed", type=int, default=0)
|
| 427 |
ap.add_argument("--output-dir", type=str, default="./training/_runs/grpo")
|
| 428 |
ap.add_argument("--wandb-project", type=str, default=None,
|