Dream-Coder/base/eval_bcb.sh at main · DreamLM/Dream-Coder · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
export LC_ALL="POSIX"

INPUT_MODEL=Dream-org/Dream-Coder-v0-Base-7B
OUTPUT_DIR=evals_code_results

export TOKENIZERS_PARALLELISM=false
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1


subset=hard
temperature=0.1
max_new_tokens=512
top_p=0.95
alg='maskgit_plus'
alg_temp=0
tag=Temp_${temperature}-Len_${max_new_tokens}-Topp_${top_p}-Alg_${alg}_${alg_temp}
# split: $subset
mkdir -p ${OUTPUT_DIR}/bigcodebench/$subset
python bigcodebench/generate.py \
    --model ${INPUT_MODEL} \
    --split complete \
    --subset $subset \
    --greedy  \
    --bs 1 \
    --n_samples 1 \
    --resume  \
    --backend dllm \
    --save_path ${OUTPUT_DIR}/bigcodebench/$subset/completion-$tag.jsonl \
    --temperature $temperature \
    --max_new_tokens $max_new_tokens \
    --steps $max_new_tokens \
    --top_p $top_p \
    --alg $alg \
    --alg_temp $alg_temp

python bigcodebench/sanitize.py \
    --samples ${OUTPUT_DIR}/bigcodebench/$subset/completion-$tag.jsonl \
    --calibrate

python bigcodebench/evaluate.py \
    --split complete \
    --subset $subset \
    --no-gt \
    --samples ${OUTPUT_DIR}/bigcodebench/$subset/completion-$tag-sanitized-calibrated.jsonl


subset=full
temperature=0.1
max_new_tokens=512
top_p=0.95
alg='entropy'
alg_temp=0
tag=Temp_${temperature}-Len_${max_new_tokens}-Topp_${top_p}-Alg_${alg}_${alg_temp}
# split: $subset
mkdir -p ${OUTPUT_DIR}/bigcodebench/$subset
python bigcodebench/generate.py \
    --model ${INPUT_MODEL} \
    --split complete \
    --subset $subset \
    --greedy  \
    --bs 1 \
    --n_samples 1 \
    --resume  \
    --backend dllm \
    --save_path ${OUTPUT_DIR}/bigcodebench/$subset/completion-$tag.jsonl \
    --temperature $temperature \
    --max_new_tokens $max_new_tokens \
    --steps $max_new_tokens \
    --top_p $top_p \
    --alg $alg \
    --alg_temp $alg_temp

python bigcodebench/sanitize.py \
    --samples ${OUTPUT_DIR}/bigcodebench/$subset/completion-$tag.jsonl \
    --calibrate

python bigcodebench/evaluate.py \
    --split complete \
    --subset $subset \
    --no-gt \
    --samples ${OUTPUT_DIR}/bigcodebench/$subset/completion-$tag-sanitized-calibrated.jsonl