RedHatAI / gemma-4-31B-it-NVFP4

from datasets import load_dataset
from transformers import AutoProcessor, Gemma4ForConditionalGeneration, ProcessorMixin

from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier

MODEL_ID = "google/gemma-4-31B-it"

model = Gemma4ForConditionalGeneration.from_pretrained(MODEL_ID, dtype="auto")
processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)

NUM_CALIBRATION_SAMPLES = 32
MAX_SEQUENCE_LENGTH = 2048

DATASET_ID = "mit-han-lab/pile-val-backup"
DATASET_SPLIT = "validation"

recipe = [
    QuantizationModifier(
        targets="Linear",
        scheme="NVFP4",
        ignore=["re:.*vision.*", "re:.*audio.*", "lm_head", "re:.*embed.*"],
    ),
]


def get_calib_dataset(processor: ProcessorMixin):
    ds = load_dataset(
        DATASET_ID,
        split=f"{DATASET_SPLIT}[:{NUM_CALIBRATION_SAMPLES * 10}]",
    )

    def preprocess(example):
        return {
            "input_ids": processor.tokenizer.encode(example["text"].strip())[
                :MAX_SEQUENCE_LENGTH
            ]
        }

    ds = (
        ds.shuffle(seed=42)
        .map(preprocess, remove_columns=ds.column_names)
        .filter(lambda example: len(example["input_ids"]) >= MAX_SEQUENCE_LENGTH)
        .select(range(NUM_CALIBRATION_SAMPLES))
    )

    return ds


oneshot(
    model=model,
    processor=processor,
    dataset=get_calib_dataset(processor),
    recipe=recipe,
    batch_size=1,
    shuffle_calibration_samples=False,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
)

SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-NVFP4"
model.save_pretrained(SAVE_DIR, save_compressed=True)
processor.save_pretrained(SAVE_DIR)

Each benchmark was run 3 times with different random seeds (42, 1234, 4158) and the scores were averaged.

vLLM server:

vllm serve RedHatAI/gemma-4-31B-it-NVFP4 --max-model-len 96000

GSM8k-Platinum (lm-eval, 5-shot, 3 repetitions)

lm_eval --model local-chat-completions \
  --tasks gsm8k_platinum_cot_llama \
  --model_args "model=RedHatAI/gemma-4-31B-it-NVFP4,max_length=96000,base_url=http://0.0.0.0:8000/v1/chat/completions,num_concurrent=128,max_retries=3,tokenized_requests=False,tokenizer_backend=None,timeout=2400" \
  --num_fewshot 5 \
  --apply_chat_template \
  --fewshot_as_multiturn \
  --output_path results_gsm8k_platinum.json \
  --seed 1234 \
  --gen_kwargs "do_sample=True,temperature=1.0,top_p=0.95,top_k=64,max_gen_toks=64000,seed=1234"

MMLU-CoT (lm-eval, 5-shot, 3 repetitions)

lm_eval --model local-chat-completions \
  --tasks mmlu_cot_llama \
  --model_args "model=RedHatAI/gemma-4-31B-it-NVFP4,max_length=96000,base_url=http://0.0.0.0:8000/v1/chat/completions,num_concurrent=128,max_retries=3,tokenized_requests=False,tokenizer_backend=None,timeout=2400" \
  --num_fewshot 5 \
  --apply_chat_template \
  --fewshot_as_multiturn \
  --output_path results_mmlu_cot.json \
  --seed 1234 \
  --gen_kwargs "do_sample=True,temperature=1.0,top_p=0.95,top_k=64,max_gen_toks=64000,seed=1234"

MMLU-Pro (lm-eval, 5-shot, 3 repetitions)

lm_eval --model local-chat-completions \
  --tasks mmlu_pro_chat \
  --model_args "model=RedHatAI/gemma-4-31B-it-NVFP4,max_length=96000,base_url=http://0.0.0.0:8000/v1/chat/completions,num_concurrent=128,max_retries=3,tokenized_requests=False,tokenizer_backend=None,timeout=2400" \
  --num_fewshot 5 \
  --apply_chat_template \
  --fewshot_as_multiturn \
  --output_path results_mmlu_pro.json \
  --seed 1234 \
  --gen_kwargs "do_sample=True,temperature=1.0,top_p=0.95,top_k=64,max_gen_toks=64000,seed=1234"

IFEval (lm-eval, 0-shot, 3 repetitions)

lm_eval --model local-chat-completions \
  --tasks ifeval \
  --model_args "model=RedHatAI/gemma-4-31B-it-NVFP4,max_length=96000,base_url=http://0.0.0.0:8000/v1/chat/completions,num_concurrent=128,max_retries=3,tokenized_requests=False,tokenizer_backend=None,timeout=2400" \
  --apply_chat_template \
  --fewshot_as_multiturn \
  --output_path results_ifeval.json \
  --seed 1234 \
  --gen_kwargs "do_sample=True,temperature=1.0,top_p=0.95,top_k=64,max_gen_toks=64000,seed=1234"