GLM-4.7

python -m sglang.launch_server \
  --model zai-org/GLM-4.7 \
  --reasoning-parser glm45 \
  --tp 8 \
  --host 0.0.0.0 \
  --port 8000

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="EMPTY"
)

# Enable streaming to see the thinking process in real-time
response = client.chat.completions.create(
    model="zai-org/GLM-4.7",
    messages=[
        {"role": "user", "content": "Solve this problem step by step: What is 15% of 240?"}
    ],
    temperature=0.7,
    max_tokens=2048,
    stream=True
)

# Process the stream
has_thinking = False
has_answer = False
thinking_started = False

for chunk in response:
    if chunk.choices and len(chunk.choices) > 0:
        delta = chunk.choices[0].delta

        # Print thinking process
        if hasattr(delta, 'reasoning_content') and delta.reasoning_content:
            if not thinking_started:
                print("=============== Thinking =================", flush=True)
                thinking_started = True
            has_thinking = True
            print(delta.reasoning_content, end="", flush=True)

        # Print answer content
        if delta.content:
            # Close thinking section and add content header
            if has_thinking and not has_answer:
                print("\n=============== Content =================", flush=True)
                has_answer = True
            print(delta.content, end="", flush=True)

print()

=============== Thinking =================
To solve this problem, I need to calculate 15% of 240.
Step 1: Convert 15% to decimal: 15% = 0.15
Step 2: Multiply 240 by 0.15
Step 3: 240 × 0.15 = 36
=============== Content =================

The answer is 36. To find 15% of 240, we multiply 240 by 0.15, which equals 36.

python -m sglang.launch_server \
  --model zai-org/GLM-4.7 \
  --reasoning-parser glm45 \
  --tool-call-parser glm47 \
  --tp 8 \
  --host 0.0.0.0 \
  --port 8000

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="EMPTY"
)

# Define available tools
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get the current weather for a location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city name"
                    },
                    "unit": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"],
                        "description": "Temperature unit"
                    }
                },
                "required": ["location"]
            }
        }
    }
]

# Make request with streaming to see thinking process
response = client.chat.completions.create(
    model="zai-org/GLM-4.7",
    messages=[
        {"role": "user", "content": "What's the weather in Beijing?"}
    ],
    tools=tools,
    temperature=0.7,
    stream=True
)

# Process streaming response
thinking_started = False
has_thinking = False

for chunk in response:
    if chunk.choices and len(chunk.choices) > 0:
        delta = chunk.choices[0].delta

        # Print thinking process
        if hasattr(delta, 'reasoning_content') and delta.reasoning_content:
            if not thinking_started:
                print("=============== Thinking =================", flush=True)
                thinking_started = True
            has_thinking = True
            print(delta.reasoning_content, end="", flush=True)

        # Print tool calls
        if hasattr(delta, 'tool_calls') and delta.tool_calls:
            # Close thinking section if needed
            if has_thinking and thinking_started:
                print("\n=============== Content =================", flush=True)
                thinking_started = False

            for tool_call in delta.tool_calls:
                if tool_call.function:
                    print(f"Tool Call: {tool_call.function.name}")
                    print(f"   Arguments: {tool_call.function.arguments}")

        # Print content
        if delta.content:
            print(delta.content, end="", flush=True)

print()

=============== Thinking =================
The user is asking about the weather in Beijing. I need to use the get_weather function to retrieve this information.
I should call the function with location="Beijing".
=============== Content =================

Tool Call: get_weather
   Arguments: {"location": "Beijing", "unit": "celsius"}

# After getting the tool call, execute the function
def get_weather(location, unit="celsius"):
    # Your actual weather API call here
    return f"The weather in {location} is 22°{unit[0].upper()} and sunny."

# Send tool result back to the model
messages = [
    {"role": "user", "content": "What's the weather in Beijing?"},
    {
        "role": "assistant",
        "content": None,
        "tool_calls": [{
            "id": "call_123",
            "type": "function",
            "function": {
                "name": "get_weather",
                "arguments": '{"location": "Beijing", "unit": "celsius"}'
            }
        }]
    },
    {
        "role": "tool",
        "tool_call_id": "call_123",
        "content": get_weather("Beijing", "celsius")
    }
]

final_response = client.chat.completions.create(
    model="zai-org/GLM-4.7",
    messages=messages,
    temperature=0.7
)

print(final_response.choices[0].message.content)
# Output: "The weather in Beijing is currently 22°C and sunny."

import openai
from sglang.srt.sampling.custom_logit_processor import Glm4MoeThinkingBudgetLogitProcessor

client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="*")
response = client.chat.completions.create(
    model="zai-org/GLM-4.7",
    messages=[{"role": "user", "content": "Is Paris the Capital of France?"}],
    max_tokens=1024,
    extra_body={
        "custom_logit_processor": Glm4MoeThinkingBudgetLogitProcessor().to_str(),
        "custom_params": {"thinking_budget": 512},
    },
)
print(response)

python -m sglang.launch_server \
  --model zai-org/GLM-4.7 \
  --tp 8

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 1000 \
  --random-output-len 1000 \
  --num-prompts 10 \
  --max-concurrency 1 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 1000 \
  --random-output-len 1000 \
  --num-prompts 80 \
  --max-concurrency 16 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 1000 \
  --random-output-len 1000 \
  --num-prompts 500 \
  --max-concurrency 100 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 1000 \
  --random-output-len 8000 \
  --num-prompts 10 \
  --max-concurrency 1 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 1000 \
  --random-output-len 8000 \
  --num-prompts 80 \
  --max-concurrency 16 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 1000 \
  --random-output-len 8000 \
  --num-prompts 320 \
  --max-concurrency 64 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 8000 \
  --random-output-len 1000 \
  --num-prompts 10 \
  --max-concurrency 1 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 8000 \
  --random-output-len 1000 \
  --num-prompts 80 \
  --max-concurrency 16 \
  --request-rate inf

python -m sglang.bench_serving \
  --backend sglang \
  --model zai-org/GLM-4.7 \
  --dataset-name random \
  --random-input-len 8000 \
  --random-output-len 1000 \
  --num-prompts 320 \
  --max-concurrency 64 \
  --request-rate inf

python -m sglang.launch_server \
  --model nvidia/GLM-4.7-NVFP4 \
  --tp-size 2 \
  --mem-fraction-static 0.85 \
  --reasoning-parser glm45 \
  --tool-call-parser glm47

python -m sglang.bench_serving \
  --backend sglang \
  --model nvidia/GLM-4.7-NVFP4 \
  --dataset-name random \
  --random-input-len 4096 \
  --random-output-len 1024 \
  --num-prompts 5 \
  --max-concurrency 1 \
  --request-rate inf

============ Serving Benchmark Result ============
Backend:                                 sglang
Max request concurrency:                 1
Successful requests:                     5
Benchmark duration (s):                  25.07
Total input tokens:                      8105
Total generated tokens:                  2674
Request throughput (req/s):              0.20
Input token throughput (tok/s):          323.25
Output token throughput (tok/s):         106.65
Total token throughput (tok/s):          429.90
Concurrency:                             1.00
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   5011.93
Median E2E Latency (ms):                 6441.44
---------------Time to First Token----------------
Mean TTFT (ms):                          179.61
Median TTFT (ms):                        169.05
P99 TTFT (ms):                           238.01
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          9.05
Median TPOT (ms):                        9.03
P99 TPOT (ms):                           9.16
---------------Inter-Token Latency----------------
Mean ITL (ms):                           9.05
Median ITL (ms):                         9.05
==================================================

python -m sglang.bench_serving \
  --backend sglang \
  --model nvidia/GLM-4.7-NVFP4 \
  --dataset-name random \
  --random-input-len 4096 \
  --random-output-len 1024 \
  --num-prompts 80 \
  --max-concurrency 16 \
  --request-rate inf

============ Serving Benchmark Result ============
Backend:                                 sglang
Max request concurrency:                 16
Successful requests:                     80
Benchmark duration (s):                  60.60
Total input tokens:                      179772
Total generated tokens:                  39657
Request throughput (req/s):              1.32
Input token throughput (tok/s):          2966.39
Output token throughput (tok/s):         654.37
Total token throughput (tok/s):          3620.76
Concurrency:                             14.01
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   10615.87
Median E2E Latency (ms):                 9985.45
---------------Time to First Token----------------
Mean TTFT (ms):                          267.39
Median TTFT (ms):                        177.26
P99 TTFT (ms):                           584.29
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          20.98
Median TPOT (ms):                        21.06
P99 TPOT (ms):                           24.88
---------------Inter-Token Latency----------------
Mean ITL (ms):                           20.92
Median ITL (ms):                         17.93
==================================================

python -m sglang.bench_serving \
  --backend sglang \
  --model nvidia/GLM-4.7-NVFP4 \
  --dataset-name random \
  --random-input-len 4096 \
  --random-output-len 1024 \
  --num-prompts 640 \
  --max-concurrency 128 \
  --request-rate inf

============ Serving Benchmark Result ============
Backend:                                 sglang
Max request concurrency:                 128
Successful requests:                     640
Benchmark duration (s):                  172.95
Total input tokens:                      1453591
Total generated tokens:                  308740
Request throughput (req/s):              3.70
Input token throughput (tok/s):          8404.67
Output token throughput (tok/s):         1785.14
Total token throughput (tok/s):          10189.80
Concurrency:                             117.85
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   31848.20
Median E2E Latency (ms):                 28554.42
---------------Time to First Token----------------
Mean TTFT (ms):                          1598.40
Median TTFT (ms):                        298.88
P99 TTFT (ms):                           11015.96
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          65.94
Median TPOT (ms):                        65.81
P99 TPOT (ms):                           137.73
---------------Inter-Token Latency----------------
Mean ITL (ms):                           62.99
Median ITL (ms):                         35.44
==================================================

python -m sglang.launch_server \
  --model nvidia/GLM-4.7-NVFP4 \
  --tp-size 2 \
  --mem-fraction-static 0.85 \
  --reasoning-parser glm45 \
  --tool-call-parser glm47

python -m sglang.bench_serving \
  --backend sglang \
  --model nvidia/GLM-4.7-NVFP4 \
  --dataset-name random \
  --random-input-len 4096 \
  --random-output-len 1024 \
  --num-prompts 5 \
  --max-concurrency 1 \
  --request-rate inf

============ Serving Benchmark Result ============
Backend:                                 sglang
Max request concurrency:                 1
Successful requests:                     5
Benchmark duration (s):                  24.74
Total input tokens:                      8105
Total generated tokens:                  2674
Request throughput (req/s):              0.20
Input token throughput (tok/s):          327.65
Output token throughput (tok/s):         108.10
Total token throughput (tok/s):          435.75
Concurrency:                             1.00
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   4944.47
Median E2E Latency (ms):                 6347.31
---------------Time to First Token----------------
Mean TTFT (ms):                          211.41
Median TTFT (ms):                        207.25
P99 TTFT (ms):                           226.46
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          8.86
Median TPOT (ms):                        8.84
P99 TPOT (ms):                           8.96
---------------Inter-Token Latency----------------
Mean ITL (ms):                           8.87
Median ITL (ms):                         8.85
==================================================

python -m sglang.bench_serving \
  --backend sglang \
  --model nvidia/GLM-4.7-NVFP4 \
  --dataset-name random \
  --random-input-len 4096 \
  --random-output-len 1024 \
  --num-prompts 80 \
  --max-concurrency 16 \
  --request-rate inf

============ Serving Benchmark Result ============
Backend:                                 sglang
Max request concurrency:                 16
Successful requests:                     80
Benchmark duration (s):                  60.40
Total input tokens:                      179772
Total generated tokens:                  39657
Request throughput (req/s):              1.32
Input token throughput (tok/s):          2976.52
Output token throughput (tok/s):         656.61
Total token throughput (tok/s):          3633.13
Concurrency:                             13.97
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   10611.51
Median E2E Latency (ms):                 9956.84
---------------Time to First Token----------------
Mean TTFT (ms):                          338.14
Median TTFT (ms):                        215.25
P99 TTFT (ms):                           915.40
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          20.87
Median TPOT (ms):                        21.36
P99 TPOT (ms):                           27.05
---------------Inter-Token Latency----------------
Mean ITL (ms):                           20.77
Median ITL (ms):                         16.53
==================================================

python -m sglang.bench_serving \
  --backend sglang \
  --model nvidia/GLM-4.7-NVFP4 \
  --dataset-name random \
  --random-input-len 4096 \
  --random-output-len 1024 \
  --num-prompts 640 \
  --max-concurrency 128 \
  --request-rate inf

============ Serving Benchmark Result ============
Backend:                                 sglang
Max request concurrency:                 128
Successful requests:                     640
Benchmark duration (s):                  181.89
Total input tokens:                      1453591
Total generated tokens:                  309221
Request throughput (req/s):              3.52
Input token throughput (tok/s):          7991.59
Output token throughput (tok/s):         1700.04
Total token throughput (tok/s):          9691.63
Concurrency:                             118.86
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   33690.47
Median E2E Latency (ms):                 30421.55
---------------Time to First Token----------------
Mean TTFT (ms):                          1353.16
Median TTFT (ms):                        383.52
P99 TTFT (ms):                           8940.53
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          69.88
Median TPOT (ms):                        71.77
P99 TPOT (ms):                           131.75
---------------Inter-Token Latency----------------
Mean ITL (ms):                           67.23
Median ITL (ms):                         33.46
==================================================

python -m sglang.test.few_shot_gsm8k \
  --num-shots 5 \
  --num-questions 1319 \
  --port 30000

Accuracy: 0.946
Latency: 178.284 s
Output throughput: 769.204 token/s

Accuracy: 0.951
Latency: 175.190 s
Invalid: 0.000