{"tag": null, "backend": "sglang", "dataset_name": "random", "request_rate": Infinity, "max_concurrency": 8, "sharegpt_output_len": null, "random_input_len": 512, "random_output_len": 128, "random_range_ratio": 1.0, "server_info": {"model_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": false, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.7, "max_running_requests": null, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 1, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "radix_eviction_policy_config": null, "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "gated_launch_port": null, "nnodes": 1, "node_rank": 0, "tp_size": 1, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 1, "attn_dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_cp_decode_attn_tp": false, "enable_kv_cache_sharding": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "shared_experts_tp_size": null, "enable_dense_mlp_attn_tp": false, "enable_layernorm_sp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 337600984, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "127.0.0.1", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "weight_version": "default", "chat_template": null, "trust_request_chat_template": false, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": null, "enable_cache_report": true, "return_input_ids": false, "return_output_ids": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "constrained_json_max_whitespace_cnt": null, "attention_backend": "flashinfer", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 256, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}, "prefill": {"backend": "breakable", "max_bs": 8192, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_prefill_max_context": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "boundary_reduction": "rs+rsv", "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "uno_lora_path": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_enable_w4a4_mxfp4_megamoe": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 1, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 8, "hicache_storage_prefetch_retry_max_attempts": 8, "enable_unified_cache_external_linker": false, "unified_cache_external_linker_backend": "mooncake", "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "sampling_mask_max_tokens": 4096, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "ple_offload_embedding": null, "ple_offload_backend": "pinned", "ple_offload_dir": null, "disaggregation_enable_kv_checksum": false, "enable_pd_role_switch": false, "speculative_domino_candidate_pool_size": 2048, "flashinfer_a2a_dispatch_type": null, "cuda_graph_max_seq_len_prefill": null, "prefill_kv_cache_dequant_dtype": "auto", "dsv4_attn_backend": "auto", "otlp_service_name": null, "_radix_eviction_policy_explicitly_set": false, "hicache_host_memory_fraction": 0.8, "enable_linker_mla_dedup": false, "_swa_full_tokens_ratio_explicitly_set": false, "swa_prefix_tails": null, "enable_encoder_swa_bounded_replay": false, "enable_decoder_swa_bounded_replay": false, "enable_response_store": false, "disaggregation_decode_host_receive_threshold": 1.0, "qsa_indexer_dtype": "auto", "disaggregation_decode_allocation_policy": "early", "speculative_boundary_reduction": "rs+rsv", "speculative_use_block_verification": false, "speculative_draft_sink_size": null, "disable_aiter_allreduce_fusion_in_prefill": false, "disable_aiter_allreduce_fusion_in_decode": false, "dsa_triton_union": 0, "enable_dsa_fused_indexer": null, "enable_cp_tp_group_sharing": false, "enable_watermark": false, "watermark_config": "<redacted>", "watermark_key": "<redacted>", "watermark_key_b": "<redacted>", "watermark_context_window": 4, "watermark_mixing_probability": 0.5, "watermark_max_probability": 1.0, "watermark_default_enabled": false, "watermark_enforce_all": false, "grpc_response_timeout_secs": 300, "launch_command": "--model-path Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --host 127.0.0.1 --port 30000 --enable-cache-report --enable-metrics --mem-fraction-static 0.7 --kv-cache-dtype fp8_e4m3", "status": "ready", "max_total_num_tokens": 1080523, "max_req_input_len": 262138, "startup_time": {"load_weight": 152.64007084499463, "kv_cache_allocation": 6.450723860994913, "scheduler_e2e": 224.92462453301414, "cuda_graph": {"prefill": 55.088790353998775, "decode": 4.267712268003379, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}, "tokenizer_e2e": 229.9016736259946}, "kv_events": null, "internal_states": [{"model_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": false, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.7, "max_running_requests": null, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 1, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "radix_eviction_policy_config": null, "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "gated_launch_port": null, "nnodes": 1, "node_rank": 0, "tp_size": 1, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 2110, "pp_async_batch_depth": 0, "dp_size": 1, "attn_dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_cp_decode_attn_tp": false, "enable_kv_cache_sharding": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "shared_experts_tp_size": null, "enable_dense_mlp_attn_tp": false, "enable_layernorm_sp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 337600984, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "127.0.0.1", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "weight_version": "default", "chat_template": null, "trust_request_chat_template": false, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": null, "enable_cache_report": true, "return_input_ids": false, "return_output_ids": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "constrained_json_max_whitespace_cnt": null, "attention_backend": "flashinfer", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 256, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}, "prefill": {"backend": "breakable", "max_bs": 8192, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_prefill_max_context": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "boundary_reduction": "rs+rsv", "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "uno_lora_path": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_enable_w4a4_mxfp4_megamoe": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 1, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 8, "hicache_storage_prefetch_retry_max_attempts": 8, "enable_unified_cache_external_linker": false, "unified_cache_external_linker_backend": "mooncake", "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "sampling_mask_max_tokens": 4096, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "ple_offload_embedding": null, "ple_offload_backend": "pinned", "ple_offload_dir": null, "disaggregation_enable_kv_checksum": false, "enable_pd_role_switch": false, "speculative_domino_candidate_pool_size": 2048, "flashinfer_a2a_dispatch_type": null, "cuda_graph_max_seq_len_prefill": null, "prefill_kv_cache_dequant_dtype": "auto", "dsv4_attn_backend": "auto", "otlp_service_name": null, "_radix_eviction_policy_explicitly_set": false, "hicache_host_memory_fraction": 0.8, "enable_linker_mla_dedup": false, "_swa_full_tokens_ratio_explicitly_set": false, "swa_prefix_tails": null, "enable_encoder_swa_bounded_replay": false, "enable_decoder_swa_bounded_replay": false, "enable_response_store": false, "disaggregation_decode_host_receive_threshold": 1.0, "qsa_indexer_dtype": "auto", "disaggregation_decode_allocation_policy": "early", "speculative_boundary_reduction": "rs+rsv", "speculative_use_block_verification": false, "speculative_draft_sink_size": null, "disable_aiter_allreduce_fusion_in_prefill": false, "disable_aiter_allreduce_fusion_in_decode": false, "dsa_triton_union": 0, "enable_dsa_fused_indexer": null, "enable_cp_tp_group_sharing": false, "enable_watermark": false, "watermark_config": "<redacted>", "watermark_key": "<redacted>", "watermark_key_b": "<redacted>", "watermark_context_window": 4, "watermark_mixing_probability": 0.5, "watermark_max_probability": 1.0, "watermark_default_enabled": false, "watermark_enforce_all": false, "grpc_response_timeout_secs": 300, "world_size": 1, "startup_time": {"load_weight": 152.64007084499463, "kv_cache_allocation": 6.450723860994913, "scheduler_e2e": 224.92462453301414, "cuda_graph": {"prefill": 55.088790353998775, "decode": 4.267712268003379, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 2110, "last_gen_throughput": 153.8184003487785, "memory_usage": {"weight": 30.139, "kvcache": 49.462, "startup_available": 26.533, "token_capacity": 1080523, "token_capacity_swa": null, "graph": {"prefill": 2.587, "decode": 1.95, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "decode_cuda_graph_bs": [], "decode_cuda_graph_memory_gb": 1.95}], "version": "0.5.22", "frontend": "python"}, "duration": 59.90566459899128, "completed": 64, "total_input_tokens": 32768, "total_input_text_tokens": 32768, "total_input_vision_tokens": 0, "total_output_tokens": 8192, "total_output_tokens_retokenized": 8176, "request_throughput": 1.068346381405101, "input_throughput": 546.9933472794118, "output_throughput": 136.74833681985294, "total_throughput": 683.7416840992647, "mean_e2e_latency_ms": 7485.3643099067995, "median_e2e_latency_ms": 7560.75530200178, "std_e2e_latency_ms": 438.8478241132452, "p90_e2e_latency_ms": 8035.841528698802, "p95_e2e_latency_ms": 8035.969649005711, "p99_e2e_latency_ms": 8036.166000880476, "mean_ttft_ms": 399.2743957346647, "median_ttft_ms": 399.17423199949553, "std_ttft_ms": 240.03865322062336, "p90_ttft_ms": 693.714334096876, "p95_ttft_ms": 694.0459619014291, "p99_ttft_ms": 694.5409949666646, "mean_tpot_ms": 55.79598357615855, "median_tpot_ms": 55.64667291340682, "std_tpot_ms": 2.78023630811408, "p90_tpot_ms": 59.53564664649624, "p95_tpot_ms": 60.132183707064826, "p99_tpot_ms": 61.57230327939147, "mean_itl_ms": 55.79591820275358, "median_itl_ms": 54.69341300340602, "std_itl_ms": 19.60033061027367, "p90_itl_ms": 58.83799600560451, "p95_itl_ms": 59.9990851507755, "p99_itl_ms": 62.462455278873676, "concurrency": 7.9969618739878205, "accept_length": null, "max_output_tokens_per_s": 160.0, "max_concurrent_requests": 16}
{"tag": null, "backend": "sglang", "dataset_name": "random", "request_rate": Infinity, "max_concurrency": 8, "sharegpt_output_len": null, "random_input_len": 512, "random_output_len": 128, "random_range_ratio": 1.0, "server_info": {"model_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": false, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.7, "max_running_requests": null, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 1, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "radix_eviction_policy_config": null, "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "gated_launch_port": null, "nnodes": 1, "node_rank": 0, "tp_size": 1, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 1, "attn_dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_cp_decode_attn_tp": false, "enable_kv_cache_sharding": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "shared_experts_tp_size": null, "enable_dense_mlp_attn_tp": false, "enable_layernorm_sp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 302506050, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "127.0.0.1", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "weight_version": "default", "chat_template": null, "trust_request_chat_template": false, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": null, "enable_cache_report": true, "return_input_ids": false, "return_output_ids": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "constrained_json_max_whitespace_cnt": null, "attention_backend": "flashinfer", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 256, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}, "prefill": {"backend": "breakable", "max_bs": 8192, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_prefill_max_context": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "boundary_reduction": "rs+rsv", "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "uno_lora_path": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_enable_w4a4_mxfp4_megamoe": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 1, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 8, "hicache_storage_prefetch_retry_max_attempts": 8, "enable_unified_cache_external_linker": false, "unified_cache_external_linker_backend": "mooncake", "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "sampling_mask_max_tokens": 4096, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "ple_offload_embedding": null, "ple_offload_backend": "pinned", "ple_offload_dir": null, "disaggregation_enable_kv_checksum": false, "enable_pd_role_switch": false, "speculative_domino_candidate_pool_size": 2048, "flashinfer_a2a_dispatch_type": null, "cuda_graph_max_seq_len_prefill": null, "prefill_kv_cache_dequant_dtype": "auto", "dsv4_attn_backend": "auto", "otlp_service_name": null, "_radix_eviction_policy_explicitly_set": false, "hicache_host_memory_fraction": 0.8, "enable_linker_mla_dedup": false, "_swa_full_tokens_ratio_explicitly_set": false, "swa_prefix_tails": null, "enable_encoder_swa_bounded_replay": false, "enable_decoder_swa_bounded_replay": false, "enable_response_store": false, "disaggregation_decode_host_receive_threshold": 1.0, "qsa_indexer_dtype": "auto", "disaggregation_decode_allocation_policy": "early", "speculative_boundary_reduction": "rs+rsv", "speculative_use_block_verification": false, "speculative_draft_sink_size": null, "disable_aiter_allreduce_fusion_in_prefill": false, "disable_aiter_allreduce_fusion_in_decode": false, "dsa_triton_union": 0, "enable_dsa_fused_indexer": null, "enable_cp_tp_group_sharing": false, "enable_watermark": false, "watermark_config": "<redacted>", "watermark_key": "<redacted>", "watermark_key_b": "<redacted>", "watermark_context_window": 4, "watermark_mixing_probability": 0.5, "watermark_max_probability": 1.0, "watermark_default_enabled": false, "watermark_enforce_all": false, "grpc_response_timeout_secs": 300, "launch_command": "--model-path Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --host 127.0.0.1 --port 30000 --enable-cache-report --enable-metrics --mem-fraction-static 0.7 --kv-cache-dtype fp8_e4m3", "status": "ready", "max_total_num_tokens": 1083008, "max_req_input_len": 262138, "startup_time": {"load_weight": 152.2862869290002, "kv_cache_allocation": 7.09510689900344, "scheduler_e2e": 196.96553649600173, "cuda_graph": {"prefill": 29.92951290199926, "decode": 4.178592932999891, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}, "tokenizer_e2e": 202.06633559600232}, "kv_events": null, "internal_states": [{"model_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_path": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": false, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.7, "max_running_requests": null, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 1, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "radix_eviction_policy_config": null, "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "gated_launch_port": null, "nnodes": 1, "node_rank": 0, "tp_size": 1, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 2115, "pp_async_batch_depth": 0, "dp_size": 1, "attn_dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_cp_decode_attn_tp": false, "enable_kv_cache_sharding": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "shared_experts_tp_size": null, "enable_dense_mlp_attn_tp": false, "enable_layernorm_sp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 302506050, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "127.0.0.1", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8", "weight_version": "default", "chat_template": null, "trust_request_chat_template": false, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": null, "enable_cache_report": true, "return_input_ids": false, "return_output_ids": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "constrained_json_max_whitespace_cnt": null, "attention_backend": "flashinfer", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 256, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}, "prefill": {"backend": "breakable", "max_bs": 8192, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192], "tc_compiler": "eager", "max_context_size": null, "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null, "max_seq_len": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_prefill_max_context": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "boundary_reduction": "rs+rsv", "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "uno_lora_path": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_enable_w4a4_mxfp4_megamoe": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 1, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 8, "hicache_storage_prefetch_retry_max_attempts": 8, "enable_unified_cache_external_linker": false, "unified_cache_external_linker_backend": "mooncake", "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "sampling_mask_max_tokens": 4096, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "ple_offload_embedding": null, "ple_offload_backend": "pinned", "ple_offload_dir": null, "disaggregation_enable_kv_checksum": false, "enable_pd_role_switch": false, "speculative_domino_candidate_pool_size": 2048, "flashinfer_a2a_dispatch_type": null, "cuda_graph_max_seq_len_prefill": null, "prefill_kv_cache_dequant_dtype": "auto", "dsv4_attn_backend": "auto", "otlp_service_name": null, "_radix_eviction_policy_explicitly_set": false, "hicache_host_memory_fraction": 0.8, "enable_linker_mla_dedup": false, "_swa_full_tokens_ratio_explicitly_set": false, "swa_prefix_tails": null, "enable_encoder_swa_bounded_replay": false, "enable_decoder_swa_bounded_replay": false, "enable_response_store": false, "disaggregation_decode_host_receive_threshold": 1.0, "qsa_indexer_dtype": "auto", "disaggregation_decode_allocation_policy": "early", "speculative_boundary_reduction": "rs+rsv", "speculative_use_block_verification": false, "speculative_draft_sink_size": null, "disable_aiter_allreduce_fusion_in_prefill": false, "disable_aiter_allreduce_fusion_in_decode": false, "dsa_triton_union": 0, "enable_dsa_fused_indexer": null, "enable_cp_tp_group_sharing": false, "enable_watermark": false, "watermark_config": "<redacted>", "watermark_key": "<redacted>", "watermark_key_b": "<redacted>", "watermark_context_window": 4, "watermark_mixing_probability": 0.5, "watermark_max_probability": 1.0, "watermark_default_enabled": false, "watermark_enforce_all": false, "grpc_response_timeout_secs": 300, "world_size": 1, "startup_time": {"load_weight": 152.2862869290002, "kv_cache_allocation": 7.09510689900344, "scheduler_e2e": 196.96553649600173, "cuda_graph": {"prefill": 29.92951290199926, "decode": 4.178592932999891, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 2115, "last_gen_throughput": 158.206827182721, "memory_usage": {"weight": 29.752, "kvcache": 49.576, "startup_available": 26.504, "token_capacity": 1083008, "token_capacity_swa": null, "graph": {"prefill": 2.483, "decode": 1.528, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "decode_cuda_graph_bs": [], "decode_cuda_graph_memory_gb": 1.528}], "version": "0.5.22", "frontend": "python"}, "duration": 59.08518128899959, "completed": 64, "total_input_tokens": 32768, "total_input_text_tokens": 32768, "total_input_vision_tokens": 0, "total_output_tokens": 8192, "total_output_tokens_retokenized": 8176, "request_throughput": 1.0831819181016111, "input_throughput": 554.5891420680249, "output_throughput": 138.64728551700622, "total_throughput": 693.236427585031, "mean_e2e_latency_ms": 7381.709340859061, "median_e2e_latency_ms": 7567.389910000202, "std_e2e_latency_ms": 413.9876765433661, "p90_e2e_latency_ms": 7844.818987501276, "p95_e2e_latency_ms": 7844.907532098841, "p99_e2e_latency_ms": 7844.995993438133, "mean_ttft_ms": 442.2748783435395, "median_ttft_ms": 572.9699429975881, "std_ttft_ms": 244.60452274535663, "p90_ttft_ms": 692.4249782998232, "p95_ttft_ms": 703.3860280998852, "p99_ttft_ms": 703.4973054714646, "mean_tpot_ms": 54.64121624027969, "median_tpot_ms": 54.69146391731656, "std_tpot_ms": 2.46007019564951, "p90_tpot_ms": 56.86731111338991, "p95_tpot_ms": 58.8856776275632, "p99_tpot_ms": 60.19941167951444, "mean_itl_ms": 54.641182869583936, "median_itl_ms": 54.228280001552776, "std_itl_ms": 14.070710917241065, "p90_itl_ms": 57.77809280043584, "p95_itl_ms": 58.6477452001418, "p99_itl_ms": 61.60952164125774, "concurrency": 7.995734082700297, "accept_length": null, "max_output_tokens_per_s": 160.0, "max_concurrent_requests": 16}