{ "leaderboardData": [ { "info": { "name": "Human Oracle", "date": "-", "type": "human_expert", "frame": "-", "link": "-" }, "validation": {}, "test": { "overall": "95.3" } }, { "info": { "name": "Human Open-book", "date": "-", "type": "human_expert", "frame": "-", "link": "-" }, "validation": {}, "test": { "overall": "86.8" } }, { "info": { "name": "Human Closed-book", "date": "-", "type": "human_expert", "frame": "-", "link": "-" }, "validation": {}, "test": { "overall": "49.7" } }, { "info": { "name": "o1", "date": "2024-12", "type": "proprietary", "frame": "32 frames", "link": "https://openai.com/o1/" }, "validation": { "overall": "75.5" }, "test": { "overall": "76.1" } }, { "info": { "name": "Gemini 2.0 Flash Thinking", "date": "2024-12", "type": "proprietary", "frame": "32 frames", "link": "https://deepmind.google/technologies/gemini/flash-thinking/" }, "validation": { "overall": "69.1" }, "test": { "overall": "69.5" } }, { "info": { "name": "GPT-4o", "date": "2024-08", "type": "proprietary", "frame": "32 frames", "link": "https://openai.com/index/hello-gpt-4o/" }, "validation": { "overall": "67.4" }, "test": { "overall": "66.7" } }, { "info": { "name": "Gemini 2.0 Flash", "date": "2024-12", "type": "proprietary", "frame": "32 frames", "link": "https://deepmind.google/technologies/gemini/flash/" }, "validation": { "overall": "65.9" }, "test": { "overall": "66.5" } }, { "info": { "name": "Gemini 1.5 Pro", "date": "2024-09", "type": "proprietary", "frame": "32 frames", "link": "https://deepmind.google/technologies/gemini/pro/" }, "validation": { "overall": "65.4" }, "test": { "overall": "65.8" } }, { "info": { "name": "Claude 3.5 Sonnet", "date": "2024-10", "type": "proprietary", "frame": "32 frames", "link": "https://www.anthropic.com/news/claude-3-5-sonnet" }, "validation": { "overall": "65.2" }, "test": { "overall": "64.1" } }, { "info": { "name": "Grok-2-Vision", "date": "2024-12", "type": "proprietary", "frame": "32 frames", "link": "https://x.ai/blog/grok-2" }, "validation": { "overall": "62.7" }, "test": { "overall": "63.4" } }, { "info": { "name": "GPT-4o mini", "date": "2024-07", "type": "proprietary", "frame": "32 frames", "link": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/" }, "validation": { "overall": "61.6" }, "test": { "overall": "61.5" } }, { "info": { "name": "Gemini 1.5 Flash", "date": "2024-09", "type": "proprietary", "frame": "32 frames", "link": "https://deepmind.google/technologies/gemini/flash/" }, "validation": { "overall": "58.8" }, "test": { "overall": "58.8" } }, { "info": { "name": "GLM-4V-Plus", "date": "2025-01", "type": "proprietary", "frame": "1 fps", "link": "https://open.bigmodel.cn/dev/api/normal-model/glm-4v" }, "validation": { "overall": "54.1" }, "test": { "overall": "56.7" } }, { "info": { "name": "Qwen2-VL-72B", "date": "2024-09", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/Qwen/Qwen2-VL-72B" }, "validation": { "overall": "53.0" }, "test": { "overall": "53.2" } }, { "info": { "name": "DeepSeek-VL2", "date": "2024-12", "type": "open_source", "frame": "2 frames", "link": "https://huggingface.co/deepseek-ai/deepseek-vl2" }, "validation": { "overall": "52.1" }, "test": { "overall": "51.5" } }, { "info": { "name": "InternVL2.5-38B", "date": "2024-11", "type": "open_source", "frame": "4 frames", "link": "https://huggingface.co/OpenGVLab/InternVL2_5-38B" }, "validation": { "overall": "50.5" }, "test": { "overall": "50.7" } }, { "info": { "name": "Aria", "date": "2024-11", "type": "open_source", "frame": "8 frames", "link": "https://huggingface.co/rhymes-ai/Aria" }, "validation": { "overall": "49.3" }, "test": { "overall": "49.3" } }, { "info": { "name": "InternVideo2.5-8B", "date": "2025-01", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/OpenGVLab/InternVideo2_5_Chat_8B" }, "validation": { "overall": "48.3" }, "test": { "overall": "48.0" } }, { "info": { "name": "Llama-3.2-90B-Vision", "date": "2024-09", "type": "open_source", "frame": "8 frames", "link": "https://huggingface.co/meta-llama/Llama-3.2-90B-Vision" }, "validation": { "overall": "47.1" }, "test": { "overall": "47.6" } }, { "info": { "name": "VideoLLaMA3-7B", "date": "2025-01", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/DAMO-NLP-SG/VideoLLaMA3-7B" }, "validation": { "overall": "45.0" }, "test": { "overall": "47.2" } }, { "info": { "name": "DeepSeek-VL2-Small", "date": "2024-12", "type": "open_source", "frame": "2 frames", "link": "https://huggingface.co/deepseek-ai/deepseek-vl2-small" }, "validation": { "overall": "46.9" }, "test": { "overall": "46.9" } }, { "info": { "name": "VideoChat-Flash-Qwen2-7B (res448)", "date": "2025-01", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2-7B_res448" }, "validation": { "overall": "45.1" }, "test": { "overall": "45.2" } }, { "info": { "name": "Qwen2-VL-7B-Instruct", "date": "2024-08", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct" }, "validation": { "overall": "42.1" }, "test": { "overall": "42.5" } }, { "info": { "name": "InternVL2.5-8B", "date": "2024-11", "type": "open_source", "frame": "4 frames", "link": "https://huggingface.co/OpenGVLab/InternVL2_5-8B" }, "validation": { "overall": "41.1" }, "test": { "overall": "41.0" } }, { "info": { "name": "VideoLLaMA2.1-7B", "date": "2024-10", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/DAMO-NLP-SG/VideoLLaMA2.1-7B-16F" }, "validation": { "overall": "39.5" }, "test": { "overall": "39.8" } }, { "info": { "name": "VideoLLaMA3-2B", "date": "2025-01", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/DAMO-NLP-SG/VideoLLaMA3-2B" }, "validation": { "overall": "38.7" }, "test": { "overall": "39.6" } }, { "info": { "name": "Llama-3.2-11B-Vision", "date": "2024-09", "type": "open_source", "frame": "8 frames", "link": "https://huggingface.co/meta-llama/Llama-3.2-11B-Vision" }, "validation": { "overall": "38.9" }, "test": { "overall": "39.0" } }, { "info": { "name": "Phi-3.5-Vision", "date": "2024-08", "type": "open_source", "frame": "16 frames", "link": "https://huggingface.co/microsoft/Phi-3.5-vision-instruct" }, "validation": { "overall": "38.1" }, "test": { "overall": "38.7" } }, { "info": { "name": "LLaVA-OneVision-7B", "date": "2024-09", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/lmms-lab/llava-onevision-qwen2-7b-ov" }, "validation": { "overall": "37.9" }, "test": { "overall": "37.7" } }, { "info": { "name": "Qwen2-VL-2B", "date": "2024-08", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/Qwen/Qwen2-VL-2B" }, "validation": { "overall": "36.5" }, "test": { "overall": "36.5" } }, { "info": { "name": "InternVL2-8B", "date": "2024-06", "type": "open_source", "frame": "4 frames", "link": "https://huggingface.co/OpenGVLab/InternVL2_5-8B" }, "validation": { "overall": "36.3" }, "test": { "overall": "36.2" } }, { "info": { "name": "Idefics3-8B", "date": "2024-08", "type": "open_source", "frame": "4 frames", "link": "https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3" }, "validation": { "overall": "35.3" }, "test": { "overall": "35.6" } }, { "info": { "name": "VideoLLaMA2-7B", "date": "2024-06", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/DAMO-NLP-SG/VideoLLaMA2-7B" }, "validation": { "overall": "34.4" }, "test": { "overall": "34.4" } }, { "info": { "name": "DeepSeek-VL2-Tiny", "date": "2024-12", "type": "open_source", "frame": "2 frames", "link": "https://huggingface.co/deepseek-ai/deepseek-vl2-tiny" }, "validation": { "overall": "33.0" }, "test": { "overall": "32.8" } }, { "info": { "name": "Pixtral-12B", "date": "2024-09", "type": "open_source", "frame": "8 frames", "link": "https://huggingface.co/mistral-community/pixtral-12b" }, "validation": { "overall": "32.3" }, "test": { "overall": "32.2" } }, { "info": { "name": "LLaVA-NeXT-Video-34B", "date": "2024-06", "type": "open_source", "frame": "8 frames", "link": "https://huggingface.co/lmms-lab/LLaVA-NeXT-Video-34B" }, "validation": { "overall": "30.5" }, "test": { "overall": "30.4" } }, { "info": { "name": "InternVideo2-8B", "date": "2024-08", "type": "open_source", "frame": "1 fps", "link": "https://huggingface.co/OpenGVLab/InternVideo2-Chat-8B" }, "validation": { "overall": "29.9" }, "test": { "overall": "29.8" } }, { "info": { "name": "H2OVL Mississippi-2B", "date": "2024-10", "type": "open_source", "frame": "4 frames", "link": "https://huggingface.co/h2oai/h2ovl-mississippi-2b" }, "validation": { "overall": "29.1" }, "test": { "overall": "28.8" } }, { "info": { "name": "LLaVA-NeXT-Video-7B", "date": "2024-06", "type": "open_source", "frame": "16 frames", "link": "https://huggingface.co/lmms-lab/LLaVA-NeXT-Video-7B" }, "validation": { "overall": "28.6" }, "test": { "overall": "28.8" } } ] }