{
  "OpenCoder-8B-Instruct": {
    "link": "https://huggingface.co/infly/OpenCoder-8B-Instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 81.7,
      "humaneval+": 77.4,
      "mbpp": 82.0,
      "mbpp+": 71.4
    },
    "prompted": true,
    "size": 8.0
  },
  "Artigenz-Coder-DS-6.7B": {
    "link": "https://huggingface.co/Artigenz/Artigenz-Coder-DS-6.7B",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 75.6,
      "humaneval+": 72.6,
      "mbpp": 80.7,
      "mbpp+": 69.6
    },
    "prompted": true,
    "size": 6.7
  },
  "Code-13B": {
    "link": "https://huggingface.co/ajibawa-2023/Code-13B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 56.1,
      "humaneval+": 52.4,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": 13.0
  },
  "Code-290k-6.7B-Instruct": {
    "link": "https://huggingface.co/ajibawa-2023/Code-290k-6.7B-Instruct",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 64.6,
      "humaneval+": 59.7,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": 6.7
  },
  "Code-33B": {
    "link": "https://huggingface.co/ajibawa-2023/Code-33B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 54.9,
      "humaneval+": 49.4,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": 33.0
  },
  "CodeGen-16B": {
    "link": "https://arxiv.org/abs/2203.13474",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 32.9,
      "humaneval+": 28.0,
      "mbpp": 54.2,
      "mbpp+": 45.5
    },
    "prompted": false,
    "size": 16.0
  },
  "CodeGen-2B": {
    "link": "https://arxiv.org/abs/2203.13474",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 24.4,
      "humaneval+": 22.6,
      "mbpp": 46.3,
      "mbpp+": 36.0
    },
    "prompted": false,
    "size": 2.0
  },
  "CodeGen-6B": {
    "link": "https://arxiv.org/abs/2203.13474",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 29.3,
      "humaneval+": 25.6,
      "mbpp": 50.8,
      "mbpp+": 42.9
    },
    "prompted": false,
    "size": 6.0
  },
  "CodeGen2-16B": {
    "link": "https://arxiv.org/abs/2305.02309",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 19.5,
      "humaneval+": 16.5,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 16.0
  },
  "CodeGen2-1B": {
    "link": "https://arxiv.org/abs/2305.02309",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 11.0,
      "humaneval+": 9.1,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 1.0
  },
  "CodeGen2-3B": {
    "link": "https://arxiv.org/abs/2305.02309",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 15.9,
      "humaneval+": 12.8,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 3.0
  },
  "CodeGen2-7B": {
    "link": "https://arxiv.org/abs/2305.02309",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 18.3,
      "humaneval+": 17.7,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 7.0
  },
  "CodeLlama-13B": {
    "link": "https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 42.7,
      "humaneval+": 38.4,
      "mbpp": 63.5,
      "mbpp+": 52.6
    },
    "prompted": false,
    "size": 13.0
  },
  "CodeLlama-34B": {
    "link": "https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 51.8,
      "humaneval+": 43.9,
      "mbpp": 69.3,
      "mbpp+": 56.3
    },
    "prompted": false,
    "size": 34.0
  },
  "CodeLlama-70B": {
    "link": "https://huggingface.co/codellama/CodeLlama-70b-Python-hf",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 55.5,
      "humaneval+": 50.6,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 70.0
  },
  "CodeLlama-70B-Instruct": {
    "link": "https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 72.0,
      "humaneval+": 65.9,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": 70.0
  },
  "CodeLlama-7B": {
    "link": "https://about.fb.com/news/2023/08/code-llama-ai-for-coding/amp/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 37.8,
      "humaneval+": 35.4,
      "mbpp": 59.5,
      "mbpp+": 46.8
    },
    "prompted": false,
    "size": 7.0
  },
  "Qwen2.5-Coder-32B-Instruct": {
    "link": "https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 92.1,
      "humaneval+": 87.2,
      "mbpp": 90.5,
      "mbpp+": 77.0
    },
    "prompted": true,
    "size": 32
  },
  "CodeQwen1.5-7B": {
    "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 51.8,
      "humaneval+": 45.7,
      "mbpp": 73.5,
      "mbpp+": 60.8
    },
    "prompted": false,
    "size": 7
  },
  "CodeQwen1.5-7B-Chat": {
    "link": "https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 83.5,
      "humaneval+": 78.7,
      "mbpp": 79.4,
      "mbpp+": 69.0
    },
    "prompted": true,
    "size": 7
  },
  "Qwen1.5-72B-Chat": {
    "link": "https://huggingface.co/Qwen/Qwen1.5-72B-Chat",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 68.3,
      "humaneval+": 59.1,
      "mbpp": 72.5,
      "mbpp+": 61.6
    },
    "prompted": true,
    "size": 72
  },
  "CodeT5+-16B": {
    "link": "https://blog.salesforceairesearch.com/codet5/",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 31.7,
      "humaneval+": 26.8,
      "mbpp": 56.6,
      "mbpp+": 47.1
    },
    "prompted": false,
    "size": 16.0
  },
  "CodeT5+-2B": {
    "link": "https://blog.salesforceairesearch.com/codet5/",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 25.0,
      "humaneval+": 22.0,
      "mbpp": 48.4,
      "mbpp+": 38.1
    },
    "prompted": false,
    "size": 2.0
  },
  "CodeT5+-6B": {
    "link": "https://blog.salesforceairesearch.com/codet5/",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 29.3,
      "humaneval+": 24.4,
      "mbpp": 52.9,
      "mbpp+": 41.5
    },
    "prompted": false,
    "size": 6.0
  },
  "Command-R+": {
    "link": "https://huggingface.co/CohereForAI/c4ai-command-r-plus",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 64,
      "humaneval+": 56.7,
      "mbpp": 74.3,
      "mbpp+": 63.5
    },
    "prompted": true,
    "size": 104
  },
  "DeepSeek-V3 (Nov 2024)": {
    "link": "https://huggingface.co/deepseek-ai/DeepSeek-V3",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 91.5,
      "humaneval+": 86.6,
      "mbpp": 87.6,
      "mbpp+": 73.0
    },
    "prompted": true,
    "size": 37
  },
  "DeepSeek-V2.5 (Nov 2024)": {
    "link": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 90.2,
      "humaneval+": 83.5,
      "mbpp": 87.6,
      "mbpp+": 74.1
    },
    "prompted": true,
    "size": 21
  },
  "Gemini 1.5 Pro 002": {
    "link": "https://ai.google.dev/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 89.0,
      "humaneval+": 79.3,
      "mbpp": 89.7,
      "mbpp+": 74.6
    },
    "prompted": true,
    "size": null
  },
  "Gemini 1.5 Flash 002": {
    "link": "https://ai.google.dev/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 82.3,
      "humaneval+": 75.6,
      "mbpp": 84.7,
      "mbpp+": 67.5
    },
    "prompted": true,
    "size": null
  },
  "DeepSeek-Coder-V2-Instruct": {
    "link": "https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 85.4,
      "humaneval+": 82.3,
      "mbpp": 89.4,
      "mbpp+": 75.1
    },
    "prompted": true,
    "size": 21
  },
  "DeepSeek-Coder-1.3B-base": {
    "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 28.7,
      "humaneval+": 25.6,
      "mbpp": 56.9,
      "mbpp+": 47.9
    },
    "prompted": false,
    "size": 1.3
  },
  "DeepSeek-Coder-1.3B-instruct": {
    "link": "https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 65.9,
      "humaneval+": 60.4,
      "mbpp": 65.3,
      "mbpp+": 54.8
    },
    "prompted": true,
    "size": 1.3
  },
  "DeepSeek-Coder-33B-base": {
    "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-base",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 51.2,
      "humaneval+": 44.5,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 33.0
  },
  "DeepSeek-Coder-33B-instruct": {
    "link": "https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 81.1,
      "humaneval+": 75.0,
      "mbpp": 80.4,
      "mbpp+": 70.1
    },
    "prompted": true,
    "size": 33.0
  },
  "DeepSeek-Coder-6.7B-base": {
    "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 47.6,
      "humaneval+": 39.6,
      "mbpp": 72.0,
      "mbpp+": 58.7
    },
    "prompted": false,
    "size": 6.7
  },
  "DeepSeek-Coder-6.7B-instruct": {
    "link": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 74.4,
      "humaneval+": 71.3,
      "mbpp": 74.9,
      "mbpp+": 65.6
    },
    "prompted": true,
    "size": 6.7
  },
  "DeepSeek-Coder-7B-instruct-v1.5": {
    "link": "https://huggingface.co/deepseek-ai/deepseek-coder-7b-instruct-v1.5",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 75.6,
      "humaneval+": 71.3,
      "mbpp": 75.2,
      "mbpp+": 62.2
    },
    "prompted": true,
    "size": 7.0
  },
  "Claude Sonnet 3.5 (June 2024)": {
    "link": "https://platform.openai.com/docs/models/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 87.2,
      "humaneval+": 81.7,
      "mbpp": 89.4,
      "mbpp+": 74.3
    },
    "prompted": true,
    "size": null
  },
  "Grok Beta": {
    "link": "https://x.ai/blog/grok-2",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 88.4,
      "humaneval+": 80.5,
      "mbpp": 86.0,
      "mbpp+": 65.6
    },
    "prompted": true,
    "size": null
  },
  "O1 Preview (Sept 2024)": {
    "link": "https://platform.openai.com/docs/models/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 96.3,
      "humaneval+": 89.0,
      "mbpp": 95.5,
      "mbpp+": 80.2
    },
    "prompted": true,
    "size": null
  },
  "O1 Mini (Sept 2024)": {
    "link": "https://platform.openai.com/docs/models/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 96.3,
      "humaneval+": 89.0,
      "mbpp": 93.1,
      "mbpp+": 78.8
    },
    "prompted": true,
    "size": null
  },
  "GPT 4o (Aug 2024)": {
    "link": "https://platform.openai.com/docs/models/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 92.7,
      "humaneval+": 87.2,
      "mbpp": 87.6,
      "mbpp+": 72.2
    },
    "prompted": true,
    "size": null
  },
  "GPT 4o Mini (July 2024)": {
    "link": "https://platform.openai.com/docs/models/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 88.4,
      "humaneval+": 83.5,
      "mbpp": 85.4,
      "mbpp+": 72.2
    },
    "prompted": true,
    "size": null
  },
  "GPT-3.5 (May 2023)": {
    "link": "https://openai.com/blog/chatgpt",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 73.2,
      "humaneval+": 66.5,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": null
  },
  "GPT-3.5-Turbo (Nov 2023)": {
    "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 76.8,
      "humaneval+": 70.7,
      "mbpp": 82.5,
      "mbpp+": 69.7
    },
    "prompted": true,
    "size": null
  },
  "GPT-4 (May 2023)": {
    "link": "https://openai.com/research/gpt-4",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 88.4,
      "humaneval+": 79.3,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": null
  },
  "GPT-4-Turbo (April 2024)": {
    "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 90.2,
      "humaneval+": 86.6,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": null
  },
  "GPT-4-Turbo (Nov 2023)": {
    "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 85.4,
      "humaneval+": 81.7,
      "mbpp": 85.7,
      "mbpp+": 73.3
    },
    "prompted": true,
    "size": null
  },
  "GPT-J-6B": {
    "link": "https://www.eleuther.ai/artifacts/gpt-j",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 12.2,
      "humaneval+": 11.0,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 6.0
  },
  "GPT-Neo-2.7B": {
    "link": "https://www.eleuther.ai/artifacts/gpt-neo",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 7.9,
      "humaneval+": 6.7,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 2.7
  },
  "Gemini Pro 1.0": {
    "link": "https://deepmind.google/technologies/gemini/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 63.4,
      "humaneval+": 55.5,
      "mbpp": 75.4,
      "mbpp+": 61.4
    },
    "prompted": true,
    "size": null
  },
  "Gemini Pro 1.5": {
    "link": "https://blog.google/technology/ai/google-gemini-next-generation-model-february-2024/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 68.3,
      "humaneval+": 61.0,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": null
  },
  "InCoder-1.3B": {
    "link": "https://arxiv.org/abs/2204.05999",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 12.2,
      "humaneval+": 11.0,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 1.3
  },
  "InCoder-6.7B": {
    "link": "https://arxiv.org/abs/2204.05999",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 15.9,
      "humaneval+": 12.2,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 6.7
  },
  "Llama3-70B-instruct": {
    "link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 77.4,
      "humaneval+": 72.0,
      "mbpp": 82.3,
      "mbpp+": 69.0
    },
    "prompted": true,
    "size": 70.0
  },
  "Llama3-8B-base": {
    "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 33.5,
      "humaneval+": 29.3,
      "mbpp": 61.4,
      "mbpp+": 51.6
    },
    "prompted": false,
    "size": 8.0
  },
  "Llama3.1-8B-instruct": {
    "link": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 69.5,
      "humaneval+": 62.8,
      "mbpp": 68.3,
      "mbpp+": 55.6
    },
    "prompted": true,
    "size": 8.0
  },
  "Llama3-8B-instruct": {
    "link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 61.6,
      "humaneval+": 56.7,
      "mbpp": 64.6,
      "mbpp+": 54.8
    },
    "prompted": true,
    "size": 8.0
  },
  "WaveCoder-Ultra-6.7B": {
    "link": "https://huggingface.co/microsoft/wavecoder-ultra-6.7b",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 75.0,
      "humaneval+": 69.5,
      "mbpp": 74.9,
      "mbpp+": 63.5
    },
    "prompted": true,
    "size": 7.0
  },
  "Magicoder-S-CL-7B": {
    "link": "https://huggingface.co/ise-uiuc/Magicoder-S-CL-7B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 70.7,
      "humaneval+": 67.7,
      "mbpp": 70.6,
      "mbpp+": 60.1
    },
    "prompted": true,
    "size": 7.0
  },
  "Magicoder-S-DS-6.7B": {
    "link": "https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 76.8,
      "humaneval+": 71.3,
      "mbpp": 79.4,
      "mbpp+": 69.0
    },
    "prompted": true,
    "size": 6.7
  },
  "Mistral Large (Mar 2024)": {
    "link": "https://mistral.ai/news/mistral-large/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 69.5,
      "humaneval+": 62.2,
      "mbpp": 72.8,
      "mbpp+": 59.5
    },
    "prompted": true,
    "size": null
  },
  "Mistral-7B": {
    "link": "https://mistral.ai/news/announcing-mistral-7b/",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 28.7,
      "humaneval+": 23.8,
      "mbpp": 51.9,
      "mbpp+": 42.1
    },
    "prompted": false,
    "size": 7.0
  },
  "Mistral-codealpaca-7B": {
    "link": "https://huggingface.co/Nondzu/Mistral-7B-codealpaca-lora",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 48.2,
      "humaneval+": 42.1,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 7.0
  },
  "MistralHermes-CodePro-7B-v1": {
    "link": "https://huggingface.co/beowolx/MistralHermes-CodePro-7B-v1",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 47.6,
      "humaneval+": 42.1,
      "mbpp": 57.4,
      "mbpp+": 46.4
    },
    "prompted": true,
    "size": 7.0
  },
  "databricks/dbrx-instruct": {
    "link": "https://huggingface.co/databricks/dbrx-instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 75.0,
      "humaneval+": 70.1,
      "mbpp": 67.2,
      "mbpp+": 55.8
    },
    "prompted": true,
    "size": 36
  },
  "Codestral-22B-v0.1": {
    "link": "https://huggingface.co/mistralai/Codestral-22B-v0.1",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 79.9,
      "humaneval+": 73.8,
      "mbpp": 72.5,
      "mbpp+": 61.9
    },
    "prompted": true,
    "size": 22
  },
  "Mistral-7B-Instruct-v0.2": {
    "link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 42.1,
      "humaneval+": 36.0,
      "mbpp": 44.7,
      "mbpp+": 37.0
    },
    "prompted": true,
    "size": 7
  },
  "Mixtral-8x22B-Instruct-v0.1": {
    "link": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 76.2,
      "humaneval+": 72.0,
      "mbpp": 73.8,
      "mbpp+": 64.3
    },
    "prompted": true,
    "size": 44
  },
  "Mixtral-8x7B-Instruct-v0.1": {
    "link": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 45.1,
      "humaneval+": 39.6,
      "mbpp": 59.5,
      "mbpp+": 49.7
    },
    "prompted": true,
    "size": 13
  },
  "OpenChat-3.5-7B-0106": {
    "link": "https://huggingface.co/openchat/openchat-3.5-0106",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 72.6,
      "humaneval+": 67.7,
      "mbpp": 63.8,
      "mbpp+": 54.5
    },
    "prompted": true,
    "size": 7.0
  },
  "OpenCodeInterpreter-DS-33B": {
    "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 79.3,
      "humaneval+": 73.8,
      "mbpp": 80.2,
      "mbpp+": 68.5
    },
    "prompted": true,
    "size": 33.0
  },
  "OpenCodeInterpreter-DS-6.7B": {
    "link": "https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 77.4,
      "humaneval+": 72.0,
      "mbpp": 76.5,
      "mbpp+": 66.4
    },
    "prompted": true,
    "size": 6.7
  },
  "OpenHermes-2.5-Code-290k-13B": {
    "link": "https://huggingface.co/ajibawa-2023/OpenHermes-2.5-Code-290k-13B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 54.3,
      "humaneval+": 48.8,
      "mbpp": 52.4,
      "mbpp+": 45.8
    },
    "prompted": true,
    "size": 13.0
  },
  "Phind-CodeLlama-34B-v2": {
    "link": "https://huggingface.co/Phind/Phind-CodeLlama-34B-v2",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 71.3,
      "humaneval+": 67.1,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 34.0
  },
  "PolyCoder-2.7B": {
    "link": "https://github.com/VHellendoorn/Code-LMs",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 6.1,
      "humaneval+": 6.1,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 2.7
  },
  "Python-Code-13B": {
    "link": "https://huggingface.co/ajibawa-2023/Python-Code-13B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 32.9,
      "humaneval+": 30.5,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": 13.0
  },
  "SOLAR-10.7B-Instruct-v1.0": {
    "link": "https://huggingface.co/upstage/SOLAR-10.7B-Instruct-v1.0",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 43.3,
      "humaneval+": 37.2,
      "mbpp": 43.9,
      "mbpp+": 36.2
    },
    "prompted": true,
    "size": 10.7
  },
  "SantaCoder-1.1B": {
    "link": "https://arxiv.org/abs/2301.03988",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 14.6,
      "humaneval+": 14.0,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 1.1
  },
  "StableLM-7B": {
    "link": "https://huggingface.co/stabilityai/stablelm-base-alpha-7b",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 2.4,
      "humaneval+": 2.4,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 7.0
  },
  "starchat2-15b-v0.1": {
    "link": "https://huggingface.co/HuggingFaceH4/starchat2-15b-v0.1",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 73.8,
      "humaneval+": 71.3,
      "mbpp": 74.9,
      "mbpp+": 64.6
    },
    "prompted": true,
    "size": 15.0
  },
  "StarCoder-15B": {
    "link": "https://huggingface.co/bigcode/starcoder",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 34.1,
      "humaneval+": 29.3,
      "mbpp": 55.1,
      "mbpp+": 46.1
    },
    "prompted": false,
    "size": 15.0
  },
  "starcoder2-15b-instruct-v0.1": {
    "link": "https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 67.7,
      "humaneval+": 60.4,
      "mbpp": 78.0,
      "mbpp+": 65.1
    },
    "prompted": true,
    "size": 15.0
  },
  "StarCoder2-15B": {
    "link": "https://huggingface.co/bigcode/starcoder2-15b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 46.3,
      "humaneval+": 37.8,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 15.0
  },
  "StarCoder2-3B": {
    "link": "https://huggingface.co/bigcode/starcoder2-3b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 31.7,
      "humaneval+": 27.4,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 3.0
  },
  "StarCoder2-7B": {
    "link": "https://huggingface.co/bigcode/starcoder2-7b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 35.4,
      "humaneval+": 29.9,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 7.0
  },
  "StarCoderBase-1B": {
    "link": "https://huggingface.co/bigcode/starcoderbase-1b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 14.6,
      "humaneval+": 12.2,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 1.0
  },
  "StarCoderBase-3B": {
    "link": "https://huggingface.co/bigcode/starcoderbase-3b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 17.7,
      "humaneval+": 15.9,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 3.0
  },
  "StarCoderBase-7B": {
    "link": "https://huggingface.co/bigcode/starcoderbase-7b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 24.4,
      "humaneval+": 21.3,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 7.0
  },
  "Vicuna-13B": {
    "link": "https://lmsys.org/blog/2023-03-30-vicuna/",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 17.1,
      "humaneval+": 15.9,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 13.0
  },
  "Vicuna-7B": {
    "link": "https://lmsys.org/blog/2023-03-30-vicuna/",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 11.6,
      "humaneval+": 11.6,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": false,
    "size": 7.0
  },
  "WhiteRabbitNeo-33B-v1": {
    "link": "https://huggingface.co/WhiteRabbitNeo/WhiteRabbitNeo-33B-v1",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 72.0,
      "humaneval+": 65.9,
      "mbpp": 79.4,
      "mbpp+": 66.9
    },
    "prompted": true,
    "size": 33.0
  },
  "WizardCoder-15B-V1.0": {
    "link": "https://huggingface.co/WizardLM/WizardCoder-15B-V1.0",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 56.7,
      "humaneval+": 50.6,
      "mbpp": 64.3,
      "mbpp+": 54.2
    },
    "prompted": true,
    "size": 15.0
  },
  "WizardCoder-33B-V1.1": {
    "link": "https://huggingface.co/WizardLM/WizardCoder-33B-V1.1",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 79.9,
      "humaneval+": 73.2,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": 33.0
  },
  "WizardCoder-Python-34B-V1.0": {
    "link": "https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 73.2,
      "humaneval+": 64.6,
      "mbpp": 75.1,
      "mbpp+": 63.2
    },
    "prompted": true,
    "size": 34.0
  },
  "WizardCoder-Python-7B-V1.0": {
    "link": "https://huggingface.co/WizardLM/WizardCoder-Python-7B-V1.0",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 50.6,
      "humaneval+": 45.1,
      "mbpp": 58.5,
      "mbpp+": 49.5
    },
    "prompted": true,
    "size": 7.0
  },
  "XwinCoder-34B": {
    "link": "https://huggingface.co/Xwin-LM/XwinCoder-34B",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 75.6,
      "humaneval+": 69.5,
      "mbpp": 77.0,
      "mbpp+": 64.8
    },
    "prompted": true,
    "size": 34.0
  },
  "Zephyr \u03b2-7B": {
    "link": "https://huggingface.co/HuggingFaceH4/zephyr-7b-beta",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 30.0,
      "humaneval+": 23.2,
      "mbpp": 42.1,
      "mbpp+": 34.7
    },
    "prompted": false,
    "size": 7.0
  },
  "claude-2 (Mar 2024)": {
    "link": "https://www.anthropic.com/news/claude-2",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 69.5,
      "humaneval+": 61.6,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": null
  },
  "claude-3-haiku (Mar 2024)": {
    "link": "https://www.anthropic.com/news/claude-3-family",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 76.8,
      "humaneval+": 68.9,
      "mbpp": 80.2,
      "mbpp+": 68.8
    },
    "prompted": true,
    "size": null
  },
  "claude-3-opus (Mar 2024)": {
    "link": "https://www.anthropic.com/news/claude-3-family",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 82.9,
      "humaneval+": 77.4,
      "mbpp": 89.4,
      "mbpp+": 73.3
    },
    "prompted": true,
    "size": null
  },
  "claude-3-sonnet (Mar 2024)": {
    "link": "https://www.anthropic.com/news/claude-3-family",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 70.7,
      "humaneval+": 64.0,
      "mbpp": 83.6,
      "mbpp+": 69.3
    },
    "prompted": true,
    "size": null
  },
  "claude-instant-1 (Mar 2024)": {
    "link": "https://www.anthropic.com/news/releasing-claude-instant-1-2",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 57.3,
      "humaneval+": 50.6,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": null
  },
  "code-millenials-34B": {
    "link": "https://huggingface.co/budecosystem/code-millenials-34b",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 74.4,
      "humaneval+": 70.7,
      "mbpp": 76.2,
      "mbpp+": 64.6
    },
    "prompted": true,
    "size": 34.0
  },
  "codegemma-2b": {
    "link": "https://huggingface.co/google/codegemma-2b",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 26.8,
      "humaneval+": 20.7,
      "mbpp": 55.6,
      "mbpp+": 46.6
    },
    "prompted": false,
    "size": 2.0
  },
  "codegemma-7b": {
    "link": "https://huggingface.co/google/codegemma-7b",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 44.5,
      "humaneval+": 41.5,
      "mbpp": 65.1,
      "mbpp+": 52.4
    },
    "prompted": false,
    "size": 7.0
  },
  "codegemma-7b-it": {
    "link": "https://huggingface.co/google/codegemma-7b-it",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 60.4,
      "humaneval+": 51.8,
      "mbpp": 70.4,
      "mbpp+": 56.9
    },
    "prompted": true,
    "size": 7.0
  },
  "dolphin-2.6-mixtral-8x7b": {
    "link": "https://huggingface.co/cognitivecomputations/dolphin-2.6-mixtral-8x7b",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 64.0,
      "humaneval+": 57.3,
      "mbpp": 70.6,
      "mbpp+": 59.0
    },
    "prompted": true,
    "size": 13
  },
  "gemma-1.1-2b-it": {
    "link": "https://huggingface.co/google/gemma-1.1-2b-it",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 22.6,
      "humaneval+": 17.7,
      "mbpp": 29.8,
      "mbpp+": 23.3
    },
    "prompted": true,
    "size": 7.0
  },
  "gemma-1.1-7b-it": {
    "link": "https://huggingface.co/google/gemma-1.1-7b-it",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 42.7,
      "humaneval+": 35.4,
      "mbpp": 57.1,
      "mbpp+": 45.0
    },
    "prompted": true,
    "size": 7.0
  },
  "gemma-2b": {
    "link": "https://huggingface.co/google/gemma-2b",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 25.0,
      "humaneval+": 20.7,
      "mbpp": 41.8,
      "mbpp+": 34.1
    },
    "prompted": false,
    "size": 2.0
  },
  "gemma-2b-it": {
    "link": "https://huggingface.co/google/gemma-2b-it",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 17.7,
      "humaneval+": 15.2,
      "mbpp": null,
      "mbpp+": null
    },
    "prompted": true,
    "size": 2.0
  },
  "gemma-7b": {
    "link": "https://huggingface.co/google/gemma-7b",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 35.4,
      "humaneval+": 28.7,
      "mbpp": 52.6,
      "mbpp+": 43.4
    },
    "prompted": false,
    "size": 7.0
  },
  "gemma-7b-it": {
    "link": "https://huggingface.co/google/gemma-7b-it",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 28.7,
      "humaneval+": 25.0,
      "mbpp": 47.1,
      "mbpp+": 36.8
    },
    "prompted": true,
    "size": 7.0
  },
  "Phi-3-mini-4k-instruct": {
    "link": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 64.6,
      "humaneval+": 59.1,
      "mbpp": 65.9,
      "mbpp+": 54.2
    },
    "prompted": true,
    "size": 3.8
  },
  "phi-2-2.7B": {
    "link": "https://huggingface.co/microsoft/phi-2",
    "open-data": "NONE",
    "pass@1": {
      "humaneval": 49.4,
      "humaneval+": 45.1,
      "mbpp": 64.0,
      "mbpp+": 54.2
    },
    "prompted": false,
    "size": 2.7
  },
  "speechless-code-mistral-7B-v1.0": {
    "link": "https://huggingface.co/uukuguy/speechless-code-mistral-7b-v1.0",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 48.2,
      "humaneval+": 41.5,
      "mbpp": 57.4,
      "mbpp+": 48.7
    },
    "prompted": true,
    "size": 7.0
  },
  "speechless-codellama-34B-v2.0": {
    "link": "https://huggingface.co/uukuguy/speechless-codellama-34b-v2.0",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 77.4,
      "humaneval+": 72.0,
      "mbpp": 73.8,
      "mbpp+": 61.4
    },
    "prompted": true,
    "size": 34.0
  },
  "speechless-coder-ds-6.7B": {
    "link": "https://huggingface.co/uukuguy/speechless-coder-ds-6.7b",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 71.3,
      "humaneval+": 65.9,
      "mbpp": 75.9,
      "mbpp+": 64.4
    },
    "prompted": true,
    "size": 6.7
  },
  "speechless-coding-7B-16k-tora": {
    "link": "https://huggingface.co/uukuguy/speechless-coding-7b-16k-tora",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 54.9,
      "humaneval+": 50.6,
      "mbpp": 64.2,
      "mbpp+": 50.6
    },
    "prompted": true,
    "size": 7.0
  },
  "speechless-starcoder2-15b": {
    "link": "https://huggingface.co/uukuguy/speechless-starcoder2-15b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 67.1,
      "humaneval+": 62.8,
      "mbpp": 73.5,
      "mbpp+": 62.4
    },
    "prompted": true,
    "size": 15.0
  },
  "speechless-starcoder2-7b": {
    "link": "https://huggingface.co/uukuguy/speechless-starcoder2-7b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 56.1,
      "humaneval+": 51.8,
      "mbpp": 66.7,
      "mbpp+": 56.3
    },
    "prompted": true,
    "size": 7.0
  },
  "stable-code-3B": {
    "link": "https://huggingface.co/stabilityai/stable-code-3b",
    "open-data": "Full",
    "pass@1": {
      "humaneval": 29.3,
      "humaneval+": 25.6,
      "mbpp": 54.8,
      "mbpp+": 45.8
    },
    "prompted": false,
    "size": 3.0
  },
  "xDAN-L1-Chat-RL-v1-7B": {
    "link": "https://huggingface.co/xDAN-AI/xDAN-L1-Chat-RL-v1",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 40.2,
      "humaneval+": 32.9,
      "mbpp": 50.3,
      "mbpp+": 41.3
    },
    "prompted": true,
    "size": 7.0
  },
  "zyte-1B": {
    "link": "https://huggingface.co/aihub-app/zyte-1B",
    "open-data": "Partial",
    "pass@1": {
      "humaneval": 2.4,
      "humaneval+": 1.8,
      "mbpp": 9.8,
      "mbpp+": 7.5
    },
    "prompted": true,
    "size": 1.0
  }
}