[
 {
  "id": "bench-01-casp",
  "kind": "benchmark",
  "nn": "01",
  "short_slug": "casp",
  "names": {
   "display": "CASP",
   "full": "Critical Assessment of protein Structure Prediction",
   "full_null_reason": null,
   "aliases": [
    "CASP",
    "CASP14",
    "CASP15",
    "CASP16",
    "CASP17"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "CASP1 to CASP17 are rounds of one challenge, not a family entry.",
  "domain": "structure",
  "domains_secondary": [
   "small-molecule",
   "genomics-rna"
  ],
  "inclusion": "original-30",
  "type": "A",
  "type_note": "Biennial blind prediction challenge",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": null,
   "qualifier": "Truth unpublished at submission"
  },
  "publisher": {
   "orgs": [
    "Protein Structure Prediction Center, University of California, Davis"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Biennial blind test of protein structure prediction; remaining gains lie in complexes, RNA, ligands and ensembles.",
  "first_release": {
   "date": "1994",
   "precision": "year",
   "year_hint": null,
   "basis": "First CASP round (CASP1), 1994; year-only evidence. The meeting month is not sourced and a conference end date is never used as first release.",
   "event_kind": "challenge-open",
   "timestamp_utc": null,
   "source_url": "https://predictioncenter.org/casp1-9/index.cgi",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": null,
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "1994",
    "precision": "year",
    "year_hint": null,
    "basis": "First CASP round (CASP1), 1994; year-only evidence. The meeting month is not sourced and a conference end date is never used as first release.",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://predictioncenter.org/casp1-9/index.cgi",
    "verification": "verified",
    "note": null
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-09-11",
   "precision": "day",
   "year_hint": null,
   "basis": "CASP17 prediction season ended (official page)",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://predictioncenter.org/casp17/index.cgi",
   "verification": "verified",
   "note": "Dates differ across sources: official page says results online 11-30, meeting 12-01 to 12-05",
   "description": "CASP17 prediction season ended 2026-09-11; assessment under way; results due online 2026-11-30, meeting in Rome 2026-12-01 to 05."
  },
  "next_expected": {
   "date": "2026-11-30",
   "precision": "day",
   "year_hint": null,
   "basis": "Organizer announcement: CASP17 results go online",
   "event_kind": "results-published",
   "timestamp_utc": null,
   "source_url": "https://predictioncenter.org/casp17/index.cgi",
   "verification": "partial",
   "note": "CASP18 presumably 2028, not announced"
  },
  "next_expected_null_reason": null,
  "status": "active",
  "status_note": "Active; CASP17 results pending; funding depends on few sponsors",
  "scale": {
   "summary": "CASP16: 210 registered groups (162 expert + 48 server); pharma ligand track 233 pose and 140 affinity targets.",
   "items": [
    {
     "label": "Registered groups",
     "value": "210",
     "unit": "groups",
     "version": "CASP16"
    },
    {
     "label": "Expert / server groups",
     "value": "162 / 48",
     "unit": "groups",
     "version": "CASP16"
    },
    {
     "label": "Protein tertiary targets",
     "value": "156",
     "unit": "targets",
     "version": "CASP16"
    },
    {
     "label": "Multimer targets",
     "value": "108",
     "unit": "targets",
     "version": "CASP16"
    },
    {
     "label": "RNA targets",
     "value": "52",
     "unit": "targets",
     "version": "CASP16"
    },
    {
     "label": "Pharma ligand pose targets",
     "value": "233",
     "unit": "targets",
     "version": "CASP16"
    },
    {
     "label": "Pharma affinity targets (stage 2)",
     "value": "140 (110)",
     "unit": "targets",
     "version": "CASP16"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "Biennial; truth comes from unpublished experimental structures donated by structural biologists; new truth only by entering the next round.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Blind prediction, before release, of protein, complex, RNA, ligand pose and affinity, and ensemble structures from sequence.",
   "truth": "X-ray, cryo-EM and NMR structures held unpublished until assessment.",
   "metric_primary": "Official per-category z-score ranking",
   "metric_secondary": [
    "Monomers: GDT_TS, lDDT",
    "Complexes: QS, DockQ, ICS",
    "RNA: TM-score, INF",
    "Ligands: LDDT-PLI and affinity ranking"
   ],
   "version_traps": "Tracks change by round and assessors rotate, so rounds are not strictly comparable; models trained after a round are not blind to it.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "CASP16 rank by official z-score (Z>-2) per category; GDT_TS (domains), DockQ (oligomers), TM-score (RNA)",
   "eval_set": "CASP16 (2024); CASP17 results pending",
   "snapshot_date": {
    "date": "2025-08-25",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://predictioncenter.org/casp17/index.cgi",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://predictioncenter.org/casp17/index.cgi",
   "entries": [
    {
     "rank": 13,
     "model_label": "AF3-server",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "rank 13 of 110 (protein domains, all); avg GDT_TS 87.1 (top-ranked models)",
     "score_unit": null,
     "metric": "CASP16 rank by official z-score (Z>-2) per category; GDT_TS (domains), DockQ (oligomers), TM-score (RNA)",
     "eval_set": "CASP16 (2024); CASP17 results pending",
     "source_type": "third",
     "date": {
      "date": "2025-08-25",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "CASP16 AF3 assessment paper (Proteins 2026), Table 2; AF2-based methods about 85.5",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 10,
     "model_label": "AF3-server",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "rank 10 of 81 (protein multimers, all); avg DockQ 0.53 (first-ranked models)",
     "score_unit": null,
     "metric": "CASP16 rank by official z-score (Z>-2) per category; GDT_TS (domains), DockQ (oligomers), TM-score (RNA)",
     "eval_set": "CASP16 (2024); CASP17 results pending",
     "source_type": "third",
     "date": {
      "date": "2025-08-25",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "Same paper, Table 2; ranks 2nd on medium-difficulty oligomers",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 8,
     "model_label": "AF3-server",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "rank 8 of 32 (RNA/DNA multimers)",
     "score_unit": null,
     "metric": "CASP16 rank by official z-score (Z>-2) per category; GDT_TS (domains), DockQ (oligomers), TM-score (RNA)",
     "eval_set": "CASP16 (2024); CASP17 results pending",
     "source_type": "third",
     "date": {
      "date": "2025-08-25",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "Same paper, Table 2; RNA monomer rank not cleanly extractable",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 40,
     "model_label": "MassiveFold",
     "model_id": "model-massivefold",
     "site_model": true,
     "score": "rank 40 of 110 (domains); rank 30 of 81 (multimers)",
     "score_unit": null,
     "metric": "CASP16 rank by official z-score (Z>-2) per category; GDT_TS (domains), DockQ (oligomers), TM-score (RNA)",
     "eval_set": "CASP16 (2024); CASP17 results pending",
     "source_type": "third",
     "date": {
      "date": "2025-08-25",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "Same paper, Table 2",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 1,
     "model_label": "trRosettaRNA-family server (academic group)",
     "model_id": "model-rna-structure-models",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "CASP16 rank by official z-score (Z>-2) per category; GDT_TS (domains), DockQ (oligomers), TM-score (RNA)",
     "eval_set": "CASP16 (2024); CASP17 results pending",
     "source_type": "third",
     "date": {
      "date": "2025-08-25",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "Named top group for domains (GDT_TS); a trRosettaRNA-family server per site note, a group not a model; RNA and oligomer tops are other academic groups",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "CASP has no single overall board and CASP17 results are not out yet.",
   "caveat": "CASP17 results due 2026-11-30, no ranking yet. CASP16 has no single overall board; other site models mostly report retrospective numbers on CASP15/16 targets, not comparable.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-massivefold",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Group took part in CASP16",
    "resolution": null
   },
   {
    "model_id": "model-opencomplex2",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://predictioncenter.org/casp16/index.cgi",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CASP16 participant with a talk at the CASP16 meeting; official ranks not found",
    "resolution": null
   },
   {
    "model_id": "model-rna-structure-models",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://www.nature.com/articles/s41592-024-02487-0",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Family entry: trRosettaRNA2 (server) took part in CASP16; RhoFold+ is a CASP15 retrospective evaluation",
    "resolution": null
   },
   {
    "model_id": "model-alphafold3",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11168924/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "AF3 paper used CASP15 RNA targets post hoc; in CASP16 AF3 is only a component of other groups, no own entry",
    "resolution": null
   },
   {
    "model_id": "model-aqaffinity",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://www.sandboxaq.com/post/aqaffinity-open-source-structure-to-affinity-built-on-openfold3-by-sandboxaq",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CASP16 L1000/L3000 post hoc, figure only, L3000 overlaps training",
    "resolution": null
   },
   {
    "model_id": "model-boltz-2",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12262699/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Retrospective CASP16 affinity data (2 targets, 140 pairs); not a blind participation",
    "resolution": null
   },
   {
    "model_id": "model-chai-1",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "CASP15 monomer LDDT, second-hand quote only; post hoc",
    "resolution": null
   },
   {
    "model_id": "model-esm3",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://www.biorxiv.org/content/10.1101/2024.07.01.600583v2.full",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "CASP14/15 time-split post hoc",
    "resolution": null
   },
   {
    "model_id": "model-isodde",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://zenodo.org/records/18606681",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CASP16 blind affinity data used post hoc, figure only",
    "resolution": null
   },
   {
    "model_id": "model-nesso-1",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://www.valencelabs.com/wp-content/uploads/2026/07/nesso1.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Added as a missing candidate in round 7 and confirmed in review.",
    "resolution": "Post-hoc use of the released affinity set (140 complexes, 2 targets), not participation in the blind round; the same rule applies to DPLM-2 × CAMEO."
   },
   {
    "model_id": "model-neuralplexer3",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2412.10743v2",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Matrix marks a self-report, but page evidence is only CASP15 RNA, duplicated from the RNA-Puzzles column",
    "resolution": null
   },
   {
    "model_id": "model-openfold3",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://github.com/aqlaboratory/openfold-3",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CASP16 protein/RNA monomers, README figure only, post hoc",
    "resolution": null
   },
   {
    "model_id": "model-profluent-e1",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://www.biorxiv.org/content/10.1101/2025.11.12.688125v1.full",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CASP15 contact maps, numbers not retrieved",
    "resolution": null
   },
   {
    "model_id": "model-terrabind",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2602.07735",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Retrospective CASP16 affinity evaluation",
    "resolution": null
   },
   {
    "model_id": "model-xtrimopglm",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2401.06199",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Post hoc on CASP15 (n=56)",
    "resolution": null
   },
   {
    "model_id": "model-enchant-v1-v2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.iambic.ai/news/enchant-3",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "disputed",
    "note": "v3 page claims held-out structures from this round, figure only, no protocol; split from a combined candidate and kept unconfirmed.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 15,
   "by_relation": {
    "official-participation": 3,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 12,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 1
   }
  },
  "how_to_use": "CASP17's prediction season is closed; the next round is presumably 2028 (not announced). Released CASP14–16 targets can serve as a retrospective set if the training cutoff predates target release; score with the official LGA, lDDT and OpenStructure tools. Report round, track, number of targets, model 1 versus best-of-5, and training cutoff.",
  "pairs_with": [
   "bench-15-cameo",
   "bench-22-capri",
   "bench-05-posebusters",
   "bench-25-rnapuzzles",
   "bench-02-proteingym"
  ],
  "caveats": [
   "Complex, RNA and ligand tracks have few targets, so rankings hinge on a handful of targets and rarely carry confidence intervals.",
   "Single-chain accuracy is near experimental uncertainty; remaining headroom is in complexes, RNA, ligands and ensembles.",
   "Models trained after a round are not blind to its targets; post hoc use must check training cutoff against target release dates.",
   "Funding is fragile: after the NIH grant lapsed in 2025-07, a one-off donation from a leading developer in the field filled the gap (press report; amount undisclosed).",
   "Does not measure wet-lab hit rates; causal evidence from structure accuracy to discovery hit rates is weak."
  ],
  "sources": [
   {
    "url": "https://predictioncenter.org/casp17/index.cgi",
    "title": "CASP17 official page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://predictioncenter.org/casp16/numbers.cgi",
    "title": "CASP16 official numbers page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://predictioncenter.org/casp1-9/index.cgi",
    "title": "CASP1–9 round list",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://pubmed.ncbi.nlm.nih.gov/41178755/",
    "title": "CASP16 overview (Proteins 2026)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750038/",
    "title": "CASP16 pharma ligand assessment (Proteins)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
    "title": "AlphaFold 3 in CASP16 (Proteins 2026)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC9299164/",
    "title": "CASP14 domain assessment (Proteins 2021)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://www.statnews.com/2025/07/21/casp-new-funding-from-alphafold-developer-google-deepmind-after-nih-grant-runs-out/",
    "title": "STAT: CASP funding report",
    "kind": "press",
    "date": null
   },
   {
    "url": "https://events.humanitix.com/casp17",
    "title": "CASP17 meeting registration page",
    "kind": "official-site",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "1994-12",
    "now": "1994",
    "reason": "Only the year of CASP1 is sourced; the December Asilomar meeting month is not, and a meeting end date is not a first release (DATE_AUDIT, FIX_LIST B).",
    "source_url": "https://predictioncenter.org/casp1-9/index.cgi",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "first CASP (CASP1) meeting at Asilomar, December 1994 (prediction season earlier in 1994)",
    "now": "first CASP round, 1994 (year-only evidence; prediction season 1994). Meeting month not sourced; conference end date never used as first release.",
    "reason": "Basis realigned to the year-only value.",
    "source_url": "https://predictioncenter.org/casp1-9/index.cgi",
    "date_found": "2026-11-16"
   },
   {
    "field": "scale",
    "was": "209 groups registered, ~300 targets, 200+ pharma co-crystals, ~160 affinity data",
    "now": "210 groups registered (162 expert + 48 server); 233 pharma pose ligand targets; 140 pharma affinity targets (110 at stage 2); 156 tertiary + 108 multimer + 52 RNA targets; no single ~300-target figure",
    "reason": "The official numbers page gives 210 registered groups, 233 pose targets and 140 affinity targets; no \"~300 targets\" figure exists there.",
    "source_url": "https://predictioncenter.org/casp16/numbers.cgi",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported.model-nesso-1",
    "was": "not listed (zero-pairing list)",
    "now": "retrospective-dataset-use",
    "reason": "Use of the released CASP16 affinity set is post-hoc data reuse, not participation.",
    "source_url": "https://www.valencelabs.com/wp-content/uploads/2026/07/nesso1.pdf",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "next_expected",
   "models_reported[0]",
   "models_reported[6]",
   "models_reported[15]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-01-casp/",
   "en": "/bio/benchmark/benchmarks/casp/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-02-proteingym",
  "kind": "benchmark",
  "nn": "02",
  "short_slug": "proteingym",
  "names": {
   "display": "ProteinGym",
   "full": null,
   "full_null_reason": "The name is not an abbreviation, so there is no expanded form.",
   "aliases": [
    "ProteinGym",
    "ProteinGym v1",
    "ProteinGym v1.3",
    "PG_v1.3"
   ],
   "ambiguous_aliases": [
    "ProteinGym v0"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "ProteinGym v0 (inside the Tranception preprint)",
     "id_or_null": null,
     "first_release": {
      "date": "2022-05-27",
      "precision": "day",
      "year_hint": null,
      "basis": "Tranception preprint, arXiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2205.13760",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "ProteinGym v1 (standalone benchmark, NeurIPS 2023 Datasets and Benchmarks)",
     "id_or_null": null,
     "first_release": {
      "date": "2023-12-08",
      "precision": "day",
      "year_hint": null,
      "basis": "Standalone benchmark paper, bioRxiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1101/2023.12.07.570727",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "v0 is the substitution set inside the Tranception preprint; v1 onward is the standalone benchmark adding indels and a clinical subset. ViroGym and the R client are excluded."
  },
  "family_null_reason": null,
  "domain": "protein",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static DMS suite with a maintainer-run leaderboard",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Retrospective; supervised positional splits near L3"
  },
  "publisher": {
   "orgs": [
    "Harvard Medical School, Department of Systems Biology",
    "University of Oxford, OATML group"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Unified suite of 200+ public deep mutational scans for protein variant-effect and fitness ranking.",
  "first_release": {
   "date": "2022-05-27",
   "precision": "day",
   "year_hint": null,
   "basis": "Earliest family member: ProteinGym v0 in the Tranception preprint (arXiv v1); the standalone benchmark paper is bioRxiv 2023-12-08.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2205.13760",
   "verification": "partial",
   "note": "Family-level date; the standalone benchmark first appeared 2023-12-08"
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2022-05-27",
    "precision": "day",
    "year_hint": null,
    "basis": "First appearance of the benchmark name, in the Tranception preprint",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2205.13760",
    "verification": "verified",
    "note": null
   },
   "paper_v1": {
    "date": "2023-12-08",
    "precision": "day",
    "year_hint": null,
    "basis": "Standalone benchmark, bioRxiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/2023.12.07.570727",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2025-04-28",
    "precision": "day",
    "year_hint": null,
    "basis": "PG_v1.3 data release on the repository releases page; the year comes from an existing site note, as the page shows only day and month",
    "event_kind": "data-release",
    "timestamp_utc": null,
    "source_url": "https://pypi.org/project/proteingym/",
    "verification": "partial",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-03-25",
   "precision": "day",
   "year_hint": null,
   "basis": "Official repository commit history",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://proteingym.org/",
   "verification": "partial",
   "note": null,
   "description": "A repository commit restored missing benchmark files; an earlier commit added a supervised clinical evaluation script. Neither carries a new release tag."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have not announced v1.4 or a next generation.",
  "status": "low-frequency",
  "status_note": "Active but low-frequency; latest data version v1.3 (2025)",
  "scale": {
   "summary": "v1.3: 217 substitution assays, 74 indel assays, clinical subset of 2,525 proteins.",
   "items": [
    {
     "label": "Substitution DMS assays",
     "value": "217",
     "unit": null,
     "version": "v1.3"
    },
    {
     "label": "Substitution variants",
     "value": "about 2.7 million",
     "unit": null,
     "version": "v1.3"
    },
    {
     "label": "Indel assays",
     "value": "74",
     "unit": null,
     "version": "v1.3"
    },
    {
     "label": "Indel variants",
     "value": "about 300,000",
     "unit": null,
     "version": "v1.3"
    },
    {
     "label": "Proteins in the clinical subset; the data card also states 2,575",
     "value": "2525",
     "unit": null,
     "version": "v1.3"
    },
    {
     "label": "Model rows on the official summary table, including sizes of one model",
     "value": "97",
     "unit": null,
     "version": "2026-11-16"
    }
   ]
  },
  "wetlab_refresh": "accumulating",
  "wetlab_note": "No new experiments; published DMS assays are added irregularly and recent releases mainly added baselines.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Zero-shot and supervised ranking of protein fitness and variant effects over public deep mutational scans, plus a clinical subset.",
   "truth": "Published DMS readouts (growth, expression, binding, catalysis, stability) and expert clinical labels for the clinical subset.",
   "metric_primary": "Average zero-shot Spearman",
   "metric_secondary": [
    "NDCG, AUC, MCC, Top-K recall",
    "Supervised Spearman and MSE",
    "AUC on the clinical subset"
   ],
   "version_traps": "State the data version and track (zero-shot, supervised, clinical): assay counts and baselines differ by version, so scores do not compare across versions.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Average Spearman (zero-shot, DMS substitutions)",
   "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
   "snapshot_date": {
    "date": "2025-06-13",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://proteingym.org/",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://proteingym.org/",
   "entries": [
    {
     "rank": 1,
     "model_label": "AIDO Protein-RAG (16B)",
     "model_id": null,
     "site_model": false,
     "score": "0.518 (SE 0.0)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": "Not a site model; tied first; summary table last changed 2025-06-13, read 2026-11-16",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "VenusREM",
     "model_id": "model-venusrem",
     "site_model": true,
     "score": "0.518 (SE 0.005)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": "Ties first on score; the gap is within bootstrap standard error",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "ProSST (K=2048)",
     "model_id": "model-prosst",
     "site_model": true,
     "score": "0.507 (SE 0.006)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": "Other K variants of this model rank 4th, 7th, 10th, 13th and 28th",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 14,
     "model_label": "ESM3 open (1.4B)",
     "model_id": "model-esm3",
     "site_model": true,
     "score": "0.466 (SE 0.006)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 17,
     "model_label": "SaProt (650M)",
     "model_id": "model-saprot",
     "site_model": true,
     "score": "0.457 (SE 0.007)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": "The 35M version of the same model ranks 48th",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 49,
     "model_label": "ESM-C (300M)",
     "model_id": "model-esm-c",
     "site_model": true,
     "score": "0.406 (SE 0.013)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": "The 600M version of the same model ranks 51st",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 54,
     "model_label": "xTrimoPGLM-3B-MLM",
     "model_id": "model-xtrimopglm",
     "site_model": true,
     "score": "0.399 (SE 0.012)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": "Best-performing variant of this model; its 100B-int4 version ranks 74th",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 59,
     "model_label": "Progen3 3B",
     "model_id": "model-progen3",
     "site_model": true,
     "score": "0.392 (SE 0.008)",
     "score_unit": null,
     "metric": "Average Spearman (zero-shot, DMS substitutions)",
     "eval_set": "ProteinGym v1.3, 217 substitution DMS assays, zero-shot",
     "source_type": "board",
     "date": {
      "date": "2025-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proteingym.org/",
      "verification": "verified",
      "note": null
     },
     "url": "https://proteingym.org/",
     "note": "The 1B version of the same model ranks 61st",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": 1,
   "headline_null_reason": null,
   "caveat": "The summary table has 97 rows and gaps below about 0.01 fall within bootstrap standard error. Some models appear only in author or second-hand claims, not on the official table.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-esm-c",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteingym.org/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-esm3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteingym.org/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-progen3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteingym.org/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-prosst",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteingym.org/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-saprot",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteingym.org/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-venusrem",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteingym.org/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-xtrimopglm",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteingym.org/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-poet-2",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2508.04724v1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Paper table claims the official protocol; not found on the official table",
    "resolution": null
   },
   {
    "model_id": "model-profluent-e1",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://github.com/Profluent-AI/E1#comparison-to-other-models",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Only a curve in the project README; no row on the official summary table",
    "resolution": null
   },
   {
    "model_id": "model-frontier-llms",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "low",
    "verification": "unverified",
    "note": "Scores from a second-hand aggregator; the entry is a family, not one model",
    "resolution": null
   }
  ],
  "non_pairings": [
   {
    "model_id": "model-evolvepro",
    "note": "Twelve of its DMS sets are used only as a data source; recorded explicitly as a non-pair so it is not re-added later.",
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11275896/"
   }
  ],
  "n_models_reported": {
   "counted": 10,
   "by_relation": {
    "official-participation": 7,
    "self-reported-official-protocol": 2,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 1,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Download the reference files, score every assay's variants, run the official scripts and submit by repository pull request; no own wet lab is needed. Report the data version, track, whether MSA or structure inputs were used, bootstrap standard errors and NDCG, split by assay type and taxon. Claims about design hit rates need a prospective assay of your own.",
  "pairs_with": [
   "bench-11-clinvar",
   "bench-13-cagi",
   "bench-26-flip",
   "bench-39-rnagym",
   "bench-37-traitgym"
  ],
  "caveats": [
   "Static and tunable: mean Spearman hides assay heterogeneity and gaps below the bootstrap standard error (about 0.01) are not meaningful.",
   "No direct evidence links scores to wet-lab hit rates, and board position does not establish design or selection ability.",
   "Public data can be tuned against and there is no time freeze; the clinical subset risks circularity with predictors trained on the same clinical database.",
   "Metrics disagree: Spearman and NDCG pick different best models within assay categories.",
   "Coverage is skewed: most methods degrade systematically on viral proteins and proteins with disordered regions."
  ],
  "sources": [
   {
    "url": "https://proteingym.org/",
    "title": "Official benchmark site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://doi.org/10.1101/2023.12.07.570727",
    "title": "Standalone benchmark paper on bioRxiv (NeurIPS 2023 Datasets and Benchmarks)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2205.13760",
    "title": "Tranception preprint, first appearance of the v0 substitution set",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://pypi.org/project/proteingym/",
    "title": "Official toolkit on PyPI",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2504.16886",
    "title": "Third-party analysis of structure-input effects",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update",
    "was": "2026-01-12",
    "now": "2026-03-25",
    "reason": "The newest repository commit restores missing benchmark files and postdates the previously recorded clinical-script commit.",
    "source_url": "https://proteingym.org/",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "start year 2023",
    "now": "2022-05-27 at family level (v0 in the Tranception preprint), with the standalone benchmark at 2023-12-08 and the family boundary stated",
    "reason": "Date audit: the family entry must separate v0 inside the preprint from the standalone benchmark, and a repository creation date cannot serve as first release.",
    "source_url": "https://arxiv.org/abs/2205.13760",
    "date_found": "2026-11-16"
   },
   {
    "field": "n_models_reported",
    "was": "9",
    "now": "10 under the new relation taxonomy, counting established relations only",
    "reason": "The count was recomputed this round under the new relation taxonomy.",
    "source_url": "https://proteingym.org/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "release_milestones.code_release",
   "latest_update",
   "scale.items[4]",
   "models_reported[8]",
   "models_reported[9]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-02-proteingym/",
   "en": "/bio/benchmark/benchmarks/proteingym/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-03-moleculenet",
  "kind": "benchmark",
  "nn": "03",
  "short_slug": "moleculenet",
  "names": {
   "display": "MoleculeNet (+OGB)",
   "full": "MoleculeNet: a benchmark suite for molecular machine learning (with OGB-derived molecular sets)",
   "full_null_reason": null,
   "aliases": [
    "MoleculeNet",
    "MolNet",
    "molnet",
    "ogbg-molhiv",
    "ogbg-molpcba"
   ],
   "ambiguous_aliases": [
    "OGB"
   ]
  },
  "family": null,
  "family_null_reason": "Single dataset suite; OGB molecular sets are derived splits recorded as a version trap, not a family.",
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static dataset suite with no official leaderboard",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Random split L2; scaffold split (weak OOD) L3"
  },
  "publisher": {
   "orgs": [
    "Stanford University (academic group)",
    "DeepChem open-source project"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Historical default suite for molecular representation learning: 17 public datasets, for literature alignment, not drug-discovery claims.",
  "first_release": {
   "date": "2017-03-02",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv 1703.00564 v1",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/1703.00564",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2017-03-02",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv 1703.00564 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/1703.00564",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2018",
    "precision": "year",
    "year_hint": null,
    "basis": "Chem. Sci. journal version (year only; exact online date not retrieved)",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1039/c7sc02664a",
    "verification": "partial",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2018-10-26",
   "precision": "day",
   "year_hint": null,
   "basis": "Last-updated date of v3 in the arXiv API",
   "event_kind": "paper-revision",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/1703.00564",
   "verification": "verified",
   "note": "DeepChem loaders are still maintained; library upkeep, not a benchmark change",
   "description": "arXiv v3 posted (final paper revision); Chem. Sci. version published 2018; no versioned data snapshot since."
  },
  "next_expected": null,
  "next_expected_null_reason": "The datasets are frozen and no new version has been announced.",
  "status": "frozen",
  "status_note": "Effectively frozen since 2018; still widely reused for literature alignment",
  "scale": {
   "summary": "17 datasets in the paper, from FreeSolv (642) to PCBA (437,929 compounds); QM9 has 133,885 molecules.",
   "items": [
    {
     "label": "Datasets",
     "value": "17",
     "unit": null,
     "version": "paper (Chem. Sci. 2018)"
    },
    {
     "label": "Smallest dataset (FreeSolv)",
     "value": "642",
     "unit": "molecules",
     "version": "paper"
    },
    {
     "label": "Largest dataset (PCBA)",
     "value": "437,929",
     "unit": "compounds",
     "version": "paper"
    },
    {
     "label": "QM9 molecules",
     "value": "133,885",
     "unit": "molecules",
     "version": "paper"
    },
    {
     "label": "DeepChem molnet loaders",
     "value": "44",
     "unit": null,
     "version": "DeepChem master, checked 2026-11-16 (includes non-paper datasets)"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No new experiments; public databases are reused and QM9 is a computed DFT oracle.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Small-molecule property prediction across 17 public datasets covering quantum, physical chemistry, biophysics and toxicity endpoints.",
   "truth": "DFT values for QM sets; others from literature compilations, PubChem screens and Tox21 cell assays.",
   "metric_primary": "ROC-AUC (classification); RMSE/MAE (regression)",
   "metric_secondary": [
    "PRC-AUC (PCBA, MUV)"
   ],
   "version_traps": "Same-name splits differ by implementation; OGB sets use fixed scaffold splits, not comparable; Tox21 here differs from the original challenge.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "ROC-AUC (classification), MAE/RMSE (regression)",
   "eval_set": "MoleculeNet 17 datasets; splits (scaffold/random) and seeds differ per paper",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "MoLFormer-XL",
     "model_id": "model-molformer-xl",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "ROC-AUC (classification), MAE/RMSE (regression)",
     "eval_set": "MoleculeNet 17 datasets; splits (scaffold/random) and seeds differ per paper",
     "source_type": "self",
     "date": {
      "date": "2021-06-17",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2106.09553v3",
      "verification": "verified",
      "note": "(arXiv v1; v3 fetched)"
     },
     "url": "https://arxiv.org/html/2106.09553v3",
     "note": "Self-reported (Table 1, scaffold splits): authors claim first on 3 of 6 sets and close second on the rest; no board, self-reported positions only.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "not-comparable: protocols/subsets differ, so no score is displayed in the ranking row; the per-protocol values live on the entity page."
    },
    {
     "rank": null,
     "model_label": "Uni-Mol2",
     "model_id": "model-uni-mol2",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "ROC-AUC (classification), MAE/RMSE (regression)",
     "eval_set": "MoleculeNet 17 datasets; splits (scaffold/random) and seeds differ per paper",
     "source_type": "self",
     "date": {
      "date": "2024-06-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2406.14969v2",
      "verification": "verified",
      "note": "(arXiv 2406.14969)"
     },
     "url": "https://arxiv.org/html/2406.14969v2",
     "note": "Reports only 8 QM9 properties with a scaffold rather than the recommended random split; per-property values not re-verified this round.",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "not-comparable: protocols/subsets differ, so no score is displayed in the ranking row; the per-protocol values live on the entity page."
    },
    {
     "rank": null,
     "model_label": "Chemprop v2",
     "model_id": "model-chemprop-v2",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "ROC-AUC (classification), MAE/RMSE (regression)",
     "eval_set": "MoleculeNet 17 datasets; splits (scaffold/random) and seeds differ per paper",
     "source_type": "third",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": null,
      "verification": "unverified",
      "note": null
     },
     "url": null,
     "note": "No v2-specific MoleculeNet numbers exist; circulating numbers belong to the 2019 D-MPNN original; pairing unverified.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Incomplete provenance (missing url); the honesty rule forbids displaying a number."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "Not comparable: no official board and each paper re-splits, so no headline number.",
   "caveat": "No official leaderboard; data frozen since 2018 and each paper re-splits, so cross-paper numbers are no ranking; OGB-derived sets are not comparable.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-molformer-xl",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2106.09553",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "MoleculeNet scaffold/random splits",
    "resolution": null
   },
   {
    "model_id": "model-uni-mol2",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2406.14969v2",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "QM9 only, scaffold split",
    "resolution": null
   },
   {
    "model_id": "model-chemprop-v2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/1904.01561",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "Page marks it unconfirmed; numbers belong to the 2019 original",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 2,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 1,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 1
   }
  },
  "how_to_use": "Run statically via DeepChem loaders or OGB fixed splits; no wet lab, retrospective evidence only. Use fixed scaffold or cluster splits, report an ECFP plus tree-model baseline, at least 10 seeds with SD, cleaned BBBP/BACE, and deduplicate pretraining data. Pair with TDC, Polaris and FS-Mol.",
  "pairs_with": [
   "bench-04-tdc",
   "bench-27-polaris",
   "bench-44-fsmol",
   "bench-06-casf"
  ],
  "caveats": [
   "Known data errors are not corrected, and simple fingerprint baselines often match pretrained models; split implementations differ, so numbers are often not comparable.",
   "Third-party analyses report unparsable SMILES and duplicates in BBBP, stereoisomer activity gaps in BACE, and an ESOL range far beyond industrial assays.",
   "A 2025 preprint reports that MoleculeNet's Tox21 training and test sets differ from the original challenge.",
   "A 2025 preprint found that of 25 pretrained molecular embedding models, all but one fail to beat an ECFP baseline.",
   "No blind or prospective test; no study links its scores to prospective hit rates."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/1703.00564",
    "title": "MoleculeNet paper (arXiv 1703.00564)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.1039/c7sc02664a",
    "title": "MoleculeNet journal version (Chem. Sci. 2018)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://github.com/deepchem/deepchem/blob/master/deepchem/molnet/__init__.py",
    "title": "DeepChem molnet loader source",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://practicalcheminformatics.blogspot.com/2023/08/we-need-better-benchmarks-for-machine.html",
    "title": "Cheminformatics blog: benchmark data-quality analysis (2023-08)",
    "kind": "second-hand",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2508.06199",
    "title": "Pretrained molecular embeddings vs ECFP baseline (arXiv 2508.06199)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2511.14744",
    "title": "Tox21 reproduction study (arXiv 2511.14744)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://ogb.stanford.edu/docs/graphprop/",
    "title": "OGB graph property prediction docs",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2106.09553",
    "title": "MoLFormer-XL paper (arXiv 2106.09553)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/html/2406.14969v2",
    "title": "Uni-Mol2 paper (arXiv 2406.14969)",
    "kind": "preprint",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "ranking.entries[2] (Chemprop v2)",
    "was": "MoleculeNet × Chemprop v2 listed as a pairing with numbers",
    "now": "withheld: incomplete provenance (missing url); numbers belong to the 2019 D-MPNN original",
    "reason": "CHANGESET §5.3: the entry lacks a url and is withheld under the honesty rule; §4.2 marks the pairing unverified.",
    "source_url": "https://arxiv.org/abs/1904.01561",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "release_milestones.journal_online",
   "ranking.entries[1]",
   "ranking.entries[2]",
   "models_reported[2]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-03-moleculenet/",
   "en": "/bio/benchmark/benchmarks/moleculenet/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-04-tdc",
  "kind": "benchmark",
  "nn": "04",
  "short_slug": "tdc",
  "names": {
   "display": "TDC (ADMET + DTI)",
   "full": "Therapeutics Data Commons",
   "full_null_reason": null,
   "aliases": [
    "TDC",
    "TDC-2",
    "PyTDC",
    "TDC ADMET Benchmark Group",
    "Therapeutics Data Commons"
   ],
   "ambiguous_aliases": []
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "PyTDC 0.0.1 (PyPI)",
     "id_or_null": null,
     "first_release": {
      "date": "2020-11-13",
      "precision": "day",
      "year_hint": null,
      "basis": "First PyPI upload, version 0.0.1",
      "event_kind": "code-release",
      "timestamp_utc": null,
      "source_url": "https://pypi.org/pypi/PyTDC/json",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "TDC paper",
     "id_or_null": null,
     "first_release": {
      "date": "2021-02-18",
      "precision": "day",
      "year_hint": null,
      "basis": "TDC paper, arXiv 2102.09548 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2102.09548",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "TDC-2",
     "id_or_null": null,
     "first_release": {
      "date": "2024-06-14",
      "precision": "day",
      "year_hint": null,
      "basis": "TDC-2 bioRxiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2024.06.12.598655",
      "verification": "partial",
      "note": null
     }
    },
    {
     "name": "PyTDC paper",
     "id_or_null": null,
     "first_release": {
      "date": "2025-05-08",
      "precision": "day",
      "year_hint": null,
      "basis": "PyTDC paper, arXiv 2505.05577 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2505.05577",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "TDC, TDC-2 and PyTDC are software and paper iterations of one platform; this entry evaluates only the ADMET and DTI-DG groups, not TDC-2 multimodal data."
  },
  "family_null_reason": null,
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static dataset suite with submission leaderboards",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Scaffold split; DTI-DG reaches L3"
  },
  "publisher": {
   "orgs": [
    "Harvard Medical School (Department of Biomedical Informatics)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "De facto ADMET leaderboard; useful for method screening, not a basis for development decisions.",
  "first_release": {
   "date": "2020-11-13",
   "precision": "day",
   "year_hint": null,
   "basis": "Earliest public evidence: first PyPI upload PyTDC 0.0.1 (2020-11-13); TDC paper arXiv v1 2021-02-18; TDC-2 bioRxiv v1 2024-06-14; PyTDC paper arXiv v1 2025-05-08.",
   "event_kind": "code-release",
   "timestamp_utc": null,
   "source_url": "https://pypi.org/pypi/PyTDC/json",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2021-02-18",
    "precision": "day",
    "year_hint": null,
    "basis": "TDC paper arXiv 2102.09548 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2102.09548",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2020-11-13",
    "precision": "day",
    "year_hint": null,
    "basis": "PyTDC 0.0.1 uploaded to PyPI",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://pypi.org/pypi/PyTDC/json",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-01-21",
   "precision": "day",
   "year_hint": null,
   "basis": "Revision of the PyTDC paper (arXiv 2505.05577)",
   "event_kind": "paper-revision",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2505.05577",
   "verification": "verified",
   "note": null,
   "description": "PyTDC paper (arXiv 2505.05577, v1 2025-05-08) revised; latest PyPI release 1.1.15 on 2025-03-31, last repo commit 2025-04-18."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no further release or ADMET group revision.",
  "status": "active",
  "status_note": "Active; ADMET group data and rules not revised since 2021",
  "scale": {
   "summary": "ADMET group: 22 datasets (475–13,130 records); full library 66 datasets across 22 tasks in the original paper.",
   "items": [
    {
     "label": "ADMET group datasets",
     "value": "22",
     "unit": "datasets",
     "version": "ADMET Benchmark Group"
    },
    {
     "label": "Records per dataset",
     "value": "475–13,130",
     "unit": "records",
     "version": "ADMET Benchmark Group"
    },
    {
     "label": "DTI-DG group datasets",
     "value": "1",
     "unit": "datasets",
     "version": "BindingDB_Patent"
    },
    {
     "label": "Datasets in original library",
     "value": "66",
     "unit": "datasets",
     "version": "TDC paper (2021)"
    },
    {
     "label": "Official benchmark groups",
     "value": "8",
     "unit": "groups",
     "version": null
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None for the ADMET group; data are reused from ChEMBL and public deposits.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "ADMET endpoint prediction under a scaffold split, plus one drug-target interaction domain-generalization task.",
   "truth": "Labels compiled from literature and databases, not measured under uniform conditions.",
   "metric_primary": "Per task: MAE, AUROC, AUPRC or Spearman",
   "metric_secondary": [
    "Mean and SD over 5 seeds",
    "DTI-DG: Pearson correlation"
   ],
   "version_traps": "TDC datasets differ from official groups: Davis and KIBA are not official groups; boards lack versions and dates, and loaders may yield different molecule counts.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "per-task: MAE / AUROC / AUPRC / Spearman (5-seed mean ± sd), scaffold split",
   "eval_set": "TDC ADMET Benchmark Group, 22 datasets",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "Board retrieval date (board pages themselves carry no dates)",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://tdcommons.ai/benchmark/admet_group/overview/",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://tdcommons.ai/benchmark/admet_group/overview/",
   "entries": [
    {
     "rank": null,
     "model_label": "MiniMol",
     "model_id": "model-minimol",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "per-task: MAE / AUROC / AUPRC / Spearman (5-seed mean ± sd), scaffold split",
     "eval_set": "TDC ADMET Benchmark Group, 22 datasets",
     "source_type": "board",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "Entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://tdcommons.ai/benchmark/admet_group/overview/",
      "verification": "unverified",
      "note": "Board pages carry no dates; a retrieval day is not an entry date"
     },
     "url": "https://tdcommons.ai/benchmark/admet_group/overview/",
     "note": "Ties with MapLight + GNN for the most first places across the 22 task boards.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Board entries carry no date, so under the honesty rule no score or rank is shown."
    },
    {
     "rank": null,
     "model_label": "MapLight + GNN",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "per-task: MAE / AUROC / AUPRC / Spearman (5-seed mean ± sd), scaffold split",
     "eval_set": "TDC ADMET Benchmark Group, 22 datasets",
     "source_type": "board",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "Entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://tdcommons.ai/benchmark/admet_group/03pgp/",
      "verification": "unverified",
      "note": "Board pages carry no dates; a retrieval day is not an entry date"
     },
     "url": "https://tdcommons.ai/benchmark/admet_group/03pgp/",
     "note": "Not a site model; ties with MiniMol for most first places; remaining firsts go to several boosting and ensemble methods.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Board entries carry no date, so under the honesty rule no score or rank is shown."
    },
    {
     "rank": null,
     "model_label": "admet_ai_v2",
     "model_id": "model-admet-ai",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "per-task: MAE / AUROC / AUPRC / Spearman (5-seed mean ± sd), scaffold split",
     "eval_set": "TDC ADMET Benchmark Group, 22 datasets",
     "source_type": "board",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "Entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://tdcommons.ai/benchmark/admet_group/01caco2/",
      "verification": "unverified",
      "note": "Board pages carry no dates; a retrieval day is not an entry date"
     },
     "url": "https://tdcommons.ai/benchmark/admet_group/01caco2/",
     "note": "The only ADMET-AI-labelled submission on the 22 boards, Caco2 only and last; value looks like a unit error; the paper's best-average-rank claim has no live counterpart.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Board entries carry no date, so under the honesty rule no score or rank is shown."
    },
    {
     "rank": null,
     "model_label": "Chemprop-RDKit / Chemprop",
     "model_id": "model-chemprop-v2",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "per-task: MAE / AUROC / AUPRC / Spearman (5-seed mean ± sd), scaffold split",
     "eval_set": "TDC ADMET Benchmark Group, 22 datasets",
     "source_type": "board",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "Entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://tdcommons.ai/benchmark/admet_group/05lipo/",
      "verification": "unverified",
      "note": "Board pages carry no dates; a retrieval day is not an entry date"
     },
     "url": "https://tdcommons.ai/benchmark/admet_group/05lipo/",
     "note": "Board rows are original Chemprop / Chemprop-RDKit submissions, not v2; no first place on any task; pairing to the v2 page is a proxy.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Board entries carry no date, so under the honesty rule no score or rank is shown."
    },
    {
     "rank": null,
     "model_label": "MolE",
     "model_id": "model-mole",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "per-task: MAE / AUROC / AUPRC / Spearman (5-seed mean ± sd), scaffold split",
     "eval_set": "TDC ADMET Benchmark Group, 22 datasets",
     "source_type": "board",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "Entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://tdcommons.ai/benchmark/admet_group/overview/",
      "verification": "unverified",
      "note": "Board pages carry no dates; a retrieval day is not an entry date"
     },
     "url": "https://tdcommons.ai/benchmark/admet_group/overview/",
     "note": "No MolE row on any of the 22 official boards in a 2026-11-16 scan; the \"10 of 22 first places\" claim belongs to its 2024 paper only.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "No row for this model on the live boards and no entry date, so no score is shown."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "None of the 22 official boards dates its entries, so every score is withheld and no entry can serve as headline.",
   "caveat": "Self-reported, no independent re-run or sanity check, no versions or dates; top ranks often within 1-2 SD; boosting-plus-fingerprint baselines beat most pretrained models.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-admet-ai",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11226862/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Submitted to the TDC ADMET group board (only one Caco2 row on the live board)",
    "resolution": null
   },
   {
    "model_id": "model-minimol",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://arxiv.org/abs/2404.14986",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self paper plus official board entries",
    "resolution": null
   },
   {
    "model_id": "model-mole",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://doi.org/10.1038/s41467-024-53751-y",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Paper compares with a 2023-09 board snapshot; no row on the live boards",
    "resolution": null
   },
   {
    "model_id": "model-txgemma",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2504.06196",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "66 TDC tasks, recommended splits",
    "resolution": null
   },
   {
    "model_id": "model-chemprop-v2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://tdcommons.ai/benchmark/admet_group/01caco2/",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "disputed",
    "note": "Unversioned board entries, linked indirectly via ADMET-AI",
    "resolution": null
   },
   {
    "model_id": "model-enchant-v1-v2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.iambic.ai/post/enchant",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "Obach human PK result not shown to use the TDC split; three site surfaces unified to unconfirmed",
    "resolution": "The developer post never names the benchmark, the official split or the protocol; the comparison is a hyperlink to a baseline page. The published surfaces disagree three ways (zh matrix 'self', zh model page 'unconfirmed', en pairs.json 'self'); all three are unified to unconfirmed. Not an established pairing."
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 4,
   "by_relation": {
    "official-participation": 3,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 2
   }
  },
  "how_to_use": "Install PyTDC, load data with admet_group get and score with evaluate_many over 5 seeds. Report seed SD, SMILES standardisation and deduplication, any external pretraining or precomputed features, and a gradient-boosting baseline; generalisation claims need time-split own data or prospective blind tests.",
  "pairs_with": [
   "bench-27-polaris",
   "bench-03-moleculenet",
   "bench-44-fsmol"
  ],
  "caveats": [
   "Submissions are self-reported with no independent re-run or sanity check; scaffold splits do not stop near-analog leakage, and gradient-boosting baselines are strong.",
   "Labels are compiled across sources; small sets such as hERG and DILI have low power, and binary thresholds come from literature.",
   "A user report notes PPBR_AZ molecule counts differ by loader and test duplicates carry conflicting labels (not an independent audit).",
   "Models pretrained on ChEMBL or ZINC may have seen test molecules with no official deduplication; rules on external data and ensembles are weak.",
   "No verifiable evidence links board gains to experimental hit rates."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2102.09548",
    "title": "TDC paper (arXiv 2102.09548)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://tdcommons.ai/benchmark/admet_group/overview/",
    "title": "ADMET group overview and leaderboards",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://tdcommons.ai/benchmark/overview/",
    "title": "Official benchmark group overview",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://tdcommons.ai/benchmark/dti_dg_group/overview/",
    "title": "DTI-DG group overview",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2505.05577",
    "title": "PyTDC paper (arXiv 2505.05577)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://pypi.org/project/PyTDC/",
    "title": "PyTDC on PyPI",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://github.com/mims-harvard/TDC/issues/377",
    "title": "TDC repository issue #377 (PPBR_AZ data issue)",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2310.00174",
    "title": "MapLight preprint (arXiv 2310.00174)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11226862/",
    "title": "ADMET-AI paper (Bioinformatics 2024)",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2021-02-18",
    "now": "2020-11-13",
    "reason": "Earliest public evidence is the first PyTDC PyPI upload (0.0.1), not arXiv v1; the entry's own basis already disclosed it.",
    "source_url": "https://pypi.org/pypi/PyTDC/json",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "arXiv v1 of TDC paper (PyPI PyTDC 0.0.1 appeared earlier, 2020-11-13)",
    "now": "first public code release: PyTDC 0.0.1 on PyPI (2020-11-13); TDC paper arXiv v1 2021-02-18; TDC-2 bioRxiv v1 2024-06-14; PyTDC paper arXiv v1 2025-05-08.",
    "reason": "Basis rewritten for the earliest-public-evidence rule.",
    "source_url": "https://pypi.org/pypi/PyTDC/json",
    "date_found": "2026-11-16"
   },
   {
    "field": "members",
    "was": "[{\"member\": \"PyTDC first PyPI\", \"date\": \"2020-11-13\"}, {\"member\": \"TDC paper (arXiv)\", \"date\": \"2021-02-18\"}, {\"member\": \"PyTDC (arXiv v1)\", \"date\": \"2025-05-08\"}]",
    "now": "[{\"member\": \"PyTDC first PyPI release 0.0.1\", \"date\": \"2020-11-13\"}, {\"member\": \"TDC paper (arXiv v1)\", \"date\": \"2021-02-18\"}, {\"member\": \"TDC-2 (bioRxiv v1)\", \"date\": \"2024-06-14\"}, {\"member\": \"PyTDC paper (arXiv v1)\", \"date\": \"2025-05-08\"}]",
    "reason": "The TDC-2 family member was missing and the third member was mislabelled as a software release.",
    "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2024.06.12.598655",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries[model-mole]",
    "was": "MolE '10 of 22 first places' presented as a board standing",
    "now": "No MolE row on any of the 22 official boards (2026-11-16 scan); claim attributed to its 2024 paper only",
    "reason": "A full scan of the 22 official task boards found no MolE row, so the claim belongs to the paper only.",
    "source_url": "https://tdcommons.ai/benchmark/admet_group/overview/",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries[model-admet-ai]",
    "was": "ADMET-AI 'best average rank over the 22 datasets' presented as a board standing",
    "now": "Only one ADMET-AI-labelled board submission (Caco2, last place, value likely a unit error); the average-rank claim has no live counterpart",
    "reason": "The only ADMET-AI-labelled board submission covers one task and sits last.",
    "source_url": "https://tdcommons.ai/benchmark/admet_group/overview/",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported.model-enchant-v1-v2",
    "was": "self-reported (zh matrix, en pairs.json)",
    "now": "disputed-or-unconfirmed",
    "reason": "The developer post names no benchmark, official split or protocol.",
    "source_url": "https://www.iambic.ai/post/enchant",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "family.members[2]",
   "ranking.entries[0]",
   "ranking.entries[1]",
   "ranking.entries[2]",
   "ranking.entries[3]",
   "ranking.entries[4]",
   "models_reported[2]",
   "models_reported[4]",
   "models_reported[5]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-04-tdc/",
   "en": "/bio/benchmark/benchmarks/tdc/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-05-posebusters",
  "kind": "benchmark",
  "nn": "05",
  "short_slug": "posebusters",
  "names": {
   "display": "PoseBusters",
   "full": null,
   "full_null_reason": "The name is not an abbreviation, so there is no expanded form.",
   "aliases": [
    "PoseBusters",
    "PoseBusters V1",
    "PoseBusters V2",
    "PoseBusters Benchmark set"
   ],
   "ambiguous_aliases": [
    "PB-valid"
   ]
  },
  "family": null,
  "family_null_reason": "V1 and V2 are two sets of one tool: versions, not family members.",
  "domain": "structure",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "C+B",
  "type_note": "Validity-checker tool plus two frozen static complex sets",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L3",
   "range": null,
   "qualifier": "Retrospective with a time split, not blind"
  },
  "publisher": {
   "orgs": [
    "University of Oxford, OPIG group"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Protein-ligand pose benchmark: an RMSD threshold plus 18 physical and chemical validity checks.",
  "first_release": {
   "date": "2023-08-10",
   "precision": "day",
   "year_hint": null,
   "basis": "Preprint arXiv 2308.05777 v1; the first public package version was uploaded to PyPI two days later.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2308.05777",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2023-08-10",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv 2308.05777 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2308.05777",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2023-08-12",
    "precision": "day",
    "year_hint": null,
    "basis": "First public PyPI version, 0.2.2",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://pypi.org/project/posebusters/",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-10-03",
   "precision": "day",
   "year_hint": null,
   "basis": "Commit history on the official repository's main branch",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://github.com/maabuu/posebusters/commits/HEAD.atom",
   "verification": "verified",
   "note": null,
   "description": "Unreleased commits on main (an HTML report, plus a version-bump commit) with no tag or release object; the last published version is still 0.6.5 and both sets are unchanged."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have not announced a new set version or a new evaluation round.",
  "status": "active",
  "status_note": "The checker is widely used; the sets are frozen since V2 and near saturation",
  "scale": {
   "summary": "V1 has 428 complexes, V2 has 308, with an 85-complex easy control set; 18 checks.",
   "items": [
    {
     "label": "Complexes",
     "value": "428",
     "unit": null,
     "version": "V1"
    },
    {
     "label": "Complexes, crystal-contact ligands removed",
     "value": "308",
     "unit": null,
     "version": "V2"
    },
    {
     "label": "Complexes in the easy control set",
     "value": "85",
     "unit": null,
     "version": "Astex Diverse set"
    },
    {
     "label": "Validity checks: 6 chemical, 6 intramolecular, 6 intermolecular",
     "value": "18",
     "unit": null,
     "version": null
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No new experiments; reuses public PDB complexes released after 2021.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Predict the ligand pose for a protein-ligand pair, judging both geometric closeness to the crystal and physical and chemical plausibility.",
   "truth": "Public PDB crystal structures, limited to post-2021 releases outside the common training set.",
   "metric_primary": "Share with RMSD under 2 Å and all checks passed",
   "metric_secondary": [
    "Success rate on RMSD under 2 Å alone",
    "Pass rates of the individual checks"
   ],
   "version_traps": "Any number must state V1 or V2, whether validity checks are included, and the sampling rule (best@N or top confidence).",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "Share with RMSD under 2 Å and checks passed; some sources give RMSD only",
   "eval_set": "Mixed: PoseBusters V2 (308) vs Pearl post-2021-10-01 subset (297); best@5 vs top-confidence",
   "snapshot_date": {
    "date": "2025-10-29",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2510.24670",
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "Pearl",
     "model_id": "model-pearl",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Share with RMSD under 2 Å and checks passed; some sources give RMSD only",
     "eval_set": "Mixed: PoseBusters V2 (308) vs Pearl post-2021-10-01 subset (297); best@5 vs top-confidence",
     "source_type": "self",
     "date": {
      "date": "2025-10-29",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2510.24670",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2510.24670",
     "note": "Self-reported on a post-2021-10-01 subset; no independent reproduction found",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: subsets and sampling rules differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "Boltz-1x",
     "model_id": "model-boltz-1-1x",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Share with RMSD under 2 Å and checks passed; some sources give RMSD only",
     "eval_set": "Mixed: PoseBusters V2 (308) vs Pearl post-2021-10-01 subset (297); best@5 vs top-confidence",
     "source_type": "third",
     "date": {
      "date": "2025-10-29",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2510.24670",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2510.24670",
     "note": "Run by the team behind another model's paper, not self-reported",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: subsets and sampling rules differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "Chai-1",
     "model_id": "model-chai-1",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Share with RMSD under 2 Å and checks passed; some sources give RMSD only",
     "eval_set": "Mixed: PoseBusters V2 (308) vs Pearl post-2021-10-01 subset (297); best@5 vs top-confidence",
     "source_type": "third",
     "date": {
      "date": "2025-10-29",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2510.24670",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2510.24670",
     "note": "Run by the team behind another model's paper, not self-reported",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: subsets and sampling rules differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "AlphaFold 3",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Share with RMSD under 2 Å and checks passed; some sources give RMSD only",
     "eval_set": "Mixed: PoseBusters V2 (308) vs Pearl post-2021-10-01 subset (297); best@5 vs top-confidence",
     "source_type": "third",
     "date": {
      "date": "2025-10-29",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2510.24670",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2510.24670",
     "note": "Pose-selection rule differs from best@5, and the training cutoff overlaps part of the set",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: subsets and sampling rules differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "NeuralPLexer3",
     "model_id": "model-neuralplexer3",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Share with RMSD under 2 Å and checks passed; some sources give RMSD only",
     "eval_set": "Mixed: PoseBusters V2 (308) vs Pearl post-2021-10-01 subset (297); best@5 vs top-confidence",
     "source_type": "self",
     "date": {
      "date": "2024-12-18",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2412.10743",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2412.10743",
     "note": "Self-reported on the full V2 set",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: subsets and sampling rules differ, so the ranking row shows no score."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "There is no official board and the sources are not comparable, so no headline number is given.",
   "caveat": "No official leaderboard; all numbers come from papers with different subsets, sampling rules and training cutoffs, and some entries overlap the training window. One model was excluded for its cutoff and two others were not re-verified.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphafold3",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11168924/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-report on the official V1 set; V2 numbers are third-party recomputations",
    "resolution": null
   },
   {
    "model_id": "model-chai-1",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1101/2024.10.10.615955",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "V1 self-report (preprint, quoted via reviews); the V2 numbers are third-party",
    "resolution": null
   },
   {
    "model_id": "model-neuralplexer3",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2412.10743",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "V2 set, with validity checks included",
    "resolution": null
   },
   {
    "model_id": "model-protenix",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2507.11839v1",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "unverified",
    "note": "Only an early version's table; current versions report no score on this set, so the matrix cell overstated it",
    "resolution": null
   },
   {
    "model_id": "model-intellifold-2",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://github.com/IntelliGen-AI/IntelliFold",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Approximate values in the repository README with a steering option; no table",
    "resolution": null
   },
   {
    "model_id": "model-nesso-1",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://www.valencelabs.com/wp-content/uploads/2026/07/nesso1.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "A missing candidate added and confirmed in the review",
    "resolution": "The same sentence names this benchmark; its poses are coarse-grained, so the physical-validity checks are not run as on all-atom predictions, hence modified protocol."
   },
   {
    "model_id": "model-pearl",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2510.24670v2",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Mixes its own best@5 with another model's top-confidence result; one model excluded for its cutoff",
    "resolution": null
   },
   {
    "model_id": "model-terrabind",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2602.07735",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Merged into a chart with two other benchmarks; no number for this benchmark alone",
    "resolution": null
   },
   {
    "model_id": "model-boltz-1-1x",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2510.24670",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Numbers come from another paper, not a self-report",
    "resolution": null
   },
   {
    "model_id": "model-diffdock",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2412.10743",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Quoted from the comparison table of another model's paper",
    "resolution": null
   },
   {
    "model_id": "model-vina",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2412.10743",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Quoted from the comparison table of another model's paper",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 8,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 4,
    "self-reported-modified-protocol": 4,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 3,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Install the official package, predict a pose for each complex and let the tool compute RMSD and the 18 checks; it runs statically with no wet lab of your own. Report the set version, whether validity checks are included, the sampling rule, the training cutoff and any overlap with the set. Generalization claims need similarity-stratified splits or a newer time split. Pair with PLINDER and PoseX.",
  "pairs_with": [
   "bench-30-plinder",
   "bench-33-posex",
   "bench-31-foldbench",
   "bench-06-casf",
   "bench-32-crossdocked"
  ],
  "caveats": [
   "Some structures fall inside newer models' training windows, so about 80% success cannot be read as generalization to new pockets.",
   "Docking and co-folding differ in inputs and sampling rules, so cross-comparisons are not like-for-like.",
   "With only 308 systems in V2, differences of a few percentage points at the top do not support a ranking.",
   "The sets are frozen since V2 and near saturation, and most new models already satisfy the validity checks.",
   "Pose accuracy is not affinity or activity, and no public study was found linking these scores to wet-lab hit rates."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2308.05777",
    "title": "Original preprint",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/maabuu/posebusters",
    "title": "Official repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://pypi.org/project/posebusters/",
    "title": "Official package on PyPI",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2510.24670",
    "title": "Third-party model paper using a subset of this benchmark",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2412.10743",
    "title": "Model paper recomputing results on the full V2 set",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11168924/",
    "title": "Co-folding model paper reporting V1 results",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update_kind",
    "was": "new-version",
    "now": "code-maintenance-only",
    "reason": "The 2026-10-03 commit is untagged on main and the version-bump commit is not a release, so new-version is unsupported.",
    "source_url": "https://github.com/maabuu/posebusters/releases.atom",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_desc",
    "was": "Repo added HTML diagnostic report (#102); 2026-09-17 version bumped 0.6.5 to 0.7.0. PyPI still shows 0.6.5 (2026-03-07).",
    "now": "Unreleased commits on main: HTML report (2026-10-03) and a 0.7.0 version-bump commit (2026-09-17); no tag or release object. Last tagged release and PyPI version: v0.6.5 (2026-03-07). Benchmark sets V1/V2 unchanged.",
    "reason": "The description was synced with the corrected event type.",
    "source_url": "https://github.com/maabuu/posebusters/commits/HEAD.atom",
    "date_found": "2026-11-16"
   },
   {
    "field": "domain",
    "was": "small-molecule",
    "now": "structure",
    "reason": "This round's decision: this entry and the co-folding benchmarks are unified under the structure domain.",
    "source_url": "https://arxiv.org/abs/2308.05777",
    "date_found": "2026-11-16"
   },
   {
    "field": "n_models_reported",
    "was": "10",
    "now": "8 counted under the new relation taxonomy, with comparison-only citations excluded",
    "reason": "The count was recomputed under the new relation taxonomy; comparison-only citations are not counted.",
    "source_url": "https://arxiv.org/abs/2510.24670",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported (one co-folding model)",
    "was": "listed as reporting this benchmark in its current versions",
    "now": "version-qualified: only an early version's table reports a score on this set",
    "reason": "Version-qualified: the current versions report no score on this set, so the matrix cell overstated it.",
    "source_url": "https://arxiv.org/html/2507.11839v1",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "models_reported[3]",
   "models_reported[4]",
   "models_reported[6]",
   "models_reported[7]",
   "ranking.entries[0]",
   "ranking.entries[3]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-05-posebusters/",
   "en": "/bio/benchmark/benchmarks/posebusters/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-06-casf",
  "kind": "benchmark",
  "nn": "06",
  "short_slug": "casf",
  "names": {
   "display": "CASF-2016",
   "full": "Comparative Assessment of Scoring Functions 2016",
   "full_null_reason": null,
   "aliases": [
    "CASF-2016",
    "CASF-2013",
    "CASF-2007",
    "CASF-07",
    "Comparative Assessment of Scoring Functions"
   ],
   "ambiguous_aliases": [
    "CASF",
    "PDBbind core set"
   ]
  },
  "family": null,
  "family_null_reason": "CASF-2007, 2013 and 2016 are successive versions of one benchmark, recorded as milestones, not a family.",
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static dataset suite with official scoring scripts",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Retrospective, not blind; heavy train-test overlap"
  },
  "publisher": {
   "orgs": [
    "Shanghai Institute of Organic Chemistry, CAS (PDBbind-CN team)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Decade-old default test of protein-ligand scoring functions across four powers; frozen and heavily overlapping with training data.",
  "first_release": {
   "date": "2009-04-09",
   "precision": "day",
   "year_hint": null,
   "basis": "CASF-2007 paper online in JCIM; the benchmark is named 'CASF-2007' but was first published in 2009",
   "event_kind": "journal-online",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1021/ci9000053",
   "verification": "verified",
   "note": "2007 is the version label, not the release date"
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": null,
   "journal_online": {
    "date": "2009-04-09",
    "precision": "day",
    "year_hint": null,
    "basis": "CASF-2007 paper online in JCIM",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1021/ci9000053",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2018-11-27",
    "precision": "day",
    "year_hint": null,
    "basis": "CASF-2016 update paper online in JCIM (current version)",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1021/acs.jcim.8b00545",
    "verification": "verified",
    "note": "CASF-2013 papers went online 2014-06-02"
   }
  },
  "latest_update": {
   "date": "2018-11-27",
   "precision": "day",
   "year_hint": null,
   "basis": "CASF-2016 paper online date in JCIM; absence of a newer version rests on web search, the official site was not fetched",
   "event_kind": "none-since-release",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1021/acs.jcim.8b00545",
   "verification": "verified",
   "note": null,
   "description": "CASF-2016 update paper published online in JCIM; no later CASF version found."
  },
  "next_expected": null,
  "next_expected_null_reason": "The publisher has announced no version after CASF-2016.",
  "status": "frozen",
  "status_note": "Frozen since 2016; being replaced by leakage-controlled splits",
  "scale": {
   "summary": "285 complexes (57 target clusters x 5 ligands), about 100 docking decoys per complex.",
   "items": [
    {
     "label": "Complexes",
     "value": "285",
     "unit": null,
     "version": "CASF-2016"
    },
    {
     "label": "Target clusters",
     "value": "57",
     "unit": null,
     "version": "CASF-2016"
    },
    {
     "label": "Ligands per cluster",
     "value": "5",
     "unit": null,
     "version": "CASF-2016"
    },
    {
     "label": "Docking decoys per complex",
     "value": "~100",
     "unit": null,
     "version": "CASF-2016"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None; reuses PDBbind crystal structures and literature Kd/Ki.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Scoring, ranking, docking and screening power of protein-ligand scoring functions on the PDBbind core set.",
   "truth": "PDBbind crystal structures and literature Kd/Ki mixed across labs and assays; screening negatives are assumed.",
   "metric_primary": "Scoring power: Pearson R, RMSE",
   "metric_secondary": [
    "Ranking power: within-cluster Spearman ρ",
    "Docking power: top-1 success (RMSD<2Å)",
    "Screening power: EF1%, success rate"
   ],
   "version_traps": "Only the test set is fixed; state the PDBbind version and whether CleanSplit or LP-PDBBind was used; never mix CASF-2013 and 2016.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "Scoring power Pearson R / RMSE",
   "eval_set": "CASF-2016 core set (285)",
   "snapshot_date": {
    "date": "2025-10-21",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "GEMS (trained on PDBbind CleanSplit)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Scoring power Pearson R / RMSE",
     "eval_set": "CASF-2016 core set (285)",
     "source_type": "self",
     "date": {
      "date": "2025-10-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
     "note": "Not a site model; self-reported, trained on CleanSplit",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "not-comparable: protocols/subsets differ, so no score is displayed in the ranking row; the per-protocol values live on the entity page."
    },
    {
     "rank": null,
     "model_label": "GenScore (CleanSplit retrain)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Scoring power Pearson R / RMSE",
     "eval_set": "CASF-2016 core set (285)",
     "source_type": "third",
     "date": {
      "date": "2025-10-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
     "note": "Not a site model; retrained on CleanSplit in a leakage assessment",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "not-comparable: protocols/subsets differ, so no score is displayed in the ranking row; the per-protocol values live on the entity page."
    },
    {
     "rank": null,
     "model_label": "Pafnucy (CleanSplit retrain)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Scoring power Pearson R / RMSE",
     "eval_set": "CASF-2016 core set (285)",
     "source_type": "third",
     "date": {
      "date": "2025-10-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
     "note": "Not a site model; retrained on CleanSplit in a leakage assessment",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "not-comparable: protocols/subsets differ, so no score is displayed in the ranking row; the per-protocol values live on the entity page."
    },
    {
     "rank": null,
     "model_label": "5-nearest-neighbour search baseline (no training)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Scoring power Pearson R / RMSE",
     "eval_set": "CASF-2016 core set (285)",
     "source_type": "third",
     "date": {
      "date": "2025-10-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
     "note": "Not a model: retrieval baseline averaging the most similar training complexes",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "not-comparable: protocols/subsets differ, so no score is displayed in the ranking row; the per-protocol values live on the entity page."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "Not comparable: no official board and scores depend mainly on the training split, so no headline number.",
   "caveat": "No site model reports CASF-2016; no official board; ~49% of test complexes have similar training pairs, so splits dominate; older high scores used leaky splits, unverified.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Register on the PDBbind site, download the CASF-2016 package and run the four powers with official scripts; no wet lab, retrospective only. Train on CleanSplit or LP-PDBBind, add a retrieval baseline and a time-split set, state the training version. Pair with PoseBusters, PLINDER, DUD-E, CACHE.",
  "pairs_with": [
   "bench-05-posebusters",
   "bench-30-plinder",
   "bench-12-dude",
   "bench-20-cache"
  ],
  "caveats": [
   "Test and PDBbind training sets overlap heavily: a no-training retrieval baseline approaches older deep models; docking power is saturated and screening negatives are unmeasured cross-pairs.",
   "A 2025 leakage study reports similar training pairs for about 49% of test complexes; most models drop after de-leaking.",
   "Ranking uses only 57 five-ligand groups and screening only 285 ligands, so noise is large; no official intervals or pose-validity checks.",
   "Boltz-2 official evaluation docs do not list CASF; no recent AI model report on it was found this round.",
   "PDBbind data now sits behind a licence gate whose terms were not verified on official pages."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1021/acs.jcim.8b00545",
    "title": "CASF-2016 paper (JCIM 2019)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1021/ci9000053",
    "title": "CASF-2007 paper (JCIM 2009)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1021/ci500080q",
    "title": "CASF-2013 paper (JCIM 2014)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1038/nprot.2017.114",
    "title": "PDBbind/CASF protocol (Nature Protocols 2018)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12552120/",
    "title": "PDBbind CleanSplit leakage study (Nat Mach Intell 2025)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2308.09639",
    "title": "LP-PDBBind re-split (arXiv 2308.09639)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://github.com/jwohlwend/boltz/blob/main/docs/evaluation.md",
    "title": "Boltz official evaluation docs",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2411.01223",
    "title": "PDBbind licensing discussion (arXiv 2411.01223)",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2007 (start year)",
    "now": "2009-04-09",
    "reason": "'2007' is the version label; the first publication is the JCIM paper online 2009-04-09 (CHANGESET §0.7).",
    "source_url": "https://doi.org/10.1021/ci9000053",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-06-casf/",
   "en": "/bio/benchmark/benchmarks/casf/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-07-uspto",
  "kind": "benchmark",
  "nn": "07",
  "short_slug": "uspto",
  "names": {
   "display": "USPTO family (+PaRoutes)",
   "full": "USPTO patent reaction dataset family",
   "full_null_reason": null,
   "aliases": [
    "USPTO-50K",
    "USPTO-MIT",
    "USPTO-480K",
    "MIT-480K",
    "USPTO-FULL",
    "USPTO-STEREO",
    "PaRoutes"
   ],
   "ambiguous_aliases": [
    "USPTO"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "Patent reaction extraction data 1976-2013",
     "id_or_null": null,
     "first_release": {
      "date": "2014-02",
      "precision": "month",
      "year_hint": null,
      "basis": "First public release of the patent-text reaction extraction (NextMove blog, 2014-02)",
      "event_kind": "data-release",
      "timestamp_utc": null,
      "source_url": "https://depth-first.com/articles/2019/01/28/the-nextmove-patent-reaction-dataset/",
      "verification": "partial",
      "note": "Month-level only; a 2012 thesis date is unverified"
     }
    },
    {
     "name": "USPTO-50K",
     "id_or_null": null,
     "first_release": {
      "date": "2016-12-08",
      "precision": "day",
      "year_hint": null,
      "basis": "JCIM paper online (curated at Novartis)",
      "event_kind": "journal-online",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1021/acs.jcim.6b00564",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "Patent reactions 1976-Sep2016 (figshare)",
     "id_or_null": null,
     "first_release": {
      "date": "2017-06-13",
      "precision": "day",
      "year_hint": null,
      "basis": "figshare release of 1976 to 2016-09 data (CC0)",
      "event_kind": "data-release",
      "timestamp_utc": null,
      "source_url": "https://figshare.com/articles/dataset/Chemical_reactions_from_US_patents_1976-Sep2016_/5104873",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "USPTO-MIT 480K",
     "id_or_null": null,
     "first_release": {
      "date": "2017-09-13",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv 1709.04555 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/1709.04555",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "PaRoutes",
     "id_or_null": null,
     "first_release": {
      "date": "2022",
      "precision": "year",
      "year_hint": null,
      "basis": "Digital Discovery paper (year only)",
      "event_kind": "journal-online",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1039/d2dd00015f",
      "verification": "partial",
      "note": null
     }
    }
   ],
   "boundary": "Several splits and a route set derived from one patent-text extraction; subsets differ in size, split and preprocessing, so numbers do not transfer across them."
  },
  "family_null_reason": null,
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static dataset family; PaRoutes adds route sets and scripts",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Random-split retrospective; OOD studies reach L3"
  },
  "publisher": {
   "orgs": [
    "NextMove Software",
    "Novartis",
    "Massachusetts Institute of Technology",
    "AstraZeneca"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Default data for retrosynthesis and reaction prediction; measures reproducing patent records, not lab feasibility.",
  "first_release": {
   "date": "2014-02",
   "precision": "month",
   "year_hint": null,
   "basis": "First public release of the patent reaction extraction data (NextMove blog, 2014-02); day not retrieved",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://depth-first.com/articles/2019/01/28/the-nextmove-patent-reaction-dataset/",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2014-02",
    "precision": "month",
    "year_hint": null,
    "basis": "First public release of the patent reaction data",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://depth-first.com/articles/2019/01/28/the-nextmove-patent-reaction-dataset/",
    "verification": "partial",
    "note": "A 2012 thesis date is unverified and not used"
   },
   "paper_v1": {
    "date": "2017-09-13",
    "precision": "day",
    "year_hint": null,
    "basis": "USPTO-MIT 480K paper arXiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/1709.04555",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2016-12-08",
    "precision": "day",
    "year_hint": null,
    "basis": "USPTO-50K JCIM paper online",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1021/acs.jcim.6b00564",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2022",
   "precision": "year",
   "year_hint": null,
   "basis": "PaRoutes paper published in Digital Discovery (year only)",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1039/d2dd00015f",
   "verification": "verified",
   "note": null,
   "description": "Newest member: PaRoutes route-benchmark paper in Digital Discovery (preprint reviews 2022-03-18); source data frozen at 2016-09."
  },
  "next_expected": null,
  "next_expected_null_reason": "Source data are frozen and no maintainer has announced a new version.",
  "status": "frozen",
  "status_note": "Source data frozen at 2016; methods papers still report it",
  "scale": {
   "summary": "Source 3.3M reactions; USPTO-50K about 50k; MIT-480K 479,035; PaRoutes two sets of 10,000 routes.",
   "items": [
    {
     "label": "Source reactions",
     "value": "3,305,795",
     "unit": "reactions",
     "version": "1976-Sep2016"
    },
    {
     "label": "USPTO-50K reactions (10 classes)",
     "value": "~50,000",
     "unit": "reactions",
     "version": "USPTO-50K"
    },
    {
     "label": "MIT split (train/valid/test)",
     "value": "409,035 / 30,000 / 40,000",
     "unit": "reactions",
     "version": "USPTO-MIT"
    },
    {
     "label": "FULL raw reactions (preprocessing varies)",
     "value": "1,808,937",
     "unit": "reactions",
     "version": "USPTO-FULL"
    },
    {
     "label": "PaRoutes route sets",
     "value": "2 × 10,000",
     "unit": "routes",
     "version": "PaRoutes v1.0.0"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None; patent text mining, not experimental validation.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Reaction outcome prediction, single-step retrosynthesis and multi-step route search on reactions mined from US patents.",
   "truth": "Records extracted from patent text; no failed reactions, no yield guarantee, conditions often missing.",
   "metric_primary": "Top-k exact match (canonical SMILES)",
   "metric_secondary": [
    "Round-trip accuracy",
    "Route solve rate (PaRoutes)"
   ],
   "version_traps": "Reaction-class disclosure, test-time augmentation, FULL pretraining and per-paper preprocessing all shift numbers; MIT has mixed and separated settings.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "top-k exact match (%), canonical SMILES",
   "eval_set": "USPTO-50K, USPTO-MIT/480K, USPTO-FULL; settings differ (reaction class, augmentation, pretraining)",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "Compilation date (entries are paper self-reports with no common board snapshot)",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "RetroDFM-R",
     "model_id": "model-retrodfm-r",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "top-k exact match (%), canonical SMILES",
     "eval_set": "USPTO-50K, USPTO-MIT/480K, USPTO-FULL; settings differ (reaction class, augmentation, pretraining)",
     "source_type": "self",
     "date": {
      "date": "2025-07-23",
      "precision": "day",
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2507.17448v1",
      "verification": "verified",
      "note": "arXiv 2507.17448 v1"
     },
     "url": "https://arxiv.org/html/2507.17448v1",
     "note": "Correction: the site cited arXiv 2510.16588 (that is C-SMILES); this model is arXiv 2507.17448; values came via a search summary, not a full read.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: protocols and subsets differ, so no score is shown in the ranking row; per-protocol values stay on the entity page."
    },
    {
     "rank": null,
     "model_label": "Uni-Mol3",
     "model_id": "model-uni-mol3",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "top-k exact match (%), canonical SMILES",
     "eval_set": "USPTO-50K, USPTO-MIT/480K, USPTO-FULL; settings differ (reaction class, augmentation, pretraining)",
     "source_type": "self",
     "date": {
      "date": "2025-08-01",
      "precision": "day",
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2508.00920v2",
      "verification": "verified",
      "note": "arXiv 2508.00920, v2 read"
     },
     "url": "https://arxiv.org/html/2508.00920v2",
     "note": "Re-verified from the paper; the same tables show it below the Chemformer baseline on 50K retrosynthesis.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: protocols and subsets differ, so no score is shown in the ranking row; per-protocol values stay on the entity page."
    },
    {
     "rank": null,
     "model_label": "Chemformer",
     "model_id": "model-chemformer",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "top-k exact match (%), canonical SMILES",
     "eval_set": "USPTO-50K, USPTO-MIT/480K, USPTO-FULL; settings differ (reaction class, augmentation, pretraining)",
     "source_type": "third",
     "date": {
      "date": "2025-08-01",
      "precision": "day",
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2508.00920v2",
      "verification": "verified",
      "note": "As tabulated in the Uni-Mol3 paper"
     },
     "url": "https://arxiv.org/html/2508.00920v2",
     "note": "Third-party tabulation (baseline row in the Uni-Mol3 paper), re-verified.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: protocols and subsets differ, so no score is shown in the ranking row; per-protocol values stay on the entity page."
    },
    {
     "rank": null,
     "model_label": "C-SMILES (not a site model)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "top-k exact match (%), canonical SMILES",
     "eval_set": "USPTO-50K, USPTO-MIT/480K, USPTO-FULL; settings differ (reaction class, augmentation, pretraining)",
     "source_type": "self",
     "date": {
      "date": "2025-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2510.16588",
      "verification": "verified",
      "note": "arXiv 2510.16588"
     },
     "url": "https://arxiv.org/abs/2510.16588",
     "note": "Not a site model; highest 50K top-1 seen this round; the site's known/unknown-class split was not confirmed on the abstract.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: protocols and subsets differ, so no score is shown in the ranking row; per-protocol values stay on the entity page."
    },
    {
     "rank": null,
     "model_label": "ether0",
     "model_id": "model-ether0",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "top-k exact match (%), canonical SMILES",
     "eval_set": "USPTO-50K, USPTO-MIT/480K, USPTO-FULL; settings differ (reaction class, augmentation, pretraining)",
     "source_type": "self",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": null,
      "verification": "unverified",
      "note": null
     },
     "url": null,
     "note": "Evaluated on its own reaction set, not an official USPTO test split; not comparable.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Incomplete provenance (no url) and a different protocol, so the honesty rule forbids showing a number."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "No official board and settings differ across papers, so entries are not comparable and no headline is given.",
   "caveat": "No official board; Papers with Code closed. Numbers depend on class disclosure, augmentation and pretraining; ~5k test reactions, 1 point ~50 reactions; few seeds or CIs.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-chemformer",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1088/2632-2153/ac3ffb",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "USPTO-50K; the Uni-Mol3 re-quote is comparison only",
    "resolution": null
   },
   {
    "model_id": "model-retrodfm-r",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2507.17448v1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "USPTO-50K/FULL; with test-time augmentation noted",
    "resolution": null
   },
   {
    "model_id": "model-scireasoner",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "unverified",
    "note": "USPTO-50K single-step retrosynthesis, matrix note only, unverified",
    "resolution": null
   },
   {
    "model_id": "model-uni-mol3",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2508.00920v2",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "USPTO-MIT/50K/Condition",
    "resolution": null
   },
   {
    "model_id": "model-ether0",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2506.17238",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-built reaction prediction, not official test set",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 5,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 4,
    "self-reported-modified-protocol": 1,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Use the CC0 figshare source data or a method repo's preprocessed split; PaRoutes ships scripts. Report subset, reaction-class disclosure, augmentation, FULL pretraining, split (prefer new or time splits), seeds with CIs and a template-retrieval baseline; round-trip must name its forward model.",
  "pairs_with": [
   "bench-10-guacamol",
   "bench-19-pmo"
  ],
  "caveats": [
   "Random splits leak templates (about 94% of USPTO-50K test reactions match training templates, per one report); exact match penalizes valid alternative routes; no failed reactions are recorded.",
   "Reaction-class disclosure, test-time augmentation and pretraining corpus can inflate numbers markedly; cross-paper comparison needs aligned settings.",
   "With about 5k test reactions, recent 1-3 point gains may be within noise (inference); most papers report no CIs.",
   "Template-free models drop sharply in OOD evaluations; single-step scores are reported to decouple from route-finding success.",
   "No evidence on lab feasibility of routes; only synthesis can establish it."
  ],
  "sources": [
   {
    "url": "https://figshare.com/articles/dataset/Chemical_reactions_from_US_patents_1976-Sep2016_/5104873",
    "title": "Chemical reactions from US patents 1976-Sep2016 (figshare, CC0)",
    "kind": "archive",
    "date": null
   },
   {
    "url": "https://doi.org/10.1021/acs.jcim.6b00564",
    "title": "USPTO-50K source paper (JCIM 2016)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/1709.04555",
    "title": "USPTO-MIT 480K paper (arXiv 1709.04555)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.1039/d2dd00015f",
    "title": "PaRoutes (Digital Discovery 2022)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://github.com/MolecularAI/PaRoutes",
    "title": "PaRoutes repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://neurips.cc/virtual/2022/58040",
    "title": "Retrosynthesis Prediction Revisited (NeurIPS 2022 workshop)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2403.03960",
    "title": "Retrosynthesis OOD evaluation (arXiv 2403.03960)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/html/2507.17448v1",
    "title": "RetroDFM-R paper (arXiv 2507.17448)",
    "kind": "preprint",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "ranking.entries[model-retrodfm-r].url",
    "was": "arXiv 2510.16588 cited for RetroDFM-R",
    "now": "arXiv 2507.17448 (2510.16588 is a different paper, C-SMILES)",
    "reason": "The cited identifier belongs to a different paper.",
    "source_url": "https://arxiv.org/html/2507.17448v1",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries[C-SMILES].note",
    "was": "known-class / unknown-class split as stated on the site",
    "now": "abstract gives a USPTO-50K figure and a USPTO-FULL figure; the known/unknown-class split is not confirmed",
    "reason": "The abstract reports a 50K and a FULL figure, not a known/unknown-class split.",
    "source_url": "https://arxiv.org/abs/2510.16588",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release.basis",
    "was": "Source data dated 2012 (thesis)",
    "now": "2014-02, first public data release; 2012 date unverified",
    "reason": "The 2012 thesis date cannot be verified; the earliest verifiable public release is 2014-02.",
    "source_url": "https://depth-first.com/articles/2019/01/28/the-nextmove-patent-reaction-dataset/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "family.members[0]",
   "family.members[4]",
   "ranking.entries[4]",
   "models_reported[2]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-07-uspto/",
   "en": "/bio/benchmark/benchmarks/uspto/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-08-enformer",
  "kind": "benchmark",
  "nn": "08",
  "short_slug": "enformer",
  "names": {
   "display": "Basenji/Enformer test set",
   "full": "The Basenji/Enformer sequence-to-function test set: the Basenji2 data and its fixed splits",
   "full_null_reason": null,
   "aliases": [
    "Basenji test set",
    "Basenji2 test split",
    "Enformer test set",
    "cross2020 data"
   ],
   "ambiguous_aliases": [
    "Enformer",
    "Basenji"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "Basenji",
     "id_or_null": null,
     "first_release": {
      "date": "2017-07-10",
      "precision": "day",
      "year_hint": null,
      "basis": "bioRxiv preprint; journal version 2018",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1101/161851",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "Basenji2 cross-species data",
     "id_or_null": null,
     "first_release": {
      "date": "2019-06-05",
      "precision": "day",
      "year_hint": null,
      "basis": "Preprint describing the cross-species data",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1371/journal.pcbi.1008050",
      "verification": "partial",
      "note": null
     }
    },
    {
     "name": "Enformer",
     "id_or_null": null,
     "first_release": {
      "date": "2021-04-08",
      "precision": "day",
      "year_hint": null,
      "basis": "bioRxiv preprint; journal version online 2021-10-04",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1101/2021.04.07.438649",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "This entry covers the data and its fixed splits, not any model; later models re-fold or change track sets, which counts as their own protocol."
  },
  "family_null_reason": null,
  "domain": "genomics-rna",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static dataset suite with fixed splits and no leaderboard",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Main task holds out regions of one genome; the variant task is near L3"
  },
  "publisher": {
   "orgs": [
    "Calico Life Sciences",
    "DeepMind"
   ],
   "org_type": "mixed"
  },
  "one_liner": "The de facto protocol for sequence-to-function models: track prediction on fixed held-out regions.",
  "first_release": {
   "date": "2017-07-10",
   "precision": "day",
   "year_hint": null,
   "basis": "Earliest family member: the Basenji bioRxiv preprint, journal version 2018-03-27; the fixed splits came later with the Basenji2 data.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1101/161851",
   "verification": "partial",
   "note": "Family-level date, not the release of the later model of the same name"
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2017-07-10",
    "precision": "day",
    "year_hint": null,
    "basis": "Earliest public preprint in the family",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/161851",
    "verification": "verified",
    "note": null
   },
   "paper_v1": {
    "date": "2021-04-08",
    "precision": "day",
    "year_hint": null,
    "basis": "The preprint that established the protocol in common use today",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/2021.04.07.438649",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2021-10-04",
    "precision": "day",
    "year_hint": null,
    "basis": "Journal online publication",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC8490152/",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-01-15",
   "precision": "day",
   "year_hint": null,
   "basis": "Commit history of the official repository holding the earlier data",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://github.com/calico/basenji/commits/master",
   "verification": "partial",
   "note": "The commit is in the earlier data repository, not in a later model's repository",
   "description": "A dependency-freezing commit in the earlier data repository: code maintenance only, with the benchmark data and splits unchanged."
  },
  "next_expected": null,
  "next_expected_null_reason": "No maintainer has announced a new version or new splits.",
  "status": "active",
  "status_note": "The data are stable and reused by newer models; no leaderboard",
  "scale": {
   "summary": "5,313 human tracks and 1,643 mouse tracks; the human test split has 1,937 sequences of 393,216 bp.",
   "items": [
    {
     "label": "Human tracks",
     "value": "5313",
     "unit": null,
     "version": "Basenji2"
    },
    {
     "label": "Mouse tracks",
     "value": "1643",
     "unit": null,
     "version": "Basenji2"
    },
    {
     "label": "Human train/valid/test sequences",
     "value": "34021/2213/1937",
     "unit": null,
     "version": "Basenji2"
    },
    {
     "label": "Mouse train/valid/test sequences",
     "value": "29295/2209/2017",
     "unit": null,
     "version": "Basenji2"
    },
    {
     "label": "Input sequence length",
     "value": "393216",
     "unit": "bp",
     "version": "Enformer"
    },
    {
     "label": "Output bin width",
     "value": "128",
     "unit": "bp",
     "version": "Enformer"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No new experiments; reuses public epigenomic tracks, with public eQTL and MPRA data for variants.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Predict thousands of epigenomic and transcriptional tracks from DNA sequence and score variant effects from reference-versus-alternate differences.",
   "truth": "Public experimental tracks, heterogeneous in batch and coverage; variant labels come from statistical fine-mapping, not experiments.",
   "metric_primary": "Track Pearson correlation at 128 bp bins",
   "metric_secondary": [
    "Gene-level Pearson correlation",
    "Variant-effect auROC"
   ],
   "version_traps": "State which data generation, folds and resolution were used: later models re-fold the data, so per-bin numbers do not compare directly.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "self-reported",
   "metric": "Track Pearson (128/32/1 bp bins), gene-level Pearson, variant auROC",
   "eval_set": "Basenji2 held-out test intervals (Borzoi/AlphaGenome use own re-folded splits)",
   "snapshot_date": {
    "date": "2026-01-28",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/s41586-025-10014-0",
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "AlphaGenome (all-folds, pre-trained)",
     "model_id": "model-alphagenome",
     "site_model": true,
     "score": "DNase 128bp r=0.87 (+2.4% vs Enformer); CAGE 128bp r=0.71 (-0.3% vs Enformer); RNA-seq 1bp r=0.59 (+28.2% vs Borzoi)",
     "score_unit": null,
     "metric": "Track Pearson (128/32/1 bp bins), gene-level Pearson, variant auROC",
     "eval_set": "Basenji2 held-out test intervals (Borzoi/AlphaGenome use own re-folded splits)",
     "source_type": "self",
     "date": {
      "date": "2026-01-28",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/google/alphagenome-all-folds",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/google/alphagenome-all-folds",
     "note": "Model card checked 2026-11-16; the paper's broader claims of leading were not re-verified",
     "second_hand": true,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Enformer",
     "model_id": null,
     "site_model": false,
     "score": "CAGE gene-level Pearson 0.85 (Basenji2 0.81; replicate ceiling 0.94)",
     "score_unit": null,
     "metric": "Track Pearson (128/32/1 bp bins), gene-level Pearson, variant auROC",
     "eval_set": "Basenji2 held-out test intervals (Borzoi/AlphaGenome use own re-folded splits)",
     "source_type": "self",
     "date": {
      "date": "2021-10-04",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC8490152/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC8490152/",
     "note": "Not a site model; reference baseline from an existing site note, not re-verified",
     "second_hand": true,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Borzoi",
     "model_id": "model-borzoi",
     "site_model": true,
     "score": "gene-level RNA-seq Pearson ~0.87",
     "score_unit": null,
     "metric": "Track Pearson (128/32/1 bp bins), gene-level Pearson, variant auROC",
     "eval_set": "Basenji2 held-out test intervals (Borzoi/AlphaGenome use own re-folded splits)",
     "source_type": "self",
     "date": {
      "date": "2025-01",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://www.biorxiv.org/content/10.1101/2023.08.30.555582.full.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://www.biorxiv.org/content/10.1101/2023.08.30.555582.full.pdf",
     "note": "Second-hand figure from an existing site note; different folds and track set",
     "second_hand": true,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "NTv3",
     "model_id": "model-ntv3",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Track Pearson (128/32/1 bp bins), gene-level Pearson, variant auROC",
     "eval_set": "Basenji2 held-out test intervals (Borzoi/AlphaGenome use own re-folded splits)",
     "source_type": "self",
     "date": {
      "date": "2025-12",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://www.biorxiv.org/content/10.64898/2025.12.22.695963.full.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://www.biorxiv.org/content/10.64898/2025.12.22.695963.full.pdf",
     "note": "Self-reported on its own task set; no number on this set, so not comparable",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "The sources contain no rank-1 entry and the entries use different protocols, so no headline number is given.",
   "caveat": "There is no leaderboard; each paper uses different track sets, folds and resolutions, so numbers do not compare across papers and relative gains are each paper's own figures.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphagenome",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1038/s41586-025-10014-0",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-reported on this data's track test set",
    "resolution": null
   },
   {
    "model_id": "model-borzoi",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1038/s41588-024-02053-6",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "The paper uses its own variant tasks and also reports on this test set",
    "resolution": null
   },
   {
    "model_id": "model-ntv3",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://instadeep.com/wp-content/uploads/2025/12/NT_v3.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-report in a technical report, fine-tuned on re-folded splits",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 3,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 3,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Download the data and fixed splits, predict tracks on held-out regions and score variants from reference-versus-alternate differences; it runs statically with no wet lab of your own. Report the data generation, folds, resolution and track subset, separate zero-shot from fine-tuned results, and give individual-level evaluation separately rather than cross-gene correlation alone. Pair with TraitGym and BEND.",
  "pairs_with": [
   "bench-37-traitgym",
   "bench-38-bend",
   "bench-16-nt_gue",
   "bench-11-clinvar"
  ],
  "caveats": [
   "Cross-gene correlation mainly reflects which genes are highly expressed, not the ability to predict across individuals.",
   "An independent study reported a near-zero mean cross-individual correlation, with some genes significantly anticorrelated.",
   "Gains on reference sequence do not carry over to variant-effect prediction, which is weaker in cell-type-specific regions.",
   "Zero-shot and fine-tuned results are often mixed, and reuse usually reports a single correlation with no confidence interval.",
   "Variant-task labels come from statistical fine-mapping, not experimentally validated causal variants."
  ],
  "sources": [
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC8490152/",
    "title": "Journal paper establishing the protocol in common use",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1101/161851",
    "title": "Earliest preprint in the family",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.1371/journal.pcbi.1008050",
    "title": "Cross-species data paper",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://github.com/calico/basenji",
    "title": "Repository holding the earlier data and code",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC10055057/",
    "title": "Third-party study of individual-level performance",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://doi.org/10.1038/s41586-025-10014-0",
    "title": "Later model paper reusing this data",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "start year 2018",
    "now": "2017-07-10 at family level (earliest preprint), with the family members and boundary stated",
    "reason": "Date audit: the family entry must separate the earliest member from the later model of the same name, and a year alone is not enough.",
    "source_url": "https://doi.org/10.1101/161851",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_kind",
    "was": "new-version",
    "now": "code-maintenance-only",
    "reason": "The latest event is a dependency-freezing commit in the earlier data repository; the benchmark data are unchanged, so it is not a new version.",
    "source_url": "https://github.com/calico/basenji/commits/master",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "latest_update",
   "family.members[1]",
   "ranking.entries[1]",
   "ranking.entries[2]",
   "ranking.entries[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-08-enformer/",
   "en": "/bio/benchmark/benchmarks/enformer/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-09-scib",
  "kind": "benchmark",
  "nn": "09",
  "short_slug": "scib",
  "names": {
   "display": "scIB (+Open Problems)",
   "full": "Single-cell integration benchmark (scIB), with the Open Problems batch-integration task",
   "full_null_reason": null,
   "aliases": [
    "scIB",
    "scib",
    "scib-metrics",
    "Open Problems batch_integration",
    "task_batch_integration"
   ],
   "ambiguous_aliases": [
    "Open Problems"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "scIB (paper and scib package)",
     "id_or_null": null,
     "first_release": {
      "date": "2020-05-23",
      "precision": "day",
      "year_hint": null,
      "basis": "scIB paper bioRxiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1101/2020.05.22.111161",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "scib-metrics (reimplementation)",
     "id_or_null": null,
     "first_release": {
      "date": "2022-10-03",
      "precision": "day",
      "year_hint": null,
      "basis": "First PyPI upload of scib-metrics",
      "event_kind": "code-release",
      "timestamp_utc": null,
      "source_url": "https://pypi.org/project/scib-metrics/",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "NeurIPS 2021 multimodal single-cell competition",
     "id_or_null": null,
     "first_release": {
      "date": "2021",
      "precision": "year",
      "year_hint": null,
      "basis": "NeurIPS 2021 competition hosted by Open Problems (year only)",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://openproblems.bio/neurips_2021",
      "verification": "partial",
      "note": null
     }
    },
    {
     "name": "Open Problems batch_integration task",
     "id_or_null": null,
     "first_release": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "Living-benchmark task page; first launch date not retrieved",
      "event_kind": "first-public-evidence",
      "timestamp_utc": null,
      "source_url": "https://openproblems.bio/benchmarks/batch_integration",
      "verification": "unverified",
      "note": "First launch date not retrieved"
     }
    }
   ],
   "boundary": "Family: scIB metrics and package, the scib-metrics reimplementation, the Open Problems batch-integration task and the NeurIPS 2021 competition; scib and scib-metrics numbers differ."
  },
  "family_null_reason": null,
  "domain": "single-cell",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B+C",
  "type_note": "Dataset suite and metrics toolkit plus a living platform",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Retrospective, no blind test; author-annotation truth"
  },
  "publisher": {
   "orgs": [
    "Helmholtz Munich",
    "Open Problems in Single-Cell Analysis (community consortium)"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Default yardstick for single-cell integration, scoring batch removal against biological conservation; continued by Open Problems.",
  "first_release": {
   "date": "2020-05-23",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv v1 of the scIB paper (Nature Methods online 2021-12-23; PyPI scib 1.0.0 2021-10-24)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1101/2020.05.22.111161",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2020-05-23",
    "precision": "day",
    "year_hint": null,
    "basis": "scIB paper bioRxiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/2020.05.22.111161",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2021-12-23",
    "precision": "day",
    "year_hint": null,
    "basis": "Nature Methods online (2022 issue)",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/s41592-021-01336-8",
    "verification": "verified",
    "note": null
   },
   "code_release": {
    "date": "2021-10-24",
    "precision": "day",
    "year_hint": null,
    "basis": "PyPI scib 1.0.0",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://pypi.org/project/scib/",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-11-01",
   "precision": "day",
   "year_hint": null,
   "basis": "Release date of scib-metrics 0.6.2 on PyPI",
   "event_kind": "new-version",
   "timestamp_utc": null,
   "source_url": "https://pypi.org/project/scib-metrics/",
   "verification": "verified",
   "note": "Site page also notes a task-repo v2.1.0-rc3 on 2026-11-03; not re-verified",
   "description": "scib-metrics 0.6.2 released on PyPI; scib GitHub release 1.2.1 on 2026-04-27 (rpy2 fix) after 1.2.0 added Moran's I."
  },
  "next_expected": null,
  "next_expected_null_reason": "Living benchmark with no announced schedule; no component announces a next-version date.",
  "status": "active",
  "status_note": "Active via Open Problems living benchmark; integration task near saturation",
  "scale": {
   "summary": "Original paper: 16 methods, 13 tasks, 85 batches, 14 metrics; Open Problems: 12 tasks in the 2025 version.",
   "items": [
    {
     "label": "Integration methods",
     "value": "16",
     "unit": null,
     "version": "Nature Methods 2022 (68 with preprocessing combinations)"
    },
    {
     "label": "Integration tasks",
     "value": "13",
     "unit": null,
     "version": "Nature Methods 2022"
    },
    {
     "label": "Batches",
     "value": "85",
     "unit": null,
     "version": "Nature Methods 2022"
    },
    {
     "label": "Metrics",
     "value": "14",
     "unit": null,
     "version": "Nature Methods 2022"
    },
    {
     "label": "Cells",
     "value": ">1.2M",
     "unit": "cells",
     "version": "Nature Methods 2022"
    },
    {
     "label": "Open Problems tasks",
     "value": "12",
     "unit": null,
     "version": "Nat Biotechnol 2025 version"
    },
    {
     "label": "Batch-integration task datasets",
     "value": "6",
     "unit": null,
     "version": "Open Problems task page, read 2026-11"
    }
   ]
  },
  "wetlab_refresh": "accumulating",
  "wetlab_note": "No new experiments; Open Problems adds datasets and tasks irregularly from public atlases.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Single-cell data integration quality: batch removal against preservation of biological variation.",
   "truth": "Author-provided cell-type annotations and batch metadata, not an experimental gold standard.",
   "metric_primary": "Overall = 0.4 × batch removal + 0.6 × bio-conservation",
   "metric_secondary": [
    "Batch: batch ASW, iLISI, kBET and others",
    "Bio: ARI/NMI, cLISI, isolated label and others"
   ],
   "version_traps": "State scib or scib-metrics (numbers not comparable); Open Problems versions and dataset sets change over time.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Open Problems batch integration: mean of 13 scaled scIB metrics per dataset, averaged over 6 datasets (failed or missing runs count as 0)",
   "eval_set": "Open Problems task_batch_integration, 6 CELLxGENE Census datasets; results run 2025-01 (website data commit 2025-02-10); page shows release 2.1.0-rc3 / stable v2.0.0",
   "snapshot_date": {
    "date": "2025-02-10",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://openproblems.bio/benchmarks/batch_integration",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://openproblems.bio/benchmarks/batch_integration",
   "entries": [
    {
     "rank": 1,
     "model_label": "embed_cell_types_jittered (positive control)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Open Problems batch integration: mean of 13 scaled scIB metrics per dataset, averaged over 6 datasets (failed or missing runs count as 0)",
     "eval_set": "Open Problems task_batch_integration, 6 CELLxGENE Census datasets; results run 2025-01 (website data commit 2025-02-10); page shows release 2.1.0-rc3 / stable v2.0.0",
     "source_type": "board",
     "date": {
      "date": "2025-02-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://openproblems.bio/benchmarks/batch_integration",
      "verification": "verified",
      "note": null
     },
     "url": "https://openproblems.bio/benchmarks/batch_integration",
     "note": "Positive control, not a model; rank derived from the data file, page shows no overall score",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "scanvi",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Open Problems batch integration: mean of 13 scaled scIB metrics per dataset, averaged over 6 datasets (failed or missing runs count as 0)",
     "eval_set": "Open Problems task_batch_integration, 6 CELLxGENE Census datasets; results run 2025-01 (website data commit 2025-02-10); page shows release 2.1.0-rc3 / stable v2.0.0",
     "source_type": "board",
     "date": {
      "date": "2025-02-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
      "verification": "verified",
      "note": null
     },
     "url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
     "note": "Not a site model; top non-control method; rank derived from the data file",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 18,
     "model_label": "scgpt_zeroshot",
     "model_id": "model-scgpt-geneformer",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Open Problems batch integration: mean of 13 scaled scIB metrics per dataset, averaged over 6 datasets (failed or missing runs count as 0)",
     "eval_set": "Open Problems task_batch_integration, 6 CELLxGENE Census datasets; results run 2025-01 (website data commit 2025-02-10); page shows release 2.1.0-rc3 / stable v2.0.0",
     "source_type": "board",
     "date": {
      "date": "2025-02-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
      "verification": "verified",
      "note": null
     },
     "url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
     "note": "Zero-shot scGPT; failed on 2 of 6 datasets, counted as 0; derived rank",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 23,
     "model_label": "geneformer",
     "model_id": "model-scgpt-geneformer",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Open Problems batch integration: mean of 13 scaled scIB metrics per dataset, averaged over 6 datasets (failed or missing runs count as 0)",
     "eval_set": "Open Problems task_batch_integration, 6 CELLxGENE Census datasets; results run 2025-01 (website data commit 2025-02-10); page shows release 2.1.0-rc3 / stable v2.0.0",
     "source_type": "board",
     "date": {
      "date": "2025-02-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
      "verification": "verified",
      "note": null
     },
     "url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
     "note": "Geneformer; 2 datasets missing (0), mean below the no-integration control; derived rank",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 25,
     "model_label": "scprint",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Open Problems batch integration: mean of 13 scaled scIB metrics per dataset, averaged over 6 datasets (failed or missing runs count as 0)",
     "eval_set": "Open Problems task_batch_integration, 6 CELLxGENE Census datasets; results run 2025-01 (website data commit 2025-02-10); page shows release 2.1.0-rc3 / stable v2.0.0",
     "source_type": "board",
     "date": {
      "date": "2025-02-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
      "verification": "verified",
      "note": null
     },
     "url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
     "note": "scPRINT v1 (not scPRINT-2); all results missing, so the rank reflects failure, not performance",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "The board orders by scaled overall mean but shows no numeric overall score; derived means are not used as a headline.",
   "caveat": "Ranks derived from the board's data file and include controls; results run 2025-01; AIDO.Cell and scPRINT-2 are not on the board; the 2022 paper has no foundation models.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-scgpt-geneformer",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://www.biorxiv.org/content/10.1101/2023.04.30.538439v2.full",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "scGPT only (PBMC10K), self-reported; Geneformer not applicable; family entry",
    "resolution": null
   },
   {
    "model_id": "model-aido-cell",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://huggingface.co/genbio-ai/GB.Cell-100M",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "disputed",
    "note": "Model card claims zero-shot clustering SOTA; scIB not named, no numbers",
    "resolution": null
   },
   {
    "model_id": "model-scprint-2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.biorxiv.org/content/10.64898/2025.12.11.693702v1",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "disputed",
    "note": "Abstract does not name scIB",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 2
   }
  },
  "how_to_use": "Install scib-metrics and call its Benchmarker with batch and cell-type keys, or submit a method through the Open Problems pipeline; no wet lab. State scib or scib-metrics, report weight sensitivity and within-type metrics; foundation models must add fitted HVG, Harmony and scVI baselines and separate zero-shot from fine-tuned. Pair with VCC perturbation.",
  "pairs_with": [
   "bench-24-vcc",
   "bench-21-dream5"
  ],
  "caveats": [
   "Ground truth is author annotation, not experimental gold standard; metrics saturate, rankings are weight-sensitive, and scib and scib-metrics numbers are not comparable.",
   "A third-party study found zero-shot scGPT and Geneformer trail HVG, Harmony and scVI; baselines are fitted on target data and models are not fine-tuned.",
   "Batch ASW is near saturation; one study built embeddings with high scIB scores but very different structure; within-type variation is missed.",
   "Pretraining corpora may overlap evaluation data; there is no train/test split, integration and scoring use the same data.",
   "Integration scores map to no experimental hit rate; at most a proxy for annotation and atlas building."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1038/s41592-021-01336-8",
    "title": "scIB paper (Nature Methods 2022)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1101/2020.05.22.111161",
    "title": "scIB preprint (bioRxiv 2020)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://openproblems.bio/benchmarks/batch_integration",
    "title": "Open Problems batch-integration board",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://raw.githubusercontent.com/openproblems-bio/website/main/results/batch_integration/data/results.json",
    "title": "Open Problems board data file",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://pypi.org/project/scib-metrics/",
    "title": "scib-metrics (PyPI)",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://pypi.org/project/scib/",
    "title": "scib (PyPI)",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11030530/",
    "title": "Open Problems platform preprint",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://link.springer.com/article/10.1186/s13059-025-03574-x",
    "title": "Zero-shot foundation-model evaluation (Genome Biol 2025)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://link.springer.com/article/10.1186/s13059-025-03869-z",
    "title": "scIB-E extended metrics (Genome Biol 2025)",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update / version",
    "was": "scib-metrics v0.5.8 latest; scib repository has no formal release",
    "now": "scib-metrics 0.6.2 (2026-11-01); scib GitHub releases 1.2.0 / 1.2.1 (2026-04)",
    "reason": "Site version info was stale: scib-metrics latest is 0.6.2 and scib now has GitHub releases (last PyPI upload still 1.1.7, 2025-01-13).",
    "source_url": "https://pypi.org/project/scib-metrics/",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.headline",
    "was": "headline pointing to a derived overall mean",
    "now": "null",
    "reason": "The board shows no numeric overall score, so derived means are not displayable scores.",
    "source_url": "https://openproblems.bio/benchmarks/batch_integration",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "family.members[2]",
   "family.members[3]",
   "ranking.entries",
   "models_reported[1]",
   "models_reported[2]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-09-scib/",
   "en": "/bio/benchmark/benchmarks/scib/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-10-guacamol",
  "kind": "benchmark",
  "nn": "10",
  "short_slug": "guacamol",
  "names": {
   "display": "GuacaMol (+MOSES)",
   "full": "GuacaMol molecular generation benchmark (with sibling MOSES)",
   "full_null_reason": null,
   "aliases": [
    "GuacaMol",
    "MOSES",
    "Molecular Sets"
   ],
   "ambiguous_aliases": []
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "GuacaMol",
     "id_or_null": null,
     "first_release": {
      "date": "2018-11-22",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv 1811.09621 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/1811.09621",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "MOSES",
     "id_or_null": null,
     "first_release": {
      "date": "2018-11-29",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv 1811.12823 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/1811.12823",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "GuacaMol has distribution and goal-directed tasks; MOSES is a sibling benchmark from another team with distribution metrics only on ZINC data; scores do not transfer."
  },
  "family_null_reason": null,
  "domain": "generation",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "D",
  "type_note": "Computed-oracle benchmark, no official board",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Retrospective on computed oracles"
  },
  "publisher": {
   "orgs": [
    "BenevolentAI",
    "Insilico Medicine"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Entry-level molecular generation benchmark; computed oracles only, unmaintained, optimization side superseded by PMO.",
  "first_release": {
   "date": "2018-11-22",
   "precision": "day",
   "year_hint": null,
   "basis": "GuacaMol paper arXiv v1 (MOSES arXiv v1 2018-11-29)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/1811.09621",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2018-11-22",
    "precision": "day",
    "year_hint": null,
    "basis": "GuacaMol arXiv 1811.09621 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/1811.09621",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2019-03-19",
    "precision": "day",
    "year_hint": null,
    "basis": "JCIM 2019 paper online",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1021/acs.jcim.8b00839",
    "verification": "partial",
    "note": "Date from audit notes; not re-checked on the DOI page"
   },
   "code_release": {
    "date": "2018-11-28",
    "precision": "day",
    "year_hint": null,
    "basis": "guacamol 0.3.1 uploaded to PyPI",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://pypi.org/project/guacamol/",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2023-09-19",
   "precision": "day",
   "year_hint": null,
   "basis": "guacamol 0.5.5 uploaded to PyPI",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://pypi.org/project/guacamol/",
   "verification": "verified",
   "note": null,
   "description": "guacamol 0.5.5 uploaded to PyPI (maintenance); MOSES repo last commit 2023-06-22."
  },
  "next_expected": null,
  "next_expected_null_reason": "The project is unmaintained and no maintainer has announced anything.",
  "status": "frozen",
  "status_note": "Unmaintained; distribution tasks partly saturated",
  "scale": {
   "summary": "5 distribution tasks plus 20 goal-directed tasks; ChEMBL 24 subset with 1,273,104 training SMILES.",
   "items": [
    {
     "label": "Distribution-learning tasks",
     "value": "5",
     "unit": "tasks",
     "version": "GuacaMol"
    },
    {
     "label": "Goal-directed tasks",
     "value": "20",
     "unit": "tasks",
     "version": "GuacaMol"
    },
    {
     "label": "Train / valid / test SMILES",
     "value": "1,273,104 / 79,568 / 238,706",
     "unit": "SMILES",
     "version": "GuacaMol (ChEMBL 24)"
    },
    {
     "label": "MOSES molecules",
     "value": "1,936,962",
     "unit": "molecules",
     "version": "MOSES"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None; computational oracles and a ChEMBL training set.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Molecule generation: five distribution-learning tasks and 20 goal-directed tasks scored by computed oracles.",
   "truth": "Public hand-written functions (similarity, descriptors, SMARTS, MPO); no experimental labels.",
   "metric_primary": "Goal-directed: summed weighted score over 20 tasks (max 20)",
   "metric_secondary": [
    "Distribution: Validity, Uniqueness, Novelty, KL, FCD",
    "MOSES: SNN, Frag, Scaf, IntDiv"
   ],
   "version_traps": "No oracle-call budget, so totals grow with calls; GuacaMol and MOSES metrics differ; PMO oracles largely derive from GuacaMol and are not independent.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "goal-directed: sum of top-1/10/100 weighted scores over 20 tasks (max 20); distribution: Validity/Uniqueness/Novelty/KL/FCD",
   "eval_set": "GuacaMol 20 goal-directed + 5 distribution-learning tasks (ChEMBL 24 subset); MOSES as sibling",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "Compilation date (no official board)",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": 1,
     "model_label": "Graph GA",
     "model_id": null,
     "site_model": false,
     "score": "20-task total 17.983 (original paper baselines: SMILES LSTM 17.340, SMILES GA 14.396, best-of-dataset 12.144, Graph MCTS 9.009)",
     "score_unit": null,
     "metric": "goal-directed: sum of top-1/10/100 weighted scores over 20 tasks (max 20); distribution: Validity/Uniqueness/Novelty/KL/FCD",
     "eval_set": "GuacaMol 20 goal-directed + 5 distribution-learning tasks (ChEMBL 24 subset); MOSES as sibling",
     "source_type": "self",
     "date": {
      "date": "2018-11-22",
      "precision": "day",
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/1811.09621",
      "verification": "verified",
      "note": "arXiv 1811.09621"
     },
     "url": "https://arxiv.org/abs/1811.09621",
     "note": "Not a site model; from the original paper baselines, not re-verified this round; later higher totals are scattered self-reports without common protocol or seed variance.",
     "second_hand": true,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "GenMol",
     "model_id": "model-genmol",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "goal-directed: sum of top-1/10/100 weighted scores over 20 tasks (max 20); distribution: Validity/Uniqueness/Novelty/KL/FCD",
     "eval_set": "GuacaMol 20 goal-directed + 5 distribution-learning tasks (ChEMBL 24 subset); MOSES as sibling",
     "source_type": "self",
     "date": {
      "date": "2025-01-10",
      "precision": "day",
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2501.06158",
      "verification": "verified",
      "note": "arXiv 2501.06158"
     },
     "url": "https://arxiv.org/abs/2501.06158",
     "note": "Reports no GuacaMol 20-task or MOSES score, using the data only for baselines; derived-oracle results sit under PMO.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "REINVENT 4",
     "model_id": "model-reinvent-4",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "goal-directed: sum of top-1/10/100 weighted scores over 20 tasks (max 20); distribution: Validity/Uniqueness/Novelty/KL/FCD",
     "eval_set": "GuacaMol 20 goal-directed + 5 distribution-learning tasks (ChEMBL 24 subset); MOSES as sibling",
     "source_type": "self",
     "date": {
      "date": "2024",
      "precision": "year",
      "year_hint": null,
      "basis": "Date given by the ranking source (paper)",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC10882833/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC10882833/",
     "note": "The paper cites GuacaMol and MOSES only as related work; the circulating number belongs to an earlier version re-tabulated by others; no pairing.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "No official board and no site model reports a GuacaMol score, so no headline is given.",
   "caveat": "No official board and no site model reports a score; unmaintained, with optimization superseded by PMO's 10K-call protocol; distribution tasks are gameable.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Install guacamol, build the data and implement a generator interface; MOSES uses molsets. Prefer the PMO 10K-budget protocol with at least 5 seeds; report oracle calls, AUC Top-10, quality-filtered scores and any non-ChEMBL data.",
  "pairs_with": [
   "bench-19-pmo",
   "bench-07-uspto"
  ],
  "caveats": [
   "All truths are cheap computed oracles open to direct optimization; there is no oracle-call budget, and trivial edits can fill distribution tasks. PMO superseded it for optimization.",
   "In the original paper the top-scoring methods pass chemical quality filters less often, so score and chemistry quality pull apart.",
   "The 20 tasks are simply summed with no CIs or seed requirement; top gaps are likely within noise.",
   "Rediscovery and similarity tasks are largely solved; isomer and some MPO tasks still discriminate.",
   "No evidence links scores to real activity or synthesizability."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/1811.09621",
    "title": "GuacaMol paper (arXiv 1811.09621)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/BenevolentAI/guacamol",
    "title": "GuacaMol repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://pypi.org/project/guacamol/",
    "title": "guacamol on PyPI",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/1811.12823",
    "title": "MOSES paper (arXiv 1811.12823)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/molecularsets/moses",
    "title": "MOSES repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2206.12411",
    "title": "PMO paper (arXiv 2206.12411)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2212.01385",
    "title": "Sample efficiency and filtering study (arXiv 2212.01385)",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_version",
    "was": "no formal release",
    "now": "PyPI releases guacamol 0.3.1 (2018-11-28) to 0.5.5 (2023-09-19)",
    "reason": "PyPI shows formal releases, contradicting the \"no formal release\" statement.",
    "source_url": "https://pypi.org/project/guacamol/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "release_milestones.journal_online",
   "ranking.entries[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-10-guacamol/",
   "en": "/bio/benchmark/benchmarks/guacamol/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-11-clinvar",
  "kind": "benchmark",
  "nn": "11",
  "short_slug": "clinvar",
  "names": {
   "display": "ClinVar hold-out protocol",
   "full": "Hold-out evaluation protocol built on the clinical variant-label database",
   "full_null_reason": null,
   "aliases": [
    "ClinVar hold-out protocol",
    "ClinVar benchmark",
    "ClinVar evaluation"
   ],
   "ambiguous_aliases": [
    "ClinVar"
   ]
  },
  "family": null,
  "family_null_reason": "This is an evaluation convention around one database, not a family with defined members.",
  "domain": "genomics-rna",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "A family of retrospective protocols with no official split",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Gene or time splits reach L3; no blind test"
  },
  "publisher": {
   "orgs": [
    "US National Center for Biotechnology Information"
   ],
   "org_type": "nonprofit-institute"
  },
  "one_liner": "Separating pathogenic from benign on held-out clinical labels, with each paper choosing its own snapshot and subset.",
  "first_release": {
   "date": "2013-04",
   "precision": "month",
   "year_hint": null,
   "basis": "The official full public release of the database per the official handbook; the November 2012 prototype is listed separately and the exact April day is unsourced.",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://www.ncbi.nlm.nih.gov/books/NBK174587/",
   "verification": "partial",
   "note": "Month only; the evaluation protocol itself has no release date"
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2012-11",
    "precision": "month",
    "year_hint": null,
    "basis": "Public record of the prototype version",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC3965032/",
    "verification": "partial",
    "note": "The prototype is not treated as first release"
   },
   "paper_v1": null,
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2013-04",
    "precision": "month",
    "year_hint": null,
    "basis": "Full public release as recorded in the official handbook",
    "event_kind": "official-announcement",
    "timestamp_utc": null,
    "source_url": "https://www.ncbi.nlm.nih.gov/books/NBK174587/",
    "verification": "partial",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-11-08",
   "precision": "day",
   "year_hint": null,
   "basis": "Official FTP release directory, read 2026-11-16",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/",
   "verification": "verified",
   "note": null,
   "description": "A weekly variant file release and a monthly full-record release; the evaluation protocol itself did not change."
  },
  "next_expected": null,
  "next_expected_null_reason": "The release cadence is observed, not announced with specific dates.",
  "status": "active",
  "status_note": "The database updates monthly; the evaluation protocol has no maintainer",
  "scale": {
   "summary": "No unified version: each paper builds its own subset, from about ten thousand to over a hundred thousand variants.",
   "items": [
    {
     "label": "Test variants in one missense model's own set",
     "value": "18924",
     "unit": null,
     "version": "2023"
    },
    {
     "label": "Missense variants in one protein language model's own set",
     "value": "about 150,000",
     "unit": null,
     "version": null
    },
    {
     "label": "Records in the database, queried 2026-11-16",
     "value": "4713415",
     "unit": null,
     "version": "2026-11-16"
    },
    {
     "label": "Records flagged as pathogenic among them",
     "value": "266706",
     "unit": null,
     "version": "2026-11-16"
    }
   ]
  },
  "wetlab_refresh": "accumulating",
  "wetlab_note": "No new experiments; the monthly additions are clinical submissions, not new measurements.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Binary classification of held-out variants as pathogenic or benign, mainly missense but also noncoding and splicing.",
   "truth": "Classifications submitted by clinical laboratories and researchers, carrying review stars, not experimental measurements.",
   "metric_primary": "auROC",
   "metric_secondary": [
    "auPRC and per-gene mean auROC",
    "Calibrated computational evidence strength"
   ],
   "version_traps": "State the snapshot date, star threshold, variant class and source of negatives: changing the negatives moves every method's score.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "auROC or auPRC, varying by paper",
   "eval_set": "Each paper builds its own ClinVar subset/snapshot",
   "snapshot_date": {
    "date": null,
    "precision": null,
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "unverified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "AlphaMissense",
     "model_id": "model-alphamissense",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "auROC or auPRC, varying by paper",
     "eval_set": "Each paper builds its own ClinVar subset/snapshot",
     "source_type": "self",
     "date": {
      "date": "2023-09-19",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://deepmind.google/research/projects/alphamissense/",
      "verification": "verified",
      "note": null
     },
     "url": "https://deepmind.google/research/projects/alphamissense/",
     "note": "Own missense test set; the figure is corroborated only second-hand, and training on this database inflates it",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "Not comparable: snapshots and subsets differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "AlphaGenome",
     "model_id": "model-alphagenome",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "auROC or auPRC, varying by paper",
     "eval_set": "Each paper builds its own ClinVar subset/snapshot",
     "source_type": "self",
     "date": {
      "date": "2026",
      "precision": "year",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1038/s41586-025-10014-0",
      "verification": "verified",
      "note": null
     },
     "url": "https://doi.org/10.1038/s41586-025-10014-0",
     "note": "Self-report on a splicing-related subset, from an existing site note, not re-verified",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "Not comparable: snapshots and subsets differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "Evo 2",
     "model_id": "model-evo-2",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "auROC or auPRC, varying by paper",
     "eval_set": "Each paper builds its own ClinVar subset/snapshot",
     "source_type": "self",
     "date": {
      "date": "2026",
      "precision": "year",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1038/s41586-026-10176-5",
      "verification": "verified",
      "note": null
     },
     "url": "https://doi.org/10.1038/s41586-026-10176-5",
     "note": "Figure-only results on its own subsets, with no quotable numbers and mixed standing by variant class",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "Not comparable: snapshots and subsets differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "popEVE",
     "model_id": "model-popeve",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "auROC or auPRC, varying by paper",
     "eval_set": "Each paper builds its own ClinVar subset/snapshot",
     "source_type": "self",
     "date": {
      "date": "2025",
      "precision": "year",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1038/s41588-025-02400-1",
      "verification": "verified",
      "note": null
     },
     "url": "https://doi.org/10.1038/s41588-025-02400-1",
     "note": "Within-gene AUROC in figures and tables, not transcribed; from an existing site note",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "Not comparable: snapshots and subsets differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "BioReason (Evo2+Qwen3-4B)",
     "model_id": "model-bioreason",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "auROC or auPRC, varying by paper",
     "eval_set": "Each paper builds its own ClinVar subset/snapshot",
     "source_type": "self",
     "date": {
      "date": "2025-05",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2505.23579",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2505.23579",
     "note": "Its own derived split; from an existing site note, not re-verified",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "Not comparable: snapshots and subsets differ, so the ranking row shows no score."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "The protocols are not comparable and there is no unified board, so no headline number is given.",
   "caveat": "This is a database, not a leaderboard: every paper picks its own snapshot, filtering and variant class, predictors trained on it risk circularity, and no cross-model ranking is possible.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphagenome",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1038/s41586-025-10014-0",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Its own variant-effect suite, including splicing classes from this database",
    "resolution": null
   },
   {
    "model_id": "model-alphamissense",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1126/science.adg7492",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Its own missense test set",
    "resolution": null
   },
   {
    "model_id": "model-bioreason",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2505.23579v2",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Its own pathway question task, not this protocol",
    "resolution": null
   },
   {
    "model_id": "model-evo-2",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC13128491/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Its own subsets, figure-only",
    "resolution": null
   },
   {
    "model_id": "model-popeve",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12695638/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "An expert-curated subset from earlier years",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 5,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 5,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Take a dated snapshot, filter by star level and variant class and score the held-out set; no wet lab of your own is needed. Report the snapshot date, star threshold, source of negatives and gene distribution, and state whether the model has seen these labels. Generalization claims need a time split or held-out genes plus calibrated evidence strength. Pair with ProteinGym and CAGI.",
  "pairs_with": [
   "bench-02-proteingym",
   "bench-13-cagi",
   "bench-37-traitgym",
   "bench-08-enformer"
  ],
  "caveats": [
   "There is no official split, predictors and labels contaminate each other, and training on this database inflates scores.",
   "Pooled coding-missense auROC is already above 0.9, so gaps of 0.01 to 0.03 do not separate methods.",
   "The gene distribution is heavily skewed, with a few frequently tested genes dominating.",
   "The truth is submitted clinical interpretation rather than measurement, and submitters sometimes conflict.",
   "High auROC does not mean clinical usability, which requires calibrated evidence strength."
  ],
  "sources": [
   {
    "url": "https://www.ncbi.nlm.nih.gov/clinvar/",
    "title": "Official database site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://www.ncbi.nlm.nih.gov/books/NBK174587/",
    "title": "Database chapter of the official handbook",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/",
    "title": "Official release directory",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://www.ncbi.nlm.nih.gov/clinvar/docs/review_status/",
    "title": "Documentation of review status stars",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC3965032/",
    "title": "Early description paper of the database",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1126/science.adg7492",
    "title": "Model paper reporting this protocol on its own subset",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release_basis",
    "was": "ClinVar NAR database paper online (public launch reportedly Apr 2013, not verified)",
    "now": "official full public release of ClinVar, April 2013 (NCBI Handbook); Nov 2012 prototype stored separately, not displayed as first release; exact April day not sourced.",
    "reason": "The review confirms April 2013 as the full public release and asks that the prototype be listed separately rather than merged.",
    "source_url": "https://www.ncbi.nlm.nih.gov/books/NBK174587/",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "2013-11-14",
    "now": "2013-04 (official launch; prototype Nov 2012)",
    "reason": "The previously recorded date is the journal paper's online date, not the database release date.",
    "source_url": "https://www.ncbi.nlm.nih.gov/books/NBK174587/",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries",
    "was": "per-model scores shown in the ranking row",
    "now": "scores withheld; each entry keeps its protocol description only",
    "reason": "Display rule: entries that are not comparable show no score; per-protocol values stay on the entity page.",
    "source_url": "https://www.ncbi.nlm.nih.gov/clinvar/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "release_milestones.first_public_evidence",
   "release_milestones.official_announcement",
   "ranking.snapshot_date",
   "ranking.entries[0]",
   "ranking.entries[1]",
   "ranking.entries[2]",
   "scale.items[1]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-11-clinvar/",
   "en": "/bio/benchmark/benchmarks/clinvar/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-12-dude",
  "kind": "benchmark",
  "nn": "12",
  "short_slug": "dude",
  "names": {
   "display": "DUD-E / LIT-PCBA",
   "full": "Directory of Useful Decoys, Enhanced / Literature-derived unbiased PubChem set for virtual screening",
   "full_null_reason": null,
   "aliases": [
    "DUD-E",
    "DUDE",
    "LIT-PCBA",
    "DUDE-Z",
    "Directory of Useful Decoys, Enhanced"
   ],
   "ambiguous_aliases": [
    "DUD",
    "MUV",
    "DEKOIS"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "MUV",
     "id_or_null": null,
     "first_release": {
      "date": "2009-01-22",
      "precision": "day",
      "year_hint": null,
      "basis": "MUV paper online",
      "event_kind": "journal-online",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC8352508",
      "verification": "partial",
      "note": "Member date from the existing record"
     }
    },
    {
     "name": "DUD-E",
     "id_or_null": null,
     "first_release": {
      "date": "2012-07-05",
      "precision": "day",
      "year_hint": null,
      "basis": "DUD-E paper online in JMC",
      "event_kind": "journal-online",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1021/jm300687e",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "LIT-PCBA",
     "id_or_null": null,
     "first_release": {
      "date": "2020-04-13",
      "precision": "day",
      "year_hint": null,
      "basis": "LIT-PCBA paper online in JCIM",
      "event_kind": "journal-online",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1021/acs.jcim.0c00155",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "DUDE-Z",
     "id_or_null": null,
     "first_release": {
      "date": "2021",
      "precision": "year",
      "year_hint": null,
      "basis": "DUDE-Z paper, JCIM 2021 (online date not retrieved)",
      "event_kind": "journal-online",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1021/acs.jcim.0c00598",
      "verification": "partial",
      "note": null
     }
    }
   ],
   "boundary": "Family: MUV, DUD-E, LIT-PCBA and successor DUDE-Z; DEKOIS 2.0 is only a related set; the original 2006 DUD was not checked or listed."
  },
  "family_null_reason": null,
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static virtual-screening dataset suite, no official board",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Retrospective, random or clustered splits"
  },
  "publisher": {
   "orgs": [
    "University of California, San Francisco (academic group)",
    "University of Strasbourg (academic group)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Two generations of virtual-screening enrichment sets: constructed decoys (DUD-E) and PubChem inactives (LIT-PCBA); both have known bias or leakage.",
  "first_release": {
   "date": "2012-07-05",
   "precision": "day",
   "year_hint": null,
   "basis": "DUD-E paper (JMC online)",
   "event_kind": "journal-online",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1021/jm300687e",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": null,
   "journal_online": {
    "date": "2012-07-05",
    "precision": "day",
    "year_hint": null,
    "basis": "DUD-E paper online in JMC",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1021/jm300687e",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-02-05",
   "precision": "day",
   "year_hint": null,
   "basis": "DUDE-Z page update note",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://dudez.docking.org",
   "verification": "partial",
   "note": "The 2025 third-party LIT-PCBA audit is not a dataset update",
   "description": "DUDE-Z page: recommended grids and ligand/decoy files re-checked against the paper with DOCK3.8; no new DUD-E or LIT-PCBA data version found."
  },
  "next_expected": null,
  "next_expected_null_reason": "Neither dataset has an announced versioning plan.",
  "status": "low-frequency",
  "status_note": "Static; LIT-PCBA audited in 2025 and found to contain duplicates across splits",
  "scale": {
   "summary": "DUD-E: 102 targets, 22,886 actives; LIT-PCBA: 15 targets, 7,761 actives, 382,674 inactives (official site count).",
   "items": [
    {
     "label": "Targets",
     "value": "102",
     "unit": null,
     "version": "DUD-E"
    },
    {
     "label": "Actives",
     "value": "22,886",
     "unit": null,
     "version": "DUD-E"
    },
    {
     "label": "Decoys per active",
     "value": "50",
     "unit": null,
     "version": "DUD-E"
    },
    {
     "label": "Targets",
     "value": "15",
     "unit": null,
     "version": "LIT-PCBA"
    },
    {
     "label": "Actives",
     "value": "7,761",
     "unit": null,
     "version": "LIT-PCBA (official site; some sources give 7,844)"
    },
    {
     "label": "Unique inactives",
     "value": "382,674",
     "unit": null,
     "version": "LIT-PCBA (official site; some sources give 407,381)"
    },
    {
     "label": "Targets",
     "value": "43",
     "unit": null,
     "version": "DUDE-Z"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None; DUD-E uses constructed decoys, LIT-PCBA uses PubChem measured data.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Virtual screening enrichment: ranking true actives above decoys or measured inactives.",
   "truth": "DUD-E: ChEMBL actives plus property-matched ZINC decoys; LIT-PCBA: PubChem dose-response labels.",
   "metric_primary": "AUROC, EF1%",
   "metric_secondary": [
    "BEDROC",
    "logAUC"
   ],
   "version_traps": "Do not mix DUD-E, DUDE-Z and LIT-PCBA; LIT-PCBA's 'validation' is a held-out test needing a separate internal split; two count versions exist.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "third-party",
   "metric": "Median EF1% (LIT-PCBA)",
   "eval_set": "LIT-PCBA, 15 targets (DUD-E: no current site-model numbers)",
   "snapshot_date": {
    "date": "2026-05-05",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "Supervised ML re-ranking (best)",
     "model_id": null,
     "site_model": false,
     "score": "median EF1% 4.49",
     "score_unit": null,
     "metric": "Median EF1% (LIT-PCBA)",
     "eval_set": "LIT-PCBA, 15 targets (DUD-E: no current site-model numbers)",
     "source_type": "third",
     "date": {
      "date": "2026-05-05",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2605.01681",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2605.01681",
     "note": "Not a site model; best in that study",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "AutoDock-GPU + GNINA rescoring",
     "model_id": null,
     "site_model": false,
     "score": "median EF1% 2.14",
     "score_unit": null,
     "metric": "Median EF1% (LIT-PCBA)",
     "eval_set": "LIT-PCBA, 15 targets (DUD-E: no current site-model numbers)",
     "source_type": "third",
     "date": {
      "date": "2026-05-05",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2605.01681",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2605.01681",
     "note": "Not a site model; best single scorer",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "DiffDock",
     "model_id": "model-diffdock",
     "site_model": true,
     "score": "median EF1% 1.17",
     "score_unit": null,
     "metric": "Median EF1% (LIT-PCBA)",
     "eval_set": "LIT-PCBA, 15 targets (DUD-E: no current site-model numbers)",
     "source_type": "third",
     "date": {
      "date": "2026-05-05",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2605.01681",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2605.01681",
     "note": "The DiffDock-GNINA variant scores lower; all variants score zero on OPRK1",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "AutoDock Vina",
     "model_id": "model-vina",
     "site_model": true,
     "score": "median EF1% 1.3, median AUROC 0.61",
     "score_unit": null,
     "metric": "Median EF1% (LIT-PCBA)",
     "eval_set": "LIT-PCBA, 15 targets (DUD-E: no current site-model numbers)",
     "source_type": "third",
     "date": {
      "date": "2026-05-05",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2605.01681",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2605.01681",
     "note": "Cited from literature in that preprint, not run by its authors; their own AutoDock-GPU run differs",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "No headline number: the ranking source gives no rank-1 entry (all entries are unranked or protocol-specific); the caveat text is shown instead, as for a not-comparable cell.",
   "caveat": "No official board for either set; source is a non-peer-reviewed preprint; AlphaFold 3, Protenix, Boltz-2 have only second-hand qualitative claims; known bias and leakage.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-diffdock",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2605.01681",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Third-party LIT-PCBA study (arXiv 2605.01681)",
    "resolution": null
   },
   {
    "model_id": "model-vina",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/pdf/2605.01681",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Third-party LIT-PCBA study ran AutoDock/GNINA",
    "resolution": null
   },
   {
    "model_id": "model-alphafold3",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "low",
    "verification": "disputed",
    "note": "Only a second-hand pose claim in an RSC 2026 paper; no numbers; the benchmark paper does not evaluate AF3",
    "resolution": null
   },
   {
    "model_id": "model-boltz-2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://pubs.rsc.org/en/content/articlelanding/2026/sc/d5sc06481c",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "disputed",
    "note": "Third-party paper uses DEKOIS 2.0 etc., not clearly DUD-E/LIT-PCBA; second-hand",
    "resolution": null
   },
   {
    "model_id": "model-protenix",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "low",
    "verification": "disputed",
    "note": "Second-hand LIT-PCBA mention, no numbers",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 2,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 2,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 3
   }
  },
  "how_to_use": "Download and run statically; no wet lab. Before LIT-PCBA, deduplicate train/validation, run a near-neighbour analysis and carve an internal validation split; report EF1% with intervals and per-target spread plus ligand-only and memorisation baselines. Novel-scaffold claims need prospective assays or CACHE; pair with CASF-2016 and PLINDER.",
  "pairs_with": [
   "bench-20-cache",
   "bench-06-casf",
   "bench-30-plinder",
   "bench-05-posebusters"
  ],
  "caveats": [
   "DUD-E has decoy and analogue biases that inflate ML results. A 2025 preprint reports cross-split duplicates and near-neighbour leakage in LIT-PCBA (single preprint, not peer-reviewed).",
   "The audit claims a no-learning memorisation baseline matches or beats SOTA; no response from the LIT-PCBA authors was found.",
   "A 2026 preprint finds docking and ML re-ranking only slightly better than random on LIT-PCBA (not peer-reviewed).",
   "LIT-PCBA has two count versions (official site vs some archives); this entry follows the official site.",
   "Retrospective; no systematic evidence that score gains raise experimental hit rates."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1021/jm300687e",
    "title": "DUD-E paper (J Med Chem 2012)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://dude.docking.org/",
    "title": "DUD-E official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://dudez.docking.org",
    "title": "DUDE-Z official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://doi.org/10.1021/acs.jcim.0c00155",
    "title": "LIT-PCBA paper (JCIM 2020)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://lab.drugdesign.unistra.fr/datasets/lit-pcba/",
    "title": "LIT-PCBA data page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2507.21404",
    "title": "LIT-PCBA audit preprint (arXiv 2507.21404)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2605.01681",
    "title": "LIT-PCBA docking benchmark preprint (arXiv 2605.01681)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6701836",
    "title": "DUD-E bias analysis (PLoS ONE 2019)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://pubs.rsc.org/en/content/articlelanding/2026/sc/d5sc06481c",
    "title": "Co-folding models in virtual screening (Chem Sci 2026, cited second-hand)",
    "kind": "second-hand",
    "date": null
   }
  ],
  "corrections": [],
  "unverified_fields": [
   "latest_update",
   "family.members[0]",
   "family.members[3]",
   "family.boundary",
   "models_reported[2]",
   "models_reported[3]",
   "models_reported[4]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-12-dude/",
   "en": "/bio/benchmark/benchmarks/dude/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-13-cagi",
  "kind": "benchmark",
  "nn": "13",
  "short_slug": "cagi",
  "names": {
   "display": "CAGI",
   "full": "Critical Assessment of Genome Interpretation",
   "full_null_reason": null,
   "aliases": [
    "CAGI",
    "CAGI5",
    "CAGI6",
    "CAGI7",
    "Critical Assessment of Genome Interpretation"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "Single recurring challenge; CAGI1–7 are editions of one challenge, not family members.",
  "domain": "genomics-rna",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "A",
  "type_note": "Recurring blind challenge with heterogeneous tasks per edition",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": [
    "L4",
    "L5"
   ],
   "qualifier": "Annotate-All-Missense near L5 (post-deadline ClinVar)"
  },
  "publisher": {
   "orgs": [
    "CAGI consortium (NIH-funded)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Long-running CASP-style blind test of variant interpretation, scored on data unpublished at submission.",
  "first_release": {
   "date": "2010",
   "precision": "year",
   "year_hint": null,
   "basis": "first CAGI season organised in 2010 (year-only evidence). The 2010-12-10 Berkeley workshop is the closing event, not the opening; no organiser page states the opening month.",
   "event_kind": "challenge-open",
   "timestamp_utc": null,
   "source_url": "https://genomeinterpretation.org/",
   "verification": "partial",
   "note": "Year sourced; opening month has no primary source"
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2010",
    "precision": "year",
    "year_hint": null,
    "basis": "Blog report on the first season (2010-12)",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://bioinfo-pioneers.blogspot.com/2010/12/mutation-prediction-software-rewarded.html",
    "verification": "partial",
    "note": "2010-12-10 workshop is the closing event, not first release"
   },
   "paper_v1": null,
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2010",
    "precision": "year",
    "year_hint": null,
    "basis": "first CAGI season organised in 2010 (year-only evidence). The 2010-12-10 Berkeley workshop is the closing event, not the opening; no organiser page states the opening month.",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://genomeinterpretation.org/",
    "verification": "partial",
    "note": "Year sourced; opening month has no primary source"
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-06-22",
   "precision": "day",
   "year_hint": null,
   "basis": "Organiser website announcement",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://genomeinterpretation.org/",
   "verification": "verified",
   "note": null,
   "description": "Organisers announced the CAGI7 prediction season complete (14 challenges; RGP-CRAM opened 2025-12-27, deadlines extended to February 2026)."
  },
  "next_expected": null,
  "next_expected_null_reason": "The organiser site does not announce CAGI8.",
  "status": "active",
  "status_note": "Active; CAGI7 ran in 2025, conference December 2025",
  "scale": {
   "summary": "CAGI7: 14 challenges (12 plus 2 off-cycle); CAGI1–5: 50 challenges, 738 submissions.",
   "items": [
    {
     "label": "Challenges",
     "value": "14",
     "unit": null,
     "version": "CAGI7 (12 + 2 off-cycle)"
    },
    {
     "label": "Challenges",
     "value": "13 + 1",
     "unit": null,
     "version": "CAGI6"
    },
    {
     "label": "Challenges",
     "value": "50",
     "unit": null,
     "version": "CAGI1–5"
    },
    {
     "label": "Submissions",
     "value": "738",
     "unit": null,
     "version": "CAGI1–5"
    },
    {
     "label": "Predictors assessed (Annotate-All-Missense)",
     "value": "60",
     "unit": null,
     "version": "CAGI6"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "Every 2–3 years, 12–14 challenges per edition; truth from data providers, no dedicated wet lab. New truth only by entering the next edition.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Blind assessment of variant interpretation across functional assays, splicing, rare-disease diagnosis and complex-trait prediction.",
   "truth": "Experimental (MAVE, enzyme activity) or clinical data unpublished at submission; Annotate-All uses ClinVar labels added after the deadline.",
   "metric_primary": "Varies by challenge: Pearson/Kendall τ, AUC",
   "metric_secondary": [
    "lr+, accuracy",
    "Top-N ranking (rare-disease diagnosis)"
   ],
   "version_traps": "No unified score across challenges; state edition, challenge and assessment-paper version (journal and preprint numbers may differ).",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "third-party",
   "metric": "AUROC (pathogenic vs benign missense)",
   "eval_set": "CAGI6 Annotate-All-Missense full evaluation set (60 predictors)",
   "snapshot_date": {
    "date": "2025-03-21",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://genomeinterpretation.ccs.neu.edu/challenges.html",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://genomeinterpretation.ccs.neu.edu/challenges.html",
   "entries": [
    {
     "rank": 1,
     "model_label": "ClinPred / MetaRNN",
     "model_id": null,
     "site_model": false,
     "score": "AUROC 0.92",
     "score_unit": null,
     "metric": "AUROC (pathogenic vs benign missense)",
     "eval_set": "CAGI6 Annotate-All-Missense full evaluation set (60 predictors)",
     "source_type": "third",
     "date": {
      "date": "2025-03-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
     "note": "Not a site model; meta-predictors using allele frequency",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "VARITY_R",
     "model_id": null,
     "site_model": false,
     "score": "AUROC 0.88",
     "score_unit": null,
     "metric": "AUROC (pathogenic vs benign missense)",
     "eval_set": "CAGI6 Annotate-All-Missense full evaluation set (60 predictors)",
     "source_type": "third",
     "date": {
      "date": "2025-03-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
     "note": "Not a site model; meta-predictor without allele frequency",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "AlphaMissense",
     "model_id": "model-alphamissense",
     "site_model": true,
     "score": "AUROC 0.88",
     "score_unit": null,
     "metric": "AUROC (pathogenic vs benign missense)",
     "eval_set": "CAGI6 Annotate-All-Missense full evaluation set (60 predictors)",
     "source_type": "third",
     "date": {
      "date": "2025-03-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
     "note": "Best individual (non-meta) predictor, tied with another; evaluated post hoc, not a CAGI submitter; no overall rank stated.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": 2,
   "headline_null_reason": null,
   "caveat": "From a third-party assessment paper (Hum Genet 144:281-293); AlphaMissense's overall rank is not stated (tie at 0.88); CAGI7 assessments unpublished; no unified CAGI board.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphamissense",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://link.springer.com/article/10.1007/s00439-025-02732-2",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Post-hoc third-party evaluation on CAGI6 (checked at abstract level)",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 1,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Enter the next edition, or test post hoc on released assessment data under a data-use agreement (no longer blind for later-cutoff models). No own wet lab needed; state edition, challenge and metric, and check training/evaluation overlap. Pair with ProteinGym (offline function) and ClinVar time hold-out (clinical).",
  "pairs_with": [
   "bench-02-proteingym",
   "bench-11-clinvar",
   "bench-17-cafa",
   "bench-01-casp"
  ],
  "caveats": [
   "Challenges are small and heterogeneous with no unified score; some have only tens of variants, assessors can clean data afterwards, and most predictors saw ClinVar or HGMD in training.",
   "AlphaMissense was the best individual predictor on CAGI6 (AUROC 0.88, third-party assessment, 2025), evaluated post hoc and behind two meta-predictors; not first of all 60.",
   "Absolute functional values are poorly calibrated and regulatory and complex-trait tasks remain weak; missense binary classification is near its empirical ceiling.",
   "No loop of selecting variants by prediction and then testing them, so no direct evidence of wet-lab translation."
  ],
  "sources": [
   {
    "url": "https://genomeinterpretation.org/",
    "title": "CAGI official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://genomeinterpretation.ccs.neu.edu/cagi7-challenge.html",
    "title": "CAGI7 page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://genomeinterpretation.ccs.neu.edu/challenges.html",
    "title": "CAGI challenge list",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC10882881/",
    "title": "CAGI summary paper, Genome Biology 25:53 (2024)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
    "title": "CAGI6 Annotate-All-Missense assessment (Hum Genet 2025)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://link.springer.com/article/10.1007/s00439-025-02732-2",
    "title": "Same assessment, journal page",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://bioinfo-pioneers.blogspot.com/2010/12/mutation-prediction-software-rewarded.html",
    "title": "Blog report on the first season (second-hand)",
    "kind": "second-hand",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2010 (fall); 2010-12-10 is the culminating Berkeley workshop, not the start",
    "now": "2010",
    "reason": "The culminating Berkeley workshop (2010-12-10) is not the opening of the first season and no month is sourced; year-only stays year-only.",
    "source_url": "https://genomeinterpretation.org/",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "Pre-CAGI/CAGI1 workshop in Berkeley 2010-12-10 (first round organized fall 2010)",
    "now": "first CAGI season organised in 2010 (year-only evidence). The 2010-12-10 Berkeley workshop is the closing event, not the opening; no organiser page states the opening month.",
    "reason": "Date field no longer carries season or conference prose (DATE_AUDIT).",
    "source_url": "https://genomeinterpretation.org/",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "2010-12-10",
    "now": "2010 (fall); 2010-12-10 is the culminating Berkeley workshop, not the start",
    "reason": "The old value used the 2010-12-10 closing workshop as first release; a closing date is not an opening date.",
    "source_url": "https://bioinfo-pioneers.blogspot.com/2010/12/mutation-prediction-software-rewarded.html",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking / models_reported (AlphaMissense)",
    "was": "AlphaMissense ranked first of 60 predictors in CAGI6",
    "now": "Best individual (non-meta) predictor, AUROC 0.88, tied with another non-meta predictor, behind two meta-predictors (0.92); evaluated post hoc, not a CAGI submitter; no overall rank stated in the paper",
    "reason": "CHANGESET §1.1: the published 'first of 60 predictors' is wrong; the third-party assessment supports only 'best individual predictor, evaluated post hoc'.",
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11976771/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "release_milestones.challenge_open",
   "release_milestones.first_public_evidence",
   "models_reported[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-13-cagi/",
   "en": "/bio/benchmark/benchmarks/cagi/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-14-pubmedqa",
  "kind": "benchmark",
  "nn": "14",
  "short_slug": "pubmedqa",
  "names": {
   "display": "PubMedQA",
   "full": null,
   "full_null_reason": "The name has no expanded form.",
   "aliases": [
    "PubMedQA",
    "PQA-L",
    "PQA-U",
    "PQA-A"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "Single static dataset; no family members.",
  "domain": "text-agents",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B+E",
  "type_note": "Static text dataset, a routine LLM eval row",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L1",
   "range": null,
   "qualifier": null
  },
  "publisher": {
   "orgs": [
    "University of Pittsburgh",
    "Carnegie Mellon University"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "500-question yes/no/maybe test on PubMed abstracts; top scores now sit within sampling noise.",
  "first_release": {
   "date": "2019-09-13",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv v1 of the PubMedQA paper (2019-09-13); GitHub initial commit 2019-08-23 kept as a note only (repo creation excluded as evidence).",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/1909.06146",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": null,
    "precision": null,
    "year_hint": 2019,
    "basis": "GitHub repository initial commit (repository event, excluded as first-release evidence)",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://github.com/pubmedqa/pubmedqa/commits/master",
    "verification": "unverified",
    "note": "Initial commit 2019-08-23, recorded only"
   },
   "paper_v1": {
    "date": "2019-09-13",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv v1 of the PubMedQA paper (2019-09-13); GitHub initial commit 2019-08-23 kept as a note only (repo creation excluded as evidence).",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/1909.06146",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2019",
    "precision": "year",
    "year_hint": null,
    "basis": "EMNLP-IJCNLP 2019 conference paper",
    "event_kind": "conference-publication",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.18653/v1/d19-1259",
    "verification": "partial",
    "note": "Conference paper; year only recorded"
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2023-04-18",
   "precision": "day",
   "year_hint": null,
   "basis": "README commit on the data repository",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://github.com/pubmedqa/pubmedqa/commits/master",
   "verification": "verified",
   "note": null,
   "description": "README edit on the data repository (2023-04-18); dataset frozen since 2019. Most recent sourced event, not a claim that nothing happened later."
  },
  "next_expected": null,
  "next_expected_null_reason": "Dataset frozen; nothing announced by the authors.",
  "status": "frozen",
  "status_note": "Saturated; newest official-board entry dated 2024-04-18",
  "scale": {
   "summary": "PQA-L 1.0k expert-labeled (500 test), PQA-U 61.2k, PQA-A 211.3k.",
   "items": [
    {
     "label": "Expert-labelled questions (PQA-L)",
     "value": "1,000",
     "unit": null,
     "version": "PQA-L"
    },
    {
     "label": "Test set",
     "value": "500",
     "unit": "questions",
     "version": "PQA-L"
    },
    {
     "label": "Unlabelled questions (PQA-U)",
     "value": "61.2k",
     "unit": null,
     "version": "PQA-U"
    },
    {
     "label": "Auto-generated questions (PQA-A)",
     "value": "211.3k",
     "unit": null,
     "version": "PQA-A"
    },
    {
     "label": "Label split yes/no/maybe",
     "value": "55.2% / 33.8% / 11.0%",
     "unit": null,
     "version": "PQA-L"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None; fixed expert-labelled text data.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Answering yes/no/maybe questions from PubMed abstracts, testing reading comprehension of study conclusions.",
   "truth": "Two annotators' classification of the authors' conclusion (PQA-L); authors estimate about 1% residual error.",
   "metric_primary": "Accuracy (reasoning-required setting)",
   "metric_secondary": [
    "Macro-F1"
   ],
   "version_traps": "State setting (reasoning-required or free), zero/few-shot, fine-tuning, retrieval or ensembling, and whether the train split was used.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Accuracy (%) (macro-F1 mostly not reported), reasoning-required setting",
   "eval_set": "PQA-L test (500 questions)",
   "snapshot_date": {
    "date": "2024-04-18",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://pubmedqa.github.io",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://pubmedqa.github.io",
   "entries": [
    {
     "rank": 1,
     "model_label": "GPT-4 (Medprompt)",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "82.0",
     "score_unit": null,
     "metric": "Accuracy (%) (macro-F1 mostly not reported), reasoning-required setting",
     "eval_set": "PQA-L test (500 questions)",
     "source_type": "board",
     "date": {
      "date": "2023-11-28",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pubmedqa.github.io",
      "verification": "verified",
      "note": null
     },
     "url": "https://pubmedqa.github.io",
     "note": "Overall top-1; prompting plus ensembling, self-submitted.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 6,
     "model_label": "GPT-4-base",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "80.4",
     "score_unit": null,
     "metric": "Accuracy (%) (macro-F1 mostly not reported), reasoning-required setting",
     "eval_set": "PQA-L test (500 questions)",
     "source_type": "board",
     "date": {
      "date": "2023-04-12",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pubmedqa.github.io",
      "verification": "verified",
      "note": null
     },
     "url": "https://pubmedqa.github.io",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 7,
     "model_label": "Claude 3",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "79.7",
     "score_unit": null,
     "metric": "Accuracy (%) (macro-F1 mostly not reported), reasoning-required setting",
     "eval_set": "PQA-L test (500 questions)",
     "source_type": "board",
     "date": {
      "date": "2024-03-04",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pubmedqa.github.io",
      "verification": "verified",
      "note": null
     },
     "url": "https://pubmedqa.github.io",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 16,
     "model_label": "GPT-4",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "75.2",
     "score_unit": null,
     "metric": "Accuracy (%) (macro-F1 mostly not reported), reasoning-required setting",
     "eval_set": "PQA-L test (500 questions)",
     "source_type": "board",
     "date": {
      "date": "2023-03-20",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pubmedqa.github.io",
      "verification": "verified",
      "note": null
     },
     "url": "https://pubmedqa.github.io",
     "note": "Zero-shot",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "MedGemma 27B (text-it)",
     "model_id": "model-medgemma-med-gemini",
     "site_model": true,
     "score": "76.8",
     "score_unit": null,
     "metric": "Accuracy (%) (macro-F1 mostly not reported), reasoning-required setting",
     "eval_set": "PQA-L test (500 questions)",
     "source_type": "self",
     "date": {
      "date": "2025-07-07",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/google/medgemma-27b-text-it",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/google/medgemma-27b-text-it",
     "note": "Not on board; HF model card (Gemma 3 27B: 73.4). Training data include the PubMedQA train split.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Gemini 3 Pro",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "82.2",
     "score_unit": null,
     "metric": "Accuracy (%) (macro-F1 mostly not reported), reasoning-required setting",
     "eval_set": "PQA-L test (500 questions)",
     "source_type": "self",
     "date": {
      "date": "2026-04",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2604.05081",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2604.05081",
     "note": "Not on board; from site note, not re-verified.",
     "second_hand": true,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "Newest board entry 2024-04-18; no later frontier reasoning models. Mixes zero-shot, few-shot, fine-tuned and ensembles, all self-submitted; top five within ~1.4 points vs ~±1.9-point binomial SE on 500 items.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-frontier-llms",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2604.05081",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Gemini 3 figures self-reported in the MedGemma 1.5 report; earlier board entries (GPT-4, Claude 3) were self-submitted.",
    "resolution": null
   },
   {
    "model_id": "model-medgemma-med-gemini",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2507.05201",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "MedGemma 4B, 27B and 1.5 reported; Med-Gemini does not report PubMedQA.",
    "resolution": null
   },
   {
    "model_id": "model-co-scientist",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/2502.18864",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-gpt-rosalind",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found",
    "resolution": null
   },
   {
    "model_id": "model-intern-s",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://github.com/InternLM/Intern-S1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-quine",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.microsoft.com/en-us/research/blog/introducing-quine-an-ai-research-system-designed-for-the-complexity-of-biology/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-scireasoner",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "Not in the paper text",
    "resolution": null
   },
   {
    "model_id": "model-verid",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 2,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 2,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 6
   }
  },
  "how_to_use": "Download PQA-L and evaluate on the 500-question test set; report accuracy and macro-F1, prompt template and setting, with bootstrap confidence intervals. Board entry requires sending predictions and a system description to the authors. Useful for coarse screening and small-model comparison, not for separating frontier models; pair with BioASQ and LitQA in LAB-Bench.",
  "pairs_with": [
   "bench-23-bioasq",
   "bench-18-labbench",
   "bench-29-bixbench"
  ],
  "caveats": [
   "The 500-question test set gives about ±1.9 points standard error, so top scores are indistinguishable; the leaderboard mixes prompting methods, and public data may be in training sets.",
   "The single-annotator human baseline (78.0) is a lower bound; most LLM entries report accuracy only, hiding whether the 'maybe' class is learned.",
   "MedGemma training data include the PubMedQA train split, so such scores are not contamination-free generalization.",
   "Aggregator figures of 87–95% for ensembles or knowledge-graph systems cannot be traced to primary sources and are not used.",
   "Tests reading comprehension of abstract conclusions only, not experimental design or evidence synthesis; no wet-lab translation evidence."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/1909.06146",
    "title": "PubMedQA paper, arXiv 1909.06146",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.18653/v1/d19-1259",
    "title": "EMNLP-IJCNLP 2019 version",
    "kind": "paper",
    "date": null
   },
   {
    "url": "https://pubmedqa.github.io",
    "title": "PubMedQA official leaderboard",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://github.com/pubmedqa/pubmedqa",
    "title": "PubMedQA data repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2507.05201",
    "title": "MedGemma technical report (developer)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://huggingface.co/google/medgemma-27b-text-it",
    "title": "MedGemma 27B model card (developer)",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2604.05081",
    "title": "MedGemma 1.5 report (includes Gemini 3 self-reported figures)",
    "kind": "preprint",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2019-08-23",
    "now": "2019-09-13",
    "reason": "Recorded value was the GitHub initial commit; repository creation or first commit is never first-release evidence, so paper v1 is used.",
    "source_url": "https://arxiv.org/abs/1909.06146",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "GitHub repo initial commit (data pushed 2019-08-24); arXiv/EMNLP 2019",
    "now": "arXiv v1 of the PubMedQA paper (2019-09-13); GitHub initial commit 2019-08-23 kept as a note only (repo creation excluded as evidence).",
    "reason": "Basis realigned with the first-release policy.",
    "source_url": "https://arxiv.org/abs/1909.06146",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_desc",
    "was": "README edit only; data frozen since 2019.",
    "now": "README edit on the data repository (2023-04-18); dataset itself frozen since 2019. 'Most recent sourced event' only - no claim that nothing happened later.",
    "reason": "'None since release' wording replaced per latest_update semantics (DATE_AUDIT).",
    "source_url": "https://github.com/pubmedqa/pubmedqa/commits/master",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_kind",
    "was": "none-since-release",
    "now": "code-maintenance-only",
    "reason": "A README edit is a documentation/maintenance event, not 'none since release'.",
    "source_url": "https://github.com/pubmedqa/pubmedqa/commits/master",
    "date_found": "2026-11-16"
   },
   {
    "field": "status_note",
    "was": "Official leaderboard last updated November 2023",
    "now": "Newest official-board entry dated 2024-04-18",
    "reason": "The ranking check found the newest official-board entry dated 2024-04-18, later than the November 2023 in the old note.",
    "source_url": "https://pubmedqa.github.io",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "release_milestones.first_public_evidence",
   "ranking.entries[5]",
   "models_reported[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-14-pubmedqa/",
   "en": "/bio/benchmark/benchmarks/pubmedqa/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-15-cameo",
  "kind": "benchmark",
  "nn": "15",
  "short_slug": "cameo",
  "names": {
   "display": "CAMEO",
   "full": "Continuous Automated Model EvaluatiOn",
   "full_null_reason": null,
   "aliases": [
    "CAMEO",
    "CAMEO3D",
    "cameo3d.org"
   ],
   "ambiguous_aliases": [
    "CAMEO 2022"
   ]
  },
  "family": null,
  "family_null_reason": "Single rolling evaluation service; static CAMEO subsets taken by papers are not family members.",
  "domain": "structure",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "A",
  "type_note": "Weekly automated rolling blind test, no human review",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": null,
   "qualifier": null
  },
  "publisher": {
   "orgs": [
    "SIB Swiss Institute of Bioinformatics",
    "University of Basel, Biozentrum"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Weekly rolling blind test of structure-prediction servers on PDB pre-release sequences.",
  "first_release": {
   "date": "2012",
   "precision": "year",
   "year_hint": null,
   "basis": "Maintainer papers state continuous weekly benchmarking since 2012; month of first targets not found",
   "event_kind": "first-public-evidence",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1002/prot.26213",
   "verification": "partial",
   "note": "Year sourced; archive week count implies early 2012 only by arithmetic"
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2012",
    "precision": "year",
    "year_hint": null,
    "basis": "Maintainer papers state continuous weekly benchmarking since 2012; month of first targets not found",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1002/prot.26213",
    "verification": "partial",
    "note": "Year sourced; archive week count implies early 2012 only by arithmetic"
   },
   "paper_v1": null,
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": null,
   "precision": null,
   "year_hint": null,
   "basis": "No sourced event: cameo3d.org has no news or changelog",
   "event_kind": "none-since-release",
   "timestamp_utc": null,
   "source_url": "https://cameo3d.org/",
   "verification": "unverified",
   "note": "2026-10-17 is only a Wayback snapshot date, not an update",
   "description": "No later sourced event found (as of 2026-11-16): cameo3d.org has no news or changelog and the weekly archive gives no dated release."
  },
  "next_expected": null,
  "next_expected_null_reason": "Runs weekly on a rolling basis; no separately announced next event.",
  "status": "active",
  "status_note": "Active but reduced; single-chain 3D track ended April 2025 (unverified)",
  "scale": {
   "summary": "Jan 2024–Jun 2025: 12,205 targets over 76 weeks; about 160 targets per week.",
   "items": [
    {
     "label": "New PDB entries screened",
     "value": "23,629",
     "unit": null,
     "version": "2024-01-06 to 2025-06-14"
    },
    {
     "label": "Targets selected",
     "value": "12,205",
     "unit": null,
     "version": "2024-01-06 to 2025-06-14"
    },
    {
     "label": "Targets medium / hard / ligand",
     "value": "2,106 / 3,266 / 6,833",
     "unit": null,
     "version": "2024-01-06 to 2025-06-14"
    },
    {
     "label": "Weeks",
     "value": "76",
     "unit": null,
     "version": "2024-01-06 to 2025-06-14"
    },
    {
     "label": "Targets per week (site arithmetic)",
     "value": "~160",
     "unit": null,
     "version": null
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "Weekly; truth is newly released PDB structures, about 160 per week, not produced for the benchmark.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Weekly blind evaluation of structure-prediction servers on PDB pre-release sequences; now mainly complex and ligand tracks.",
   "truth": "PDB structures released afterwards (X-ray, NMR, EM; resolution ≤4.0 Å).",
   "metric_primary": "lDDT family (iLDDT, oligo-lDDT)",
   "metric_secondary": [
    "QS-score",
    "DockQ",
    "lDDT-PLI (ligands)"
   ],
   "version_traps": "Papers use different windows and filters, so state the window; a static 'CAMEO 2022' subset is not the live service.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "third-party",
   "metric": "lDDT / ligand success rate (CAMEO complex and ligand tracks)",
   "eval_set": "Mixed windows: CAMEO 2024-01-06 to 2025-06-14 (AF3 baseline run by CAMEO); CAMEO 2022-04 to 2022-06 static set (xTrimoPGLM)",
   "snapshot_date": {
    "date": "2025-09-28",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://cameo3d.org/",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://cameo3d.org/",
   "entries": [
    {
     "rank": null,
     "model_label": "AlphaFold 3 (CAMEO baseline)",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "drug-like ligand success rate ~40%; antibody median LDDT 0.81",
     "score_unit": null,
     "metric": "lDDT / ligand success rate (CAMEO complex and ligand tracks)",
     "eval_set": "CAMEO 2024-01-06 to 2025-06-14 (76 weeks), AF3 v3.0.1 run by CAMEO as baseline",
     "source_type": "third",
     "date": {
      "date": "2025-09-28",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750032/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750032/",
     "note": "CAMEO assessment paper (Proteins 2025); best baseline in most ligand categories; behind stoichiometry-aware servers on oligomers; window end date inconsistent in paper.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "xTrimoPGLM-Fold",
     "model_id": "model-xtrimopglm",
     "site_model": true,
     "score": "TM-score 0.86 / GDT_TS 0.83 (ESMFold 0.85/0.81; AF2 0.87/0.85)",
     "score_unit": null,
     "metric": "TM-score / GDT_TS",
     "eval_set": "CAMEO 2022-04 to 2022-06 static set (n=194)",
     "source_type": "self",
     "date": {
      "date": "2024-01",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2401.06199",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2401.06199",
     "note": "Developer report on a static subset; from site note, not re-verified.",
     "second_hand": true,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Profluent-E1",
     "model_id": "model-profluent-e1",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Long-range contact precision",
     "eval_set": "CAMEO targets (window not stated in README)",
     "source_type": "self",
     "date": {
      "date": "2025-11",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/Profluent-AI/E1#comparison-to-other-models",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/Profluent-AI/E1#comparison-to-other-models",
     "note": "Shown only as a figure; no numeric score.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "No headline number: the ranking source has no rank-1 entry (entries are unranked or protocol-specific); the caveat text is shown instead.",
   "caveat": "The live site renders server results dynamically, with no retrievable text board; single-chain 3D closed April 2025 (site note); entries use different windows and are not comparable.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphafold3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12750032/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CAMEO operators ran AF3 as a baseline (2025 assessment paper); maintainer-run, not a developer submission.",
    "resolution": null
   },
   {
    "model_id": "model-dplm-2",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2410.13782v1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Uses a frozen 2022 target set; no live submission, no service ranking.",
    "resolution": "The paper uses a frozen 2022 target set derived from the service, with no live submission and no service-assigned ranking. Recorded as a relation rather than deleted, so the rule matches Nesso-1 x CASP16."
   },
   {
    "model_id": "model-profluent-e1",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://github.com/Profluent-AI/E1#comparison-to-other-models",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CAMEO targets used post hoc; curve only.",
    "resolution": null
   },
   {
    "model_id": "model-xtrimopglm",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2401.06199",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "CAMEO 2022-04 to 06 static set, n=194.",
    "resolution": null
   },
   {
    "model_id": "model-enchant-v1-v2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.iambic.ai/news/enchant-3",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "disputed",
    "note": "Same source; static 2022 set, figure only. Unconfirmed; would count as retrospective use, not participation.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 4,
   "by_relation": {
    "official-participation": 1,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 3,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 1
   }
  },
  "how_to_use": "No wet lab needed. Register as a server (receive targets automatically, return models on time), or replay contemporaneous new PDB structures offline and score with OpenStructure; the training cutoff must precede the window, and the window and filters must be stated. Not a basis for a single SOTA claim; pair with CASP, PoseBusters and PLINDER.",
  "pairs_with": [
   "bench-01-casp",
   "bench-05-posebusters",
   "bench-30-plinder",
   "bench-31-foldbench"
  ],
  "caveats": [
   "Windows differ between papers, so numbers are not comparable; difficulty tiers drift as servers improve, no confidence intervals are given, and new tools are hard to enter as servers.",
   "Pre-release data carry no stoichiometry, so homo-oligomers and monomers cannot be told apart, which affects complex scoring.",
   "Tools such as AlphaFold 3 have usage limits that hinder automated submission (second-hand report), so CAMEO runs it itself as a baseline.",
   "Truth is structure, not function or affinity; no direct evidence links lDDT gains to drug-discovery hit rates."
  ],
  "sources": [
   {
    "url": "https://cameo3d.org/",
    "title": "CAMEO official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12750032/",
    "title": "CAMEO assessment paper, Proteins 2025 (beyond single chains)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1002/prot.26213",
    "title": "CAMEO maintainer paper (Proteins)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pubmed.ncbi.nlm.nih.gov/29178137/",
    "title": "CAMEO 2018 paper (PubMed)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2401.06199",
    "title": "xTrimoPGLM paper (developer)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "http://web.archive.org/web/20261017060750/https://cameo3d.org/",
    "title": "cameo3d.org archive snapshot (2026-10-17)",
    "kind": "archive",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2012",
    "now": "2012",
    "reason": "Maintainer paper states continuous weekly benchmarking since 2012; the year is now sourced, month and day remain unsourced and are not invented.",
    "source_url": "https://d197for5662m48.cloudfront.net/documents/publicationstatus/64100/preprint_pdf/966131911343976eaa700d3011102243.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update",
    "was": "2026-10-17",
    "now": null,
    "reason": "2026-10-17 was only a Wayback snapshot date, not an update date; the live site carries no dated news or changelog.",
    "source_url": "https://cameo3d.org/",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_desc",
    "was": "Wayback snapshot of cameo3d.org shows weekly benchmarking live; no dated news/changes retrievable, last substantive change undatable.",
    "now": "No dated update found: cameo3d.org carries no news or changelog and the weekly target archive gives no dated release. Must display as unverified.",
    "reason": "Description synced with the removed date.",
    "source_url": "https://cameo3d.org/",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_kind",
    "was": "code-maintenance-only",
    "now": "none-since-release",
    "reason": "No sourced event exists; the kind renders as 'no later sourced event found'.",
    "source_url": "https://cameo3d.org/",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported (DPLM-2)",
    "was": "No explained relation on the site",
    "now": "retrospective-dataset-use",
    "reason": "CHANGESET §4.1: DPLM-2 uses a frozen 2022 target set and is recorded as retrospective dataset use, the same rule as Nesso-1 x CASP16.",
    "source_url": "https://arxiv.org/html/2410.13782v1",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "release_milestones.first_public_evidence",
   "latest_update",
   "status_note",
   "ranking.entries[1]",
   "models_reported[0]",
   "models_reported[2]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-15-cameo/",
   "en": "/bio/benchmark/benchmarks/cameo/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-16-nt_gue",
  "kind": "benchmark",
  "nn": "16",
  "short_slug": "nt-gue",
  "names": {
   "display": "GUE + Nucleotide Transformer benchmark",
   "full": "GUE (Genome Understanding Evaluation), Nucleotide Transformer downstream tasks and Genomic Benchmarks",
   "full_null_reason": null,
   "aliases": [
    "GUE",
    "GUE+",
    "Genome Understanding Evaluation",
    "NT benchmark",
    "Nucleotide Transformer downstream tasks",
    "Genomic Benchmarks",
    "NTv3 Benchmark"
   ],
   "ambiguous_aliases": [
    "Nucleotide Transformer",
    "NT"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "Genomic Benchmarks",
     "id_or_null": null,
     "first_release": {
      "date": "2022-06-10",
      "precision": "day",
      "year_hint": null,
      "basis": "Genomic Benchmarks bioRxiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1101/2022.06.08.495248",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "Nucleotide Transformer downstream tasks (original)",
     "id_or_null": null,
     "first_release": {
      "date": "2023-01-15",
      "precision": "day",
      "year_hint": null,
      "basis": "NT paper bioRxiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1101/2023.01.11.523679",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "GUE / GUE+ (DNABERT-2)",
     "id_or_null": null,
     "first_release": {
      "date": "2023-06",
      "precision": "month",
      "year_hint": null,
      "basis": "DNABERT-2 paper arXiv 2306.15006 v1 (identifier implies 2023-06)",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2306.15006",
      "verification": "partial",
      "note": "Day unverified; the earlier 2023-06-27 came from a different paper, 2306.15794"
     }
    },
    {
     "name": "Nucleotide Transformer downstream tasks (revised)",
     "id_or_null": null,
     "first_release": {
      "date": null,
      "precision": null,
      "year_hint": 2024,
      "basis": "First commits of the revised Hugging Face dataset (repository event, not a release date)",
      "event_kind": "data-release",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/datasets/InstaDeepAI/nucleotide_transformer_downstream_tasks_revised/commits/main",
      "verification": "unverified",
      "note": "Repository commits from 2024-04; not a release date"
     }
    },
    {
     "name": "NTv3 Benchmark",
     "id_or_null": null,
     "first_release": {
      "date": "2025-12-25",
      "precision": "day",
      "year_hint": null,
      "basis": "NTv3 preprint bioRxiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://api.biorxiv.org/details/biorxiv/10.64898/2025.12.22.695963",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "Family entry: GUE/GUE+, NT downstream tasks (original and revised not comparable), Genomic Benchmarks and NTv3 Benchmark; BEND and DART-Eval are separate."
  },
  "family_null_reason": null,
  "domain": "genomics-rna",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static dataset suite",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Revised NT (chromosome hold-out) near L3"
  },
  "publisher": {
   "orgs": [
    "InstaDeep",
    "Northwestern University",
    "CEITEC, Masaryk University"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Short-sequence regulatory-element classification suites for DNA language models; a smoke test, not proof of quality.",
  "first_release": {
   "date": "2022-06-10",
   "precision": "day",
   "year_hint": null,
   "basis": "Earliest family member: Genomic Benchmarks bioRxiv v1 (journal version BMC Genomic Data 2023-05-01)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1101/2022.06.08.495248",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2022-06-10",
    "precision": "day",
    "year_hint": null,
    "basis": "Earliest family member: Genomic Benchmarks bioRxiv v1 (journal version BMC Genomic Data 2023-05-01)",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/2022.06.08.495248",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2023-05-01",
    "precision": "day",
    "year_hint": null,
    "basis": "Genomic Benchmarks journal version (BMC Genomic Data)",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "partial",
    "note": "Also: NT paper online in Nature Methods 2024-11-28"
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-12-25",
   "precision": "day",
   "year_hint": null,
   "basis": "NTv3 preprint bioRxiv v1",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://api.biorxiv.org/details/biorxiv/10.64898/2025.12.22.695963",
   "verification": "verified",
   "note": null,
   "description": "NTv3 Benchmark (106 tasks, 7 species) released with the NTv3 preprint (bioRxiv v1 2025-12-25); an InstaDeep blog of 2026-02-18 later announced the same release."
  },
  "next_expected": null,
  "next_expected_null_reason": "No next version announced for any suite.",
  "status": "low-frequency",
  "status_note": "Still used but saturated; the larger GENEB suite (2026-06) partly replaces it",
  "scale": {
   "summary": "GUE: 28 datasets (+8 GUE+); NT: 18 tasks, 532,064 rows in the revised version; Genomic Benchmarks: 9 datasets.",
   "items": [
    {
     "label": "Datasets",
     "value": "28 + 8",
     "unit": null,
     "version": "GUE + GUE+"
    },
    {
     "label": "Tasks",
     "value": "18",
     "unit": null,
     "version": "NT downstream"
    },
    {
     "label": "Rows",
     "value": "532,064",
     "unit": null,
     "version": "NT revised"
    },
    {
     "label": "Datasets",
     "value": "9",
     "unit": null,
     "version": "Genomic Benchmarks"
    },
    {
     "label": "Tasks (7 species)",
     "value": "106",
     "unit": null,
     "version": "NTv3 Benchmark"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None; reuses public data such as ENCODE.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Whether a DNA language model can classify short (up to 1 kb) sequences for regulatory features such as promoters, enhancers, splice sites and histone marks.",
   "truth": "Element labels from public annotations such as ENCODE; negatives differ by version (shuffled, random or real genomic sequence).",
   "metric_primary": "MCC",
   "metric_secondary": [
    "F1 (GUE virus-variant task)",
    "10-fold CV MCC (revised NT)"
   ],
   "version_traps": "Original and revised NT scores are not comparable; tokenizer, fine-tuning vs frozen probe and pooling all change scores and must be stated.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "MCC (GUE average over 28 datasets; NT 10-fold CV MCC)",
   "eval_set": "GUE / NT downstream (original & revised) / Genomic Benchmarks",
   "snapshot_date": {
    "date": null,
    "precision": null,
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "unverified",
    "note": "No board, so no snapshot date"
   },
   "board_url": null,
   "entries": [],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "No official board and no site model reports on these suites, so there is no headline number.",
   "caveat": "No official board and no site model reports on these suites (NTv3 uses its own 106-task benchmark; Evo 2 does not report). Two non-site models differ by 0.13 GUE average MCC, within seed noise (not re-verified).",
   "retrieved": "2026-11-16"
  },
  "models_reported": [],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Download and run; no wet lab needed. Use the revised NT version; report confidence intervals over several seeds; include randomly initialised same-architecture, small-CNN and k-mer-plus-linear baselines, and state tokenizer and fine-tuning or probing setup. Add BEND (long-range and variants); for variants see ClinVar and Enformer.",
  "pairs_with": [
   "bench-38-bend",
   "bench-11-clinvar",
   "bench-08-enformer",
   "bench-37-traitgym"
  ],
  "caveats": [
   "Randomly initialised same-architecture models and small CNNs often match or beat pretrained models; seed noise reaches 5 points; original and revised NT versions are not comparable.",
   "Mostly measures local motifs and GC content; only in-distribution short-sequence classification, with no variant, cross-cell-type or cross-species OOD and no prospective test.",
   "NTv3 and Evo 2 do not report on the GUE/NT suites; the main users are not site models.",
   "The GUE paper gives inconsistent dataset counts (28, 36 and others); this page uses 28 + 8."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2306.15006",
    "title": "GUE / DNABERT-2 paper, arXiv 2306.15006",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.1101/2023.01.11.523679",
    "title": "Nucleotide Transformer paper, bioRxiv",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.1038/s41592-024-02523-z",
    "title": "Nucleotide Transformer journal version (Nature Methods)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://huggingface.co/datasets/InstaDeepAI/nucleotide_transformer_downstream_tasks_revised",
    "title": "Revised NT dataset card",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://doi.org/10.1101/2022.06.08.495248",
    "title": "Genomic Benchmarks, bioRxiv",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://pypi.org/project/genomic-benchmarks/",
    "title": "genomic-benchmarks on PyPI",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://api.biorxiv.org/details/biorxiv/10.64898/2025.12.22.695963",
    "title": "NTv3 preprint (includes NTv3 Benchmark)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/html/2606.04525v3",
    "title": "GENEB suite, arXiv 2606.04525",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update",
    "was": "2025-07-30",
    "now": "2025-12-25",
    "reason": "Later family event missed: the NTv3 Benchmark (106 tasks, 7 species) was released with the NTv3 preprint, v1 posted 2025-12-25.",
    "source_url": "https://api.biorxiv.org/details/biorxiv/10.64898/2025.12.22.695963",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_desc",
    "was": "Genomic Benchmarks repo: cache moved to Zenodo and version bump; NT revised HF dataset got 'task' column 2025-06-30.",
    "now": "NTv3 Benchmark (106 tasks, 7 species) released with the Nucleotide Transformer v3 preprint (bioRxiv v1 2025-12-25); InstaDeep blog 2026-02-18 is a later announcement of the same release.",
    "reason": "Description synced with the corrected date.",
    "source_url": "https://api.biorxiv.org/details/biorxiv/10.64898/2025.12.22.695963",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_kind",
    "was": "code-maintenance-only",
    "now": "data-release",
    "reason": "The event is a benchmark/data release of a family member.",
    "source_url": "https://api.biorxiv.org/details/biorxiv/10.64898/2025.12.22.695963",
    "date_found": "2026-11-16"
   },
   {
    "field": "family.members[GUE].first_release",
    "was": "2023-06-27 (arXiv 2306.15794)",
    "now": "2023-06 (arXiv 2306.15006; day unverified)",
    "reason": "2306.15794 is a different paper (HyenaDNA), not DNABERT-2/GUE; the identifier supports only the month.",
    "source_url": "https://arxiv.org/abs/2306.15006",
    "date_found": "2026-11-16"
   },
   {
    "field": "publisher (NT journal date)",
    "was": "Nature Methods 2025-02",
    "now": "Nature Methods online 2024-11-28",
    "reason": "The 2025-02 figure is likely the issue date; the online date is 2024-11-28.",
    "source_url": "https://doi.org/10.1038/s41592-024-02523-z",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "family.members[2].first_release",
   "family.members[3].first_release",
   "release_milestones.journal_online",
   "ranking.caveat"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-16-nt_gue/",
   "en": "/bio/benchmark/benchmarks/nt-gue/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-17-cafa",
  "kind": "benchmark",
  "nn": "17",
  "short_slug": "cafa",
  "names": {
   "display": "CAFA",
   "full": "Critical Assessment of Functional Annotation",
   "full_null_reason": null,
   "aliases": [
    "CAFA",
    "CAFA1",
    "CAFA2",
    "CAFA3",
    "CAFA4",
    "CAFA5",
    "CAFA6",
    "Critical Assessment of Functional Annotation"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "Single recurring challenge; CAFA1–6 are editions of one challenge, not family members.",
  "domain": "protein",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "A",
  "type_note": "Time-delayed recurring blind challenge",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": [
    "L4",
    "L5"
   ],
   "qualifier": "CAFA3 added dedicated screens (L5)"
  },
  "publisher": {
   "orgs": [
    "CAFA organisers (ISCB Automated Function Prediction SIG)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Time-delayed blind test of protein function (GO) prediction against experimental annotations added after submission.",
  "first_release": {
   "date": "2010-09",
   "precision": "month",
   "year_hint": null,
   "basis": "challenge_open: CAFA1 targets released 2010-09-15 (organisers' paper, p. 9). Registration opened 2010-07-15, stored separately and not used as first release.",
   "event_kind": "challenge-open",
   "timestamp_utc": null,
   "source_url": "https://access.archive-ouverte.unige.ch/access/metadata/05c000b5-6e95-49fa-8c64-22e83ebd7758/download",
   "verification": "partial",
   "note": "Day 2010-09-15 from the organisers' paper; kept at month"
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2010-07-15",
    "precision": "day",
    "year_hint": null,
    "basis": "Registration opened (second-hand quote of a 2010 announcement)",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://www.biofunctionprediction.org/cafa/",
    "verification": "unverified",
    "note": "Registration date; not used as first release"
   },
   "paper_v1": null,
   "journal_online": {
    "date": "2013-01-27",
    "precision": "day",
    "year_hint": null,
    "basis": "CAFA1 results paper online in Nature Methods (second-hand quote)",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/nmeth.2340",
    "verification": "partial",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2010-09",
    "precision": "month",
    "year_hint": null,
    "basis": "challenge_open: CAFA1 targets released 2010-09-15 (organisers' paper, p. 9). Registration opened 2010-07-15, stored separately and not used as first release.",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://access.archive-ouverte.unige.ch/access/metadata/05c000b5-6e95-49fa-8c64-22e83ebd7758/download",
    "verification": "partial",
    "note": "Day 2010-09-15 from the organisers' paper; kept at month"
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-10-16",
   "precision": "day",
   "year_hint": null,
   "basis": "Organisers' mailing-list announcement",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://groups.google.com/a/iscb.org/g/mlcsb.cosi/c/Cvc_Vn8e4Jc",
   "verification": "unverified",
   "note": "No official dated record; shown as unverified",
   "description": "CAFA6 launched on Kaggle ($50k prizes) per the organisers' mailing-list post; final results not found."
  },
  "next_expected": null,
  "next_expected_null_reason": "CAFA6 final evaluation timing (about 2026-05) appears only on aggregator sites; nothing announced by the organisers.",
  "status": "active",
  "status_note": "CAFA6 opened on Kaggle 2025-10-16; no official results as of 2026-11-16",
  "scale": {
   "summary": "CAFA2: 126 methods, 3,681 proteins; CAFA5: 1,600+ teams; CAFA6 evaluation-set size not published.",
   "items": [
    {
     "label": "Methods",
     "value": "126",
     "unit": null,
     "version": "CAFA2"
    },
    {
     "label": "Proteins evaluated",
     "value": "3,681",
     "unit": null,
     "version": "CAFA2"
    },
    {
     "label": "Teams",
     "value": "1,600+",
     "unit": null,
     "version": "CAFA5"
    },
    {
     "label": "Submissions",
     "value": "2,850",
     "unit": null,
     "version": "CAFA5"
    },
    {
     "label": "Evaluation-set size",
     "value": "not published",
     "unit": null,
     "version": "CAFA6"
    }
   ]
  },
  "wetlab_refresh": "accumulating",
  "wetlab_note": "Every 2–3 years; new truth is experimental GO annotation accumulated in databases, not new experiments, and comes only by entering the next edition.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Time-delayed prediction of protein function as Gene Ontology terms, scored against experimental annotations added after submission.",
   "truth": "Experimental-evidence GO annotations added to UniProt/GOA between t0 and t1, propagated up the GO hierarchy.",
   "metric_primary": "Fmax",
   "metric_secondary": [
    "Smin (IA-weighted)",
    "IA-weighted F-measure (CAFA5/6 Kaggle)"
   ],
   "version_traps": "CAFA1–4 used academic submissions with independent assessment, CAFA5/6 are Kaggle contests with different metrics, so scores are not comparable across editions; separate no-knowledge and limited-knowledge.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "IA-weighted F-measure (CAFA-evaluator, Kaggle)",
   "eval_set": "CAFA6 (Kaggle, launched 2025-10-16)",
   "snapshot_date": {
    "date": "2026-02-02",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://www.kaggle.com/competitions/cafa-6-protein-function-prediction/leaderboard",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://www.kaggle.com/competitions/cafa-6-protein-function-prediction/leaderboard",
   "entries": [
    {
     "rank": 1,
     "model_label": "(individual Kaggle participant; interim)",
     "model_id": null,
     "site_model": false,
     "score": "0.36",
     "score_unit": null,
     "metric": "IA-weighted F-measure (CAFA-evaluator, Kaggle)",
     "eval_set": "CAFA6 (Kaggle, launched 2025-10-16)",
     "source_type": "board",
     "date": {
      "date": "2026-02-02",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://clist.by/standings/cafa-6-protein-function-prediction-healthcare-biology-tabular-custom-metric-62574283/",
      "verification": "verified",
      "note": null
     },
     "url": "https://clist.by/standings/cafa-6-protein-function-prediction-healthcare-biology-tabular-custom-metric-62574283/",
     "note": "Not a site model; interim public-board snapshot via a third-party aggregator with 57 days left, not final; participant name omitted.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "CAFA6 final board and winners were not found; the only entry is an interim snapshot, so there is no headline number.",
   "caveat": "CAFA6 final board and winners not retrievable; no site model reports CAFA (SciReasoner pairing unconfirmed); CAFA5 scores are not comparable.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-scireasoner",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/html/2607.07708",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "disputed",
    "note": "Self-reports on DeepFRI-GO; CAFA-3 only in a figure caption, a modified protocol if kept; paper identity unresolved (two same-name papers).",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 1
   }
  },
  "how_to_use": "Enter the next edition, or build a time split (train on an old snapshot, evaluate on a newer one) and compute Fmax and Smin with CAFA-evaluator, next to Naive and BLAST baselines, reporting no-knowledge and limited-knowledge and IA-weighted versions. No own wet lab needed, but scores reflect only documented function.",
  "pairs_with": [
   "bench-01-casp",
   "bench-13-cagi",
   "bench-02-proteingym"
  ],
  "caveats": [
   "Unannotated terms count as negatives (open-world problem); CAFA5/6 moved to Kaggle, so scores are not comparable across editions; text-mining entries are allowed.",
   "Fmax favours shallow, frequent terms, and rankings shift substantially with the metric.",
   "Top CAFA5 teams used text mining of new papers as features, which is legal in time but changes what the task measures.",
   "Top-method gains from CAFA2 to CAFA3 were smaller than the previous step and CCO did not improve; the bottleneck is annotation, not models.",
   "CAFA3 included prospective experiments, but the mapping from experimental hit rate to Fmax is not quantified."
  ],
  "sources": [
   {
    "url": "https://www.kaggle.com/competitions/cafa-6-protein-function-prediction",
    "title": "CAFA6 on Kaggle",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://groups.google.com/a/iscb.org/g/mlcsb.cosi/c/Cvc_Vn8e4Jc",
    "title": "CAFA6 organisers' mailing-list announcement",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://www.biofunctionprediction.org/cafa/",
    "title": "CAFA official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://doi.org/10.1038/nmeth.2340",
    "title": "CAFA1 results paper (Nature Methods 2013)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pubmed.ncbi.nlm.nih.gov/27604469/",
    "title": "CAFA2 paper (PubMed)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://link.springer.com/article/10.1186/s13059-019-1835-8",
    "title": "CAFA3 paper (Genome Biology 2019)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://access.archive-ouverte.unige.ch/access/metadata/05c000b5-6e95-49fa-8c64-22e83ebd7758/download",
    "title": "Archived organisers' paper (CAFA1 dates)",
    "kind": "archive",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2412.04529",
    "title": "ProtBoost (CAFA5 team paper)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://clist.by/standings/cafa-6-protein-function-prediction-healthcare-biology-tabular-custom-metric-62574283/",
    "title": "CAFA6 interim standings aggregator (second-hand)",
    "kind": "second-hand",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2010-09 (challenge began 2010-09-15; registration 2010-07-15)",
    "now": "2010-09",
    "reason": "The organisers' archived paper (p. 9) supports target release 2010-09-15; July 2010 is registration, not the challenge opening.",
    "source_url": "https://access.archive-ouverte.unige.ch/access/metadata/05c000b5-6e95-49fa-8c64-22e83ebd7758/download",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "CAFA1 organized fall 2010 to spring 2012 per secondary summaries; month not found",
    "now": "challenge_open: CAFA1 targets released 2010-09-15 (organisers' paper, p. 9). Registration opened 2010-07-15, stored separately and not used as first release.",
    "reason": "Basis now names the milestone actually used.",
    "source_url": "https://access.archive-ouverte.unige.ch/access/metadata/05c000b5-6e95-49fa-8c64-22e83ebd7758/download",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "2010-fall",
    "now": "2010-09 (challenge began 2010-09-15; registration 2010-07-15)",
    "reason": "'2010-fall' is not a valid date; refined to the month from organiser material.",
    "source_url": "https://www.biofunctionprediction.org/cafa/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "release_milestones.challenge_open",
   "release_milestones.first_public_evidence",
   "release_milestones.journal_online",
   "latest_update",
   "ranking.entries[0]",
   "models_reported[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-17-cafa/",
   "en": "/bio/benchmark/benchmarks/cafa/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-18-labbench",
  "kind": "benchmark",
  "nn": "18",
  "short_slug": "labbench",
  "names": {
   "display": "LAB-Bench / LABBench2",
   "full": "Language Agent Biology Benchmark",
   "full_null_reason": null,
   "aliases": [
    "LAB-Bench",
    "LABBench2",
    "LitQA2",
    "LitQA3",
    "FigQA",
    "ProtocolQA",
    "SeqQA",
    "SeqQA2",
    "CloningScenarios",
    "CloningQA",
    "DbQA",
    "SuppQA",
    "TableQA"
   ],
   "ambiguous_aliases": [
    "LitQA"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "LAB-Bench",
     "id_or_null": null,
     "first_release": {
      "date": "2024-07-14",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv 2407.10362 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2407.10362",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "LABBench2",
     "id_or_null": null,
     "first_release": {
      "date": "2026-02-04",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv 2604.09554 v1 (API date)",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2604.09554",
      "verification": "partial",
      "note": "Identifier prefix 2604 conflicts with the February API date; unresolved"
     }
    }
   ],
   "boundary": "Family entry: original LAB-Bench (multiple choice) and LABBench2 (open answers, tools allowed); question sets and scores are not comparable."
  },
  "family_null_reason": null,
  "domain": "text-agents",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "E",
  "type_note": "Research-skill question suites; LABBench2 adds executable verifiers",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L1",
   "range": [
    "L1",
    "L2"
   ],
   "qualifier": "CloningScenarios near L2"
  },
  "publisher": {
   "orgs": [
    "FutureHouse",
    "Edison Scientific"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Biology research-skill benchmark for LLMs and agents; the multiple-choice original is partly saturated and superseded by LABBench2.",
  "first_release": {
   "date": "2024-07-14",
   "precision": "day",
   "year_hint": null,
   "basis": "LAB-Bench arXiv v1 (2407.10362)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2407.10362",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2024-07-14",
    "precision": "day",
    "year_hint": null,
    "basis": "LAB-Bench arXiv v1 (2407.10362)",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2407.10362",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-10-23",
   "precision": "day",
   "year_hint": null,
   "basis": "labbench2 repository commit",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://github.com/EdisonScientific/labbench2/commits/main",
   "verification": "verified",
   "note": null,
   "description": "labbench2 harness commit: grading seqqa2 genes from annotated CDS files; several grader and runner fixes in October 2026."
  },
  "next_expected": null,
  "next_expected_null_reason": "No next version or board update announced.",
  "status": "active",
  "status_note": "LABBench2 has an official board (updated 2026-03-23); original subsets partly saturated",
  "scale": {
   "summary": "Original: 2,457 questions, 30 subtasks; LABBench2: 1,912 tasks, 11 categories, no multiple choice.",
   "items": [
    {
     "label": "Questions (paper Table 1)",
     "value": "2,457",
     "unit": null,
     "version": "LAB-Bench"
    },
    {
     "label": "Subtasks",
     "value": "30",
     "unit": null,
     "version": "LAB-Bench"
    },
    {
     "label": "Public HF rows (about 20% held out)",
     "value": "1,967",
     "unit": null,
     "version": "LAB-Bench"
    },
    {
     "label": "Tasks (paper Table 1; release page says 1,892)",
     "value": "1,912",
     "unit": null,
     "version": "LABBench2"
    },
    {
     "label": "Categories",
     "value": "11",
     "unit": null,
     "version": "LABBench2"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "None; questions written in-house and revised by version.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Biology research skills of LLMs and agents: literature, databases, figures, sequences, cloning and protocol troubleshooting.",
   "truth": "Expert-written answers, partly programmatic (ProtocolQA, SeqQA); LABBench2 sequence and cloning tasks use executable verifiers.",
   "metric_primary": "Accuracy (LABBench2: verifier or expert grading)",
   "metric_secondary": [
    "Precision (original)",
    "Coverage (original)"
   ],
   "version_traps": "Original and LABBench2 differ and cannot share a table; state version, tool use, inline context and board date; vendor system-card subitems are often derived variants.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "LABBench2 aggregate score (mean across sub-benchmarks), settings 'Tools + high' and 'Base'",
   "eval_set": "LABBench2 (1,912 tasks, 11 categories)",
   "snapshot_date": {
    "date": "2026-03-23",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
   "entries": [
    {
     "rank": 1,
     "model_label": "gpt-5-2",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.609 (Tools + high); Base 0.345",
     "score_unit": null,
     "metric": "LABBench2 aggregate score (mean across sub-benchmarks), settings 'Tools + high' and 'Base'",
     "eval_set": "LABBench2 (1,912 tasks, 11 categories)",
     "source_type": "board",
     "date": {
      "date": "2026-03-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
     "note": "Overall top-1 (Tools + high); Base top-1 is gemini-3-pro-preview (0.404).",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "gpt-5-2-pro",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.583 (Tools + high); Base 0.382",
     "score_unit": null,
     "metric": "LABBench2 aggregate score (mean across sub-benchmarks), settings 'Tools + high' and 'Base'",
     "eval_set": "LABBench2 (1,912 tasks, 11 categories)",
     "source_type": "board",
     "date": {
      "date": "2026-03-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "gemini-3-pro-preview",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.564 (Tools + high); Base 0.404",
     "score_unit": null,
     "metric": "LABBench2 aggregate score (mean across sub-benchmarks), settings 'Tools + high' and 'Base'",
     "eval_set": "LABBench2 (1,912 tasks, 11 categories)",
     "source_type": "board",
     "date": {
      "date": "2026-03-13",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "claude-opus-4-6",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.519 (Tools + high); Base 0.349",
     "score_unit": null,
     "metric": "LABBench2 aggregate score (mean across sub-benchmarks), settings 'Tools + high' and 'Base'",
     "eval_set": "LABBench2 (1,912 tasks, 11 categories)",
     "source_type": "board",
     "date": {
      "date": "2026-03-23",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 5,
     "model_label": "claude-opus-4-5",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.471 (Tools + high); Base 0.325",
     "score_unit": null,
     "metric": "LABBench2 aggregate score (mean across sub-benchmarks), settings 'Tools + high' and 'Base'",
     "eval_set": "LABBench2 (1,912 tasks, 11 categories)",
     "source_type": "board",
     "date": {
      "date": "2026-03-23",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Biomni",
     "model_id": "model-biomni",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "LABBench2 aggregate score (mean across sub-benchmarks), settings 'Tools + high' and 'Base'",
     "eval_set": "Original LAB-Bench DbQA/SeqQA subsets (not LABBench2)",
     "source_type": "self",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": null,
      "verification": "unverified",
      "note": null
     },
     "url": null,
     "note": "Original LAB-Bench subsets, not LABBench2; from site note (second-hand), not re-verified; not comparable to the board.",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "Incomplete provenance (missing url and date): the honesty rule forbids displaying this number."
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "Board lists only 5 models (released 2026-02-05, updated 2026-03-23), no GPT-Rosalind or agent systems. The original has no board; vendor system-card numbers are self-reported subsets.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-frontier-llms",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://futurehouse.org/articles/lab-bench-measuring-capabilities-of-language-models-for-biology-research",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "LABBench2 official board entries; family entry.",
    "resolution": null
   },
   {
    "model_id": "model-biomni",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1101/2025.05.30.656746",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "unverified",
    "note": "Original DbQA/SeqQA subsets with 45 dev / 315 hidden test questions and 3 runs.",
    "resolution": null
   },
   {
    "model_id": "model-gpt-rosalind",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://openai.com/index/introducing-gpt-rosalind/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "11 of 15 LABBench2 tasks, qualitative only.",
    "resolution": null
   },
   {
    "model_id": "model-co-scientist",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/2502.18864",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-intern-s",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://github.com/InternLM/Intern-S1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-quine",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.microsoft.com/en-us/research/blog/introducing-quine-an-ai-research-system-designed-for-the-complexity-of-biology/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-scireasoner",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "Not in the paper text",
    "resolution": null
   },
   {
    "model_id": "model-verid",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 3,
   "by_relation": {
    "official-participation": 1,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 2,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 5
   }
  },
  "how_to_use": "Run from the repository or Hugging Face data; no wet lab needed; the original needs an agent function returning the option letter. Prefer LABBench2 and state tool use (search, code execution), inline context and board date; the private 20% hold-out cannot be checked for contamination. Pair with BixBench and wet-lab blind tests such as Adaptyv.",
  "pairs_with": [
   "bench-29-bixbench",
   "bench-28-adaptyv",
   "bench-14-pubmedqa",
   "bench-23-bioasq"
  ],
  "caveats": [
   "Multiple choice inflated original scores; small subsets (41 and 135 questions) are noisy; the public 80% may be contaminated; no wet-lab evidence.",
   "The LABBench2 paper reports accuracy 26–46 points below the original; original scores do not represent open-answer ability.",
   "Vendor system-card subitems (e.g. open-ended ProtocolQA) are derived variants, and the same subitem circulates with several numbers; use the primary document.",
   "ProtocolQA is paper-based error finding and involves no actual experiment."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2407.10362",
    "title": "LAB-Bench paper, arXiv 2407.10362",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2604.09554",
    "title": "LABBench2 paper, arXiv 2604.09554",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
    "title": "LABBench2 official board",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://edisonscientific.com/news/labbench2-an-improved-benchmark",
    "title": "LABBench2 release page",
    "kind": "company-blog",
    "date": null
   },
   {
    "url": "https://github.com/EdisonScientific/labbench2",
    "title": "labbench2 repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://github.com/Future-House/LAB-Bench",
    "title": "LAB-Bench repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://huggingface.co/datasets/futurehouse/lab-bench",
    "title": "LAB-Bench on Hugging Face",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://www-cdn.anthropic.com/bf10f64990cfda0ba858290be7b8cc6317685f47.pdf",
    "title": "Claude Opus 4.5 system card (vendor self-report)",
    "kind": "company-blog",
    "date": null
   },
   {
    "url": "https://doi.org/10.1101/2025.05.30.656746",
    "title": "Biomni preprint (developer)",
    "kind": "preprint",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "ranking.entries[Biomni]",
    "was": "Biomni original LAB-Bench subset scores shown as a ranking entry",
    "now": "Score withheld: entry lacks url and date",
    "reason": "CHANGESET §5.3: entries with incomplete provenance (no url or date) are not shown with a score.",
    "source_url": "https://advances.edisonscientific.com/benchmarks/labbench2/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "family.members[1].first_release",
   "ranking.entries[5]",
   "models_reported[1]",
   "models_reported[2]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-18-labbench/",
   "en": "/bio/benchmark/benchmarks/labbench/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-19-pmo",
  "kind": "benchmark",
  "nn": "19",
  "short_slug": "pmo",
  "names": {
   "display": "PMO (Practical Molecular Optimization)",
   "full": "Practical Molecular Optimization",
   "full_null_reason": null,
   "aliases": [
    "PMO",
    "Practical Molecular Optimization",
    "mol_opt",
    "PMO-1K"
   ],
   "ambiguous_aliases": [
    "PMO-Dock"
   ]
  },
  "family": null,
  "family_null_reason": "Single benchmark; PMO-1K and PMO-Dock are derivative extensions, not family members.",
  "domain": "small-molecule",
  "domains_secondary": [
   "generation"
  ],
  "inclusion": "original-30",
  "type": "D",
  "type_note": "Computational-oracle benchmark with a fixed call budget",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Computational oracles only, no wet-lab evidence"
  },
  "publisher": {
   "orgs": [
    "Georgia Institute of Technology",
    "Massachusetts Institute of Technology"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Compares sample efficiency of molecular optimization within 10,000 oracle calls",
  "first_release": {
   "date": "2022-06-22",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv 2206.12411 v1 (v2 2022-10-09)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2206.12411",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2022-06-22",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv 2206.12411 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2206.12411",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2022",
    "precision": "year",
    "year_hint": null,
    "basis": "NeurIPS 2022 Datasets and Benchmarks track; year only",
    "event_kind": "conference-publication",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2206.12411",
    "verification": "partial",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-11-09",
   "precision": "day",
   "year_hint": null,
   "basis": "PMO-Dock preprint arXiv 2610.19011 v2, a docking-oracle extension",
   "event_kind": "new-version",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2610.19011",
   "verification": "partial",
   "note": "Task count inconsistent: the ICLR page says 23, a secondary index says 25.",
   "description": "PMO-Dock (ICLR 2026), a docking-oracle extension, arXiv 2610.19011 v2; the mol_opt repository's last commit is 2024-05-23."
  },
  "next_expected": null,
  "next_expected_null_reason": "No next version or round announced by the organizers.",
  "status": "low-frequency",
  "status_note": "Paper final; repository lightly maintained; no official online leaderboard",
  "scale": {
   "summary": "23 oracles, a 10,000-call budget, 5 seeds, 25 algorithms compared in the paper.",
   "items": [
    {
     "label": "Oracles",
     "value": "23",
     "unit": null,
     "version": "paper (2022)"
    },
    {
     "label": "Call budget",
     "value": "10000",
     "unit": "oracle calls",
     "version": "paper (2022)"
    },
    {
     "label": "Seeds",
     "value": "5",
     "unit": null,
     "version": "paper (2022)"
    },
    {
     "label": "Algorithms compared",
     "value": "25",
     "unit": null,
     "version": "paper (2022)"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No wet lab; all ground truth comes from computational oracle functions.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Given black-box scalar oracles and a fixed call budget, measure how efficiently an optimizer finds high-scoring molecules.",
   "truth": "Computational oracle scores: QED, DRD2, GSK3B, JNK3 and 19 objectives taken from GuacaMol, served through TDC.",
   "metric_primary": "AUC top-10, summed over 23 tasks (max 23)",
   "metric_secondary": [
    "Per-task AUC top-10",
    "Final top-10 mean score"
   ],
   "version_traps": "State the budget (10,000 / 1,000 / 100 calls), seed count, whether surrogate training counts as calls, and whether the original suite or PMO-Dock was used.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "self-reported",
   "metric": "AUC top-10, summed over 23 oracles (max 23), 10,000-call budget",
   "eval_set": "PMO 23 oracles (QED, DRD2, GSK3B, JNK3 plus 19 GuacaMol-derived), oracle functions via TDC",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "partial",
    "note": "No official online board; the snapshot day is this round's check date."
   },
   "board_url": null,
   "entries": [
    {
     "rank": 1,
     "model_label": "GenMol",
     "model_id": "model-genmol",
     "site_model": true,
     "score": "Sum 18.362 (Table 3, 'Goal-directed hit generation results', 23 PMO tasks, AUC top-10)",
     "score_unit": null,
     "metric": "AUC top-10, summed over 23 oracles (max 23), 10,000-call budget",
     "eval_set": "PMO 23 oracles (QED, DRD2, GSK3B, JNK3 plus 19 GuacaMol-derived), oracle functions via TDC",
     "source_type": "self",
     "date": {
      "date": "2025-01-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2501.06158v2",
      "verification": "verified",
      "note": "arXiv 2501.06158, v2 fetched"
     },
     "url": "https://arxiv.org/html/2501.06158v2",
     "note": "Re-verified from the primary table (23 tasks, mean of 3 runs). Same table: f-RAG 16.928, Genetic GFN 16.213, REINVENT 14.196. The repository README lists a V2 score of 17.943 (site note, not re-verified). Self-reported.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "f-RAG",
     "model_id": null,
     "site_model": false,
     "score": "Sum 16.928",
     "score_unit": null,
     "metric": "AUC top-10, summed over 23 oracles (max 23), 10,000-call budget",
     "eval_set": "PMO 23 oracles (QED, DRD2, GSK3B, JNK3 plus 19 GuacaMol-derived), oracle functions via TDC",
     "source_type": "third",
     "date": {
      "date": "2025-01-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2501.06158v2",
      "verification": "verified",
      "note": "As tabulated in the GenMol paper's Table 3"
     },
     "url": "https://arxiv.org/html/2501.06158v2",
     "note": "Not a site model; re-verified as a baseline row.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "Genetic GFN",
     "model_id": null,
     "site_model": false,
     "score": "Sum 16.213",
     "score_unit": null,
     "metric": "AUC top-10, summed over 23 oracles (max 23), 10,000-call budget",
     "eval_set": "PMO 23 oracles (QED, DRD2, GSK3B, JNK3 plus 19 GuacaMol-derived), oracle functions via TDC",
     "source_type": "third",
     "date": {
      "date": "2025-01-10",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2501.06158v2",
      "verification": "verified",
      "note": "As tabulated in the GenMol paper's Table 3"
     },
     "url": "https://arxiv.org/html/2501.06158v2",
     "note": "Not a site model; 16.213 is self-reported in its own paper and re-tabulated here.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 5,
     "model_label": "REINVENT (prior generation; the site page covers REINVENT 4)",
     "model_id": "model-reinvent-4",
     "site_model": true,
     "score": "Sum 14.196 (top of the original PMO paper's 25-method comparison; Graph GA 13.751, SELFIES-REINVENT 13.471, GP BO 13.156, lowest MolDQN 5.620)",
     "score_unit": null,
     "metric": "AUC top-10, summed over 23 oracles (max 23), 10,000-call budget",
     "eval_set": "PMO 23 oracles (QED, DRD2, GSK3B, JNK3 plus 19 GuacaMol-derived), oracle functions via TDC",
     "source_type": "third",
     "date": {
      "date": "2022-06-22",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2501.06158v2",
      "verification": "verified",
      "note": "arXiv 2206.12411; re-tabulated 2025-01-10"
     },
     "url": "https://arxiv.org/html/2501.06158v2",
     "note": "The 14.196 value is re-verified in GenMol's Table 3; the other totals from the PMO paper come from the site note and are not re-verified. REINVENT 4 itself reports no PMO score, so the pairing points at the prior-generation REINVENT.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "ReaSyn v2",
     "model_id": "model-reasyn-v2",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "AUC top-10, summed over 23 oracles (max 23), 10,000-call budget",
     "eval_set": "PMO 23 oracles (QED, DRD2, GSK3B, JNK3 plus 19 GuacaMol-derived), oracle functions via TDC",
     "source_type": "self",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "The source gives no date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": null,
      "verification": "unverified",
      "note": null
     },
     "url": null,
     "note": "No 23-task total was located this round, so the score is null.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Incomplete provenance (no url, no date), so the honesty rule forbids showing a number."
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "No official board; each paper's own table serves as the ranking, with inconsistent budgets (10k/1k/100 calls), no significance tests, and 1-7% gaps at seed-noise level.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-genmol",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://github.com/NVIDIA-Digital-Bio/genmol",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "PMO 23 tasks, original protocol",
    "resolution": null
   },
   {
    "model_id": "model-reasyn-v2",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2509.16084v2",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Uses 15 TDC oracle tasks rather than the full 23",
    "resolution": null
   },
   {
    "model_id": "model-reinvent-4",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2206.12411",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "The PMO paper used the prior-generation REINVENT as a baseline; REINVENT 4 itself reports no PMO score.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 2,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 1,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 1,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Clone mol_opt, build per-method environments, pull oracles through TDC and run 5 seeds for AUC top-10; no wet lab needed. Report the budget, the seed count and whether surrogate training counts as calls, and re-run baselines instead of copying other papers' tables; claims about real value need synthetic-accessibility, QED and diversity filters plus a docking or free-energy oracle.",
  "pairs_with": [
   "bench-10-guacamol",
   "bench-20-cache",
   "bench-04-tdc",
   "bench-32-crossdocked"
  ],
  "caveats": [
   "All oracles are cheap computational functions and most come from GuacaMol, so the two are not independent evidence; no docking, no assays.",
   "Self-reported gains of 1-7% sit at the noise level of 5 seeds, and most papers run no significance test.",
   "Summing 23 tasks is an arbitrary weighting; task difficulty varies and a single task can dominate the total.",
   "Budgets are mixed (10,000 / 1,000 / 100 calls), so numbers across papers are often not comparable.",
   "No evidence links PMO scores to measured hit rates; rediscovery tasks can be solved from pretraining memory."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2206.12411",
    "title": "PMO paper, arXiv 2206.12411",
    "kind": "preprint",
    "date": {
     "date": "2022-06-22",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2206.12411",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://github.com/wenhao-gao/mol_opt",
    "title": "mol_opt repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/html/2501.06158v2",
    "title": "GenMol paper with the 23-task PMO table",
    "kind": "preprint",
    "date": {
     "date": "2025-01-10",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv 2501.06158 v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2501.06158",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://arxiv.org/abs/2610.19011",
    "title": "PMO-Dock preprint",
    "kind": "preprint",
    "date": {
     "date": "2026-11-09",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv 2610.19011 v2",
     "event_kind": "paper-revision",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2610.19011",
     "verification": "partial",
     "note": null
    }
   },
   {
    "url": "https://iclr.cc/virtual/2026/10022790",
    "title": "PMO-Dock page at ICLR 2026",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2407.12186",
    "title": "Third-party analysis of docking-oracle exploitation",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update",
    "was": "2026",
    "now": "2026-11-09 (PMO-Dock preprint arXiv 2610.19011 v2; v1 2026-10-13)",
    "reason": "The record held a year only; the PMO-Dock preprint supplies a citable day, and the mol_opt repository's last commit is 2024-05-23.",
    "source_url": "https://arxiv.org/abs/2610.19011",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries",
    "was": "ReaSyn v2 shown with a PMO score",
    "now": "score withheld: no url and no date for the figure",
    "reason": "Every displayed score needs metric, eval set, source type, date and url; this entry has neither url nor date.",
    "source_url": "https://arxiv.org/html/2509.16084v2",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "REINVENT 4 counted as reporting PMO",
    "now": "cited-as-comparison-only: only the prior-generation REINVENT appears, as a baseline in the PMO paper",
    "reason": "The REINVENT 4 paper reports no PMO score and only cites the sample-efficiency conclusion, so it is not counted.",
    "source_url": "https://arxiv.org/abs/2206.12411",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "ranking.entries[4]",
   "ranking.snapshot_date",
   "release_milestones.journal_online"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-19-pmo/",
   "en": "/bio/benchmark/benchmarks/pmo/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-20-cache",
  "kind": "benchmark",
  "nn": "20",
  "short_slug": "cache",
  "names": {
   "display": "CACHE",
   "full": "Critical Assessment of Computational Hit-finding Experiments",
   "full_null_reason": null,
   "aliases": [
    "CACHE",
    "CACHE challenge",
    "CACHE #1",
    "CACHE #5",
    "CACHE #8"
   ],
   "ambiguous_aliases": [
    "DREAM x CACHE",
    "Target 2035"
   ]
  },
  "family": null,
  "family_null_reason": "Successive rounds of one challenge rather than a family entry; per-round volumes are stored in the scale items.",
  "domain": "small-molecule",
  "domains_secondary": [
   "structure"
  ],
  "inclusion": "original-30",
  "type": "A",
  "type_note": "Periodic prospective blind test, one target per round",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L5",
   "range": null,
   "qualifier": "Prospective wet lab; no independent replication"
  },
  "publisher": {
   "orgs": [
    "Structural Genomics Consortium",
    "Conscience"
   ],
   "org_type": "nonprofit-institute"
  },
  "one_liner": "Teams submit purchasable compounds and the organizers measure binding: a blind test with real assays",
  "first_release": {
   "date": "2021-12-01",
   "precision": "day",
   "year_hint": null,
   "basis": "Organizer page announcing challenge #1 targets and open participation; no same-day archive",
   "event_kind": "official-announcement",
   "timestamp_utc": null,
   "source_url": "https://cache-challenge.org/join-competition",
   "verification": "partial",
   "note": "The 2022-01-31 application deadline is kept as a note only and is never a first release."
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2022-01-31",
    "precision": "day",
    "year_hint": null,
    "basis": "Application deadline for challenge #1, recorded as a note and not a release",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://cache-challenge.org/join-competition",
    "verification": "verified",
    "note": null
   },
   "paper_v1": null,
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2021-12-01",
    "precision": "day",
    "year_hint": null,
    "basis": "Organizer announcement of open participation for challenge #1",
    "event_kind": "official-announcement",
    "timestamp_utc": null,
    "source_url": "https://cache-challenge.org/join-competition",
    "verification": "partial",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-08",
   "precision": "month",
   "year_hint": null,
   "basis": "Host organization's press release PDF on the challenge #5 results, uploaded in August 2026",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://conscience.ca/wp-content/uploads/2026/08/CACHE-5-Press-Release-EN.pdf",
   "verification": "partial",
   "note": "The exact press-release day was not seen; the site note says 2026-08-18.",
   "description": "The host organization issued a press release on the challenge #5 results (PDF uploaded August 2026); DREAM x CACHE Target 2035 launched in June 2026."
  },
  "next_expected": {
   "date": "2026",
   "precision": "year",
   "year_hint": null,
   "basis": "Host press release says challenge #9 is expected late 2026; DREAM x CACHE applications close 2026-11-30",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://conscience.ca/wp-content/uploads/2026/08/CACHE-5-Press-Release-EN.pdf",
   "verification": "partial",
   "note": null
  },
  "next_expected_null_reason": null,
  "status": "active",
  "status_note": "Active: challenge #8 under way, challenge #5 results published 2026-08",
  "scale": {
   "summary": "Rounds #1-#8, one target each and up to 100 compounds per team; round #5 totalled 2,031 compounds and round #6 1,344.",
   "items": [
    {
     "label": "Round #1 compounds, 23 teams",
     "value": "1955",
     "unit": "compounds",
     "version": "CACHE #1"
    },
    {
     "label": "Round #5 compounds in total, 24 teams",
     "value": "2031",
     "unit": "compounds",
     "version": "CACHE #5"
    },
    {
     "label": "Round #5, stage 1",
     "value": "1455",
     "unit": "compounds",
     "version": "CACHE #5"
    },
    {
     "label": "Round #5, stage 2, 15 participants",
     "value": "576",
     "unit": "compounds",
     "version": "CACHE #5"
    },
    {
     "label": "Round #6 compounds, 18 participants",
     "value": "1344",
     "unit": "compounds",
     "version": "CACHE #6"
    },
    {
     "label": "Per-team submission cap",
     "value": "100",
     "unit": "compounds",
     "version": null
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "Yes: about two rounds a year, 1,300-2,000 compounds plus 300-600 analogues measured in laboratories each round.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "One target per round: teams submit up to 100 purchasable compounds, the organizers screen and orthogonally confirm them, and a second stage takes analogues.",
   "truth": "Newly measured binding data from the organizers (SPR and similar) plus functional or orthogonal assays in stage 2.",
   "metric_primary": "Experimentally confirmed hits and hit rate",
   "metric_secondary": [
    "Confirmed analogues and series in stage 2",
    "Affinity range of the hits"
   ],
   "version_traps": "Targets, assays and hit criteria differ by round (round #1 counted KD below 150 uM as a hit), so hit rates are not comparable across rounds.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "Experimentally confirmed hits and hit rate (SPR, dose-response, functional assays); no single score",
   "eval_set": "CACHE challenges #1-#8, one target per round, up to 100 purchasable compounds per team",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://cache-challenge.org/",
    "verification": "partial",
    "note": "There is no ranking board; the snapshot day is this round's check date."
   },
   "board_url": "https://cache-challenge.org/",
   "entries": [
    {
     "rank": null,
     "model_label": "CACHE #5 (MCHR1) round results, workflow level rather than model level",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Experimentally confirmed hits and hit rate (SPR, dose-response, functional assays); no single score",
     "eval_set": "CACHE challenges #1-#8, one target per round, up to 100 purchasable compounds per team",
     "source_type": "board",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "The official results page carries no date; data file names are prefixed 260624 and the host press release is from August 2026",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://cache-challenge.org/node/3000",
      "verification": "unverified",
      "note": null
     },
     "url": "https://cache-challenge.org/node/3000",
     "note": "Not a site model. The official results page gives hit counts by workflow id, discloses no models and publishes no rank.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "The page has no date and the organizers publish no rank, so the honesty rule keeps this out of the ranking row."
    },
    {
     "rank": null,
     "model_label": "Boltz-2",
     "model_id": "model-boltz-2",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Experimentally confirmed hits and hit rate (SPR, dose-response, functional assays); no single score",
     "eval_set": "CACHE challenges #1-#8, one target per round, up to 100 purchasable compounds per team",
     "source_type": "third",
     "date": {
      "date": null,
      "precision": null,
      "year_hint": null,
      "basis": "The source gives no date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12262699/",
      "verification": "unverified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12262699/",
     "note": "The Boltz-2 paper reports no CACHE participation and CACHE result pages do not name the models inside each workflow, so its use is not disclosed.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "No number and no date, so nothing is shown in the ranking row."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "The organizers publish no ranking and never name the models inside each workflow, so no number can serve as a headline.",
   "caveat": "CACHE prospectively evaluates whole team workflows rather than single models and publishes hit counts only; per-round hit rates of 0.7-4% leave samples too small for significance testing.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [],
  "non_pairings": [
   {
    "model_id": "model-boltz-2",
    "note": "Its paper reports no CACHE result and the official pages do not list the models used by each workflow, so there is no pairing.",
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12262699/"
   }
  ],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "A real blind test means entering the next round; compound-level data from released rounds can serve as a retrospective set but is no longer blind for models with later training cutoffs. Report the round, target, hit criterion and assay, and separate stage 1 from stage 2. A real generalization claim needs prospective measurement, whose cost the organizers carry in that round.",
  "pairs_with": [
   "bench-05-posebusters",
   "bench-30-plinder",
   "bench-12-dude",
   "bench-27-polaris"
  ],
  "caveats": [
   "It evaluates whole team workflows, not single models, and never discloses which models a workflow used.",
   "Each round yields only tens of hits at 0.7-4%, and without significance tests ML and physics methods cannot be compared statistically.",
   "Most hits are weak binders in the tens of micromolar, and binding is not functional activity or developability.",
   "Biases: compounds come from a single supplier, entry is voluntary, and assay sensitivity limits detection of weak binders.",
   "Rounds #7 and #8 have no published results yet, and peer-reviewed papers for rounds #4-#6 lag behind."
  ],
  "sources": [
   {
    "url": "https://cache-challenge.org/",
    "title": "CACHE official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://cache-challenge.org/results-cache-challenge-5",
    "title": "Official results page for challenge #5",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://conscience.ca/wp-content/uploads/2026/08/CACHE-5-Press-Release-EN.pdf",
    "title": "Host organization press release on the challenge #5 results",
    "kind": "press",
    "date": {
     "date": "2026-08",
     "precision": "month",
     "year_hint": null,
     "basis": "Month the press-release PDF was uploaded",
     "event_kind": "results-published",
     "timestamp_utc": null,
     "source_url": "https://conscience.ca/wp-content/uploads/2026/08/CACHE-5-Press-Release-EN.pdf",
     "verification": "partial",
     "note": null
    }
   },
   {
    "url": "https://cache-challenge.org/challenges/finding-selective-inhibitors-of-phosphoglycerate-kinase-2",
    "title": "Rules page for challenge #7",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://www.biorxiv.org/content/10.1101/2024.07.18.603797.full.pdf",
    "title": "Challenge #1 results preprint",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12892310/",
    "title": "Paper on a challenge #3 participant workflow",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2022-01-31",
    "now": "2021-12-01",
    "reason": "2022-01-31 was the application deadline for challenge #1, and a registration deadline is never a first release; the organizer page gives 2021-12-01 as the announced opening.",
    "source_url": "https://cache-challenge.org/join-competition",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "challenge #1 application deadline (launch date not stated on site)",
    "now": "official announcement of challenge #1 targets and open participation, 2021-12-01 (organiser page). This is the announced arrangement; no same-day archive of the live opening was obtained, so the day stays flagged. Application deadline 2022-01-31 is stored as a note only.",
    "reason": "The basis was realigned to the announced opening arrangement, with the application deadline demoted to a note.",
    "source_url": "https://cache-challenge.org/join-competition",
    "date_found": "2026-11-16"
   },
   {
    "field": "members",
    "was": "[{\"member\": \"CACHE #1 (deadline)\", \"date\": \"2022-01-31\"}, {\"member\": \"CACHE #8 (deadline)\", \"date\": \"2026-04-30\"}]",
    "now": "[{\"member\": \"CACHE #1 announced open (2021-12-01)\", \"date\": \"2021-12-01\"}, {\"member\": \"CACHE #1 application deadline\", \"date\": \"2022-01-31\"}, {\"member\": \"CACHE #8 application deadline\", \"date\": \"2026-04-30\"}]",
    "reason": "The opening date was separated from the per-round application deadlines so a deadline cannot be read as a release.",
    "source_url": "https://cache-challenge.org/join-competition",
    "date_found": "2026-11-16"
   },
   {
    "field": "scale",
    "was": "round #5: 2,031 compounds; round #6: 1,344",
    "now": "unchanged, with provenance: round #5's 2,031 is the sum of stage 1 (1,455 compounds, 24 teams) and stage 2 (576); round #6 is 1,344 compounds from 18 participants",
    "reason": "The wet-lab check requires every figure to carry its parts and stage, so a two-stage sum is not read as one stage.",
    "source_url": "https://cache-challenge.org/results-cache-challenge-5",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.status",
    "was": "round results shown as a ranking row",
    "now": "no-ranking: hit counts at workflow level, with the undated results page withheld from display",
    "reason": "The organizers publish no rank and the results page is undated, while every displayed score needs a date and a source type.",
    "source_url": "https://cache-challenge.org/results-cache-challenge-5",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "latest_update",
   "next_expected",
   "ranking.entries[0]",
   "ranking.entries[1]",
   "ranking.snapshot_date"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-20-cache/",
   "en": "/bio/benchmark/benchmarks/cache/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-21-dream5",
  "kind": "benchmark",
  "nn": "21",
  "short_slug": "dream5",
  "names": {
   "display": "DREAM5 / BEELINE / CausalBench",
   "full": "Gene regulatory network inference family: the DREAM5 network inference challenge, BEELINE and CausalBench",
   "full_null_reason": null,
   "aliases": [
    "DREAM5",
    "DREAM5 network inference challenge",
    "BEELINE",
    "Beeline",
    "CausalBench"
   ],
   "ambiguous_aliases": [
    "DREAM",
    "DREAM challenges"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "DREAM5 network inference challenge",
     "id_or_null": null,
     "first_release": {
      "date": "2010",
      "precision": "year",
      "year_hint": null,
      "basis": "Year of the first DREAM5 challenge; results paper 2012-07-15",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://dreamchallenges.org/dream-5-network-inference-challenge/",
      "verification": "partial",
      "note": null
     }
    },
    {
     "name": "DREAM5 results paper",
     "id_or_null": null,
     "first_release": {
      "date": "2012-07-15",
      "precision": "day",
      "year_hint": null,
      "basis": "Nature Methods online publication",
      "event_kind": "journal-online",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1038/nmeth.2016",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "BEELINE",
     "id_or_null": null,
     "first_release": {
      "date": "2019-05-20",
      "precision": "day",
      "year_hint": null,
      "basis": "bioRxiv v1; Nature Methods 2020-01-06",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1038/s41592-019-0690-6",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "CausalBench",
     "id_or_null": null,
     "first_release": {
      "date": "2022-10-31",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv 2210.17283 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2210.17283",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "Three different protocols: DREAM5 is a 2010 blind challenge, BEELINE a single-cell evaluation framework and CausalBench a CRISPRi perturbation causal suite; scores do not compare across members."
  },
  "family_null_reason": null,
  "domain": "genomics-rna",
  "domains_secondary": [
   "single-cell"
  ],
  "inclusion": "original-30",
  "type": "A+B+C",
  "type_note": "A closed blind challenge plus an evaluation framework and a perturbation suite",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L5"
   ],
   "qualifier": "Mostly retrospective comparison; one DREAM5 experimental check reaches L5"
  },
  "publisher": {
   "orgs": [
    "DREAM Challenges",
    "Virginia Tech",
    "GSK.ai"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Three generations of GRN inference evaluation whose scores mainly reflect gold-standard quality",
  "first_release": {
   "date": "2010",
   "precision": "year",
   "year_hint": null,
   "basis": "Earliest family member: the year of the DREAM5 challenge; the conference was in November 2010 and the launch month is unsourced",
   "event_kind": "challenge-open",
   "timestamp_utc": null,
   "source_url": "https://dreamchallenges.org/dream-5-network-inference-challenge/",
   "verification": "partial",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2019-05-20",
    "precision": "day",
    "year_hint": null,
    "basis": "BEELINE bioRxiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/s41592-019-0690-6",
    "verification": "partial",
    "note": null
   },
   "journal_online": {
    "date": "2012-07-15",
    "precision": "day",
    "year_hint": null,
    "basis": "DREAM5 results paper, Nature Methods online",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/nmeth.2016",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2010",
    "precision": "year",
    "year_hint": null,
    "basis": "Year of the DREAM5 challenge",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://dreamchallenges.org/dream-5-network-inference-challenge/",
    "verification": "partial",
    "note": null
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-03-11",
   "precision": "day",
   "year_hint": null,
   "basis": "CausalBench published in Communications Biology",
   "event_kind": "paper-revision",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1038/s42003-025-07764-y",
   "verification": "partial",
   "note": "The site note cites a 2025-05 PMLR version; only the journal version was verified this round.",
   "description": "CausalBench was published in Communications Biology; its arXiv v1 is from 2022-10-31 and the challenge ran in 2023."
  },
  "next_expected": null,
  "next_expected_null_reason": "None of the three members has announced a new round or version.",
  "status": "low-frequency",
  "status_note": "DREAM5 frozen; BEELINE lightly maintained; the CausalBench challenge ended in 2023",
  "scale": {
   "summary": "DREAM5: 4 networks, 35 methods; BEELINE: 12 algorithms, 400+ simulated datasets; CausalBench: 2 CRISPRi datasets, over 200,000 samples.",
   "items": [
    {
     "label": "Networks",
     "value": "4",
     "unit": null,
     "version": "DREAM5"
    },
    {
     "label": "Methods: 29 submissions plus 6 off-the-shelf tools",
     "value": "35",
     "unit": null,
     "version": "DREAM5"
    },
    {
     "label": "Algorithms",
     "value": "12",
     "unit": null,
     "version": "BEELINE"
    },
    {
     "label": "Simulated datasets",
     "value": "400+",
     "unit": null,
     "version": "BEELINE"
    },
    {
     "label": "Experimental scRNA-seq datasets",
     "value": "5",
     "unit": null,
     "version": "BEELINE"
    },
    {
     "label": "CRISPRi datasets, K562 and RPE1",
     "value": "2",
     "unit": null,
     "version": "CausalBench"
    },
    {
     "label": "Interventional samples",
     "value": "200000+",
     "unit": "cells",
     "version": "CausalBench"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No: DREAM5 once tested 53 new predictions and 23 were supported; nothing has been added since.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Given expression data, infer directed regulator-to-target edges; unsupervised, with no train/test split.",
   "truth": "Known networks and pathway libraries: GeneNetWeaver simulations, RegulonDB, ChIP plus motifs, and interaction libraries such as STRING and CORUM.",
   "metric_primary": "Combined AUPR/AUROC score in DREAM5",
   "metric_secondary": [
    "AUPRC ratio and EPR in BEELINE",
    "Mean Wasserstein and false omission rate in CausalBench",
    "Precision and recall against pathway libraries"
   ],
   "version_traps": "State which generation, dataset and gold-standard library: BEELINE data are public and tunable while DREAM5 was blind, so the two do not mix.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "DREAM5: combined AUPR/AUROC; BEELINE: AUPRC ratio and EPR",
   "eval_set": "DREAM5 network inference (frozen), BEELINE and CausalBench suites",
   "snapshot_date": {
    "date": "2012-07-15",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://dreamchallenges.org/dream-5-network-inference-challenge/",
    "verification": "partial",
    "note": "The DREAM5 results paper's online date serves as the snapshot."
   },
   "board_url": "https://dreamchallenges.org/dream-5-network-inference-challenge/",
   "entries": [
    {
     "rank": null,
     "model_label": "Community (consensus) network",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "DREAM5: combined AUPR/AUROC; BEELINE: AUPRC ratio and EPR",
     "eval_set": "DREAM5 network inference (frozen), BEELINE and CausalBench suites",
     "source_type": "board",
     "date": {
      "date": "2012-07-15",
      "precision": "day",
      "year_hint": null,
      "basis": "Online date of the DREAM5 results paper",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://www.nature.com/articles/nmeth.2016",
      "verification": "verified",
      "note": null
     },
     "url": "https://www.nature.com/articles/nmeth.2016",
     "note": "Not a site model; averaging edge ranks across methods gave a consensus network that robustly beat any single method. No value was re-verified, so no score is shown.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Protocols are not comparable and the value is unverified, so no score is shown."
    },
    {
     "rank": null,
     "model_label": "scGPT / Geneformer, attention-edge GRN inference",
     "model_id": "model-scgpt-geneformer",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "DREAM5: combined AUPR/AUROC; BEELINE: AUPRC ratio and EPR",
     "eval_set": "DREAM5 network inference (frozen), BEELINE and CausalBench suites",
     "source_type": "third",
     "date": {
      "date": "2026-02",
      "precision": "month",
      "year_hint": null,
      "basis": "Month of the preprint",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2602.17532",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2602.17532",
     "note": "A third-party preprint using the BEELINE protocol: attention edges mostly reflect co-expression, with no numeric score.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "The source gives no number and is a preprint without peer review."
    },
    {
     "rank": null,
     "model_label": "scPRINT-2",
     "model_id": "model-scprint-2",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "DREAM5: combined AUPR/AUROC; BEELINE: AUPRC ratio and EPR",
     "eval_set": "DREAM5 network inference (frozen), BEELINE and CausalBench suites",
     "source_type": "self",
     "date": {
      "date": "2025-12",
      "precision": "month",
      "year_hint": null,
      "basis": "Month of the preprint",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://www.biorxiv.org/content/10.64898/2025.12.11.693702.full.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://www.biorxiv.org/content/10.64898/2025.12.11.693702.full.pdf",
     "note": "scPRINT-2 self-reports gains on its own GRN benchmark rather than the DREAM5 or BEELINE protocol, and no number was located this round.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "A different protocol, with no citable number."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "Protocols are not comparable and no entry carries a displayable score.",
   "caveat": "DREAM5 has been frozen since 2012 with no site model entered, BEELINE and CausalBench have no live board, and foundation-model results use their own splits, so no comparable ranking exists.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-scgpt-geneformer",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/2602.17532",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "disputed",
    "note": "A third-party attention-edge study, read in abstract only.",
    "resolution": null
   },
   {
    "model_id": "model-scprint-2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.biorxiv.org/content/10.64898/2025.12.11.693702v1",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "disputed",
    "note": "The preprint mentions its own GRN benchmark with scores in the body, not under this family's protocol.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 2
   }
  },
  "how_to_use": "Run it statically with no wet lab: BEELINE ships Docker images and configs, so run one curated dataset and compare AUPRC ratios, always alongside a co-expression or correlation baseline, and use CausalBench to test scalability of causal methods. Proving an edge is real regulation needs a CRISPRi or ChIP spot check; do not report AUROC alone.",
  "pairs_with": [
   "bench-24-vcc",
   "bench-09-scib",
   "bench-08-enformer"
  ],
  "caveats": [
   "Gold standards contain only known edges, so a predicted false positive may be a real unknown edge and scores mainly reflect truth quality.",
   "Most methods are only slightly better than random; on BEELINE's experimental data most EPR values sit near 1 and none exceeds 1.5.",
   "CausalBench found interventional methods no better than observational ones, contradicting synthetic benchmarks.",
   "Foundation models pretrain on these public datasets and ChIP libraries, and the contamination risk has not been systematically assessed.",
   "The only wet-lab translation is DREAM5's 53 predictions, of which 23 were supported, with no later evidence."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1038/nmeth.2016",
    "title": "DREAM5 results paper in Nature Methods",
    "kind": "journal",
    "date": {
     "date": "2012-07-15",
     "precision": "day",
     "year_hint": null,
     "basis": "Journal online",
     "event_kind": "journal-online",
     "timestamp_utc": null,
     "source_url": "https://doi.org/10.1038/nmeth.2016",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://dreamchallenges.org/dream-5-network-inference-challenge/",
    "title": "Official DREAM5 challenge page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC7098173",
    "title": "BEELINE full text",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://doi.org/10.1038/s41592-019-0690-6",
    "title": "BEELINE journal version",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2210.17283",
    "title": "CausalBench paper",
    "kind": "preprint",
    "date": {
     "date": "2022-10-31",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2210.17283",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://doi.org/10.1038/s42003-025-07764-y",
    "title": "CausalBench journal version",
    "kind": "journal",
    "date": {
     "date": "2025-03-11",
     "precision": "day",
     "year_hint": null,
     "basis": "Journal online",
     "event_kind": "journal-online",
     "timestamp_utc": null,
     "source_url": "https://doi.org/10.1038/s42003-025-07764-y",
     "verification": "partial",
     "note": null
    }
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2012",
    "now": "2010 (the DREAM5 challenge year; 2012-07-15 is the results paper)",
    "reason": "The site used the results paper's year as first release; the first release is the year the challenge ran, with the paper's online date kept as a separate milestone.",
    "source_url": "https://dreamchallenges.org/dream-5-network-inference-challenge/",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update",
    "was": "CausalBench PMLR paper, 2025-05",
    "now": "2025-03-11, CausalBench published in Communications Biology (kind: paper-revision)",
    "reason": "Only the journal version was verified this round; the PMLR date could not be reproduced, so a sourced event is used and flagged unverified.",
    "source_url": "https://doi.org/10.1038/s42003-025-07764-y",
    "date_found": "2026-11-16"
   },
   {
    "field": "family",
    "was": "three generations described in prose only",
    "now": "explicit family entry with four members and a boundary statement",
    "reason": "The family boundary must be stated so scores are not compared across members.",
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC7098173",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "scPRINT-2 counted as reporting this family",
    "now": "disputed-or-unconfirmed: its own GRN benchmark, not the DREAM5/BEELINE protocol",
    "reason": "This moves the model onto the zero-pairing list, since an own benchmark is not a pairing with this family.",
    "source_url": "https://www.biorxiv.org/content/10.64898/2025.12.11.693702v1",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "latest_update",
   "ranking.entries[0]",
   "ranking.entries[1]",
   "ranking.entries[2]",
   "ranking.snapshot_date",
   "release_milestones.challenge_open"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-21-dream5/",
   "en": "/bio/benchmark/benchmarks/dream5/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-22-capri",
  "kind": "benchmark",
  "nn": "22",
  "short_slug": "capri",
  "names": {
   "display": "CAPRI",
   "full": "Critical Assessment of PRedicted Interactions",
   "full_null_reason": null,
   "aliases": [
    "CAPRI",
    "CAPRI Round 57",
    "CAPRI Round 65",
    "CASP16-CAPRI",
    "CASP15-CAPRI"
   ],
   "ambiguous_aliases": [
    "DockQ"
   ]
  },
  "family": null,
  "family_null_reason": "Successive rounds of one challenge rather than a family entry; joint rounds with CASP share targets but are assessed separately.",
  "domain": "structure",
  "domains_secondary": [
   "protein"
  ],
  "inclusion": "original-30",
  "type": "A",
  "type_note": "Periodic blind test with a scorer track",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": null,
   "qualifier": "Truth unpublished at submission; no wet-lab functional validation"
  },
  "publisher": {
   "orgs": [
    "CAPRI management group",
    "EMBL-EBI Protein Data Bank in Europe"
   ],
   "org_type": "mixed"
  },
  "one_liner": "The oldest prospective blind test for protein complexes; its classes and DockQ are the common protocol",
  "first_release": {
   "date": "2001",
   "precision": "year",
   "year_hint": null,
   "basis": "The official site confirms only the year of the first round; the month is unverifiable, with a secondary paper saying July 2001 and the design meeting held in June 2001",
   "event_kind": "challenge-open",
   "timestamp_utc": null,
   "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
   "verification": "partial",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2001",
    "precision": "year",
    "year_hint": null,
    "basis": "The official round list gives no dates for the early rounds",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/round/1",
    "verification": "unverified",
    "note": null
   },
   "paper_v1": null,
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2001",
    "precision": "year",
    "year_hint": null,
    "basis": "Year of the first round, per the official site",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
    "verification": "partial",
    "note": null
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-11-02",
   "precision": "day",
   "year_hint": null,
   "basis": "Official round page: Round 66 opened for predictions, with registration from 2026-10-20 to 11-19",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
   "verification": "verified",
   "note": null,
   "description": "Round 66 opened for predictions, with registration to 2026-11-19, submissions closing 11-20 and scoring on 11-23 to 11-27; three nanobody targets, co-organized with a 2026 meeting."
  },
  "next_expected": {
   "date": "2026-11-23",
   "precision": "day",
   "year_hint": null,
   "basis": "Start of the Round 66 scoring window as published on the official round page",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
   "verification": "verified",
   "note": null
  },
  "next_expected_null_reason": null,
  "status": "round-in-progress",
  "status_note": "Round in progress: Round 65 is closed pending publication and Round 66 is open for predictions",
  "scale": {
   "summary": "Over 60 rounds and 160+ targets cumulatively; Round 57 (CASP16-CAPRI) had 34 targets and Round 65 has 54.",
   "items": [
    {
     "label": "Rounds cumulatively",
     "value": "60+",
     "unit": null,
     "version": null
    },
    {
     "label": "Targets cumulatively, per a 2020 organizer statement",
     "value": "160+",
     "unit": null,
     "version": null
    },
    {
     "label": "Targets",
     "value": "37",
     "unit": null,
     "version": "Round 54 (CASP15-CAPRI)"
    },
    {
     "label": "Models submitted",
     "value": "21941",
     "unit": null,
     "version": "Round 54 (CASP15-CAPRI)"
    },
    {
     "label": "Targets",
     "value": "34",
     "unit": null,
     "version": "Round 57 (CASP16-CAPRI)"
    },
    {
     "label": "Targets",
     "value": "54",
     "unit": null,
     "version": "Round 65 (CASP17-CAPRI)"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "About one joint round every two years with 30-40 targets; truth comes from experimental structures supplied before publication, not measured for the benchmark.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Two tracks: predict complex structures from component sequences, or pick near-native models out of the pool of all submissions in the scorer track.",
   "truth": "X-ray or cryo-EM structures, supplied by structural biologists and unpublished at prediction time.",
   "metric_primary": "The four CAPRI quality classes from Fnat, L-RMSD and i-RMSD",
   "metric_secondary": [
    "DockQ, a continuous 0-1 score",
    "Per-round counts of targets by quality class"
   ],
   "version_traps": "State the round, target class and number of submitted models: the protocol allows external MSAs, templates and unlimited compute, so rankings mix in sampling effort.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "CAPRI quality classes and DockQ",
   "eval_set": "Round 65 (CASP17-CAPRI) closed in Aug 2026 with results unpublished; Round 66 open with scoring 2026-11-23 to 11-27; entries come from Round 57 (CASP16-CAPRI, 2024)",
   "snapshot_date": {
    "date": "2025-08-25",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/round/65/",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/round/65/",
   "entries": [
    {
     "rank": 10,
     "model_label": "AF3-server",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "rank 10 of 81 (CASP16 protein multimers); avg DockQ 0.53 first-ranked",
     "score_unit": null,
     "metric": "CAPRI quality classes and DockQ",
     "eval_set": "Round 65 (CASP17-CAPRI) closed in Aug 2026 with results unpublished; Round 66 open with scoring 2026-11-23 to 11-27; entries come from Round 57 (CASP16-CAPRI, 2024)",
     "source_type": "third",
     "date": {
      "date": "2025-08-25",
      "precision": "day",
      "year_hint": null,
      "basis": "Online date of the third-party assessment paper",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "From the CASP16 oligomer ranking assessed jointly with CAPRI, not a CAPRI-own ranking.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 30,
     "model_label": "MassiveFold",
     "model_id": "model-massivefold",
     "site_model": true,
     "score": "rank 30 of 81 (CASP16 protein multimers)",
     "score_unit": null,
     "metric": "CAPRI quality classes and DockQ",
     "eval_set": "Round 65 (CASP17-CAPRI) closed in Aug 2026 with results unpublished; Round 66 open with scoring 2026-11-23 to 11-27; entries come from Round 57 (CASP16-CAPRI, 2024)",
     "source_type": "third",
     "date": {
      "date": "2025-08-25",
      "precision": "day",
      "year_hint": null,
      "basis": "Online date of the third-party assessment paper",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "Same third-party assessment table; the MassiveFold paper on CASP16-CAPRI (31 targets, 65 interfaces) gives no rank.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Top-performing academic group (CASP16 oligomers)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "CAPRI quality classes and DockQ",
     "eval_set": "Round 65 (CASP17-CAPRI) closed in Aug 2026 with results unpublished; Round 66 open with scoring 2026-11-23 to 11-27; entries come from Round 57 (CASP16-CAPRI, 2024)",
     "source_type": "third",
     "date": {
      "date": "2025",
      "precision": "year",
      "year_hint": null,
      "basis": "The secondary summary gives a year only",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
      "verification": "unverified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "note": "Not a site model; the official assessment calls complex prediction unsolved, with over 30% of targets, especially antibody-antigen, highly difficult.",
     "second_hand": true,
     "withheld": true,
     "withheld_reason": "The figure appears only in a secondary summary and was not re-verified in primary material, so it is not shown."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "There is no ranking this round, and the entries come from the CASP16 assessment of the joint round rather than a CAPRI-own standing.",
   "caveat": "Round 65 results are to appear in the CASP17 special issue and no ranking exists yet; CAPRI publishes per-target class counts rather than one board.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-af2-multimer",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://pubmed.ncbi.nlm.nih.gov/37905971/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "The official round assessment lists it as the assessors' yardstick baseline, counted as an official entry; no per-target numbers.",
    "resolution": null
   },
   {
    "model_id": "model-massivefold",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://doi.org/10.1101/2025.05.26.653955",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Took part in Round 57 (CASP16-CAPRI); self-reports 31 targets and 65 interfaces, with no rank.",
    "resolution": null
   },
   {
    "model_id": "model-alphafold3",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "disputed",
    "note": "No standalone entry appears in the official CAPRI assessment material; only secondary sources call it a component of most pipelines.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 2,
   "by_relation": {
    "official-participation": 2,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 1
   }
  },
  "how_to_use": "A real blind test means entering the next round. Historical targets can be replayed statically: take joint-round targets released after the training cutoff, submit 5 models, score with DockQ, report per target rather than only a mean, and keep antibody-antigen separate. Models trained on the full PDB risk leakage on old targets; for a de-leaked static complement see the PLINDER family.",
  "pairs_with": [
   "bench-01-casp",
   "bench-30-plinder",
   "bench-31-foldbench",
   "bench-15-cameo"
  ],
  "caveats": [
   "Each round has only tens of targets with an uncontrolled class mix, so a single target can dominate aggregate scores and confidence intervals are usually absent.",
   "It shares targets and submissions with the CASP complex track, so it is not independent evidence.",
   "The protocol allows external MSAs, templates and unlimited compute, so rankings mix in sampling and compute, and there is no common baseline suite.",
   "Easy homo- and heterodimers are near saturation while antibody-antigen, flexible complexes and large assemblies remain unsolved.",
   "No study links the scores to affinity, specificity or design hit rates, and targets and submissions are scattered with no versioned dataset."
  ],
  "sources": [
   {
    "url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
    "title": "Official CAPRI site with the round list",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/round/65/",
    "title": "Round 65 page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://pubmed.ncbi.nlm.nih.gov/37905971/",
    "title": "Assessment paper for Round 54, CASP15-CAPRI",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
    "title": "Third-party assessment paper on CASP16 structure prediction",
    "kind": "third-party-paper",
    "date": {
     "date": "2025-08-25",
     "precision": "day",
     "year_hint": null,
     "basis": "Journal online",
     "event_kind": "journal-online",
     "timestamp_utc": null,
     "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC4999177",
    "title": "DockQ paper",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2407.16580v2",
    "title": "DockQ v2",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://pubmed.ncbi.nlm.nih.gov/12784359/",
    "title": "Review on the origins of CAPRI",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2001-07",
    "now": "2001",
    "reason": "The official site confirms only the year of the first round; the month is unverifiable between June and July 2001 and must not be upgraded on plausibility.",
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "Round 1 began soon after Charleston meeting (2001-06-16..19) per CAPRI site; a paper says July 2001; EBI round page shows odd 2000-11-29 registration, unreconciled",
    "now": "first CAPRI round, 2001 (year-only evidence on the official site). Month unverifiable: a secondary paper says July 2001, the Charleston design meeting was 2001-06-16..19, and the EBI round-1 page shows placeholder 2000 dates.",
    "reason": "The basis was reduced to match, naming the three conflicting clues.",
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "2001-06",
    "now": "2001-07",
    "reason": "An earlier round moved the first release from 2001-06 to 2001-07; this round reduces it further to the year.",
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/round/1",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update",
    "was": "Round 57 (CASP16-CAPRI, 2024) described as the latest round",
    "now": "2026-11-02: Round 66 opened for predictions; Round 65 (54 targets) closed in Aug 2026 with results unpublished",
    "reason": "The site treated the 2024 joint round as the latest event, while Rounds 65 and 66 have since run.",
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "AF2-Multimer listed as having no CAPRI pairing",
    "now": "official-participation: the assessors' baseline in the official round assessment",
    "reason": "This takes the model off the zero-pairing list, since an assessor-run yardstick baseline counts as an organizer-confirmed entry.",
    "source_url": "https://pubmed.ncbi.nlm.nih.gov/37905971/",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries",
    "was": "a top group's success rate above 60% shown as a number",
    "now": "withheld: the figure appears only in a secondary summary",
    "reason": "A second-hand figure cannot be displayed in a ranking cell, and the organizers published no such rank.",
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12750027/",
    "date_found": "2026-11-16"
   },
   {
    "field": "status",
    "was": "active",
    "now": "round-in-progress (Round 66 predictions open, scoring 2026-11-23 to 11-27)",
    "reason": "The status was realigned with the round currently under way.",
    "source_url": "https://www.ebi.ac.uk/pdbe/complex-pred/capri/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "release_milestones.challenge_open",
   "release_milestones.first_public_evidence",
   "ranking.entries[2]",
   "scale.items[1]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-22-capri/",
   "en": "/bio/benchmark/benchmarks/capri/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-23-bioasq",
  "kind": "benchmark",
  "nn": "23",
  "short_slug": "bioasq",
  "names": {
   "display": "BioASQ",
   "full": "BioASQ challenge on biomedical semantic indexing and question answering",
   "full_null_reason": null,
   "aliases": [
    "BioASQ",
    "BioASQ 14",
    "BioASQ 13",
    "Task 14b",
    "BioASQ-QA",
    "Synergy task"
   ],
   "ambiguous_aliases": [
    "Task a",
    "Task b"
   ]
  },
  "family": null,
  "family_null_reason": "Successive yearly editions of one challenge rather than a family entry; past questions are released separately as static datasets.",
  "domain": "text-agents",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "A+B",
  "type_note": "Yearly challenge whose past questions are public static sets",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L1",
   "range": [
    "L1",
    "L4"
   ],
   "qualifier": "Reproduces literature knowledge, while each new batch approximates a blind test"
  },
  "publisher": {
   "orgs": [
    "BioASQ consortium",
    "National Centre for Scientific Research Demokritos",
    "Athens University of Economics and Business"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "The only yearly biomedical retrieval and QA challenge with new expert-written questions and expert grading",
  "first_release": {
   "date": "2013",
   "precision": "year",
   "year_hint": null,
   "basis": "First BioASQ challenge at the CLEF 2013 lab, with the EU project starting in October 2012; the month has no primary source",
   "event_kind": "challenge-open",
   "timestamp_utc": null,
   "source_url": "https://bioasq.org/",
   "verification": "partial",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": null,
   "journal_online": {
    "date": "2015",
    "precision": "year",
    "year_hint": null,
    "basis": "Results paper for the first challenge; year only",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC4450488",
    "verification": "partial",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2013",
    "precision": "year",
    "year_hint": null,
    "basis": "Earliest edition listed on the official workshop page",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://bioasq.org/",
    "verification": "partial",
    "note": null
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-09",
   "precision": "month",
   "year_hint": null,
   "basis": "Publication of the BioASQ 14 overview paper in CEUR-WS Vol-4283 with preliminary Task 14b results",
   "event_kind": "results-published",
   "timestamp_utc": null,
   "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
   "verification": "verified",
   "note": "The CLEF meeting dates of 2026-09-21 to 24 are a conference event, not a benchmark event.",
   "description": "The BioASQ 14 overview paper appeared in CEUR Vol-4283 with preliminary Task 14b results; the tasks ran in spring 2026."
  },
  "next_expected": null,
  "next_expected_null_reason": "No BioASQ 15 announcement was found; the official site has only a placeholder page.",
  "status": "active",
  "status_note": "Active: the 14th edition published its overview and preliminary results in 2026-09",
  "scale": {
   "summary": "Task 14b: 5,729 training questions, 280 test questions in 4 batches, 151 systems and 821 submissions.",
   "items": [
    {
     "label": "Training questions",
     "value": "5729",
     "unit": null,
     "version": "14b"
    },
    {
     "label": "Test questions in 4 batches of 80/80/60/60",
     "value": "280",
     "unit": null,
     "version": "14b"
    },
    {
     "label": "Systems",
     "value": "151",
     "unit": null,
     "version": "14b"
    },
    {
     "label": "Submissions",
     "value": "821",
     "unit": null,
     "version": "14b"
    },
    {
     "label": "Teams, of which 48 in the QA part",
     "value": "87",
     "unit": null,
     "version": "BioASQ 14"
    }
   ]
  },
  "wetlab_refresh": "accumulating",
  "wetlab_note": "No: about 280 new expert-written questions a year, with expert-annotated literature as truth rather than wet-lab data.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Task b runs in phases: retrieve PubMed documents and snippets, or answer given gold material; question types are yes/no, factoid, list and summary.",
   "truth": "Expert-written questions with gold documents, snippets and answers, finalized after experts add missed items post-evaluation.",
   "metric_primary": "Yes/no macro-F1 and retrieval MAP/F1",
   "metric_secondary": [
    "Factoid MRR",
    "List F1",
    "Manual scores and ROUGE for summaries"
   ],
   "version_traps": "Scores depend on which experts wrote the questions, so years do not compare; state the edition, batch and phase (A, A+, B) and distinguish the BLURB yes/no slice.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Yes/no macro-F1 as the ranking key of Tables 7 and 8, plus factoid MRR and list F1",
   "eval_set": "BioASQ Task 14b batch 2 exact answers, phase A+ (Table 7) and phase B (Table 8); preliminary results",
   "snapshot_date": {
    "date": "2026-09",
    "precision": "month",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "verification": "partial",
    "note": "The overview paper gives a month; the exact publication day was not extracted."
   },
   "board_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
   "entries": [
    {
     "rank": 1,
     "model_label": "mckpt2",
     "model_id": null,
     "site_model": false,
     "score": "Yes/No F1 0.9481, Acc. 0.9524; Factoid MRR 0.2000; List F1 0.1537",
     "score_unit": null,
     "metric": "Yes/no macro-F1 as the ranking key of Tables 7 and 8, plus factoid MRR and list F1",
     "eval_set": "BioASQ Task 14b batch 2 exact answers, phase A+ (Table 7) and phase B (Table 8); preliminary results",
     "source_type": "board",
     "date": {
      "date": "2026-09",
      "precision": "month",
      "year_hint": null,
      "basis": "Month the overview paper was published",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
     "note": "Not a site model; top of Table 7 for phase A+ batch 2, tied on yes/no F1 with another system.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "MedQA-1 (university team; phase B uses a frontier-LLM ensemble)",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "Yes/No F1 0.9443, Acc. 0.9524; Factoid MRR 0.4000; List F1 0.3551",
     "score_unit": null,
     "metric": "Yes/no macro-F1 as the ranking key of Tables 7 and 8, plus factoid MRR and list F1",
     "eval_set": "BioASQ Task 14b batch 2 exact answers, phase A+ (Table 7) and phase B (Table 8); preliminary results",
     "source_type": "board",
     "date": {
      "date": "2026-09",
      "precision": "month",
      "year_hint": null,
      "basis": "Month the overview paper was published",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
     "note": "Third row of Table 7, phase A+ batch 2; 0.9443 is tied across many systems so the rank is only a listing position. The board ranks submitted systems and the model attribution comes from the system description.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 1,
     "model_label": "dictycite-baseline",
     "model_id": null,
     "site_model": false,
     "score": "Yes/No F1 1.0000, Acc. 1.0000; Factoid MRR 0.3000; List F1 0.4301",
     "score_unit": null,
     "metric": "Yes/no macro-F1 as the ranking key of Tables 7 and 8, plus factoid MRR and list F1",
     "eval_set": "BioASQ Task 14b batch 2 exact answers, phase A+ (Table 7) and phase B (Table 8); preliminary results",
     "source_type": "board",
     "date": {
      "date": "2026-09",
      "precision": "month",
      "year_hint": null,
      "basis": "Month the overview paper was published",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
     "note": "Not a site model; top of Table 8 for phase B batch 2.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "MedQA-1",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "Yes/No F1 0.9443, Acc. 0.9524; Factoid MRR 0.4000; List F1 0.4433",
     "score_unit": null,
     "metric": "Yes/no macro-F1 as the ranking key of Tables 7 and 8, plus factoid MRR and list F1",
     "eval_set": "BioASQ Task 14b batch 2 exact answers, phase A+ (Table 7) and phase B (Table 8); preliminary results",
     "source_type": "board",
     "date": {
      "date": "2026-09",
      "precision": "month",
      "year_hint": null,
      "basis": "Month the overview paper was published",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
     "note": "Table 8, phase B batch 2; the yes/no F1 of 0.9443 is tied across many systems, so there is no unique rank.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "MedGemma-27B-IT as a component of participant systems",
     "model_id": "model-medgemma-med-gemini",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Yes/no macro-F1 as the ranking key of Tables 7 and 8, plus factoid MRR and list F1",
     "eval_set": "BioASQ Task 14b batch 2 exact answers, phase A+ (Table 7) and phase B (Table 8); preliminary results",
     "source_type": "third",
     "date": {
      "date": "2026-09",
      "precision": "month",
      "year_hint": null,
      "basis": "Month the overview paper was published",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
     "note": "No standalone model score exists: the official board scores whole systems and the batch-2 indicative tables read here contain no such row.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "No standalone model figure can be cited, so no score is shown."
    }
   ],
   "headline": 0,
   "best_site_model": 1,
   "headline_null_reason": null,
   "caveat": "The official board ranks submitted systems per batch rather than underlying models, and 14b results are preliminary until expert re-assessment; scores do not compare across years.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-co-scientist",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/2502.18864",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report by it was found in the official material.",
    "resolution": null
   },
   {
    "model_id": "model-frontier-llms",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "The overview ranks submitted systems, not the underlying LLMs.",
    "resolution": null
   },
   {
    "model_id": "model-gpt-rosalind",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report was found.",
    "resolution": null
   },
   {
    "model_id": "model-intern-s",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://github.com/InternLM/Intern-S1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report by it was found in the official material.",
    "resolution": null
   },
   {
    "model_id": "model-medgemma-med-gemini",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "The overview scores whole systems and contains no standalone figure for it, so the board marking in the matrix is unsupported.",
    "resolution": null
   },
   {
    "model_id": "model-quine",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.microsoft.com/en-us/research/blog/introducing-quine-an-ai-research-system-designed-for-the-complexity-of-biology/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report by it was found in the official material.",
    "resolution": null
   },
   {
    "model_id": "model-scireasoner",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "Not found in the paper text.",
    "resolution": null
   },
   {
    "model_id": "model-verid",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report by it was found in the official material.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 8
   }
  },
  "how_to_use": "No wet lab needed. A strict comparison means submitting in the official window; offline, the newest batch can serve as an uncontaminated hold-out, but only for the current edition. Report phases A, A+ and B together, since phase B alone hides accumulated retrieval errors; registration is free and in-batch submission is time-limited.",
  "pairs_with": [
   "bench-14-pubmedqa",
   "bench-18-labbench",
   "bench-42-biomysterybench"
  ],
  "caveats": [
   "Scores depend on the question writers: 64% of 14b questions came from new experts and the top list and factoid means fell below the previous edition, so years do not compare.",
   "Yes/no has only 11-21 questions per batch, so one question is worth 5-9 points, and the organizers give no confidence intervals or significance tests.",
   "Past questions are public and LLM training corpora very likely contain them, with the contamination unquantified.",
   "The board ranks whole systems including retrieval, reranking and ensembling, so single-model performance cannot be read off it.",
   "The corpus is PubMed abstracts rather than full texts, and with no wet-lab translation it only proxies literature skill."
  ],
  "sources": [
   {
    "url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "title": "BioASQ 14 overview paper, CEUR Vol-4283",
    "kind": "paper",
    "date": {
     "date": "2026-09",
     "precision": "month",
     "year_hint": null,
     "basis": "Month the overview paper was published",
     "event_kind": "results-published",
     "timestamp_utc": null,
     "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
     "verification": "partial",
     "note": null
    }
   },
   {
    "url": "https://arxiv.org/abs/2508.20554",
    "title": "BioASQ 13 overview paper",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://bioasq.org/",
    "title": "BioASQ official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC4450488",
    "title": "Results paper for the first challenge",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC10042099/",
    "title": "BioASQ-QA dataset paper",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2210.06852",
    "title": "Statement on the discontinuation of Task a",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2509.18843",
    "title": "Third-party LLM evaluation on BioASQ",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update",
    "was": "2026-09-21",
    "now": "2026-09",
    "reason": "2026-09-21 was the CLEF meeting's opening day rather than a data or benchmark event; the sourced event is the overview paper, at month precision.",
    "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_desc",
    "was": "BioASQ 14 (CLEF 2026, Jena, 21-24 Sep) concluded; overview paper in CEUR-WS Vol-4283; six tasks incl. new MultiClinSum-2, BioNNE-R, ELCardioCC, GutBrainIE.",
    "now": "BioASQ 14 (CLEF 2026) overview paper published in CEUR Vol-4283 with preliminary Task 14b results; tasks ran spring 2026. CLEF meeting (Jena, 2026-09-21..24) is a conference date, not a benchmark event.",
    "reason": "The description was rewritten with the date, making clear that a conference date is not a benchmark event.",
    "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_kind",
    "was": "new-round",
    "now": "results-published",
    "reason": "The event is the publication of round results, so the kind becomes results-published.",
    "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries",
    "was": "MedQA-1 list F1 0.325",
    "now": "0.3551 (Table 7, phase A+, batch 2)",
    "reason": "The site figure disagrees with Table 7 of the overview, so the primary table's value is used.",
    "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "MedGemma / Med-Gemini marked as a board pairing in the matrix",
    "now": "disputed-or-unconfirmed: the overview scores whole systems and reports no standalone figure",
    "reason": "The official board gives no standalone model score, leaving the matrix's board marking unsupported, an item left open in R7.",
    "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "frontier LLMs marked as a board pairing",
    "now": "disputed-or-unconfirmed: systems are ranked, not base LLMs",
    "reason": "Participant systems include retrieval and ensembling, so they cannot be read as a single LLM's board result.",
    "source_url": "https://ceur-ws.org/Vol-4283/paper1.pdf",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "release_milestones.challenge_open",
   "release_milestones.journal_online",
   "ranking.snapshot_date",
   "ranking.entries[4]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-23-bioasq/",
   "en": "/bio/benchmark/benchmarks/bioasq/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-24-vcc",
  "kind": "benchmark",
  "nn": "24",
  "short_slug": "vcc",
  "names": {
   "display": "Virtual Cell Challenge and perturbation benchmarks",
   "full": "The Virtual Cell Challenge plus PerturBench and related single-cell perturbation benchmarks",
   "full_null_reason": null,
   "aliases": [
    "Virtual Cell Challenge",
    "VCC",
    "VCC 2025",
    "VCC 2026",
    "PerturBench",
    "cell-eval"
   ],
   "ambiguous_aliases": [
    "Norman",
    "Replogle",
    "Adamson"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "PerturBench",
     "id_or_null": null,
     "first_release": {
      "date": "2024-08-20",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv 2408.10609 v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2408.10609",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "Virtual Cell Challenge 2025",
     "id_or_null": null,
     "first_release": {
      "date": "2025-06-26",
      "precision": "day",
      "year_hint": null,
      "basis": "Challenge launch with datasets and leaderboard opened, per the organizer announcement",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "Virtual Cell Challenge 2026",
     "id_or_null": null,
     "first_release": {
      "date": "2026-08-20",
      "precision": "day",
      "year_hint": null,
      "basis": "Announcement of the 2026 edition with registration open",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2026",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "VCC is a yearly blind challenge while PerturBench and the Norman, Replogle and Adamson suites are static benchmarks and cell-eval is only a tool; the first release is the earliest member, PerturBench, not the VCC launch."
  },
  "family_null_reason": null,
  "domain": "single-cell",
  "domains_secondary": [
   "genomics-rna"
  ],
  "inclusion": "original-30",
  "type": "A+B+C",
  "type_note": "A blind challenge plus static perturbation suites and an evaluation tool",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": [
    "L2",
    "L4"
   ],
   "qualifier": "VCC uses a hidden test set; the static suites and baseline critiques are L2-L3"
  },
  "publisher": {
   "orgs": [
    "Arc Institute",
    "Altos Labs"
   ],
   "org_type": "nonprofit-institute"
  },
  "one_liner": "The only challenge generating Perturb-seq data for evaluation, yet linear and mean baselines remain hard to beat",
  "first_release": {
   "date": "2024-08-20",
   "precision": "day",
   "year_hint": null,
   "basis": "Earliest family member: PerturBench's arXiv v1; VCC itself launched on 2025-06-26",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2408.10609",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2024-08-20",
    "precision": "day",
    "year_hint": null,
    "basis": "PerturBench arXiv 2408.10609 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2408.10609",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2025-06-26",
    "precision": "day",
    "year_hint": null,
    "basis": "VCC 2025 launch with data and leaderboard opened",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025",
    "verification": "verified",
    "note": null
   },
   "official_announcement": {
    "date": "2026-08-20",
    "precision": "day",
    "year_hint": null,
    "basis": "VCC 2026 announcement with registration open",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2026",
    "verification": "verified",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-10-22",
   "precision": "day",
   "year_hint": null,
   "basis": "Release of the VCC 2026 final test set, per the organizer's published schedule",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2026",
   "verification": "partial",
   "note": "The date could not be matched to a dated official passage, so it is shown as unverified.",
   "description": "The VCC 2026 final test set was released and final submissions closed on 2026-11-05; as of 2026-11-16 no results have been published."
  },
  "next_expected": {
   "date": "2026-11",
   "precision": "month",
   "year_hint": null,
   "basis": "The organizer schedule says the 2026 results come in mid-to-late November, while a press release says December",
   "event_kind": "results-published",
   "timestamp_utc": null,
   "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2026",
   "verification": "unverified",
   "note": null
  },
  "next_expected_null_reason": null,
  "status": "active",
  "status_note": "Active: VCC 2026 has closed and its results are not yet published",
  "scale": {
   "summary": "VCC 2025: about 300,000 cells, 300 perturbations and 300+ teams in the final round; VCC 2026: six cell lines with cell counts unpublished.",
   "items": [
    {
     "label": "Cells, H1 hESC",
     "value": "~300000",
     "unit": "cells",
     "version": "VCC 2025"
    },
    {
     "label": "CRISPRi perturbations",
     "value": "300",
     "unit": null,
     "version": "VCC 2025"
    },
    {
     "label": "Registrations",
     "value": "5000+",
     "unit": "people",
     "version": "VCC 2025"
    },
    {
     "label": "Teams in the final round",
     "value": "300+",
     "unit": null,
     "version": "VCC 2025"
    },
    {
     "label": "Cell lines, 3 validation and 3 final, not re-confirmed",
     "value": "6",
     "unit": null,
     "version": "VCC 2026"
    },
    {
     "label": "Cells",
     "value": null,
     "unit": null,
     "version": "VCC 2026"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "Yes, yearly: the organizers run their own Perturb-seq; the 2025 edition had about 300,000 cells and the 2026 cell count is unpublished.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Predict how a cell's transcriptome changes when a gene is knocked down, including unseen perturbations and unseen cell backgrounds, with the 2026 edition zero-shot.",
   "truth": "Newly measured CRISPRi Perturb-seq data from the organizers; the static suites use published Norman, Replogle and Adamson data.",
   "metric_primary": "2025: aggregate rank over PDS, DES and MAE",
   "metric_secondary": [
    "Generalist Prize: mean rank across 7 metrics",
    "2026: a six-metric aggregate in the new cell-eval",
    "Pearson delta and AUPRC"
   ],
   "version_traps": "The 2025 data are now public and no longer blind, the two editions use different metric sets, and the L1/L2 forms of PDS are sensitive to prediction scale, so ranks do not compare across editions.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "VCC 2025 final rank aggregating PDS, DES and MAE, with the Generalist Prize as mean rank over 7 metrics",
   "eval_set": "Virtual Cell Challenge 2025 final test set (H1 hESC, held-out CRISPRi perturbations)",
   "snapshot_date": {
    "date": "2025-12-06",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
   "entries": [
    {
     "rank": 1,
     "model_label": "Team BM_xTVC, xTrimoSCPerturb (improved scFoundation architecture with protein-model embeddings)",
     "model_id": "model-xtrimoscperturb-scfoundation",
     "site_model": true,
     "score": "rank 1 of 300+ final-round teams (board publishes ranks only, no numeric scores)",
     "score_unit": null,
     "metric": "VCC 2025 final rank aggregating PDS, DES and MAE, with the Generalist Prize as mean rank over 7 metrics",
     "eval_set": "Virtual Cell Challenge 2025 final test set (H1 hESC, held-out CRISPRi perturbations)",
     "source_type": "board",
     "date": {
      "date": "2025-12-06",
      "precision": "day",
      "year_hint": null,
      "basis": "Publication date of the organizer wrap-up page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
      "verification": "verified",
      "note": null
     },
     "url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
     "note": "Overall winner; the organizer wrap-up publishes ranks and prizes only, with no numeric score for any team.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "Team XLearning Lab (fully connected network, ESM-2 embeddings, residual delta prediction)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "VCC 2025 final rank aggregating PDS, DES and MAE, with the Generalist Prize as mean rank over 7 metrics",
     "eval_set": "Virtual Cell Challenge 2025 final test set (H1 hESC, held-out CRISPRi perturbations)",
     "source_type": "board",
     "date": {
      "date": "2025-12-06",
      "precision": "day",
      "year_hint": null,
      "basis": "Publication date of the organizer wrap-up page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
      "verification": "verified",
      "note": null
     },
     "url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
     "note": "Not a site model; no numeric score published.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "Team Outlier, TransPert (summary-level statistical framework)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "VCC 2025 final rank aggregating PDS, DES and MAE, with the Generalist Prize as mean rank over 7 metrics",
     "eval_set": "Virtual Cell Challenge 2025 final test set (H1 hESC, held-out CRISPRi perturbations)",
     "source_type": "board",
     "date": {
      "date": "2025-12-06",
      "precision": "day",
      "year_hint": null,
      "basis": "Publication date of the organizer wrap-up page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
      "verification": "verified",
      "note": null
     },
     "url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
     "note": "Not a site model; no numeric score published.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Team Altos Labs, go-with-the-flow (flow matching with a custom U-Net), Generalist Prize",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "VCC 2025 final rank aggregating PDS, DES and MAE, with the Generalist Prize as mean rank over 7 metrics",
     "eval_set": "Virtual Cell Challenge 2025 final test set (H1 hESC, held-out CRISPRi perturbations)",
     "source_type": "board",
     "date": {
      "date": "2025-12-06",
      "precision": "day",
      "year_hint": null,
      "basis": "Publication date of the organizer wrap-up page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
      "verification": "verified",
      "note": null
     },
     "url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
     "note": "Not a site model; the prize goes by mean rank over 7 metrics among the top 50 final entries, with no numbers.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "STATE",
     "model_id": "model-state",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "VCC 2025 final rank aggregating PDS, DES and MAE, with the Generalist Prize as mean rank over 7 metrics",
     "eval_set": "Virtual Cell Challenge 2025 final test set (H1 hESC, held-out CRISPRi perturbations)",
     "source_type": "self",
     "date": {
      "date": "2025-06",
      "precision": "month",
      "year_hint": null,
      "basis": "Month of the preprint",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.06.26.661135",
      "verification": "verified",
      "note": null
     },
     "url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.06.26.661135",
     "note": "Not an entry: the organizers used four metrics from its paper to define the Generalist Prize, and its self-reported Tahoe-100M results use a different protocol and are not board results.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "A different protocol and not a board result, so nothing is shown in the ranking row."
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "The 2026 edition closed on 2026-11-05 but had announced no standing as of 2026-11-16, so the current standing is still 2025; that board publishes ranks only, never scores.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-xtrimoscperturb-scfoundation",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Overall winner of VCC 2025 per the organizer wrap-up; scores unpublished.",
    "resolution": null
   },
   {
    "model_id": "model-c2s-scale",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://www.biorxiv.org/content/10.1101/2025.04.14.648850v1.full",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Uses its own datasets and metrics, not the VCC protocol.",
    "resolution": null
   },
   {
    "model_id": "model-cellfm",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://www.nature.com/articles/s41467-025-59926-5",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Adamson and Norman data through GEARS, not VCC.",
    "resolution": null
   },
   {
    "model_id": "model-stack",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.64898/2026.01.09.698608",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Own perturbation sets; its VCC claim is rejected on the model page.",
    "resolution": null
   },
   {
    "model_id": "model-state",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arcinstitute.org/news/virtual-cell-model-state",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-reported with cell-eval on Tahoe-100M; its VCC blind score is unpublished.",
    "resolution": null
   },
   {
    "model_id": "model-tahoe-x1",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1101/2025.10.23.683759",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "An own benchmark, explicitly not VCC.",
    "resolution": null
   },
   {
    "model_id": "model-x-cell",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://xaira-therapeutics.github.io/X-Cell",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Own held-out iPSC set; VCC participation unconfirmed.",
    "resolution": null
   },
   {
    "model_id": "model-lingshu-cell",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/pdf/2603.25240",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "A post-hoc evaluation on the VCC H1 leaderboard rather than participation.",
    "resolution": null
   },
   {
    "model_id": "model-scgpt-geneformer",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://doi.org/10.1038/s41592-025-02772-6",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Independent third-party evaluations on Norman and Adamson data.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 9,
   "by_relation": {
    "official-participation": 1,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 6,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 2,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "The 2025 data are public and can be run statically, but a real blind test means entering the next edition. Always report mean, no-change and linear baselines, use several metrics such as cosine PDS, Pearson delta and AUPRC plus PerturBench ranks and RMSD, and never rely on a single PDS. Generalization to one's own cell lines needs new Perturb-seq, whose cost is undisclosed.",
  "pairs_with": [
   "bench-21-dream5",
   "bench-09-scib",
   "bench-08-enformer"
  ],
  "caveats": [
   "The 2025 metrics can be gamed by scale and transformations: a third-party analysis shows L1/L2 PDS is sensitive to prediction scale and almost all submissions scored worse than baseline on MAE.",
   "Several independent evaluations find deep and foundation models do not reliably beat mean, no-change or linear baselines on perturbation prediction.",
   "The board gives ranks without numbers, and with 300 perturbations the organizers never state whether rank gaps exceed noise.",
   "The 2025 edition used a single cell line so cross-cell generalization was untested, and the full data are now public and no longer blind.",
   "External pretraining and public Perturb-seq may overlap the test targets, and there is no prospective wet-lab validation nor evidence linking scores to target-discovery hit rates."
  ],
  "sources": [
   {
    "url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
    "title": "Organizer wrap-up page for VCC 2025",
    "kind": "official-site",
    "date": {
     "date": "2025-12-06",
     "precision": "day",
     "year_hint": null,
     "basis": "Publication date of the wrap-up page",
     "event_kind": "results-published",
     "timestamp_utc": null,
     "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://arcinstitute.org/news/virtual-cell-challenge-2026",
    "title": "VCC 2026 announcement",
    "kind": "official-site",
    "date": {
     "date": "2026-08-20",
     "precision": "day",
     "year_hint": null,
     "basis": "Publication date of the announcement",
     "event_kind": "challenge-open",
     "timestamp_utc": null,
     "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2026",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://arxiv.org/abs/2408.10609",
    "title": "PerturBench paper",
    "kind": "preprint",
    "date": {
     "date": "2024-08-20",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2408.10609",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12328236/",
    "title": "Third-party comparison of deep models against linear baselines",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/html/2511.16954v1",
    "title": "Third-party analysis of the PDS metric",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://proceedings.mlr.press/v267/wenteler25a.html",
    "title": "PertEval-scFM",
    "kind": "paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2025 (VCC launch treated as the family's first release)",
    "now": "2024-08-20, PerturBench arXiv v1, with the VCC 2025 launch (2025-06-26) kept as a member",
    "reason": "A family entry's first release is its earliest member and the boundary must be stated, so PerturBench's date is not read as the VCC launch.",
    "source_url": "https://arxiv.org/abs/2408.10609",
    "date_found": "2026-11-16"
   },
   {
    "field": "scale",
    "was": "2026 edition: six cell lines, zero-shot, cell counts implied",
    "now": "2025 figures verified; the 2026 cell count is unpublished and 'six cell lines' was not re-confirmed",
    "reason": "The organizer announcement gives no 2026 cell count and the six-cell-line figure was not re-confirmed, so no definite value may be shown.",
    "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2025-wrap-up",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update",
    "was": "VCC 2026 in progress, without a dated event",
    "now": "2026-10-22 final test set released; submissions closed 2026-11-05; winners unannounced as of 2026-11-16 (flagged unverified)",
    "reason": "The event could not be matched to a dated official passage, so it is flagged unverified and must not be described as results published.",
    "source_url": "https://arcinstitute.org/news/virtual-cell-challenge-2026",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "STATE, Stack, X-Cell and others listed as VCC results",
    "now": "self-reported-modified-protocol: own datasets and metrics, not the VCC board",
    "reason": "An own protocol is not a VCC board result, and the Tahoe-100M perturbation count is standardized at 1,100 rather than the press release's roughly 1,200.",
    "source_url": "https://arcinstitute.org/news/virtual-cell-model-state",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "next_expected",
   "scale.items[4]",
   "scale.items[5]",
   "ranking.entries[4]",
   "family.members[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-24-vcc/",
   "en": "/bio/benchmark/benchmarks/vcc/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-25-rnapuzzles",
  "kind": "benchmark",
  "nn": "25",
  "short_slug": "rnapuzzles",
  "names": {
   "display": "RNA-Puzzles",
   "full": "RNA-Puzzles: community blind assessment of RNA 3D structure prediction",
   "full_null_reason": null,
   "aliases": [
    "RNA-Puzzles",
    "RNA-Puzzles Round V",
    "Stanford RNA 3D Folding"
   ],
   "ambiguous_aliases": [
    "Ribonanza",
    "bpRNA",
    "ArchiveII"
   ]
  },
  "family": null,
  "family_null_reason": "RNA-Puzzles rounds are editions of one blind test; the Kaggle and CASP RNA tracks are related evaluations recorded in scale and notes, not family members.",
  "domain": "structure",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "A+B",
  "type_note": "Periodic blind test plus static secondary-structure sets",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": [
    "L2",
    "L4"
   ],
   "qualifier": "3D blind test L4; static 2D sets L2"
  },
  "publisher": {
   "orgs": [
    "RNA-Puzzles consortium (academic groups)",
    "Stanford University (Kaggle RNA 3D Folding)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Community blind test of RNA 3D structure; accuracy tracks template availability more than model type.",
  "first_release": {
   "date": "2012-02-23",
   "precision": "day",
   "year_hint": null,
   "basis": "RNA-Puzzles founding paper (RNA 2012, PubMed epub 2012-02-23); puzzles began around 2010-11",
   "event_kind": "journal-online",
   "timestamp_utc": null,
   "source_url": "https://pubmed.ncbi.nlm.nih.gov/22361291/",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": null,
   "journal_online": {
    "date": "2012-02-23",
    "precision": "day",
    "year_hint": null,
    "basis": "Founding paper online (RNA 2012)",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://pubmed.ncbi.nlm.nih.gov/22361291/",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2010-11",
    "precision": "month",
    "year_hint": null,
    "basis": "First puzzles began running (per founding paper and site notes)",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://pubmed.ncbi.nlm.nih.gov/22361291/",
    "verification": "unverified",
    "note": "Approximate month only, not separately verified"
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-04-02",
   "precision": "day",
   "year_hint": null,
   "basis": "Official-site item \"PZ74-PZ79 update: DNA residue names\", date derived from its URL only",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://www.rnapuzzles.org/",
   "verification": "unverified",
   "note": "Date not reproduced on re-check; the last round, Round V, went online 2024-12-02",
   "description": "Official-site PZ74-PZ79 DNA residue-name update, date URL-derived and unverified; last formal round is Round V (paper online 2024-12-02)."
  },
  "next_expected": null,
  "next_expected_null_reason": "No Round VI announced as of 2026-11-16.",
  "status": "low-frequency",
  "status_note": "Round V published 2025; CASP RNA and Kaggle rounds continue",
  "scale": {
   "summary": "Round V: 23 puzzles, 18 groups; CASP16: 42 targets; Kaggle: 20 private targets.",
   "items": [
    {
     "label": "Puzzles",
     "value": "23",
     "unit": "puzzles",
     "version": "Round V"
    },
    {
     "label": "Participating groups",
     "value": "18",
     "unit": "groups",
     "version": "Round V"
    },
    {
     "label": "CASP15 RNA targets / groups",
     "value": "12 / 42",
     "unit": null,
     "version": "CASP15"
    },
    {
     "label": "CASP16 nucleic-acid targets / groups",
     "value": "42 / 65",
     "unit": null,
     "version": "CASP16"
    },
    {
     "label": "Kaggle private targets (10 from RNA-Puzzles)",
     "value": "20",
     "unit": "targets",
     "version": "Kaggle RNA 3D Folding Part 1"
    },
    {
     "label": "Kaggle teams",
     "value": "1,700+",
     "unit": "teams",
     "version": "Kaggle RNA 3D Folding Part 1"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "Round-based, 12-42 targets per round from pre-publication experimental structures; new truth only by entering the next round.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "RNA 3D structure prediction from sequence, with related static sets for secondary structure and chemical reactivity.",
   "truth": "X-ray, cryo-EM or NMR structures unpublished before assessment.",
   "metric_primary": "RMSD, INF and DI (RNA-Puzzles)",
   "metric_secondary": [
    "TM-score (US-align, C1')",
    "GDT_TS, lDDT (CASP and Kaggle)"
   ],
   "version_traps": "RNA-Puzzles, CASP RNA and Kaggle differ in targets, metrics and rules; human groups may use literature and constraints, unlike servers.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Mean TM-align (US-align, C1')",
   "eval_set": "Kaggle Stanford RNA 3D Folding Part 1 private board: 20 targets incl. 10 RNA-Puzzles",
   "snapshot_date": {
    "date": "2025-12-30",
    "precision": "day",
    "year_hint": null,
    "basis": "Date of the competition paper (preprint) that reproduces the Kaggle private board",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://www.kaggle.com/competitions/stanford-rna-3d-folding",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://www.kaggle.com/competitions/stanford-rna-3d-folding",
   "entries": [
    {
     "rank": 1,
     "model_label": "Kaggle private-board rank 1 team",
     "model_id": null,
     "site_model": false,
     "score": "0.578 (all, n=20); 0.491 (RNA-Puzzles, n=10)",
     "score_unit": null,
     "metric": "Mean TM-align (US-align, C1')",
     "eval_set": "Kaggle Stanford RNA 3D Folding Part 1 private board: 20 targets incl. 10 RNA-Puzzles",
     "source_type": "board",
     "date": {
      "date": "2025-12-30",
      "precision": "day",
      "year_hint": null,
      "basis": "Competition paper preprint date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
     "note": "Not a site model; overall first; a template-discovery pipeline without deep learning.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "Kaggle private-board rank 2 team",
     "model_id": null,
     "site_model": false,
     "score": "0.562 (all, n=20); 0.418 (RNA-Puzzles, n=10)",
     "score_unit": null,
     "metric": "Mean TM-align (US-align, C1')",
     "eval_set": "Kaggle Stanford RNA 3D Folding Part 1 private board: 20 targets incl. 10 RNA-Puzzles",
     "source_type": "board",
     "date": {
      "date": "2025-12-30",
      "precision": "day",
      "year_hint": null,
      "basis": "Competition paper preprint date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
     "note": "Not a site model.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "AlphaFold 3",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "0.464 (all, n=20); 0.396 (RNA-Puzzles, n=10)",
     "score_unit": null,
     "metric": "Mean TM-align (US-align, C1')",
     "eval_set": "Kaggle Stanford RNA 3D Folding Part 1 private board: 20 targets incl. 10 RNA-Puzzles",
     "source_type": "third",
     "date": {
      "date": "2025-12-30",
      "precision": "day",
      "year_hint": null,
      "basis": "Competition paper preprint date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
     "note": "Reference comparison row in Table 2, not a ranked competition entry.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "trRosettaRNA server",
     "model_id": "model-rna-structure-models",
     "site_model": true,
     "score": "0.438 (all, n=20); 0.340 (RNA-Puzzles, n=10)",
     "score_unit": null,
     "metric": "Mean TM-align (US-align, C1')",
     "eval_set": "Kaggle Stanford RNA 3D Folding Part 1 private board: 20 targets incl. 10 RNA-Puzzles",
     "source_type": "third",
     "date": {
      "date": "2025-12-30",
      "precision": "day",
      "year_hint": null,
      "basis": "Competition paper preprint date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
     "note": "Reference comparison row in Table 2, not a ranked competition entry.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "RNAPro",
     "model_id": "model-rnapro",
     "site_model": true,
     "score": "0.648 (all, n=20); 0.541 (RNA-Puzzles, n=10)",
     "score_unit": null,
     "metric": "Mean TM-align (US-align, C1')",
     "eval_set": "Kaggle Stanford RNA 3D Folding Part 1 private board: 20 targets incl. 10 RNA-Puzzles",
     "source_type": "self",
     "date": {
      "date": "2025-12-30",
      "precision": "day",
      "year_hint": null,
      "basis": "Competition paper preprint date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
     "note": "Post-competition retrospective model, not a board entry; needs prospective validation.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "NeuralPLexer3",
     "model_id": "model-neuralplexer3",
     "site_model": true,
     "score": "mean RNA lDDT 46.5 (AF3 47.3, AIchemy_RNA2 54.8)",
     "score_unit": null,
     "metric": "Mean RNA lDDT",
     "eval_set": "CASP15 RNA, 8-target subset used in the AlphaFold 3 paper",
     "source_type": "self",
     "date": {
      "date": "2024-12",
      "precision": "month",
      "year_hint": null,
      "basis": "Model paper arXiv month",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2412.10743",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2412.10743",
     "note": "CASP15 RNA 8-target subset with a different protocol and metric; not comparable with the Kaggle TM-align column.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": 2,
   "headline_null_reason": null,
   "caveat": "Numbers from Table 2 of the competition preprint, reproducing the Kaggle Part 1 private board; Part 2 standings unknown; RNA-Puzzles itself has no standing board.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-neuralplexer3",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2412.10743v2",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "CASP15 RNA 8-target subset; AF3 baseline copied from its paper, not rerun",
    "resolution": null
   },
   {
    "model_id": "model-rnapro",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://www.biorxiv.org/content/10.64898/2025.12.30.696949.full.pdf",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "10-target RNA-Puzzles subset inside Kaggle private leaderboard",
    "resolution": null
   },
   {
    "model_id": "model-rna-structure-models",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://www.nature.com/articles/s41592-024-02487-0",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "RhoFold+ retrospective on 24 RNA-Puzzles; DRfold2/trRosettaRNA2 unconfirmed; family cell",
    "resolution": null
   },
   {
    "model_id": "model-alphafold3",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/abs/2412.10743",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "CASP15 RNA value from its own paper, quoted by NeuralPLexer3; not RNA-Puzzles proper",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 3,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 2,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 1,
    "cited-as-comparison-only": 1,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Run released targets statically with the RNA_assessment scripts and US-align; true blind tests require the next CASP or Kaggle round. Stratify by template availability, add a nearest-PDB-template baseline, state target set and models per target; do not claim 2D generalization from random splits.",
  "pairs_with": [
   "bench-01-casp",
   "bench-39-rnagym",
   "bench-31-foldbench"
  ],
  "caveats": [
   "Accuracy depends on available templates rather than model type; the 2025 Kaggle winner was a non-deep-learning template pipeline; secondary-structure sets leak same-family sequences.",
   "Very few targets (at most 23 per round, 20 on the Kaggle private board), with no confidence intervals.",
   "The CASP16 assessment reports no significant progress once templates are controlled, almost no success on template-free targets; RNA-protein complexes still fail.",
   "TM-score uses C1' atoms only and ignores non-canonical interactions.",
   "The latest-update date is URL-derived and unverified; Kaggle Part 2 results were not retrieved."
  ],
  "sources": [
   {
    "url": "https://pubmed.ncbi.nlm.nih.gov/22361291/",
    "title": "RNA-Puzzles founding paper (RNA 2012)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://pubmed.ncbi.nlm.nih.gov/39623050/",
    "title": "RNA-Puzzles Round V (Nat Methods 2025)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://www.rnapuzzles.org/",
    "title": "RNA-Puzzles official site",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12776560/",
    "title": "Kaggle Stanford RNA 3D Folding competition paper (preprint)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://www.kaggle.com/competitions/stanford-rna-3d-folding",
    "title": "Kaggle RNA 3D Folding Part 1",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://www.kaggle.com/competitions/stanford-rna-3d-folding-2",
    "title": "Kaggle RNA 3D Folding Part 2",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC10168427/",
    "title": "CASP15 RNA assessment",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12248019/",
    "title": "CASP16 nucleic-acid assessment",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://github.com/RNA-Puzzles/RNA_assessment",
    "title": "RNA_assessment scripts",
    "kind": "repository",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update_desc",
    "was": "Kaggle Stanford RNA 3D Folding Part 2 final submission deadline; RNA-Puzzles proper last round V paper (online 2024-12-02). No Round VI announced.",
    "now": "Site item 'PZ74-PZ79 update: DNA residue names' dated 2026-04-02 from its URL only; the official site did not return this date or a dated news entry on re-check, so the date must display as unverified. Last RNA-Puzzles round proper: Round V paper online 2024-12-02. The 2026-03-25 Kaggle deadline belongs to the Kaggle component, not RNA-Puzzles.",
    "reason": "Review: date not reproducible from the site; URL-derived dates are not evidence.",
    "source_url": "https://www.rnapuzzles.org/",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update",
    "was": "2026-03-25",
    "now": "2026-04-02",
    "reason": "The 2026-03-25 date is a Kaggle component deadline, not an RNA-Puzzles round; replaced by the site item date, which is URL-derived and shown as unverified.",
    "source_url": "https://www.rnapuzzles.org/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "release_milestones.challenge_open",
   "models_reported[1]",
   "models_reported[2]",
   "models_reported[3]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-25-rnapuzzles/",
   "en": "/bio/benchmark/benchmarks/rnapuzzles/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-26-flip",
  "kind": "benchmark",
  "nn": "26",
  "short_slug": "flip",
  "names": {
   "display": "FLIP / FLIP2",
   "full": "Fitness Landscape Inference for Proteins (FLIP) and its second generation",
   "full_null_reason": null,
   "aliases": [
    "FLIP",
    "FLIP2",
    "FLIPv2"
   ],
   "ambiguous_aliases": [
    "flip"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "FLIP",
     "id_or_null": null,
     "first_release": {
      "date": "2021-11-11",
      "precision": "day",
      "year_hint": null,
      "basis": "bioRxiv v1 (NeurIPS 2021 Datasets and Benchmarks)",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1101/2021.11.09.467890",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "FLIP2",
     "id_or_null": null,
     "first_release": {
      "date": "2026-01-30",
      "precision": "day",
      "year_hint": null,
      "basis": "Publication date of the archived dataset; the preprint and repository release came later",
      "event_kind": "data-release",
      "timestamp_utc": null,
      "source_url": "https://zenodo.org/records/18433203",
      "verification": "partial",
      "note": null
     }
    }
   ],
   "boundary": "The two generations share a repository and splitting philosophy but differ in datasets and splits, so numbers do not compare across generations."
  },
  "family_null_reason": null,
  "domain": "protein",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "B",
  "type_note": "Static dataset suite for supervised extrapolation, with no leaderboard",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Deliberate extrapolation splits, but within one protein and one assay"
  },
  "publisher": {
   "orgs": [
    "FLIP benchmark maintainers, an academic consortium"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Supervised protein fitness benchmark using deliberately shifted splits to test low-resource learning and extrapolation.",
  "first_release": {
   "date": "2021-11-11",
   "precision": "day",
   "year_hint": null,
   "basis": "The first generation's bioRxiv v1, published the same year in the NeurIPS Datasets and Benchmarks track.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1101/2021.11.09.467890",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2021-11-11",
    "precision": "day",
    "year_hint": null,
    "basis": "First generation, bioRxiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/2021.11.09.467890",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2026-04-23",
    "precision": "day",
    "year_hint": null,
    "basis": "Second-generation release pull request merged into the main branch",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://github.com/J-SNACKKB/FLIP",
    "verification": "partial",
    "note": "The merge date was not re-retrieved this round"
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-04-23",
   "precision": "day",
   "year_hint": null,
   "basis": "Release merge record in the official repository",
   "event_kind": "new-version",
   "timestamp_utc": null,
   "source_url": "https://github.com/J-SNACKKB/FLIP",
   "verification": "partial",
   "note": "The merge date was not re-retrieved this round, and the archive date is a separate event",
   "description": "The second-generation release was merged into the main branch, bringing data and code; later commits only refresh documentation pages."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have not announced a third generation or new splits.",
  "status": "low-frequency",
  "status_note": "The first generation is maintained infrequently; the second shipped in 2026; no official board",
  "scale": {
   "summary": "First generation: 3 landscapes and 15 splits; second generation: 7 new datasets and 16 predefined splits.",
   "items": [
    {
     "label": "Landscapes",
     "value": "3",
     "unit": null,
     "version": "FLIP"
    },
    {
     "label": "Predefined splits",
     "value": "15",
     "unit": null,
     "version": "FLIP"
    },
    {
     "label": "Variants across the three landscapes",
     "value": "8733 / 82583 / about 17,000",
     "unit": null,
     "version": "FLIP"
    },
    {
     "label": "New datasets",
     "value": "7",
     "unit": null,
     "version": "FLIP2"
    },
    {
     "label": "Predefined splits",
     "value": "16",
     "unit": null,
     "version": "FLIP2"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No new experiments; reuses published deep mutational scans and thermostability data.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Supervised prediction of protein fitness under deliberately shifted splits, testing low-resource learning and extrapolation.",
   "truth": "Published deep mutational scan readouts and thermostability measurements, organised per protein and assay.",
   "metric_primary": "Spearman rho per split",
   "metric_secondary": [
    "Variance across splits and sensitivity to the training recipe"
   ],
   "version_traps": "State the generation, split name and training recipe: some older splits are flagged by the authors as unsuitable for comparison, and averaging splits hides differences.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "Spearman rho, per split",
   "eval_set": "FLIP splits (GB1, AAV, Meltome); FLIP2 adds 7 datasets",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://github.com/J-SNACKKB/FLIP",
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "xTrimoPGLM (LoRA fine-tuned, not zero-shot)",
     "model_id": "model-xtrimopglm",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Spearman rho, per split",
     "eval_set": "FLIP splits (GB1, AAV, Meltome); FLIP2 adds 7 datasets",
     "source_type": "self",
     "date": {
      "date": "2024-01",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2401.06199",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2401.06199",
     "note": "Fine-tuned on the random control split, near the ceiling, not an extrapolation split",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: splits and training recipes differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "SaProt",
     "model_id": "model-saprot",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Spearman rho, per split",
     "eval_set": "FLIP splits (GB1, AAV, Meltome); FLIP2 adds 7 datasets",
     "source_type": "self",
     "date": {
      "date": "2024",
      "precision": "year",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://proceedings.iclr.cc/paper_files/paper/2024/file/1c42513b8895ab11fbbb5b7e8e6b6b02-Paper-Conference.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://proceedings.iclr.cc/paper_files/paper/2024/file/1c42513b8895ab11fbbb5b7e8e6b6b02-Paper-Conference.pdf",
     "note": "Self-report on a thermostability split; another dataset is figure-only with no quotable number",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: splits and training recipes differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "SaProt",
     "model_id": "model-saprot",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Spearman rho, per split",
     "eval_set": "FLIP splits (GB1, AAV, Meltome); FLIP2 adds 7 datasets",
     "source_type": "third",
     "date": {
      "date": "2025-01",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2501.18223v1",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2501.18223v1",
     "note": "Third-party re-evaluation on extrapolation splits, a different protocol from the self-reports",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: splits and training recipes differ, so the ranking row shows no score."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "There is no official board and the entries differ in split and recipe, so no headline number is given.",
   "caveat": "There is no official online board and all numbers come from per-task tables in papers; a third-party re-evaluation shows recipe noise exceeding model differences, and random splits do not mix with extrapolation splits.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-saprot",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://proceedings.iclr.cc/paper_files/paper/2024/file/1c42513b8895ab11fbbb5b7e8e6b6b02-Paper-Conference.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-reported on two datasets; a third-party re-evaluation also exists",
    "resolution": null
   },
   {
    "model_id": "model-xtrimopglm",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2401.06199",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-reported after fine-tuning on the random control split",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 2,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 2,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Download one generation's data and predefined splits, train per split and report Spearman rho; it runs statically with no wet lab of your own. State the generation, split name, training recipe and seed variance, report per split rather than averaged, and compare against simple baselines. Cross-protein generalization needs a fresh assay. Pair with ProteinGym and the few-shot setting of FS-Mol.",
  "pairs_with": [
   "bench-02-proteingym",
   "bench-13-cagi",
   "bench-44-fsmol",
   "bench-39-rnagym"
  ],
  "caveats": [
   "Scores swing with training recipe and split, and averaging splits hides large differences.",
   "Extrapolation stays within one protein and one assay, not across families.",
   "The second generation reports that simple models often match fine-tuned protein language models.",
   "There is no official online board, reuse protocols vary, and larger suites have overtaken its influence.",
   "Some first-generation splits are flagged by the authors as unsuitable for comparison or outdated."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1101/2021.11.09.467890",
    "title": "First-generation preprint",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/J-SNACKKB/FLIP",
    "title": "Official repository and data documentation",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://zenodo.org/records/18433203",
    "title": "Second-generation data archive",
    "kind": "archive",
    "date": null
   },
   {
    "url": "https://doi.org/10.64898/2026.02.23.707496",
    "title": "Second-generation preprint",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/html/2501.18223v1",
    "title": "Third-party re-evaluation on these splits",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "FLIP2 release date",
    "was": "2026-05",
    "now": "archive publication 2026-01-30; repository release commit 2026-03-03; release pull request merged 2026-04-23",
    "reason": "Date audit: the recorded release month matches no event; the three sourced events must be listed separately.",
    "source_url": "https://zenodo.org/records/18433203",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries",
    "was": "per-model scores shown in the ranking row",
    "now": "scores withheld; splits and training recipes differ",
    "reason": "Display rule: entries that are not comparable show no score; per-protocol values stay on the entity page.",
    "source_url": "https://arxiv.org/html/2501.18223v1",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "release_milestones.code_release",
   "family.members[1]",
   "scale.items[2]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-26-flip/",
   "en": "/bio/benchmark/benchmarks/flip/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-27-polaris",
  "kind": "benchmark",
  "nn": "27",
  "short_slug": "polaris",
  "names": {
   "display": "Polaris / ASAP / OpenADMET",
   "full": "Polaris benchmarking hub and its hosted ASAP antiviral blind challenge",
   "full_null_reason": null,
   "aliases": [
    "Polaris Hub",
    "polaris-lib",
    "ASAP-Polaris-OpenADMET antiviral challenge",
    "antiviral-drug-discovery-2025"
   ],
   "ambiguous_aliases": [
    "Polaris",
    "ASAP",
    "OpenADMET"
   ]
  },
  "family": null,
  "family_null_reason": "Single hub entry; only Polaris-hosted challenges count, and OpenADMET's own challenges are not members.",
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "A+C",
  "type_note": "Blind challenge plus hosting hub and method guidelines",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": null,
   "qualifier": "Hidden blind test on real project data; no L5"
  },
  "publisher": {
   "orgs": [
    "Polaris (industry-led steering committee)",
    "Valence Labs",
    "ASAP Discovery Consortium",
    "OpenADMET"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Industry-led benchmark hub, guidelines and blind challenges; its flagship is an antiviral challenge on unpublished project data.",
  "first_release": {
   "date": "2023-11-23",
   "precision": "day",
   "year_hint": null,
   "basis": "First polaris-lib PyPI release 0.2.0 (public hub launch date not verified)",
   "event_kind": "code-release",
   "timestamp_utc": null,
   "source_url": "https://pypi.org/project/polaris-lib/",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": null,
    "precision": null,
    "year_hint": null,
    "basis": "Public hub launch date not found",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://github.com/polaris-hub/polaris",
    "verification": "unverified",
    "note": "GitHub repository created 2023-06-05; repository dates are not a release"
   },
   "paper_v1": null,
   "journal_online": null,
   "code_release": {
    "date": "2023-11-23",
    "precision": "day",
    "year_hint": null,
    "basis": "polaris-lib 0.2.0 released on PyPI",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://pypi.org/project/polaris-lib/",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": {
    "date": "2025-01-13",
    "precision": "day",
    "year_hint": null,
    "basis": "Antiviral challenge opened (from the existing record, not independently re-verified)",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://polaris-hub.vercel.app/competitions/asap-discovery/antiviral-drug-discovery-2025",
    "verification": "partial",
    "note": null
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-06-06",
   "precision": "day",
   "year_hint": null,
   "basis": "Latest polaris-lib release on PyPI",
   "event_kind": "new-version",
   "timestamp_utc": null,
   "source_url": "https://pypi.org/project/polaris-lib/",
   "verification": "partial",
   "note": "OpenADMET's CYP (winners 2026-11-05) and PXR challenges ran off the hub; note only",
   "description": "polaris-lib 0.13.0 released (last release); last main-branch commit 2025-07-30; Polaris blog has nothing after 2025-01-29."
  },
  "next_expected": null,
  "next_expected_null_reason": "Polaris has announced no date for its next hosted challenge.",
  "status": "low-frequency",
  "status_note": "Hub online; few Polaris-run events since 2025",
  "scale": {
   "summary": "Antiviral challenge: 3 tracks, 66 participants, 381 submissions; later OpenADMET challenges are not counted here.",
   "items": [
    {
     "label": "Tracks (pose, potency, ADMET)",
     "value": "3",
     "unit": null,
     "version": "antiviral challenge 2025"
    },
    {
     "label": "Independent participants",
     "value": "66",
     "unit": null,
     "version": "antiviral challenge 2025 (summary paper)"
    },
    {
     "label": "Submissions",
     "value": "381",
     "unit": null,
     "version": "antiviral challenge 2025 (summary paper)"
    },
    {
     "label": "Final-board submissions",
     "value": "93",
     "unit": null,
     "version": "antiviral challenge 2025 (summary paper)"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "Hosted-challenge truth is unpublished project assays and new crystal structures, unblinded afterwards; new truth only by entering the next challenge.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Prospective prediction on unpublished industry project data: ADMET endpoints, potency and ligand poses.",
   "truth": "Consortium-measured biochemical and ADMET assays and crystal structures, hidden until unblinding.",
   "metric_primary": "Pose success (RMSD<2Å); MAE for potency and ADMET",
   "metric_secondary": [
    "Mean RMSD",
    "Kendall τ ranking",
    "Bootstrap plus Tukey HSD ranking"
   ],
   "version_traps": "Competition page and summary paper use different pose metrics, so cite the source; after unblinding the set is static and not comparable to blind results.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "Antiviral challenge: pose success rate, MAE for potency and ADMET",
   "eval_set": "ASAP-Polaris antiviral challenge 2025 (Polaris-hosted); OpenADMET CYP/ExpansionRx/PXR boards out of scope",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date; this round's check date when no readable ranking exists",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://polaris-hub.vercel.app/competitions/asap-discovery/antiviral-drug-discovery-2025",
    "verification": "partial",
    "note": "No readable ranking; snapshot day is this round's check date"
   },
   "board_url": "https://polaris-hub.vercel.app/competitions/asap-discovery/antiviral-drug-discovery-2025",
   "entries": [],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "Polaris-hosted boards are JS-rendered and unreadable, and the summary paper anonymises entries; the OpenADMET CYP board is outside this entry.",
   "caveat": "Only Polaris-hosted challenges count; their boards are unreadable and the summary paper anonymises entries, so no ranks; OpenADMET boards such as CYP are not this entry's ranking.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-chemprop-v2",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": "https://chemrxiv.org/engage/chemrxiv/article-details/6841f19f",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Anonymous baseline in the organiser report; comparison only",
    "resolution": null
   },
   {
    "model_id": "model-beacon",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://openadmet.ghost.io/lessons-learned-from-the-openadmet-expansionrx-blind-challenge/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "Press release only; organiser anonymises entries; model identity unverified",
    "resolution": "Two reasons. (1) Identity: organiser boards are pseudonymous, so the public model version cannot be matched to entries; first-place claims rest on a company press release, only one corroborated by a third-party write-up. (2) Scope: this entry covers Polaris-hosted challenges only, and ExpansionRx and PXR were run by OpenADMET off the hub. Not established."
   },
   {
    "model_id": "model-boltz-2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "low",
    "verification": "disputed",
    "note": "Site page says it matched the antiviral top 5 without fine-tuning; no numbers; participation not openly verified",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 1,
    "disputed-or-unconfirmed": 2
   }
  },
  "how_to_use": "Unblinded sets run statically with no wet lab: install polaris-lib, download the unblinded data, fit a fingerprint plus GBM baseline and apply Tukey tests; check poses with PoseBusters. A true blind test needs the next challenge. Use as external validation for TDC and compare with CACHE.",
  "pairs_with": [
   "bench-04-tdc",
   "bench-05-posebusters",
   "bench-20-cache",
   "bench-03-moleculenet"
  ],
  "caveats": [
   "Each challenge covers a single project and chemical series; once unblinded it becomes a static set; no evidence linking scores to wet-lab decisions.",
   "The summary paper anonymises all entries, so public material cannot map ranks to companies or models; rank claims mostly come from press releases.",
   "The competition page sets no formal rules, so external and private data were allowed; it notes the bootstrap may underestimate variance.",
   "OpenADMET's CYP, ExpansionRx and PXR challenges ran outside the Polaris hub and are notes only, not this entry's updates or rankings."
  ],
  "sources": [
   {
    "url": "https://polaris-hub.vercel.app/competitions/asap-discovery/antiviral-drug-discovery-2025",
    "title": "Antiviral challenge competition page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://doi.org/10.1021/acs.jcim.5c02106",
    "title": "Antiviral challenge summary paper (JCIM 2026)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://www.repository.cam.ac.uk/handle/1810/401850",
    "title": "Summary paper open version (University of Cambridge repository, CC BY)",
    "kind": "archive",
    "date": null
   },
   {
    "url": "https://pypi.org/project/polaris-lib/",
    "title": "polaris-lib (PyPI)",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://github.com/polaris-hub/polaris",
    "title": "Polaris client repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://polaris-hub.vercel.app/guidelines/small-molecules",
    "title": "Polaris small-molecule guidelines",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://doi.org/10.1038/s42256-024-00911-w",
    "title": "Correspondence on industry-led blind evaluation (Nature Machine Intelligence 2024)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://openadmet.org/community/events/blind-challenges",
    "title": "OpenADMET blind challenges page (off-hub events)",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://openadmet.ghost.io/take-me-heme-binding-modes-wrapping-up-the-cyp-inhibition-challenge/",
    "title": "OpenADMET CYP challenge results post (note only)",
    "kind": "company-blog",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update",
    "was": "2026-11-05",
    "now": "2025-06-06",
    "reason": "Owner decision: this entry covers Polaris-hosted challenges only. The 2026-11-05 OpenADMET CYP winners post never mentions Polaris, so it is another project's event kept as a note; the latest Polaris-own sourced event is polaris-lib 0.13.0.",
    "source_url": "https://pypi.org/pypi/polaris-lib/json",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_desc",
    "was": "OpenADMET CYP Inhibition Blind Challenge winners announced (2 months, 6,000+ submissions); polaris-lib last release 0.13.0 on 2025-06-06.",
    "now": "polaris-lib 0.13.0 released 2025-06-06 (last release object); last main-branch commit 2025-07-30 (multi-test-set benchmark support); Polaris blog shows nothing after 2025-01-29. Note, not a Polaris update: OpenADMET ran the CYP Inhibition Blind Challenge (winners posted 2026-11-05) and the PXR challenge on Hugging Face, outside the Polaris hub.",
    "reason": "Description rewritten to keep the OpenADMET events visible as a note instead of as a Polaris update.",
    "source_url": "https://github.com/polaris-hub/polaris/releases.atom",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_kind",
    "was": "leaderboard-entry",
    "now": "new-version",
    "reason": "The event is a library release.",
    "source_url": "https://pypi.org/pypi/polaris-lib/json",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking",
    "was": "board: OpenADMET CYP Inhibition Blind Challenge final leaderboards (top-1 per track, Boltz-2-named participant entry)",
    "now": "no-ranking; entries []",
    "reason": "CHANGESET §0.6: the OpenADMET CYP board is not a Polaris ranking; Polaris-hosted boards are unreadable and the summary paper anonymises entries, so no ranking can be shown.",
    "source_url": "https://openadmet.org/community/events/blind-challenges",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported (Beacon)",
    "was": "Beacon first in three blind challenges (self-reported)",
    "now": "disputed-or-unconfirmed",
    "reason": "CHANGESET §4.1: identity cannot be matched and the challenges are outside the Polaris-hosted scope; both reasons make it unconfirmed.",
    "source_url": "https://openadmet.ghost.io/lessons-learned-from-the-openadmet-expansionrx-blind-challenge/",
    "date_found": "2026-11-16"
   },
   {
    "field": "status",
    "was": "active (new challenge results published in 2026-11)",
    "now": "low-frequency",
    "reason": "The 2026-11 results belong to OpenADMET's off-hub events; Polaris's own last release was 2025-06 and its last blog post 2025-01.",
    "source_url": "https://pypi.org/project/polaris-lib/",
    "date_found": "2026-11-16"
   },
   {
    "field": "scale",
    "was": "ExpansionRx 370+ teams; PXR 513 test compounds; CYP 750 compounds",
    "now": "antiviral challenge only: 3 tracks, 66 participants, 381 submissions",
    "reason": "Per §0.6, OpenADMET challenge sizes are not counted here; participation counts follow the summary paper.",
    "source_url": "https://www.repository.cam.ac.uk/handle/1810/401850",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "release_milestones.first_public_evidence",
   "release_milestones.challenge_open",
   "models_reported[1]",
   "models_reported[2]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-27-polaris/",
   "en": "/bio/benchmark/benchmarks/polaris/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-28-adaptyv",
  "kind": "benchmark",
  "nn": "28",
  "short_slug": "adaptyv",
  "names": {
   "display": "Adaptyv competitions / BenchBB",
   "full": "Adaptyv protein design competition series and the BenchBB target set",
   "full_null_reason": null,
   "aliases": [
    "BenchBB",
    "Adaptyv protein design competition"
   ],
   "ambiguous_aliases": []
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "Competition round 1",
     "id_or_null": null,
     "first_release": {
      "date": "2024-07",
      "precision": "month",
      "year_hint": null,
      "basis": "Organiser announcement post (month-level evidence only)",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://adaptyvbio.substack.com/archive",
      "verification": "partial",
      "note": null
     }
    },
    {
     "name": "Competition round 2",
     "id_or_null": null,
     "first_release": {
      "date": "2024-11",
      "precision": "month",
      "year_hint": null,
      "basis": "Existing site record, day not re-verified",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://adaptyvbio.substack.com/p/protein-design-competition-results",
      "verification": "partial",
      "note": null
     }
    },
    {
     "name": "BenchBB",
     "id_or_null": null,
     "first_release": {
      "date": "2025-04",
      "precision": "month",
      "year_hint": null,
      "basis": "Community paper, bioRxiv 2025.04.17.648362",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://www.biorxiv.org/content/10.1101/2025.04.17.648362v1",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "2025 competition round",
     "id_or_null": null,
     "first_release": {
      "date": "2025-10-21",
      "precision": "day",
      "year_hint": null,
      "basis": "Organiser announcement 2025-10-21, results 2026-01-05",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://adaptyvbio.substack.com/archive",
      "verification": "verified",
      "note": null
     }
    },
    {
     "name": "2026 joint competition",
     "id_or_null": null,
     "first_release": {
      "date": "2026-10-26",
      "precision": "day",
      "year_hint": null,
      "basis": "Organiser page: challenge 5 window opened 2026-10-26",
      "event_kind": "challenge-open",
      "timestamp_utc": null,
      "source_url": "https://proteinbase.com/competitions/anthropic-adaptyv-2026",
      "verification": "verified",
      "note": null
     }
    }
   ],
   "boundary": "Several rounds from one organiser plus a static target set; targets and definitions differ, so hit rates do not compare across rounds."
  },
  "family_null_reason": null,
  "domain": "protein",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "A+B",
  "type_note": "Periodic prospective wet-lab competition plus a target set",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L5",
   "range": null,
   "qualifier": "Prospective wet lab; no independent replication"
  },
  "publisher": {
   "orgs": [
    "Adaptyv Bio"
   ],
   "org_type": "platform-company"
  },
  "one_liner": "Multi-round prospective protein design evaluation with public data; run by a commercial service provider, not a board to optimise against.",
  "first_release": {
   "date": "2024-07-18",
   "precision": "day",
   "year_hint": null,
   "basis": "Organiser post announcing the first competition round; the day was inferred from a post title, only the month is supported",
   "event_kind": "challenge-open",
   "timestamp_utc": null,
   "source_url": "https://adaptyvbio.substack.com/archive",
   "verification": "unverified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2025-04",
    "precision": "month",
    "year_hint": null,
    "basis": "BenchBB community paper, bioRxiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://www.biorxiv.org/content/10.1101/2025.04.17.648362v1",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": {
    "date": "2024-07",
    "precision": "month",
    "year_hint": null,
    "basis": "First competition round announced (month only)",
    "event_kind": "challenge-open",
    "timestamp_utc": null,
    "source_url": "https://adaptyvbio.substack.com/archive",
    "verification": "partial",
    "note": null
   },
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-10-26",
   "precision": "day",
   "year_hint": null,
   "basis": "Organiser competition page: final challenge window of the 2026 joint competition opened",
   "event_kind": "new-round",
   "timestamp_utc": null,
   "source_url": "https://proteinbase.com/competitions/anthropic-adaptyv-2026",
   "verification": "verified",
   "note": null,
   "description": "Challenge 5 of the 2026 joint competition opened on 2026-10-26; five weekly challenges all closed to submissions by 2026-11-09."
  },
  "next_expected": {
   "date": "2027-01",
   "precision": "month",
   "year_hint": null,
   "basis": "Organiser schedule: lab data to be published in early January 2027 (schedule marked subject to change)",
   "event_kind": "results-published",
   "timestamp_utc": null,
   "source_url": "https://proteinbase.com/competitions/anthropic-adaptyv-2026",
   "verification": "partial",
   "note": "The same page also states testing completes 2026-12-07; the site's earlier 2026-12-15 has no counterpart"
  },
  "next_expected_null_reason": null,
  "status": "active",
  "status_note": "Active with a new round each quarter; results of the 2026 joint competition pending",
  "scale": {
   "summary": "Round 1: 1,857 submissions, about 600 tested; 2025 round: 1,196 tested; 2026 joint run: 1,320 sequences across 16 targets; BenchBB: 7 targets.",
   "items": [
    {
     "label": "Round 1 submissions / tested",
     "value": "1,857 / ~600",
     "unit": null,
     "version": "Round 1"
    },
    {
     "label": "2025 round sequences tested",
     "value": "1,196",
     "unit": "sequences",
     "version": "2025 round"
    },
    {
     "label": "2026 joint run sequences / targets",
     "value": "1,320 / 16",
     "unit": null,
     "version": "2026 joint"
    },
    {
     "label": "BenchBB targets",
     "value": "7",
     "unit": "targets",
     "version": "BenchBB"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "One round per quarter, about 300-1,200 sequences per round, measured in the organiser automated lab.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Whether submitted designed sequences express and bind the stated target.",
   "truth": "Binding measurements from the organiser lab under one protocol, including measured negatives.",
   "metric_primary": "Binding hit rate",
   "metric_secondary": [
    "Affinity ranking",
    "Expression rate"
   ],
   "version_traps": "Targets, hit thresholds and denominators differ by round; tested sets are not random samples but pre-screened by computational metrics.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "no-ranking",
   "metric": "Binding hit rate and affinity ranking, plus expression rate",
   "eval_set": "Anthropic x Adaptyv 2026 competition, Challenges 1-5 (EGFR, TNF-a, GLP1R, TfR1, KRAS G12D pMHC)",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "Retrieval date (the current round publishes no board)",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://proteinbase.com/competitions/anthropic-adaptyv-2026",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://proteinbase.com/competitions/anthropic-adaptyv-2026",
   "entries": [],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "The current round has no results and the organiser page shows no ranks or scores.",
   "caveat": "The current round has no results: all five challenges are closed, the organiser page shows no board, ranks or scores, and data are scheduled for early 2027; past results sit in organiser posts with differing definitions.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-bindcraft",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://adaptyvbio.substack.com/p/protein-design-competition-results",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "The developers took part; an early-round top-10 placement comes from an organiser post (second-hand)",
    "resolution": null
   },
   {
    "model_id": "model-mosaic",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://proteinbase.com/collections/nipah-binder-competition-results",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entrant in the 2025 de novo track with results on the organiser site; hit rates are self-reported",
    "resolution": null
   },
   {
    "model_id": "model-proteina-complexa",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://research.nvidia.com/labs/genair/proteina-complexa/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Added by R7/fc_zero as a missing candidate and confirmed in the review.",
    "resolution": "The project page states participation and links the organiser's competition page, but the organiser page does not name the model, so this is self-reported participation that the organiser has not confirmed - which the schema routes to self-reported-modified-protocol, not official-participation. Established, so this model also leaves the zero-pairing list."
   },
   {
    "model_id": "model-claude-protein-design",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://adaptyvbio.substack.com/p/case-study-benchmarking-claudes-protein",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Organizer-run anonymous external test on 16 prior-competition targets; not a public competition",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 4,
   "by_relation": {
    "official-participation": 2,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 1,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 1,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "It cannot be run statically: there is no reusable train/test split, so designs must be sent for measurement or entered in the next round. Report hit threshold, denominator, expression rate and target, add orthogonal validation; a single round cannot rank methods or speak to function, specificity or developability.",
  "pairs_with": [
   "bench-36-aintibody",
   "bench-35-flab",
   "bench-34-abbibench"
  ],
  "caveats": [
   "One target and a few hundred designs per round; the organiser is a commercial service provider; computational proxies correlate only weakly with measured binding; rounds are not comparable.",
   "Tested sets are pre-screened by computational metrics, so correlations are computed on a truncated range and understate the true value.",
   "Rounds mix redesign of known antibodies with de novo design; these differ in difficulty and the former is a strong baseline.",
   "Replication requires paying for measurement, there is no independent lab replication, and the organiser has a commercial interest.",
   "BenchBB has no common scoring protocol and its board does not publish submission counts."
  ],
  "sources": [
   {
    "url": "https://proteinbase.com/competitions/anthropic-adaptyv-2026",
    "title": "Organiser page for the 2026 joint competition",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://adaptyvbio.substack.com/p/protein-design-competition-results",
    "title": "Round 2 results (organiser post)",
    "kind": "company-blog",
    "date": null
   },
   {
    "url": "https://adaptyvbio.substack.com/p/introducing-benchbb-and-the-community",
    "title": "BenchBB and community introduction (organiser post)",
    "kind": "company-blog",
    "date": null
   },
   {
    "url": "https://www.biorxiv.org/content/10.1101/2025.04.17.648362v1",
    "title": "BenchBB community paper (bioRxiv)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.1101/2025.08.14.670059",
    "title": "Meta-analysis of computational proxies (bioRxiv 2025)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://adaptyvbio.substack.com/archive",
    "title": "Organiser post archive",
    "kind": "company-blog",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "members",
    "was": "[{\"name\": \"EGFR round 1\", \"first_release\": \"2024-07-18\", \"basis\": \"announcement post\"}, {\"name\": \"EGFR round 2\", \"first_release\": \"2024-10\", \"basis\": \"launched Oct 2024, results Dec 2024\"}, {\"name\": \"Anthropic x Adaptyv 2026\", \"first_release\": \"2026-09-28\", \"basis\": \"Proteinbase competition page\"}]",
    "now": "[{\"name\": \"Binder design competition round 1 (EGFR)\", \"first_release\": \"2024-07\", \"basis\": \"announced July 2024; exact day inferred from a post title only, flagged\"}, {\"name\": \"Round 2 (EGFR)\", \"first_release\": \"2024-11\", \"basis\": \"existing note, day not re-verified\"}, {\"name\": \"BenchBB / community paper\", \"first_release\": \"2025-04\", \"basis\": \"bioRxiv 2025.04.17.648362\"}, {\"name\": \"Nipah protein design competition\", \"first_release\": \"2025-10-21\", \"basis\": \"announced 2025-10-21; submissions open 2025-11-01; leaderboard 2025-11-19; results 2026-01-05\"}, {\"name\": \"Anthropic x Adaptyv 2026\", \"first_release\": \"2026-10-26\", \"basis\": \"challenge 5 window opened 2026-10-26 (organiser page)\"}]",
    "reason": "Family omitted the 2025 Nipah round and BenchBB (R5 high).",
    "source_url": "https://adaptyvbio.substack.com/archive",
    "date_found": "2026-11-16"
   },
   {
    "field": "wetlab_note",
    "was": "about $149 per sequence",
    "now": "not stated by the operator; the figure appears only in secondary sources and no pricing page was located",
    "reason": "The per-sequence price appears only in secondary sources and could not be verified on operator pages, so it is removed.",
    "source_url": "https://adaptyvbio.substack.com/p/introducing-benchbb-and-the-community",
    "date_found": "2026-11-16"
   },
   {
    "field": "next_expected",
    "was": "results published 2026-12-15",
    "now": "organiser schedule: testing by 2026-12-07, data published early January 2027 (subject to change)",
    "reason": "The organiser page schedule does not match the earlier site note.",
    "source_url": "https://proteinbase.com/competitions/anthropic-adaptyv-2026",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported.model-proteina-complexa",
    "was": "not listed (zero-pairing list)",
    "now": "self-reported-modified-protocol",
    "reason": "Participation is self-reported on the project page and not confirmed by the organiser, so it is recorded as self-reported.",
    "source_url": "https://research.nvidia.com/labs/genair/proteina-complexa/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "family.members[0]",
   "family.members[1]",
   "next_expected",
   "models_reported[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-28-adaptyv/",
   "en": "/bio/benchmark/benchmarks/adaptyv/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-29-bixbench",
  "kind": "benchmark",
  "nn": "29",
  "short_slug": "bixbench",
  "names": {
   "display": "BixBench",
   "full": "A benchmark for bioinformatics analysis agents (BixBench)",
   "full_null_reason": null,
   "aliases": [
    "BixBench",
    "BixBench v1.5",
    "BixBench3"
   ],
   "ambiguous_aliases": [
    "Verified-50"
   ]
  },
  "family": null,
  "family_null_reason": "The successor is run by a different organisation with different tasks and metrics, so it is listed separately rather than as one family.",
  "domain": "text-agents",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "E",
  "type_note": "Executable containerised evaluation environment, graded on conclusions",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Truth is the conclusions of published analyses, not experimental validation"
  },
  "publisher": {
   "orgs": [
    "FutureHouse",
    "ScienceMachine"
   ],
   "org_type": "nonprofit-institute"
  },
  "one_liner": "Agents write and run code on real data in a container to do bioinformatics analyses, graded on their conclusions.",
  "first_release": {
   "date": "2025-02-28",
   "precision": "day",
   "year_hint": null,
   "basis": "The paper's arXiv v1 (2025-02-28); the dataset hosting page was created a day earlier, but creation does not prove public availability.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2503.00096",
   "verification": "partial",
   "note": "The hosting page's creation date is not used as first release"
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2025-02-28",
    "precision": "day",
    "year_hint": null,
    "basis": "The paper's arXiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2503.00096",
    "verification": "partial",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2025-10-06",
    "precision": "day",
    "year_hint": null,
    "basis": "Last merge in the official repository",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://github.com/Future-House/BixBench",
    "verification": "partial",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-10-16",
   "precision": "day",
   "year_hint": null,
   "basis": "Entry dates on the successor's official leaderboard page",
   "event_kind": "leaderboard-entry",
   "timestamp_utc": null,
   "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
   "verification": "partial",
   "note": "The board page carries per-entry publish dates but no overall snapshot date",
   "description": "A new model entry appeared on the successor's leaderboard; the original's latest events are a paper revision and a repository merge."
  },
  "next_expected": null,
  "next_expected_null_reason": "The organisers have not announced a new version or round for the original benchmark.",
  "status": "active",
  "status_note": "Active; the successor has an official board that keeps updating",
  "scale": {
   "summary": "First version: 53 capsules and 296 questions; the revision: about 61 capsules and 205 questions.",
   "items": [
    {
     "label": "Capsules",
     "value": "53",
     "unit": null,
     "version": "v1"
    },
    {
     "label": "Questions",
     "value": "296",
     "unit": null,
     "version": "v1"
    },
    {
     "label": "Capsules",
     "value": "about 61",
     "unit": null,
     "version": "v3 / repository v1.5"
    },
    {
     "label": "Questions",
     "value": "205",
     "unit": null,
     "version": "v3 / repository v1.5"
    },
    {
     "label": "Tasks in the successor",
     "value": "20",
     "unit": null,
     "version": "successor"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No experiments; version updates are filtering revisions rather than new data.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Agents write and run code on real datasets to complete multi-step bioinformatics analyses, graded on their conclusions.",
   "truth": "Conclusions from expert analysis notebooks, not wet-lab measurements.",
   "metric_primary": "Open-answer accuracy, graded by a model",
   "metric_secondary": [
    "Multiple-choice accuracy, with and without a refusal option"
   ],
   "version_traps": "State the version and subset: the first version, the revision, the third-party verified subset and the successor differ in question count and metric.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
   "eval_set": "BixBench3 (successor benchmark, 20 research-study-scale tasks)",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
   "entries": [
    {
     "rank": 1,
     "model_label": "gemini-4-argon",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.523",
     "score_unit": null,
     "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
     "eval_set": "BixBench3 (successor benchmark, 20 research-study-scale tasks)",
     "source_type": "board",
     "date": {
      "date": "2026-10-16",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
     "note": "Board top entry; the entry's published date is 2026-10-16",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "claude-fable-5.1 (with fallback)",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.515",
     "score_unit": null,
     "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
     "eval_set": "BixBench3 (successor benchmark, 20 research-study-scale tasks)",
     "source_type": "board",
     "date": {
      "date": "2026-09-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
     "note": "Model name copied verbatim from the board",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "claude-opus-5.5",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.502",
     "score_unit": null,
     "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
     "eval_set": "BixBench3 (successor benchmark, 20 research-study-scale tasks)",
     "source_type": "board",
     "date": {
      "date": "2026-09-30",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "gpt-6-astra",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.492",
     "score_unit": null,
     "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
     "eval_set": "BixBench3 (successor benchmark, 20 research-study-scale tasks)",
     "source_type": "board",
     "date": {
      "date": "2026-09-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 5,
     "model_label": "grok-4.6",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "0.488",
     "score_unit": null,
     "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
     "eval_set": "BixBench3 (successor benchmark, 20 research-study-scale tasks)",
     "source_type": "board",
     "date": {
      "date": "2026-09-21",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
      "verification": "verified",
      "note": null
     },
     "url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
     "note": "The board lists 20 models, with the last at 0.000",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Biomni",
     "model_id": "model-biomni",
     "site_model": true,
     "score": "52.2% (full set) / 88.7% (Verified-50)",
     "score_unit": null,
     "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
     "eval_set": "Original BixBench: full set, plus a third-party verified subset of 50 questions",
     "source_type": "self",
     "date": {
      "date": "2026-02-03",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://phylo.bio/blog/evaluating-ai-agents-in-biology",
      "verification": "verified",
      "note": null
     },
     "url": "https://phylo.bio/blog/evaluating-ai-agents-in-biology",
     "note": "On the original, not the successor; run by its own developer, so not ranked against the board",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Kosmos/Edison Analysis",
     "model_id": "model-kosmos",
     "site_model": true,
     "score": "46% (Edison self-report, full set); 42.4% full set / 78.0% Verified-50 (Phylo re-run)",
     "score_unit": null,
     "metric": "Average score over the successor's 20 tasks, scored programmatically against source papers",
     "eval_set": "Original BixBench: full set, plus a third-party verified subset of 50 questions",
     "source_type": "third",
     "date": {
      "date": "2026-02-03",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://phylo.bio/blog/evaluating-ai-agents-in-biology",
      "verification": "verified",
      "note": null
     },
     "url": "https://phylo.bio/blog/evaluating-ai-agents-in-biology",
     "note": "On the original; the self-reporting party is tied to the successor's operator",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "The only official board belongs to the successor, whose tasks and metric differ and do not compare with the original's accuracy; the original has no official board and no official verified subset, so those rows are self-reported or third-party numbers.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-frontier-llms",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://arxiv.org/html/2503.00096v3",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Successor's official board; counted as one family cell",
    "resolution": null
   },
   {
    "model_id": "model-kosmos",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://edisonscientific.com/news/introducing-edison-analysis",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "unverified",
    "note": "Self-report on the original; the product's mapping to this site entry is unclear and it is not on the successor's board",
    "resolution": null
   },
   {
    "model_id": "model-biomni",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://phylo.bio/blog/evaluating-ai-agents-in-biology",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Run by its developer on the full set and a verified subset with its own agent harness",
    "resolution": null
   },
   {
    "model_id": "model-co-scientist",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/2502.18864",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-gpt-rosalind",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://the-decoder.com/openai-launches-gpt-rosalind-a-reasoning-model-built-for-life-sciences-research/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "disputed",
    "note": "Version unspecified and the numbers appear only in second-hand reports",
    "resolution": null
   },
   {
    "model_id": "model-intern-s",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://github.com/InternLM/Intern-S1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-quine",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.microsoft.com/en-us/research/blog/introducing-quine-an-ai-research-system-designed-for-the-complexity-of-biology/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   },
   {
    "model_id": "model-scireasoner",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "Not found in the paper text",
    "resolution": null
   },
   {
    "model_id": "model-verid",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found in official materials",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 3,
   "by_relation": {
    "official-participation": 1,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 1,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 6
   }
  },
  "how_to_use": "Run the agent inside the official container environment, produce conclusions per capsule and score them with the official grader; no wet lab of your own is needed. Report the version, question count, whether a verified subset was used, the grading model and the agent harness, and separate open answers from multiple choice. Successor scores must not be mixed with the original's. Pair with LAB-Bench and BioML-bench.",
  "pairs_with": [
   "bench-18-labbench",
   "bench-41-bioml-bench",
   "bench-42-biomysterybench",
   "bench-23-bioasq"
  ],
  "caveats": [
   "The truth comes from a single analyst, with ambiguous questions and some wrong answers; scores on a third-party verified subset are markedly higher.",
   "The multiple-choice version carries little signal, and open answers are graded by a model, which is itself a source of error.",
   "The data and questions are public, so contamination is a risk and there is no time freeze.",
   "Scores must state the version: question counts and subsets differ, so numbers do not compare directly.",
   "The successor is operated by a commercial organisation that also self-reports results, which is a conflict of interest."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2503.00096",
    "title": "Original paper",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/Future-House/BixBench",
    "title": "Official repository and evaluation environment",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://huggingface.co/datasets/futurehouse/BixBench",
    "title": "Official dataset hosting page",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
    "title": "Official board page of the successor",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://phylo.bio/blog/evaluating-ai-agents-in-biology",
    "title": "Blog describing a third-party verified subset and re-runs",
    "kind": "company-blog",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "ranking.eval_set",
    "was": "all ranking rows labelled as the successor's task set",
    "now": "rows on the original benchmark are labelled as the original's full set plus a third-party verified subset",
    "reason": "The successor and the original differ in tasks and metric, so the labels must be separated or the two sets of scores read as comparable.",
    "source_url": "https://advances.edisonscientific.com/benchmarks/bixbench3/",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "2025-03",
    "now": "2025-02-28 (paper arXiv v1)",
    "reason": "Date audit: the paper's v1 date precedes the month implied by its identifier, and a hosting-page creation date is not a first release.",
    "source_url": "https://arxiv.org/abs/2503.00096",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "latest_update",
   "release_milestones.code_release",
   "scale.items[2]",
   "models_reported[1]",
   "ranking.snapshot_date"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-29-bixbench/",
   "en": "/bio/benchmark/benchmarks/bixbench/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-30-plinder",
  "kind": "benchmark",
  "nn": "30",
  "short_slug": "plinder",
  "names": {
   "display": "PLINDER / PINDER / Runs N' Poses",
   "full": "PLINDER (protein-ligand interaction dataset and evaluation), PINDER (protein-protein interaction dataset and evaluation) and Runs N' Poses (co-folding generalization test set)",
   "full_null_reason": null,
   "aliases": [
    "PLINDER",
    "PINDER",
    "Runs N' Poses",
    "RnP",
    "PLINDER-PL50",
    "PLINDER-TIME",
    "PLINDER-ECOD",
    "plinder",
    "pinder"
   ],
   "ambiguous_aliases": [
    "runs-n-poses"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "PLINDER",
     "id_or_null": null,
     "first_release": {
      "date": "2024-07-17",
      "precision": "day",
      "year_hint": null,
      "basis": "PLINDER bioRxiv v1",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2024.07.17.603955",
      "verification": "partial",
      "note": "Member dates rely on repositories and dynamic pages; not fully reproduced"
     }
    },
    {
     "name": "PINDER",
     "id_or_null": null,
     "first_release": {
      "date": "2024-07-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Recorded basis is bioRxiv v1; the date equals the first PyPI upload",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2024.07.17.603980",
      "verification": "partial",
      "note": "DOI string suggests 2024-07-17/18; the bioRxiv API did not resolve it"
     }
    },
    {
     "name": "Runs N' Poses",
     "id_or_null": null,
     "first_release": {
      "date": "2025-02-07",
      "precision": "day",
      "year_hint": null,
      "basis": "Runs N' Poses bioRxiv v1 (DOI string 2025.02.03; posted 2025-02-07)",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.02.03.636309",
      "verification": "partial",
      "note": null
     }
    }
   ],
   "boundary": "All three are leakage-aware infrastructure from one organization: PLINDER and PINDER provide train/test splits; Runs N' Poses stratifies trained co-folding models post hoc."
  },
  "family_null_reason": null,
  "domain": "structure",
  "domains_secondary": [],
  "inclusion": "original-30",
  "type": "C+B",
  "type_note": "Split tooling and datasets, plus a stratified static test set",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L3",
   "range": null,
   "qualifier": "Strict OOD retrospective splits, no wet lab"
  },
  "publisher": {
   "orgs": [
    "VantAI",
    "NVIDIA",
    "MIT CSAIL",
    "University of Basel",
    "SIB Swiss Institute of Bioinformatics"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Docking and co-folding infrastructure that makes training-set similarity a first-class split criterion.",
  "first_release": {
   "date": "2024-07-17",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv v1 of PLINDER, the earliest member; PINDER is recorded as 2024-07-19 and Runs N' Poses as 2025-02-07.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2024.07.17.603955",
   "verification": "partial",
   "note": "Member dates rely on dynamic pages; see unverified_fields"
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2024-07-17",
    "precision": "day",
    "year_hint": null,
    "basis": "PLINDER bioRxiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2024.07.17.603955",
    "verification": "partial",
    "note": null
   },
   "journal_online": {
    "date": "2026-05",
    "precision": "month",
    "year_hint": null,
    "basis": "Runs N' Poses final version in Nature Structural & Molecular Biology",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/s41594-026-01797-5",
    "verification": "verified",
    "note": null
   },
   "code_release": {
    "date": "2024-08-30",
    "precision": "day",
    "year_hint": null,
    "basis": "First PyPI upload of the plinder package (pinder first upload 2024-07-19)",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://pypi.org/project/plinder/",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-10-09",
   "precision": "day",
   "year_hint": null,
   "basis": "plinder 1.0.0 released on PyPI and GitHub",
   "event_kind": "new-version",
   "timestamp_utc": null,
   "source_url": "https://pypi.org/project/plinder/",
   "verification": "partial",
   "note": null,
   "description": "plinder 1.0.0 released (Metaflow removed); later commits are docs-only. pinder 0.6.0 (2026-09-14) moved dataset hosting to R2."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no date for a next release or a new challenge round.",
  "status": "active",
  "status_note": "Active; plinder 1.0.0 released 2026-10-09",
  "scale": {
   "summary": "PLINDER: 449,383 systems; PINDER: 2,319,564 dimers; Runs N' Poses: 2,073–2,600 systems.",
   "items": [
    {
     "label": "PLINDER protein-ligand systems",
     "value": "449,383",
     "unit": "systems",
     "version": null
    },
    {
     "label": "PLINDER annotations per system",
     "value": ">500",
     "unit": "annotations",
     "version": null
    },
    {
     "label": "PINDER dimers",
     "value": "2,319,564",
     "unit": "dimers",
     "version": null
    },
    {
     "label": "PINDER apo monomers",
     "value": "47,717",
     "unit": "monomers",
     "version": null
    },
    {
     "label": "PINDER AFDB monomers",
     "value": "42,827",
     "unit": "monomers",
     "version": null
    },
    {
     "label": "PINDER test set",
     "value": "1,955",
     "unit": "systems",
     "version": null
    },
    {
     "label": "Runs N' Poses systems (preprint)",
     "value": "2,600",
     "unit": "systems",
     "version": "preprint"
    },
    {
     "label": "Runs N' Poses systems (Polaris release)",
     "value": "2,073",
     "unit": "systems",
     "version": "Polaris"
    },
    {
     "label": "Systems in the lowest-similarity bin (0–20)",
     "value": "76",
     "unit": "systems",
     "version": null
    }
   ]
  },
  "wetlab_refresh": "accumulating",
  "wetlab_note": "No new experimental data; follows weekly PDB updates and reuses experimental structures.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Docking and co-folding generalization to systems dissimilar from training data in protein, pocket, interaction and ligand, reported by similarity bin.",
   "truth": "Published experimental PDB structures; Runs N' Poses uses high-resolution systems released after 2021-09-30.",
   "metric_primary": "Pose success rate (LDDT-PLI > 0.8 and BiSyRMSD < 2 Å)",
   "metric_secondary": [
    "PINDER: DockQ and iRMSD-type metrics",
    "Results stratified into 8 training-similarity bins"
   ],
   "version_traps": "The PL50 split's system definition differs between v1 (2024-04) and the v2 used in the MLSB challenge (2024-06), which relies on ligand distance only; numbers across versions are not comparable.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "success_rate: share of predictions with LDDT-PLI > 0.8 and BiSyRMSD < 2; the page also lists rmsd_success_rate, mean_lddt_pli and coverage",
   "eval_set": "Runs N' Poses (Polaris preview benchmark, plinder-org/runs-n-poses)",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
   "entries": [
    {
     "rank": 1,
     "model_label": "AlphaFold3",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "success_rate 15.068; rmsd_success_rate 17.808; mean_lddt_pli 0.304; coverage 96.053",
     "score_unit": null,
     "metric": "success_rate: share of predictions with LDDT-PLI > 0.8 and BiSyRMSD < 2; the page also lists rmsd_success_rate, mean_lddt_pli and coverage",
     "eval_set": "Runs N' Poses (Polaris preview benchmark, plinder-org/runs-n-poses)",
     "source_type": "board",
     "date": {
      "date": "2025-02-04",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
      "verification": "verified",
      "note": null
     },
     "url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
     "note": "Board top-1; benchmark created 2025-02-03, figure updated 2025-02-04; no per-entry date",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "Boltz-1",
     "model_id": "model-boltz-1-1x",
     "site_model": true,
     "score": "success_rate 5.556; rmsd_success_rate 9.259; mean_lddt_pli 0.250; coverage 71.053",
     "score_unit": null,
     "metric": "success_rate: share of predictions with LDDT-PLI > 0.8 and BiSyRMSD < 2; the page also lists rmsd_success_rate, mean_lddt_pli and coverage",
     "eval_set": "Runs N' Poses (Polaris preview benchmark, plinder-org/runs-n-poses)",
     "source_type": "board",
     "date": {
      "date": "2025-02-04",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
      "verification": "verified",
      "note": null
     },
     "url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "Chai-1",
     "model_id": "model-chai-1",
     "site_model": true,
     "score": "success_rate 4.688; rmsd_success_rate 18.750; mean_lddt_pli 0.281; coverage 84.211",
     "score_unit": null,
     "metric": "success_rate: share of predictions with LDDT-PLI > 0.8 and BiSyRMSD < 2; the page also lists rmsd_success_rate, mean_lddt_pli and coverage",
     "eval_set": "Runs N' Poses (Polaris preview benchmark, plinder-org/runs-n-poses)",
     "source_type": "board",
     "date": {
      "date": "2025-02-04",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
      "verification": "verified",
      "note": null
     },
     "url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "Protenix",
     "model_id": "model-protenix",
     "site_model": true,
     "score": "success_rate 1.471; rmsd_success_rate 11.765; mean_lddt_pli 0.251; coverage 89.474",
     "score_unit": null,
     "metric": "success_rate: share of predictions with LDDT-PLI > 0.8 and BiSyRMSD < 2; the page also lists rmsd_success_rate, mean_lddt_pli and coverage",
     "eval_set": "Runs N' Poses (Polaris preview benchmark, plinder-org/runs-n-poses)",
     "source_type": "board",
     "date": {
      "date": "2025-02-04",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
      "verification": "verified",
      "note": null
     },
     "url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Pearl",
     "model_id": "model-pearl",
     "site_model": true,
     "score": "85.2% (<2 Å, PoseBusters-valid, best@5); 70.0% at <1 Å",
     "score_unit": null,
     "metric": "Share with RMSD < 2 Å and PoseBusters-valid (best@5)",
     "eval_set": "Runs N' Poses (702-structure filtered subset, Pearl paper)",
     "source_type": "self",
     "date": {
      "date": "2025-10",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/pdf/2510.24670",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/pdf/2510.24670",
     "note": "Different protocol (best@5, filtered 702-structure subset); not comparable with the Polaris board",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "AlphaFold3",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "74.4% (<2 Å, best@5); 61.5% at <1 Å",
     "score_unit": null,
     "metric": "Share with RMSD < 2 Å (best@5)",
     "eval_set": "Runs N' Poses (702-structure filtered subset, Pearl paper)",
     "source_type": "third",
     "date": {
      "date": "2025-10",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/pdf/2510.24670",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/pdf/2510.24670",
     "note": "Second-hand, quoted from the Pearl paper's comparison table, which also lists Boltz, Chai-1 and Protenix",
     "second_hand": true,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "PLINDER and PINDER have no board; the only board is a Polaris preview of Runs N' Poses whose values sit far below the paper and whose bin counts conflict, so it is not a true ordering. Newer models are self-reported or second-hand on own subsets, unranked.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-isodde",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://zenodo.org/records/18606681",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Tech report claims a large gain in the hardest Runs N' Poses bin; no numeric page",
    "resolution": null
   },
   {
    "model_id": "model-openfold3",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://portal.openfold.omsf.io/reports/of3p2_technical_report.pdf",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Runs N' Poses appears in a README bar chart; no numbers in text",
    "resolution": null
   },
   {
    "model_id": "model-pearl",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/html/2510.24670v2",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Runs N' Poses best@5; baselines rerun by the Pearl authors",
    "resolution": null
   },
   {
    "model_id": "model-nesso-1",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://www.valencelabs.com/wp-content/uploads/2026/07/nesso1.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "A missing candidate added and confirmed in the review",
    "resolution": "Third structural benchmark named in the same sentence, with the same metrics, baseline sourcing and filtering caveat; classed as a modified-protocol self-report."
   },
   {
    "model_id": "model-protenix",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2510.24670",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "v1 self-report on the subset where all models have results, figure only; plus a third-party figure in the Pearl paper",
    "resolution": null
   },
   {
    "model_id": "model-alphafold3",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/pdf/2510.24670",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Numbers quoted second-hand from the Pearl paper",
    "resolution": null
   },
   {
    "model_id": "model-boltz-1-1x",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Numbers from the Pearl paper",
    "resolution": null
   },
   {
    "model_id": "model-boltz-2",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Quoted in the Pearl paper and flagged not directly comparable; its official evaluation docs omit this benchmark",
    "resolution": null
   },
   {
    "model_id": "model-chai-1",
    "relation_type": "cited-as-comparison-only",
    "source_type": "third",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Quoted from the Pearl paper",
    "resolution": null
   }
  ],
  "non_pairings": [
   {
    "model_id": "model-proteina-complexa",
    "note": "This family appears only as protein-ligand training data, which is not a pairing",
    "source_url": "https://arxiv.org/abs/2603.27950"
   }
  ],
  "n_models_reported": {
   "counted": 5,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 3,
    "self-reported-modified-protocol": 2,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 4,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Runs statically with no wet lab. Train own models on the PLINDER-PL50 split and report by similarity bin, not just overall; stratify trained co-folding models with Runs N' Poses. State data version and split files, seeds, pose selection (top-1 confidence or best-of-N) and pretraining cutoffs; add PoseBusters validity checks. Not for screening enrichment, affinity ranking or ADMET.",
  "pairs_with": [
   "bench-05-posebusters",
   "bench-31-foldbench",
   "bench-33-posex",
   "bench-22-capri",
   "bench-06-casf"
  ],
  "caveats": [
   "Binned numbers are mostly developer-reported with differing definitions; system definitions changed between versions; the lowest-similarity bin has only 76 systems.",
   "PLINDER splits only constrain models trained on them; fixed-training co-folding models can only be stratified post hoc against an approximate shared training set, with slight mismatch.",
   "The Polaris preview board diverges widely from the paper and has inconsistent bin counts, so it should not be cited as a true ordering.",
   "Several subsets and best@5/best@15 protocols coexist under one name, so numbers across papers are not comparable.",
   "High-similarity bins are near saturation, and there is no direct evidence that score gains translate into wet-lab success."
  ],
  "sources": [
   {
    "url": "https://www.biorxiv.org/content/10.1101/2024.07.17.603955",
    "title": "PLINDER preprint (bioRxiv)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://www.biorxiv.org/content/10.1101/2024.07.17.603980",
    "title": "PINDER preprint (bioRxiv)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://www.biorxiv.org/content/10.1101/2025.02.03.636309v3.full",
    "title": "Runs N' Poses preprint (bioRxiv v3)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://doi.org/10.1038/s41594-026-01797-5",
    "title": "Runs N' Poses final version (Nature Structural & Molecular Biology 2026)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://plinder-org.github.io/plinder/tutorial/dataset.html",
    "title": "PLINDER dataset documentation",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://github.com/plinder-org/plinder",
    "title": "PLINDER official repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://pypi.org/project/plinder/",
    "title": "plinder on PyPI",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://pypi.org/project/pinder/",
    "title": "pinder on PyPI",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://polaris-hub.vercel.app/benchmarks/plinder-org/runs-n-poses",
    "title": "Runs N' Poses preview benchmark on Polaris",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://plinder-org.github.io/plinder/examples/mlsb_challenge.html",
    "title": "MLSB 2024 challenge page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2510.24670",
    "title": "Pearl model paper (Runs N' Poses comparison table)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2402.18396",
    "title": "DockGen paper (earlier ECOD leakage-aware split)",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "n_models_reported",
    "was": "8",
    "now": "5 counted under the new relation taxonomy (4 comparison-only citations excluded)",
    "reason": "The count was recomputed under the new relation taxonomy; comparison-only citations are not counted.",
    "source_url": "https://arxiv.org/abs/2510.24670",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported (Nesso-1)",
    "was": "not listed",
    "now": "self-reported-modified-protocol",
    "reason": "The review confirmed its report discloses filtering that differs from the cited paper, so it is a modified-protocol self-report and leaves the zero-pairing list.",
    "source_url": "https://www.valencelabs.com/wp-content/uploads/2026/07/nesso1.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "non_pairings (Proteina-Complexa)",
    "was": "candidate pairing",
    "now": "non-pairing: used only as training data",
    "reason": "This family appears in that model's paper only as training data, so it is recorded as a non-pairing.",
    "source_url": "https://arxiv.org/abs/2603.27950",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "latest_update",
   "family.members[0]",
   "family.members[1]",
   "family.members[2]",
   "release_milestones.paper_v1",
   "models_reported[0]",
   "models_reported[1]",
   "models_reported[2]",
   "models_reported[4]",
   "models_reported[5]",
   "models_reported[6]",
   "models_reported[7]",
   "models_reported[8]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-30-plinder/",
   "en": "/bio/benchmark/benchmarks/plinder/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-31-foldbench",
  "kind": "benchmark",
  "nn": "31",
  "short_slug": "foldbench",
  "names": {
   "display": "FoldBench",
   "full": "FoldBench all-atom biomolecular structure prediction benchmark",
   "full_null_reason": null,
   "aliases": [
    "FoldBench",
    "FoldBench-AB"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "One dataset with one maintainer board; the paper and repository are milestones of a single entry, not family members.",
  "domain": "structure",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Fixed retrospective set with a maintainer board",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L3",
   "range": null,
   "qualifier": "Time split with low-homology filtering, not blind"
  },
  "publisher": {
   "orgs": [
    "Fudan University",
    "Shanghai Jiao Tong University",
    "The Chinese University of Hong Kong",
    "Shanghai AI Laboratory",
    "Lingang Laboratory"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "All-atom co-folding evaluation on low-homology PDB assemblies; the antibody-antigen task discriminates most.",
  "first_release": {
   "date": "2025-05-27",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv v1 (10.1101/2025.05.22.655600); the repository creation date is not used as first release",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://www.biorxiv.org/content/10.1101/2025.05.22.655600v1",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2025-05-27",
    "precision": "day",
    "year_hint": null,
    "basis": "bioRxiv v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://www.biorxiv.org/content/10.1101/2025.05.22.655600v1",
    "verification": "verified",
    "note": null
   },
   "journal_online": {
    "date": "2025-12-04",
    "precision": "day",
    "year_hint": null,
    "basis": "Nature Communications paper published",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12800276/",
    "verification": "verified",
    "note": null
   },
   "code_release": {
    "date": "2025-05-22",
    "precision": "day",
    "year_hint": null,
    "basis": "Official repository made public (MIT licence)",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://github.com/BEAM-Labs/FoldBench",
    "verification": "partial",
    "note": "Repository creation date, recorded as a milestone only, never as first release"
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-12-31",
   "precision": "day",
   "year_hint": null,
   "basis": "Results update on the repository board",
   "event_kind": "leaderboard-entry",
   "timestamp_utc": null,
   "source_url": "https://github.com/BEAM-Labs/FoldBench",
   "verification": "verified",
   "note": null,
   "description": "Board updated with RF3 figures on 2025-12-31 and with Boltz-2 and the OpenFold3 preview on 2025-12-05; May 2026 commits were later reverted."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no new version or data expansion.",
  "status": "low-frequency",
  "status_note": "Last repository results update 2025-12-31; no newer official version seen by 2026-11-16",
  "scale": {
   "summary": "Abstract: 1,522 assemblies, 9 tasks; text and README task counts sum to 1,823 and 1,819.",
   "items": [
    {
     "label": "Assemblies (abstract)",
     "value": "1,522",
     "unit": "assemblies",
     "version": "paper"
    },
    {
     "label": "Summed task counts (text / README)",
     "value": "1,823 / 1,819",
     "unit": null,
     "version": "paper / repo"
    },
    {
     "label": "Tasks",
     "value": "9",
     "unit": "tasks",
     "version": null
    },
    {
     "label": "Antibody-antigen interfaces",
     "value": "172",
     "unit": "targets",
     "version": "paper"
    },
    {
     "label": "RNA / DNA monomers",
     "value": "15 / 14",
     "unit": "targets",
     "version": "paper"
    },
    {
     "label": "PDB deposition window",
     "value": "2023-01-13 – 2024-11-01",
     "unit": null,
     "version": null
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Fixed set of already deposited PDB structures; no wet lab and no planned expansion.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "All-atom structure prediction on low-homology PDB assemblies deposited after the reference cutoff: three monomer tasks and six interface types, including antibody-antigen.",
   "truth": "Experimental PDB structures deposited in the window, filtered for low homology to training data.",
   "metric_primary": "Interface success rate",
   "metric_secondary": [
    "Monomer lDDT",
    "Joint pose and interface criteria for protein-ligand"
   ],
   "version_traps": "Two subsets coexist (post-2023-01 full set and post-2024-01); models with later training cutoffs are asterisked and must not be ranked against the rest; abstract and repo counts differ.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
   "eval_set": "FoldBench official repository board, full set (targets released after 2023-01)",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "Latest results-update date stated in the repository README",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://github.com/BEAM-Labs/FoldBench",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://github.com/BEAM-Labs/FoldBench",
   "entries": [
    {
     "rank": 1,
     "model_label": "AlphaFold 3",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "Protein-Protein 72.93% / Antibody-Antigen 47.90% / Protein-Ligand 64.90%; Protein-RNA 62.32%, Protein-DNA 79.18%, RNA Monomer 0.61, DNA Monomer 0.53",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench official repo leaderboard, full set (targets released after 2023-01)",
     "source_type": "board",
     "date": {
      "date": "2025-12-31",
      "precision": "day",
      "year_hint": null,
      "basis": "Results-update date for this entry as stated in the repository README",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/BEAM-Labs/FoldBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/BEAM-Labs/FoldBench",
     "note": "Board top-1 on every listed interface task; ordered here by the protein-protein column.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "OpenFold 3 (preview)",
     "model_id": "model-openfold3",
     "site_model": true,
     "score": "69.96% / 28.83% / 44.49%; Protein-RNA 18.84%, Protein-DNA 5.88%",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench official repo leaderboard, full set (targets released after 2023-01)",
     "source_type": "board",
     "date": {
      "date": "2025-12-05",
      "precision": "day",
      "year_hint": null,
      "basis": "Results-update date for this entry as stated in the repository README",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/BEAM-Labs/FoldBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/BEAM-Labs/FoldBench",
     "note": "Ranked 2nd on protein-protein only; weakest on nucleic-acid interfaces.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "Chai-1",
     "model_id": "model-chai-1",
     "site_model": true,
     "score": "68.53% / 23.64% / 51.23%",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench official repo leaderboard, full set (targets released after 2023-01)",
     "source_type": "board",
     "date": {
      "date": "2025-12-31",
      "precision": "day",
      "year_hint": null,
      "basis": "Results-update date for this entry as stated in the repository README",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/BEAM-Labs/FoldBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/BEAM-Labs/FoldBench",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "Boltz-1",
     "model_id": "model-boltz-1-1x",
     "site_model": true,
     "score": "68.25% / 33.54% / 55.04%",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench official repo leaderboard, full set (targets released after 2023-01)",
     "source_type": "board",
     "date": {
      "date": "2025-12-31",
      "precision": "day",
      "year_hint": null,
      "basis": "Results-update date for this entry as stated in the repository README",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/BEAM-Labs/FoldBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/BEAM-Labs/FoldBench",
     "note": "The board lists Boltz-1, not the 1x variant.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 5,
     "model_label": "Protenix",
     "model_id": "model-protenix",
     "site_model": true,
     "score": "68.18% / 34.13% / 50.70%",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench official repo leaderboard, full set (targets released after 2023-01)",
     "source_type": "board",
     "date": {
      "date": "2025-12-31",
      "precision": "day",
      "year_hint": null,
      "basis": "Results-update date for this entry as stated in the repository README",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/BEAM-Labs/FoldBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/BEAM-Labs/FoldBench",
     "note": "Figures for HelixFold 3, not a site model, appear on the same board for comparison.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "RosettaFold3*",
     "model_id": "model-rf3",
     "site_model": true,
     "score": "Protein-Protein 72.44% / Antibody-Antigen 37.50% / Protein-Ligand 57.28% (post-2024-01 subset)",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench official repo leaderboard, full set (targets released after 2023-01)",
     "source_type": "board",
     "date": {
      "date": "2025-12-31",
      "precision": "day",
      "year_hint": null,
      "basis": "Results-update date for this entry as stated in the repository README",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/BEAM-Labs/FoldBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/BEAM-Labs/FoldBench",
     "note": "Asterisked: training cutoff later than the reference date, reference only, not directly rankable.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Boltz-2*",
     "model_id": "model-boltz-2",
     "site_model": true,
     "score": "Protein-Protein 70.54% / Antibody-Antigen 25.00% / Protein-Ligand 53.90% (post-2024-01 subset)",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench official repo leaderboard, full set (targets released after 2023-01)",
     "source_type": "board",
     "date": {
      "date": "2025-12-05",
      "precision": "day",
      "year_hint": null,
      "basis": "Results-update date for this entry as stated in the repository README",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/BEAM-Labs/FoldBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/BEAM-Labs/FoldBench",
     "note": "Asterisked, reference only, not directly rankable.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "SeedFold",
     "model_id": "model-seedfold",
     "site_model": true,
     "score": "74.03 / 53.21 / 63.12 (SeedFold-Linear 74.14 / 46.91 / 66.48)",
     "score_unit": null,
     "metric": "Interface success rate (% ; DockQ>=0.23, protein-ligand requires LRMSD<2 A and LDDT-PLI>0.8); monomers use LDDT",
     "eval_set": "FoldBench tasks as reported in the model paper",
     "source_type": "self",
     "date": {
      "date": "2025-12",
      "precision": "month",
      "year_hint": null,
      "basis": "Model paper arXiv month",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2512.24354",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2512.24354",
     "note": "Author-reported and not on the official board; if protocols match it would exceed the board leader on protein-protein and antibody-antigen.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "The repository board gives no explicit snapshot date, so README update dates are used; two subsets coexist and asterisked entries have later training cutoffs that the README says must not be ranked alongside the rest.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphafold3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/BEAM-Labs/FoldBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Maintainer leaderboard (FoldBench repo)",
    "resolution": null
   },
   {
    "model_id": "model-boltz-1-1x",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/BEAM-Labs/FoldBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "The maintainer board lists Boltz-1",
    "resolution": null
   },
   {
    "model_id": "model-boltz-2",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/BEAM-Labs/FoldBench",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Maintainer ran Boltz-2 on post-2024-01 subset (training cutoff later; reference only)",
    "resolution": null
   },
   {
    "model_id": "model-chai-1",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/BEAM-Labs/FoldBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Maintainer leaderboard",
    "resolution": null
   },
   {
    "model_id": "model-openfold3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/BEAM-Labs/FoldBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Maintainer leaderboard (preview)",
    "resolution": null
   },
   {
    "model_id": "model-protenix",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/BEAM-Labs/FoldBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Maintainer leaderboard",
    "resolution": null
   },
   {
    "model_id": "model-rf3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/BEAM-Labs/FoldBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Maintainer leaderboard (2024-01+ subset)",
    "resolution": null
   },
   {
    "model_id": "model-esmfold2",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://biohub.org/news/world-model-of-protein-biology/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Model-card figures are second-hand; another paper reports figures on its own antibody subset",
    "resolution": null
   },
   {
    "model_id": "model-intellifold-2",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.64898/2026.02.09.704787",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Published as a figure only; the predecessor reports numbers",
    "resolution": null
   },
   {
    "model_id": "model-seedfold",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2512.24354",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-reported on FoldBench; arXiv 2512.24354",
    "resolution": null
   },
   {
    "model_id": "model-nesso-1",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://www.valencelabs.com/wp-content/uploads/2026/07/nesso1.pdf",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Added by R7/fc_zero as a missing candidate and confirmed in the review.",
    "resolution": "Technical report names the benchmark and copies baselines from another paper while admitting its filtering criteria differ, so it is self-reported on a modified protocol, not an official entry. Moves this model off the zero-pairing list."
   },
   {
    "model_id": "model-opendde",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/abs/2607.03787",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Self-defined FoldBench-AB antibody-antigen subset; not comparable to official board",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 12,
   "by_relation": {
    "official-participation": 7,
    "self-reported-official-protocol": 3,
    "self-reported-modified-protocol": 2,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Run the repository scripts and emit the board format; models with later training cutoffs should report only the post-2024-01 subset and state their cutoff. Report subset, sampling settings, seeds and per-task target counts, and give both success rate and the DockQ distribution for antibody-antigen; generalisation claims need newer structures.",
  "pairs_with": [
   "bench-05-posebusters",
   "bench-30-plinder",
   "bench-33-posex",
   "bench-22-capri",
   "bench-01-casp"
  ],
  "caveats": [
   "A retrospective set, not a blind test; the window starts early, so models with later training cutoffs may have seen some targets.",
   "RNA and DNA monomers have only 14-15 targets and protein-peptide 51, so small-sample rank gaps should not be read.",
   "Target counts differ across abstract, text and repository (1,522 vs 1,823 vs 1,819), so citations must state which.",
   "Self-reported entries differ in sampling settings and versions, so same-named models need not be comparable.",
   "It measures structural accuracy only, not function, affinity or developability."
  ],
  "sources": [
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12800276/",
    "title": "FoldBench paper (Nature Communications 2025)",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://www.biorxiv.org/content/10.1101/2025.05.22.655600v1",
    "title": "FoldBench preprint (bioRxiv v1)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/BEAM-Labs/FoldBench",
    "title": "FoldBench official repository and board",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://doi.org/10.5281/zenodo.17180806",
    "title": "FoldBench dataset (Zenodo)",
    "kind": "archive",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2512.24354",
    "title": "Model paper reporting FoldBench results (arXiv 2512.24354)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2607.03787",
    "title": "Model paper reporting a custom antibody subset (arXiv 2607.03787)",
    "kind": "preprint",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "models_reported.model-nesso-1",
    "was": "not listed (zero-pairing list)",
    "now": "self-reported-modified-protocol",
    "reason": "The report states filtering differs from the cited paper, so it counts as a modified-protocol self-report.",
    "source_url": "https://www.valencelabs.com/wp-content/uploads/2026/07/nesso1.pdf",
    "date_found": "2026-11-16"
   },
   {
    "field": "n_models_reported",
    "was": "11",
    "now": "recomputed under the new relation-type rules (established pairings only)",
    "reason": "The reported-model count is recomputed under the new relation-type rules.",
    "source_url": "https://github.com/BEAM-Labs/FoldBench",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "release_milestones.code_release",
   "ranking.snapshot_date",
   "models_reported[2]",
   "models_reported[7]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-31-foldbench/",
   "en": "/bio/benchmark/benchmarks/foldbench/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-32-crossdocked",
  "kind": "benchmark",
  "nn": "32",
  "short_slug": "crossdocked",
  "names": {
   "display": "CrossDocked2020",
   "full": "The CrossDocked2020 cross-docked pose dataset and its generative-task split",
   "full_null_reason": null,
   "aliases": [
    "CrossDocked2020",
    "CrossDocked",
    "CrossDocked2020 v1.3"
   ],
   "ambiguous_aliases": [
    "CrossDocked split"
   ]
  },
  "family": {
   "is_family": true,
   "members": [
    {
     "name": "CrossDocked2020 dataset",
     "id_or_null": null,
     "first_release": {
      "date": "2020-02-20",
      "precision": "day",
      "year_hint": null,
      "basis": "Preprint of the dataset paper; journal online 2020-08-31",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1021/acs.jcim.0c00411",
      "verification": "partial",
      "note": null
     }
    },
    {
     "name": "2021 generative split",
     "id_or_null": null,
     "first_release": {
      "date": "2021",
      "precision": "year",
      "year_hint": null,
      "basis": "A filtered split introduced in a 2021 conference paper; exact date unverified",
      "event_kind": "data-release",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/pdf/2303.03543",
      "verification": "unverified",
      "note": null
     }
    },
    {
     "name": "CBGBench unified re-implementation",
     "id_or_null": null,
     "first_release": {
      "date": "2024-06-16",
      "precision": "day",
      "year_hint": null,
      "basis": "arXiv v1 of the unified re-implementation",
      "event_kind": "paper-v1",
      "timestamp_utc": null,
      "source_url": "https://github.com/EDAPINENUT/CBGBench",
      "verification": "partial",
      "note": null
     }
    }
   ],
   "boundary": "The dataset, the widely reused generative split and the unified re-implementation are three different artefacts, and any number must say which one it uses."
  },
  "family_null_reason": null,
  "domain": "generation",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "D",
  "type_note": "Generative evaluation with docking software as the oracle, no official board",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Retrospective evaluation against a computational oracle, no experimental truth"
  },
  "publisher": {
   "orgs": [
    "University of Pittsburgh"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "The de facto default set for pocket-conditioned molecule generation, scored by docking software.",
  "first_release": {
   "date": "2020-02-20",
   "precision": "day",
   "year_hint": null,
   "basis": "The dataset paper's preprint (2020-02-20); the journal online date of 2020-08-31 is kept as a separate milestone.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1021/acs.jcim.0c00411",
   "verification": "partial",
   "note": "The preprint date was not re-retrieved from its original page this round"
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2020-02-20",
    "precision": "day",
    "year_hint": null,
    "basis": "Preprint of the dataset paper",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1021/acs.jcim.0c00411",
    "verification": "partial",
    "note": null
   },
   "journal_online": {
    "date": "2020-08-31",
    "precision": "day",
    "year_hint": null,
    "basis": "Journal online publication date from the citation record",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1021/acs.jcim.0c00411",
    "verification": "verified",
    "note": null
   },
   "code_release": {
    "date": "2024-06-16",
    "precision": "day",
    "year_hint": null,
    "basis": "First public version of the unified re-implementation",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://github.com/EDAPINENUT/CBGBench",
    "verification": "partial",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2022-11",
   "precision": "month",
   "year_hint": null,
   "basis": "Data-version record in a third-party database catalogue",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://ngdc.cncb.ac.cn/databasecommons/database/id/10847",
   "verification": "partial",
   "note": "The version date appears only in a third-party catalogue",
   "description": "A data-version update recorded in the catalogue, not checked against a primary release log; the evaluation code repository has since only changed documentation."
  },
  "next_expected": null,
  "next_expected_null_reason": "No maintainer has announced a new data version or new split.",
  "status": "frozen",
  "status_note": "The dataset has not changed for years; the 2021 split is still in use",
  "scale": {
   "summary": "22.5 million docked poses; the generative split has about 100,000 training complexes and 100 test pockets.",
   "items": [
    {
     "label": "Docked poses",
     "value": "22500000",
     "unit": null,
     "version": "original dataset"
    },
    {
     "label": "Training complexes",
     "value": "about 100000",
     "unit": null,
     "version": "2021 generative split"
    },
    {
     "label": "Test pockets",
     "value": "100",
     "unit": null,
     "version": "2021 generative split"
    },
    {
     "label": "Split filtering criteria",
     "value": "RMSD<1Å; sequence identity<30%",
     "unit": null,
     "version": "2021 generative split"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No experiments; everything is computational docked poses and software scores.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Generate ligands for a given protein pocket, scored by docking, drug-likeness, synthetic accessibility and diversity.",
   "truth": "There is no experimental truth; the reference is docking-software scores and the reference ligands.",
   "metric_primary": "Docking score, including the high-affinity share",
   "metric_secondary": [
    "Drug-likeness and synthetic accessibility",
    "Diversity",
    "Pose validity and interaction recovery"
   ],
   "version_traps": "State which split and preprocessing were used: the widely reused one is looser than the dataset authors' and numbers compare only within one split.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "Docking scores, high-affinity share, drug-likeness, synthetic accessibility, diversity",
   "eval_set": "CrossDocked2020, the 2021 NeurIPS generative split (~100k training complexes, 100 unseen test pockets)",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/pdf/2303.03543",
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "TamGen",
     "model_id": "model-tamgen",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Docking scores, high-affinity share, drug-likeness, synthetic accessibility, diversity",
     "eval_set": "CrossDocked2020, the 2021 NeurIPS generative split (~100k training complexes, 100 unseen test pockets)",
     "source_type": "self",
     "date": {
      "date": "2024-10",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11522292/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11522292/",
     "note": "The main text gives only a placing statement; per-metric values are in the supplement",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: splits and preprocessing differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "TargetDiff",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Docking scores, high-affinity share, drug-likeness, synthetic accessibility, diversity",
     "eval_set": "CrossDocked2020, the 2021 NeurIPS generative split (~100k training complexes, 100 unseen test pockets)",
     "source_type": "self",
     "date": {
      "date": "2023-03",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/pdf/2303.03543",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/pdf/2303.03543",
     "note": "Not a site model; the best-documented strong baseline on this split",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: splits and preprocessing differ, so the ranking row shows no score."
    },
    {
     "rank": null,
     "model_label": "Pocket2Mol",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Docking scores, high-affinity share, drug-likeness, synthetic accessibility, diversity",
     "eval_set": "CrossDocked2020, the 2021 NeurIPS generative split (~100k training complexes, 100 unseen test pockets)",
     "source_type": "third",
     "date": {
      "date": "2023-03",
      "precision": "month",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/pdf/2303.03543",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/pdf/2303.03543",
     "note": "Not a site model; a baseline row re-run in another paper",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: splits and preprocessing differ, so the ranking row shows no score."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "There is no official board and no comparable rank-1 entry, so no headline number is given.",
   "caveat": "There is no official board and papers differ in split and preprocessing, so numbers do not compare; the common split is reported to be looser than the dataset authors' and may leak, and the unified re-implementation is a re-run, not a ranked board.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-tamgen",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11522292/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Self-reported on the 100 test pockets, with values in the supplement",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Download the data for one named split, generate ligands for each of the 100 test pockets and score them with the unified re-implementation for docking, drug-likeness, synthetic accessibility and diversity; no wet lab of your own is needed. Report the split source, preprocessing, number of generated molecules, sampling settings and pose validity. Docking is only a software proxy, so activity claims need experiments or an experimentally grounded benchmark. Pair with PoseBusters and PMO.",
  "pairs_with": [
   "bench-05-posebusters",
   "bench-19-pmo",
   "bench-06-casf",
   "bench-12-dude",
   "bench-10-guacamol"
  ],
  "caveats": [
   "The common split is looser than the dataset authors' and may leak homologous information into training.",
   "Docking scores are only a software proxy with no wet-lab check.",
   "Pockets are treated as rigid and the ligand chemistry drifts from drug-like space, which has pushed later work to other sets.",
   "Docking scores are easy to optimise against, and a high score implies neither synthesizability nor activity.",
   "There is no maintainer or official results table, so cross-paper numbers compare only within one split."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1021/acs.jcim.0c00411",
    "title": "Original dataset paper",
    "kind": "journal",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2303.03543",
    "title": "Model paper reusing and documenting the generative split",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://github.com/EDAPINENUT/CBGBench",
    "title": "Unified re-implementation repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://ngdc.cncb.ac.cn/databasecommons/database/id/10847",
    "title": "Third-party database catalogue entry",
    "kind": "second-hand",
    "date": null
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11522292/",
    "title": "Model paper self-reporting on this split",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2020-08-31",
    "now": "2020-02-20",
    "reason": "Aligned with other entries in using the preprint v1 as first release, with the journal online date listed separately.",
    "source_url": "https://doi.org/10.1021/acs.jcim.0c00411",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "CrossDocked2020 dataset paper (JCIM, Europe PMC first publication date; Crossref online date not retrieved)",
    "now": "paper_v1: ChemRxiv preprint of the CrossDocked2020 paper, 2020-02-20; JCIM journal online 2020-08-31 (Crossref) stored as a separate milestone.",
    "reason": "The basis no longer says the record was not retrieved and names the milestone used instead.",
    "source_url": "https://doi.org/10.1021/acs.jcim.0c00411",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "2020-09-10",
    "now": "2020-08-31",
    "reason": "The previously recorded date came from a literature aggregator; the citation record gives an earlier online date.",
    "source_url": "https://doi.org/10.1021/acs.jcim.0c00411",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries",
    "was": "per-model scores shown in the ranking row",
    "now": "scores withheld; splits and preprocessing differ and one entry has no main-text numbers",
    "reason": "Display rule: entries that are not comparable show no score, and supplement-only per-metric values are not used as board numbers.",
    "source_url": "https://arxiv.org/pdf/2303.03543",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "latest_update",
   "release_milestones.paper_v1",
   "family.members[1]",
   "family.members[2]",
   "ranking.entries[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-32-crossdocked/",
   "en": "/bio/benchmark/benchmarks/crossdocked/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-33-posex",
  "kind": "benchmark",
  "nn": "33",
  "short_slug": "posex",
  "names": {
   "display": "PoseX",
   "full": "PoseX: a protein-ligand docking and co-folding benchmark (self-docking and cross-docking)",
   "full_null_reason": null,
   "aliases": [
    "PoseX",
    "PoseX-SD",
    "PoseX-CD"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "Single benchmark; SD and CD are two subsets of it.",
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Fixed test sets plus an author-run online board",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L3",
   "range": null,
   "qualifier": "Retrospective time-window evaluation, not blind"
  },
  "publisher": {
   "orgs": [
    "PoseX authors (academic consortium; code under the CataAI GitHub organisation)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Compares 23 physics, AI-docking and co-folding methods under one protocol, separating self-docking from cross-docking.",
  "first_release": {
   "date": "2025-05-03",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv 2505.01700 v1",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2505.01700v1",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2025-05-03",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv 2505.01700 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2505.01700v1",
    "verification": "verified",
    "note": "GitHub repository created 2025-04-24; not a release date"
   },
   "journal_online": null,
   "code_release": {
    "date": "2025-06-03",
    "precision": "day",
    "year_hint": null,
    "basis": "Repository v1.0 release",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://github.com/CataAI/PoseX",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2026",
    "precision": "year",
    "year_hint": null,
    "basis": "Accepted as an ICLR 2026 poster (year only)",
    "event_kind": "conference-publication",
    "timestamp_utc": null,
    "source_url": "https://www.iclr.cc/virtual/2026/poster/10007208",
    "verification": "partial",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-04-26",
   "precision": "day",
   "year_hint": null,
   "basis": "v3 date in the arXiv API",
   "event_kind": "paper-revision",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2505.01700",
   "verification": "verified",
   "note": null,
   "description": "arXiv v3 revision (ICLR 2026 version); last repository commits are 2026-03 README/BibTeX edits; online board changes are undated."
  },
  "next_expected": null,
  "next_expected_null_reason": "The authors have announced no new version or board schedule.",
  "status": "active",
  "status_note": "Active; accepted at ICLR 2026; online leaderboard running, no public changelog",
  "scale": {
   "summary": "PoseX-SD 718 entries; PoseX-CD 1,312 entries (109 targets, 371 structures, 362 ligands).",
   "items": [
    {
     "label": "Self-docking entries",
     "value": "718",
     "unit": null,
     "version": "PoseX-SD, arXiv v3"
    },
    {
     "label": "Cross-docking entries",
     "value": "1,312",
     "unit": null,
     "version": "PoseX-CD, arXiv v3"
    },
    {
     "label": "Targets",
     "value": "109",
     "unit": null,
     "version": "PoseX-CD"
    },
    {
     "label": "Structures",
     "value": "371",
     "unit": null,
     "version": "PoseX-CD"
    },
    {
     "label": "Ligands",
     "value": "362",
     "unit": null,
     "version": "PoseX-CD"
    },
    {
     "label": "Methods evaluated",
     "value": "23",
     "unit": null,
     "version": "arXiv v3 (5 physics, 11 AI docking, 7 co-folding)"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Public PDB entries from 2022-2025; no wet lab.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Protein-ligand pose prediction in self- and cross-docking, with pocket-given vs blind and before/after relax settings.",
   "truth": "High-quality PDB crystal structures released 2022-01 to 2025-01.",
   "metric_primary": "Top-1 pose success at RMSD<2Å",
   "metric_secondary": [
    "PoseBusters validity (PB-valid)",
    "Chirality errors and steric clashes"
   ],
   "version_traps": "State SD or CD, pocket-given or blind, before or after relax, and paper version (v1-v3 numbers may differ).",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "third-party",
   "metric": "Docking success rate, top-1 pose RMSD<2Å",
   "eval_set": "PoseX-SD (718 entries, self-docking) and PoseX-CD (1,312 entries, cross-docking), arXiv v3 (ICLR 2026) text",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "http://dock-lab.tech",
    "verification": "partial",
    "note": "Online board could not be loaded this round; snapshot day is the check date"
   },
   "board_url": "http://dock-lab.tech",
   "entries": [
    {
     "rank": null,
     "model_label": "SurfDock",
     "model_id": null,
     "site_model": false,
     "score": "SD 78.0% / CD 77.0%",
     "score_unit": null,
     "metric": "Docking success rate, top-1 pose RMSD<2Å",
     "eval_set": "PoseX-SD (718 entries, self-docking) and PoseX-CD (1,312 entries, cross-docking), arXiv v3 (ICLR 2026) text",
     "source_type": "third",
     "date": {
      "date": "2026-04-26",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2505.01700v3",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2505.01700v3",
     "note": "Not a site model; paper text says overall first on SD and CD (pocket-given track; relax setting not stated)",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "AlphaFold3",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "SD 60.5% overall / 60.3% blind track; CD 68.6% overall / 68.8% blind track",
     "score_unit": null,
     "metric": "Docking success rate, top-1 pose RMSD<2Å",
     "eval_set": "PoseX-SD (718 entries, self-docking) and PoseX-CD (1,312 entries, cross-docking), arXiv v3 (ICLR 2026) text",
     "source_type": "third",
     "date": {
      "date": "2026-04-26",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2505.01700v3",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2505.01700v3",
     "note": "Best co-folding method in the blind track on both sets; paper sections give slightly different values",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Protenix",
     "model_id": "model-protenix",
     "site_model": true,
     "score": "SD 56.3% (blind-docking track); CD not stated in text",
     "score_unit": null,
     "metric": "Docking success rate, top-1 pose RMSD<2Å",
     "eval_set": "PoseX-SD (718 entries, self-docking) and PoseX-CD (1,312 entries, cross-docking), arXiv v3 (ICLR 2026) text",
     "source_type": "third",
     "date": {
      "date": "2026-04-26",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2505.01700v3",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2505.01700v3",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Chai-1",
     "model_id": "model-chai-1",
     "site_model": true,
     "score": "SD 56.1%; CD 67.0% (blind-docking track)",
     "score_unit": null,
     "metric": "Docking success rate, top-1 pose RMSD<2Å",
     "eval_set": "PoseX-SD (718 entries, self-docking) and PoseX-CD (1,312 entries, cross-docking), arXiv v3 (ICLR 2026) text",
     "source_type": "third",
     "date": {
      "date": "2026-04-26",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2505.01700v3",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2505.01700v3",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Boltz-1x",
     "model_id": "model-boltz-1-1x",
     "site_model": true,
     "score": "CD 64.4% (blind-docking track); SD not stated in text",
     "score_unit": null,
     "metric": "Docking success rate, top-1 pose RMSD<2Å",
     "eval_set": "PoseX-SD (718 entries, self-docking) and PoseX-CD (1,312 entries, cross-docking), arXiv v3 (ICLR 2026) text",
     "source_type": "third",
     "date": {
      "date": "2026-04-26",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/html/2505.01700v3",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/html/2505.01700v3",
     "note": "The co-folding exception on chirality, thanks to its built-in physical potential",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "No headline number: the ranking source gives no rank-1 entry (all entries are unranked or protocol-specific); the caveat text is shown instead, as for a not-comparable cell.",
   "caveat": "The online board could not be loaded, so values come from the arXiv v3 text (benchmark authors running others' models); tracks and settings are mixed, so unranked; author-maintained, newer co-folding models absent.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphafold3",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2505.01700v3",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "PoseX authors ran AF3 under the same protocol (third party)",
    "resolution": null
   },
   {
    "model_id": "model-boltz-1-1x",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2505.01700v3",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "The benchmark paper's authors evaluated this model on their own sets.",
    "resolution": null
   },
   {
    "model_id": "model-chai-1",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2505.01700v3",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Same table as above.",
    "resolution": null
   },
   {
    "model_id": "model-diffdock",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2505.01700v3",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Evaluated by the benchmark paper.",
    "resolution": null
   },
   {
    "model_id": "model-protenix",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2505.01700v3",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "PoseX authors ran Protenix under the same protocol",
    "resolution": null
   },
   {
    "model_id": "model-vina",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://arxiv.org/html/2505.01700v3",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Evaluated by the benchmark paper on its Astex subset.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 6,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 6,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Run the repository scripts on both SD and CD, reporting pocket-given vs blind and before/after relax, plus chirality accuracy and PB-valid, not only RMSD. Models trained past 2022 should use post-cutoff entries or flag leakage; no wet lab. Report alongside PoseBusters and Runs N' Poses similarity bins.",
  "pairs_with": [
   "bench-05-posebusters",
   "bench-30-plinder",
   "bench-31-foldbench",
   "bench-06-casf",
   "bench-20-cache"
  ],
  "caveats": [
   "Retrospective on PDB entries up to 2025-01, so newer models may have seen them. Leaderboard is maintained by the paper's authors. Pose accuracy does not measure affinity.",
   "The paper reports most AI methods improve after relax, but relax cannot fix chirality errors.",
   "No train-similarity stratification; memorisation must be checked with a stratified benchmark.",
   "The paper's 23 methods exclude newer models such as Boltz-2 and Pearl."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2505.01700",
    "title": "PoseX paper (arXiv 2505.01700)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://arxiv.org/html/2505.01700v3",
    "title": "PoseX paper v3 full text",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/CataAI/PoseX",
    "title": "PoseX official repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://www.iclr.cc/virtual/2026/poster/10007208",
    "title": "ICLR 2026 poster page",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "http://dock-lab.tech",
    "title": "PoseX online leaderboard (could not be loaded this round)",
    "kind": "leaderboard",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update / repository",
    "was": "Repository has no release",
    "now": "v1.0 release on 2025-06-03; arXiv v3 on 2026-04-26 is the latest sourced event",
    "reason": "The existing record shows a v1.0 repository release, so the site's 'no release' statement is stale.",
    "source_url": "https://github.com/CataAI/PoseX",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "ranking.snapshot_date",
   "release_milestones.official_announcement"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-33-posex/",
   "en": "/bio/benchmark/benchmarks/posex/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-34-abbibench",
  "kind": "benchmark",
  "nn": "34",
  "short_slug": "abbibench",
  "names": {
   "display": "AbBiBench",
   "full": "AbBiBench: antibody-antigen binding benchmark",
   "full_null_reason": null,
   "aliases": [
    "AbBiBench",
    "Antibody Binding Benchmark Dataset"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "One dataset with one maintainer board; no family members.",
  "domain": "antibody",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Fixed set with a zero-shot correlation board",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Retrospective published measurements, evaluated zero-shot"
  },
  "publisher": {
   "orgs": [
    "Academic biomedical informatics group (repository MSBMI-SAFE)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Zero-shot scoring correlations evaluated on antibody-antigen complexes, filling a gap in antibody function evaluation.",
  "first_release": {
   "date": "2025-05-23",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv 2506.04235 v1 (2025-05-23); the Hugging Face dataset and repository creation dates are not used as first release",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2506.04235",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2025-05-23",
    "precision": "day",
    "year_hint": null,
    "basis": "Paper v1 is the earliest verifiable public evidence",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2506.04235",
    "verification": "partial",
    "note": "The Hugging Face dataset was created 2025-04-22, but creation does not prove public availability and is recorded as a note only"
   },
   "paper_v1": {
    "date": "2025-05-23",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv 2506.04235 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2506.04235",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-11-03",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv paper v3 revision",
   "event_kind": "paper-revision",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2506.04235",
   "verification": "verified",
   "note": null,
   "description": "Paper revised to arXiv v3; the repository board was aligned to the manuscript evaluation on 2026-10-19."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no new version or evaluation round.",
  "status": "active",
  "status_note": "Active; the repository board is a snapshot matching the paper",
  "scale": {
   "summary": "216,560 measurements, 13 antibodies, 9 antigens (v3 abstract); Hugging Face copy about 215,699 rows.",
   "items": [
    {
     "label": "Measurements",
     "value": "216,560",
     "unit": "measurements",
     "version": "arXiv v3"
    },
    {
     "label": "Antibodies / antigens",
     "value": "13 / 9",
     "unit": null,
     "version": "arXiv v3"
    },
    {
     "label": "Assays",
     "value": "16",
     "unit": "assays",
     "version": "board"
    },
    {
     "label": "Base methods / implementations",
     "value": "20 / 23",
     "unit": null,
     "version": "board"
    },
    {
     "label": "Hugging Face dataset rows",
     "value": "~215,699",
     "unit": "rows",
     "version": "Hugging Face copy"
    },
    {
     "label": "Design validation candidates",
     "value": "21",
     "unit": "candidates",
     "version": "arXiv v3"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Reuses published binding measurements; the only new experiment tests 21 candidates for reactivity.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Zero-shot correlation between model scores on antibody-antigen complexes and measured binding, plus one small design validation.",
   "truth": "Published binding measurements kept separate by assay, with different units never mixed.",
   "metric_primary": "Per-assay correlation between score and measurement",
   "metric_secondary": [
    "Approximate mean correlation over 16 assays",
    "Reactivity signal for design candidates"
   ],
   "version_traps": "Older circulating size figures are superseded by v3; the board header and the page note describe the statistic differently, and the page note governs.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
   "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
   "snapshot_date": {
    "date": "2026-10-19",
    "precision": "day",
    "year_hint": null,
    "basis": "Date of the \"manuscript-matched snapshot\" stated on the repository board",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://github.com/MSBMI-SAFE/AbBiBench",
   "entries": [
    {
     "rank": 1,
     "model_label": "ProteinMPNN",
     "model_id": null,
     "site_model": false,
     "score": "0.29",
     "score_unit": null,
     "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
     "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
     "source_type": "board",
     "date": {
      "date": "2026-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Repository board snapshot date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/MSBMI-SAFE/AbBiBench",
     "note": "Not a site model; overall first; an inverse-folding method.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "Boltz-2 (ipSAE)",
     "model_id": "model-boltz-2",
     "site_model": true,
     "score": "0.14",
     "score_unit": null,
     "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
     "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
     "source_type": "board",
     "date": {
      "date": "2026-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Repository board snapshot date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/MSBMI-SAFE/AbBiBench",
     "note": "Structure-prediction category; the highest-ranked site model.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 10,
     "model_label": "ESM-3",
     "model_id": "model-esm3",
     "site_model": true,
     "score": "0.05",
     "score_unit": null,
     "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
     "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
     "source_type": "board",
     "date": {
      "date": "2026-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Repository board snapshot date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/MSBMI-SAFE/AbBiBench",
     "note": "Masked language model category.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 11,
     "model_label": "SaProt",
     "model_id": "model-saprot",
     "site_model": true,
     "score": "0.04",
     "score_unit": null,
     "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
     "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
     "source_type": "board",
     "date": {
      "date": "2026-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Repository board snapshot date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/MSBMI-SAFE/AbBiBench",
     "note": "Masked language model category.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 12,
     "model_label": "ESMFold2 (ipSAE)",
     "model_id": "model-esmfold2",
     "site_model": true,
     "score": "0.04",
     "score_unit": null,
     "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
     "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
     "source_type": "board",
     "date": {
      "date": "2026-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Repository board snapshot date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/MSBMI-SAFE/AbBiBench",
     "note": "Structure-prediction category.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 16,
     "model_label": "ProSST",
     "model_id": "model-prosst",
     "site_model": true,
     "score": "-0.00",
     "score_unit": null,
     "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
     "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
     "source_type": "board",
     "date": {
      "date": "2026-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Repository board snapshot date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/MSBMI-SAFE/AbBiBench",
     "note": "Masked language model category.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 19,
     "model_label": "AF3 (ipSAE)",
     "model_id": "model-alphafold3",
     "site_model": true,
     "score": "-0.03",
     "score_unit": null,
     "metric": "Approx. mean rho (header reads 'Approx. mean ρ² ↑'; page states values are two-decimal Spearman correlations from manuscript Figure 3a), each of 16 assays evaluated separately",
     "eval_set": "AbBiBench manuscript-matched snapshot, 20 base methods / 23 implementations across 16 assays",
     "source_type": "board",
     "date": {
      "date": "2026-10-19",
      "precision": "day",
      "year_hint": null,
      "basis": "Repository board snapshot date",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://github.com/MSBMI-SAFE/AbBiBench",
     "note": "Structure-prediction category.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": 1,
   "headline_null_reason": null,
   "caveat": "The board sits in the repository README and states it is a manuscript-matched snapshot: values are copied from a paper figure, not an official rerun; some runners and per-variant results are not public, and rounded means can tie.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphafold3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Benchmark-registered ipSAE correlation (not structure accuracy), read from paper figure, not rerun",
    "resolution": null
   },
   {
    "model_id": "model-boltz-2",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Benchmark-registered ipSAE entry; not structure accuracy",
    "resolution": null
   },
   {
    "model_id": "model-esm3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Benchmark-registered score",
    "resolution": null
   },
   {
    "model_id": "model-esmfold2",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Benchmark-registered ipSAE entry",
    "resolution": null
   },
   {
    "model_id": "model-prosst",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Benchmark-registered score",
    "resolution": null
   },
   {
    "model_id": "model-saprot",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "medium",
    "verification": "unverified",
    "note": "Benchmark-registered score",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 6,
   "by_relation": {
    "official-participation": 6,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Score the public complex set zero-shot with the repository scripts and report per-assay correlations rather than a pooled mean; state method category, score definition, any antibody-specific fine-tuning and the structure source. Claims of replacing experimental triage need prospective measurements.",
  "pairs_with": [
   "bench-36-aintibody",
   "bench-35-flab",
   "bench-31-foldbench",
   "bench-28-adaptyv"
  ],
  "caveats": [
   "Correlations are low (the top method averages about 0.29 across 16 assays); assays must be reported separately and units never mixed.",
   "The design validation covers 21 candidates with no reported controls, replicates or thresholds, so it shows signal only.",
   "Board values are copied from a paper figure rather than an official rerun; some runners and per-variant results are unpublished, limiting replication.",
   "The paper states the ranking suits prioritising promising variants rather than excluding weak ones.",
   "The data are reused published measurements shaped by the original studies rather than one uniform assay campaign."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2506.04235",
    "title": "AbBiBench paper (arXiv 2506.04235)",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/MSBMI-SAFE/AbBiBench",
    "title": "AbBiBench official repository and board",
    "kind": "leaderboard",
    "date": null
   },
   {
    "url": "https://huggingface.co/datasets/AbBibench/Antibody_Binding_Benchmark_Dataset",
    "title": "AbBiBench dataset (Hugging Face, CC-BY-4.0)",
    "kind": "repository",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "scale",
    "was": "about 184,500 measurements, 14 antibodies, 15 models",
    "now": "216,560 measurements, 13 antibodies, 9 antigens (arXiv v3); 20 base methods / 23 implementations on the board",
    "reason": "Older figures are superseded by the v3 abstract.",
    "source_url": "https://arxiv.org/abs/2506.04235",
    "date_found": "2026-11-16"
   },
   {
    "field": "wetlab_note",
    "was": "one wet-lab round of 21 candidates described as affinity validation",
    "now": "the 216,560 values are reused published measurements and the only new experiment is a 21-candidate reactivity test, not an affinity measurement",
    "reason": "The new experiment measures a reactivity signal, not affinity.",
    "source_url": "https://arxiv.org/abs/2506.04235",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "six entries labelled as board results",
    "now": "board values are correlations copied from a paper figure, so the entries are flagged unverified",
    "reason": "Board values come from a paper figure without an official rerun and are flagged unverified.",
    "source_url": "https://github.com/MSBMI-SAFE/AbBiBench",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "release_milestones.first_public_evidence",
   "ranking.snapshot_date",
   "ranking.entries[0]",
   "ranking.entries[1]",
   "ranking.entries[2]",
   "ranking.entries[3]",
   "ranking.entries[4]",
   "ranking.entries[5]",
   "ranking.entries[6]",
   "models_reported[0]",
   "models_reported[1]",
   "models_reported[2]",
   "models_reported[3]",
   "models_reported[4]",
   "models_reported[5]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-34-abbibench/",
   "en": "/bio/benchmark/benchmarks/abbibench/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-35-flab",
  "kind": "benchmark",
  "nn": "35",
  "short_slug": "flab",
  "names": {
   "display": "FLAb / FLAb2",
   "full": null,
   "full_null_reason": "Public materials give no single expanded name.",
   "aliases": [
    "FLAb",
    "FLAb2"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "FLAb2 is treated as a new version of the same benchmark, not a separate family member.",
  "domain": "antibody",
  "domains_secondary": [
   "protein"
  ],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Aggregated public datasets, zero-shot and few-shot settings.",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L3"
   ],
   "qualifier": "Heterogeneous retrospective data; a few sets carry held-out splits."
  },
  "publisher": {
   "orgs": [
    "Johns Hopkins University"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Aggregated retrospective antibody developability benchmark with a negative headline result.",
  "first_release": {
   "date": "2024-01-15",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv preprint v1 (10.1101/2024.01.13.575504)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1101/2024.01.13.575504",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": null,
    "precision": null,
    "year_hint": null,
    "basis": "Repository creation is excluded as first-release evidence, so no date is recorded.",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://github.com/Graylab/FLAb",
    "verification": "unverified",
    "note": null
   },
   "paper_v1": {
    "date": "2024-01-15",
    "precision": "day",
    "year_hint": null,
    "basis": "bioRxiv preprint v1 of FLAb v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/2024.01.13.575504",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2025-12-23",
    "precision": "day",
    "year_hint": null,
    "basis": "GitHub release v1.0.0 for FLAb2",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://github.com/Graylab/FLAb/releases/tag/v1.0.0",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-05-12",
   "precision": "day",
   "year_hint": null,
   "basis": "commits on the GitHub main branch (repository cleanup)",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://github.com/Graylab/FLAb/commits/main",
   "verification": "verified",
   "note": null,
   "description": "Main-branch cleanup: directories removed and README edited; no release since FLAb2 v1.0.0 (2025-12-23)."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no next version or round.",
  "status": "active",
  "status_note": "Repository still receiving commits; FLAb2 is a December 2025 preprint, not peer reviewed.",
  "scale": {
   "summary": "FLAb2 abstract: 7 properties, 32 studies, over 4M records, 30 models; the README gives different figures.",
   "items": [
    {
     "label": "Property classes",
     "value": "7",
     "unit": null,
     "version": "FLAb2"
    },
    {
     "label": "Studies included",
     "value": "32",
     "unit": null,
     "version": "abstract"
    },
    {
     "label": "Antibody records",
     "value": ">4,000,000",
     "unit": null,
     "version": "abstract"
    },
    {
     "label": "Models evaluated",
     "value": "30",
     "unit": null,
     "version": "abstract"
    },
    {
     "label": "Datasets",
     "value": "241",
     "unit": null,
     "version": "repository README"
    },
    {
     "label": "Assay records",
     "value": ">3,000,000",
     "unit": null,
     "version": "repository README"
    }
   ]
  },
  "wetlab_refresh": "accumulating",
  "wetlab_note": "Community contributes published assay data; the maintainers run no new assays.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Given antibody sequences (sometimes structures), predict developability values: expression, thermostability, aggregation, polyreactivity, binding, immunogenicity, pharmacokinetics.",
   "truth": "Published antibody assay measurements from public studies, pooled across labs and assays.",
   "metric_primary": "Spearman correlation between model score and measurement.",
   "metric_secondary": [
    "Few-shot correlation after fine-tuning."
   ],
   "version_traps": "FLAb v1 covers six properties and FLAb2 seven; abstract and README scales differ, so cite version and source.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "third-party",
   "metric": "Spearman rank correlation (zero-shot and few-shot, per property)",
   "eval_set": "FLAb2 (241 datasets, 7 therapeutic properties), FLAb2 preprint evaluation",
   "snapshot_date": {
    "date": "2025-12-27",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "IgFold",
     "model_id": null,
     "site_model": false,
     "score": "ρ = 0.12",
     "score_unit": null,
     "metric": "Spearman rank correlation (zero-shot and few-shot, per property)",
     "eval_set": "FLAb2 (241 datasets, 7 therapeutic properties); FLAb2 preprint evaluation",
     "source_type": "third",
     "date": {
      "date": "2025-12-27",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
     "note": "Not a site model; best overall zero-shot result, only 0.12.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Chai-1",
     "model_id": "model-chai-1",
     "site_model": true,
     "score": "ρ = 0.45",
     "score_unit": null,
     "metric": "Spearman rank correlation (zero-shot and few-shot, per property)",
     "eval_set": "FLAb2 (241 datasets, 7 therapeutic properties); FLAb2 preprint evaluation",
     "source_type": "third",
     "date": {
      "date": "2025-12-27",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
     "note": "Best zero-shot predictor of thermostability; same paper reports -0.17 for aggregation; no overall value given.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "ESM2 3B and ISM 3B (tied)",
     "model_id": null,
     "site_model": false,
     "score": "ρ = 0.56",
     "score_unit": null,
     "metric": "Spearman rank correlation (zero-shot and few-shot, per property)",
     "eval_set": "FLAb2 (241 datasets, 7 therapeutic properties); FLAb2 preprint evaluation",
     "source_type": "third",
     "date": {
      "date": "2025-12-27",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
     "note": "Not site models; joint best overall few-shot result.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "The source has no rank-1 entry: values belong to different properties and settings, so only the caveat is shown.",
   "caveat": "No live leaderboard; figures come from the FLAb2 preprint (benchmark authors scoring third-party models, not peer reviewed) and cover one site model on one property.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-chai-1",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "The FLAb2 paper evaluates this model zero-shot on thermostability; two site surfaces label the source type differently.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 1,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Take data and scoring scripts from the repository; report zero-shot and few-shot separately, with Spearman correlation per property and the dataset count; first run a germline edit-distance baseline to rule out germline bias; state whether FLAb v1 or FLAb2 is used and any training overlap. Claims about new antibodies need a uniform-assay held-out blind test or own experiments.",
  "pairs_with": [
   "bench-43-gdpa1",
   "bench-34-abbibench",
   "bench-36-aintibody",
   "bench-02-proteingym",
   "bench-26-flip"
  ],
  "caveats": [
   "Data are heterogeneous across labs and assays, so one property pools different protocols.",
   "FLAb2 reports no significant zero-shot correlation on about 80% of datasets; the best overall value is 0.12.",
   "FLAb2 reports that germline edit distance explains about 40% of language models' apparent skill.",
   "Scale is inconsistent: abstract and repository README give different dataset and record counts.",
   "FLAb2 is a preprint, not peer reviewed; uniform-assay blind evidence comes from GDPa1 instead."
  ],
  "sources": [
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
    "title": "FLAb2 preprint (bioRxiv, PMC12767642)",
    "kind": "preprint",
    "date": {
     "date": "2025-12-27",
     "precision": "day",
     "year_hint": null,
     "basis": "bioRxiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://doi.org/10.1101/2024.01.13.575504",
    "title": "FLAb v1 preprint (bioRxiv 10.1101/2024.01.13.575504)",
    "kind": "preprint",
    "date": {
     "date": "2024-01-15",
     "precision": "day",
     "year_hint": null,
     "basis": "bioRxiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://doi.org/10.1101/2024.01.13.575504",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://github.com/Graylab/FLAb",
    "title": "Official repository Graylab/FLAb",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://github.com/Graylab/FLAb/releases/tag/v1.0.0",
    "title": "FLAb2 release v1.0.0",
    "kind": "repository",
    "date": {
     "date": "2025-12-23",
     "precision": "day",
     "year_hint": null,
     "basis": "GitHub release v1.0.0",
     "event_kind": "code-release",
     "timestamp_utc": null,
     "source_url": "https://github.com/Graylab/FLAb/releases/tag/v1.0.0",
     "verification": "verified",
     "note": null
    }
   }
  ],
  "corrections": [
   {
    "field": "latest_update_kind",
    "was": "new-version",
    "now": "code-maintenance-only",
    "reason": "The 2026-05-12 commits remove directories and edit the README; there has been no release since v1.0.0 (2025-12-23), and untagged commits are not a new version.",
    "source_url": "https://github.com/Graylab/FLAb/commits/HEAD.atom",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_desc",
    "was": "FLAb2 models, README and figure merged into Graylab/FLAb main from flab2-dev; fitness unit fixes; legacy dirs removed. Later only README fix 2026-08-17.",
    "now": "Repository cleanup/restructure on main (2026-05-12: envs/analysis/sbatch directories removed, README edited); no release since FLAb2 v1.0.0 (2025-12-23).",
    "reason": "Description synced with the date; the unsourced claim that FLAb2 models were merged is removed.",
    "source_url": "https://github.com/Graylab/FLAb/commits/HEAD.atom",
    "date_found": "2026-11-16"
   },
   {
    "field": "members",
    "was": "[{\"name\": \"FLAb\", \"first_release\": \"2024-01-15\", \"basis\": \"bioRxiv v1\"}, {\"name\": \"FLAb2\", \"first_release\": \"2025-12-27\", \"basis\": \"bioRxiv v1 10.64898/2025.12.27.696706 (GitHub release v1.0.0 2025-12-23)\"}]",
    "now": "[{\"name\": \"FLAb\", \"first_release\": \"2024-01-28\", \"basis\": \"existing record\"}, {\"name\": \"FLAb2\", \"first_release\": \"2025-12-23\", \"basis\": \"GitHub release v1.0.0, earliest public evidence (precedes bioRxiv v1)\"}]",
    "reason": "The FLAb2 member date now uses the earliest sourced public release, the GitHub release preceding preprint v1.",
    "source_url": "https://github.com/Graylab/FLAb/releases/tag/v1.0.0",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.headline",
    "was": "headline number shown for FLAb",
    "now": "no headline number; the caveat text is shown instead",
    "reason": "The source has no rank-1 entry; none of the six benchmarks in this situation shows a headline number.",
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12767642/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "models_reported[0].source_type",
   "ranking.entries[1].note"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-35-flab/",
   "en": "/bio/benchmark/benchmarks/flab/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-36-aintibody",
  "kind": "benchmark",
  "nn": "36",
  "short_slug": "aintibody",
  "names": {
   "display": "AIntibody",
   "full": null,
   "full_null_reason": "The organizers use only the name AIntibody; no expanded form exists.",
   "aliases": [
    "AIntibody",
    "AIntibody challenge"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "Only one completed round exists, so there are no family members.",
  "domain": "antibody",
  "domains_secondary": [
   "protein"
  ],
  "inclusion": "supplementary",
  "type": "A",
  "type_note": "Prospective blind challenge scored per task.",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L5",
   "range": null,
   "qualifier": "Blind test with prospective wet-lab work under uniform protocols; no independent replication."
  },
  "publisher": {
   "orgs": [
    "Specifica (IQVIA)"
   ],
   "org_type": "platform-company"
  },
  "one_liner": "Antibody-side prospective blind test: submitted designs get uniform affinity and developability measurement.",
  "first_release": {
   "date": "2024-11-04",
   "precision": "day",
   "year_hint": null,
   "basis": "Challenge design paper online (Nature Biotechnology, 10.1038/s41587-024-02469-9)",
   "event_kind": "official-announcement",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1038/s41587-024-02469-9",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": null,
   "journal_online": {
    "date": "2026-08-19",
    "precision": "day",
    "year_hint": null,
    "basis": "Results paper online (Nature Biotechnology)",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2024-11-04",
    "precision": "day",
    "year_hint": null,
    "basis": "Challenge design paper online",
    "event_kind": "official-announcement",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/s41587-024-02469-9",
    "verification": "verified",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-08-19",
   "precision": "day",
   "year_hint": null,
   "basis": "Results paper online (Nature Biotechnology)",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
   "verification": "verified",
   "note": null,
   "description": "Round-1 results paper online: 29 organizations submitted 511 sequences, measured for affinity and developability; vendor releases followed on 2026-08-27."
  },
  "next_expected": null,
  "next_expected_null_reason": "Secondary reports say a society will run the next round, but the organizers have announced no date or target.",
  "status": "low-frequency",
  "status_note": "First round finished; the next round's host appears only in secondary reports, with no date or target.",
  "scale": {
   "summary": "29 organizations submitted 511 sequences across 3 tasks; fewer than 511 unique sequences were actually tested.",
   "items": [
    {
     "label": "Organizations",
     "value": "29",
     "unit": null,
     "version": "round 1"
    },
    {
     "label": "Submitted sequences",
     "value": "511",
     "unit": null,
     "version": "round 1"
    },
    {
     "label": "Unique sequences tested",
     "value": "<511",
     "unit": null,
     "version": "round 1"
    },
    {
     "label": "Tasks",
     "value": "3",
     "unit": null,
     "version": "round 1"
    },
    {
     "label": "Developability assays",
     "value": "5",
     "unit": null,
     "version": "round 1"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "New measurements each round: affinity by both HT-SPR and KinExA, and a five-assay developability panel run by independent laboratories under uniform protocols.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Three tasks: computational affinity maturation from sequencing output, ranking affinity within HCDR3 clusters, and de novo CDR design against a protein absent from the selection output.",
   "truth": "Newly measured experimental results under uniform protocols: affinity plus five developability assays, against SARS-CoV-2 spike RBD.",
   "metric_primary": "Measured equilibrium dissociation constant KD.",
   "metric_secondary": [
    "Whether the five developability thresholds are passed."
   ],
   "version_traps": "Placements cannot be pooled across the three tasks; headline numbers need the task and assay method, and participation counts differ between vendor releases and the paper.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Measured KD (HT-SPR and KinExA) plus five developability screens; placement per task",
   "eval_set": "AIntibody round 1, 3 challenges, 511 submitted sequences from 29 organizations, SARS-CoV-2 spike RBD",
   "snapshot_date": {
    "date": "2026-08-19",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://doi.org/10.1038/s41587-026-03238-6",
   "entries": [
    {
     "rank": 1,
     "model_label": "AuraIDE / AuraBind (Aureka Biotechnologies, internal model)",
     "model_id": null,
     "site_model": false,
     "score": "KD 94.7 pM (KinExA); best experimental antibody 113 pM",
     "score_unit": null,
     "metric": "Measured KD (HT-SPR and KinExA) plus five developability screens; placement per task",
     "eval_set": "AIntibody round 1, 3 challenges, 511 designs from 29 organizations, SARS-CoV-2 spike RBD; all designs made and measured in one lab",
     "source_type": "board",
     "date": {
      "date": "2026-08-19",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
      "verification": "verified",
      "note": null
     },
     "url": "https://doi.org/10.1038/s41587-026-03238-6",
     "note": "Not a site model; first in task 1, with second and fifth also reported by the same organization; any link to open weights is a company statement only.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "IgGM (Tencent AI4S)",
     "model_id": "model-iggm",
     "site_model": true,
     "score": null,
     "score_unit": null,
     "metric": "Measured KD (HT-SPR and KinExA) plus five developability screens; placement per task",
     "eval_set": "AIntibody round 1, 3 challenges, 511 designs from 29 organizations, SARS-CoV-2 spike RBD; all designs made and measured in one lab",
     "source_type": "self",
     "date": {
      "date": "2025-08-22",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/TencentAI4S/IgGM",
      "verification": "unverified",
      "note": "The date is the repository entry's, not an organizer announcement."
     },
     "url": "https://github.com/TencentAI4S/IgGM",
     "note": "Developer claims a top-three placement in a 2025-08 repository entry; no value, no task named, not confirmed in official materials.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "The results paper is paywalled, so figures come from vendor releases and the site's earlier note; placements sit with organizations or internal models, vendor claims conflict, and task 3 values must not be mixed with task 1.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-iggm",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/TencentAI4S/IgGM",
    "evidence_site_page_only": false,
    "confidence": "low",
    "verification": "unverified",
    "note": "The team claims a top-three placement in the affinity-optimization task; no per-team standing is cited and official materials do not confirm it.",
    "resolution": null
   },
   {
    "model_id": "model-opendde",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.thailand-business-news.com/pr-news/nature-biotechnology-aureka-wins-the-global-blinded-ai-antibody-benchmark-ai-design-surpasses-the-best-experimental-result",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "The placement belongs to that company's internal model and cannot be credited to the open weights; the site matrix and model page disagree.",
    "resolution": "Ruled unconfirmed: the public weights cannot be matched to the competition entry."
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 1,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 1
   }
  },
  "how_to_use": "There is no open submission: either wait for the next round or reproduce the same pipeline. Screen internally on the AbBiBench and FLAb retrospective sets first, then keep a prospective arm. Report the three tasks separately without pooling placements, and give the assay method, KD and all five developability results with the number of sequences tested.",
  "pairs_with": [
   "bench-43-gdpa1",
   "bench-34-abbibench",
   "bench-35-flab",
   "bench-31-foldbench",
   "bench-28-adaptyv"
  ],
  "caveats": [
   "The paper's conclusion is conservative: results do not transfer across tasks and most designs did not beat conventional screening.",
   "Except for one model, predicting high-affinity clones within HCDR3 clusters did worse than picking at random.",
   "Participation and submission counts differ between sources, and much of the coverage is vendor press releases.",
   "A single 2.9 pM winning claim appears only on social media, with no public confirmation that it passed developability screening.",
   "Placements sit with organizations or internal models, open weights have no attributable placement, and there is no independent replication."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1038/s41587-026-03238-6",
    "title": "AIntibody results paper (Nature Biotechnology 2026)",
    "kind": "journal",
    "date": {
     "date": "2026-08-19",
     "precision": "day",
     "year_hint": null,
     "basis": "journal online date",
     "event_kind": "journal-online",
     "timestamp_utc": null,
     "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://doi.org/10.1038/s41587-024-02469-9",
    "title": "AIntibody challenge design paper (Nature Biotechnology 2024)",
    "kind": "journal",
    "date": {
     "date": "2024-11-04",
     "precision": "day",
     "year_hint": null,
     "basis": "journal online date",
     "event_kind": "journal-online",
     "timestamp_utc": null,
     "source_url": "https://doi.org/10.1038/s41587-024-02469-9",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://www.sovereignmagazine.com/article/aintibody-ai-antibody-design-nature-biotech-212438",
    "title": "Press coverage: results and next-round arrangements",
    "kind": "press",
    "date": null
   },
   {
    "url": "https://www.drugdiscoverynews.com/zymo-research-places-top-four-in-ai-antibody-benchmark-17531",
    "title": "Press coverage: vendor placement claims and the next round's host",
    "kind": "press",
    "date": null
   },
   {
    "url": "https://www.thailand-business-news.com/pr-news/nature-biotechnology-aureka-wins-the-global-blinded-ai-antibody-benchmark-ai-design-surpasses-the-best-experimental-result",
    "title": "Company press release (syndicated)",
    "kind": "press",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "scale",
    "was": "511 designs from 29 organizations; same-lab KinExA plus five developability assays; 14 days / max 10 sequences per task",
    "now": "511 submitted sequences from 29 organizations; affinity by HT-SPR and KinExA; five-assay developability panel run by independent laboratories under uniform protocols; fewer than 511 unique sequences were actually tested",
    "reason": "Per the results paper: two assay methods, run by independent laboratories under uniform protocols, and fewer than 511 unique sequences tested.",
    "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
    "date_found": "2026-11-16"
   },
   {
    "field": "scale",
    "was": "14 days and at most 10 sequences per team per task",
    "now": "removed",
    "reason": "The limit cannot be found in the paper and is removed.",
    "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
    "date_found": "2026-11-16"
   },
   {
    "field": "wetlab_note",
    "was": "fresh KinExA affinity and five developability assays in one lab",
    "now": "HT-SPR and KinExA affinity, five-assay developability panel run by independent laboratories under uniform protocols",
    "reason": "The site's “same lab, KinExA only” does not match the paper.",
    "source_url": "https://doi.org/10.1038/s41587-026-03238-6",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "leaderboard entry recorded for the open-weight model",
    "now": "disputed-or-unconfirmed; the placement belongs to a company-internal model",
    "reason": "The placement belongs to a company-internal model; the open weights have no attributable entry, and the matrix contradicts the model page.",
    "source_url": "https://www.thailand-business-news.com/pr-news/nature-biotechnology-aureka-wins-the-global-blinded-ai-antibody-benchmark-ai-design-surpasses-the-best-experimental-result",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "ranking.entries[1]",
   "ranking.caveat",
   "next_expected",
   "models_reported[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-36-aintibody/",
   "en": "/bio/benchmark/benchmarks/aintibody/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-37-traitgym",
  "kind": "benchmark",
  "nn": "37",
  "short_slug": "traitgym",
  "names": {
   "display": "TraitGym",
   "full": null,
   "full_null_reason": "The authors use only the name TraitGym; no expanded form exists.",
   "aliases": [
    "TraitGym"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "The two subsets are configurations of one dataset, not a family.",
  "domain": "genomics-rna",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Fixed variant set plus a public leaderboard.",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L3",
   "range": null,
   "qualifier": "Retrospective evaluation with carefully matched controls; not blind."
  },
  "publisher": {
   "orgs": [
    "University of California, Berkeley"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Benchmark classifying causal non-coding variants against matched controls.",
  "first_release": {
   "date": "2025-02-12",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv preprint v1 (10.1101/2025.02.11.637758); Hugging Face dataset and leaderboard creation dates are not used as first-release evidence",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://doi.org/10.1101/2025.02.11.637758",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": null,
    "precision": null,
    "year_hint": null,
    "basis": "The Hugging Face dataset (2025-01-26) and leaderboard (2025-02-06) creation dates are excluded as first-release evidence.",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://huggingface.co/datasets/songlab/TraitGym",
    "verification": "unverified",
    "note": null
   },
   "paper_v1": {
    "date": "2025-02-12",
    "precision": "day",
    "year_hint": null,
    "basis": "bioRxiv preprint v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://doi.org/10.1101/2025.02.11.637758",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2025-03-25",
    "precision": "day",
    "year_hint": null,
    "basis": "GitHub tag 0.1 (commit date)",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": "https://github.com/songlab-cal/TraitGym/releases/tag/0.1",
    "verification": "verified",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-03-25",
   "precision": "day",
   "year_hint": null,
   "basis": "commit date of GitHub tag 0.1; since then main has only a CI commit",
   "event_kind": "code-maintenance-only",
   "timestamp_utc": null,
   "source_url": "https://github.com/songlab-cal/TraitGym",
   "verification": "partial",
   "note": "No release object was found; the date is the tag's commit date.",
   "description": "Code tag 0.1; since then main has only a CI commit (2026-01-19) plus unmerged revision branches; paper v2 appeared on 2025-03-04."
  },
  "next_expected": null,
  "next_expected_null_reason": "The authors have announced no new version; the revision branches carry no announced date.",
  "status": "low-frequency",
  "status_note": "The leaderboard is still online but has had no new entries since 2025-03; the paper is still a preprint.",
  "scale": {
   "summary": "338 Mendelian variants (113 traits); 1,140 complex-trait variants (83 traits); nine matched controls each.",
   "items": [
    {
     "label": "Mendelian causal variants",
     "value": "338",
     "unit": null,
     "version": "matched_9"
    },
    {
     "label": "Mendelian controls",
     "value": "3,042",
     "unit": null,
     "version": "matched_9"
    },
    {
     "label": "Complex-trait variants",
     "value": "1,140",
     "unit": null,
     "version": "matched_9"
    },
    {
     "label": "Complex-trait controls",
     "value": "10,260",
     "unit": null,
     "version": "matched_9"
    },
    {
     "label": "Traits covered",
     "value": "113 / 83",
     "unit": null,
     "version": "Mendelian / complex"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Reuses public OMIM and UK Biobank fine-mapping results; no variants are added.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Binary classification separating known or strong-candidate causal non-coding regulatory variants from controls matched on position, consequence and related covariates.",
   "truth": "Mendelian positives come from OMIM causal variants; complex-trait positives come from statistical fine-mapping candidates.",
   "metric_primary": "AUPRC weighted-averaged by chromosome.",
   "metric_secondary": [
    "Bootstrap standard error."
   ],
   "version_traps": "State the subset (Mendelian or complex traits), the configuration (matched_9 or all) and whether the track is zero-shot or linear probing; standings are not interchangeable.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "AUPRC weighted-averaged by chromosome (main metric), with standard error",
   "eval_set": "TraitGym Mendelian traits (matched_9) and complex traits (matched_9), zero-shot and linear-probing tracks",
   "snapshot_date": {
    "date": "2025-03-07",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
   "entries": [
    {
     "rank": 1,
     "model_label": "CADD",
     "model_id": null,
     "site_model": false,
     "score": "0.712 (se 0.038)",
     "score_unit": null,
     "metric": "AUPRC weighted-averaged by chromosome (main metric), with standard error",
     "eval_set": "TraitGym Mendelian traits (matched_9) and Complex traits (matched_9), subset 'all', zero-shot and linear-probing tracks",
     "source_type": "board",
     "date": {
      "date": "2025-03-07",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
     "note": "Not a site model; first on the Mendelian set, zero-shot.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 5,
     "model_label": "Evo2 40B",
     "model_id": "model-evo-2",
     "site_model": true,
     "score": "0.557 (se 0.051)",
     "score_unit": null,
     "metric": "AUPRC weighted-averaged by chromosome (main metric), with standard error",
     "eval_set": "TraitGym Mendelian traits (matched_9) and Complex traits (matched_9), subset 'all', zero-shot and linear-probing tracks",
     "source_type": "board",
     "date": {
      "date": "2025-03-07",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
     "note": "Mendelian set, zero-shot; the 7B checkpoint scores 0.385 (rank 10) and the 1B 0.254 (rank 13).",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 7,
     "model_label": "Borzoi",
     "model_id": "model-borzoi",
     "site_model": true,
     "score": "0.436 (se 0.052)",
     "score_unit": null,
     "metric": "AUPRC weighted-averaged by chromosome (main metric), with standard error",
     "eval_set": "TraitGym Mendelian traits (matched_9) and Complex traits (matched_9), subset 'all', zero-shot and linear-probing tracks",
     "source_type": "board",
     "date": {
      "date": "2025-03-07",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
     "note": "Mendelian set, zero-shot.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 1,
     "model_label": "Enformer",
     "model_id": null,
     "site_model": false,
     "score": "0.303 (se 0.010)",
     "score_unit": null,
     "metric": "AUPRC weighted-averaged by chromosome (main metric), with standard error",
     "eval_set": "TraitGym Mendelian traits (matched_9) and Complex traits (matched_9), subset 'all', zero-shot and linear-probing tracks",
     "source_type": "board",
     "date": {
      "date": "2025-03-07",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
     "note": "No site model page; first on complex traits with linear probing, where the site has only the same-named benchmark entry.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "Borzoi",
     "model_id": "model-borzoi",
     "site_model": true,
     "score": "0.297 (se 0.014)",
     "score_unit": null,
     "metric": "AUPRC weighted-averaged by chromosome (main metric), with standard error",
     "eval_set": "TraitGym Mendelian traits (matched_9) and Complex traits (matched_9), subset 'all', zero-shot and linear-probing tracks",
     "source_type": "board",
     "date": {
      "date": "2025-03-07",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
     "note": "Complex traits, linear probing; the best standing of any site model on the board.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 9,
     "model_label": "Evo2 40B",
     "model_id": "model-evo-2",
     "site_model": true,
     "score": "0.153 (se 0.007)",
     "score_unit": null,
     "metric": "AUPRC weighted-averaged by chromosome (main metric), with standard error",
     "eval_set": "TraitGym Mendelian traits (matched_9) and Complex traits (matched_9), subset 'all', zero-shot and linear-probing tracks",
     "source_type": "board",
     "date": {
      "date": "2025-03-07",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
      "verification": "verified",
      "note": null
     },
     "url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
     "note": "Complex traits, linear probing.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": 1,
   "headline_null_reason": null,
   "caveat": "The board snapshot is 2025-03-07 with no models added since; ranks are per subset and track and cannot be compared across slices; values keep the board's own rounding.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-alphagenome",
    "relation_type": "self-reported-modified-protocol",
    "source_type": "self",
    "evidence_url": "https://doi.org/10.1038/s41586-025-10014-0",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Uses the variant set and matched controls but reports enrichment rather than AUPRC, so it is not board-comparable.",
    "resolution": null
   },
   {
    "model_id": "model-borzoi",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://doi.org/10.1101/2025.02.11.637758",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Evaluated on this set by the benchmark's authors.",
    "resolution": null
   },
   {
    "model_id": "model-evo-2",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://www.biorxiv.org/content/10.1101/2025.02.11.637758",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Evaluated on this set by the benchmark's authors.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 3,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 1,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 2,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Take the data from Hugging Face and score with the provided scripts. Report chromosome-weighted AUPRC with standard errors separately for the Mendelian and complex-trait matched_9 configurations; table alignment-based methods alongside and give the gap to an ensemble baseline; split by variant class so one class cannot dominate; state whether the track is zero-shot or linear probing. Using the variant set is not the same as entering the board, and new-trait claims need independent functional validation or a prospective variant set.",
  "pairs_with": [
   "bench-08-enformer",
   "bench-11-clinvar",
   "bench-16-nt_gue",
   "bench-38-bend",
   "bench-13-cagi"
  ],
  "caveats": [
   "Complex-trait positives come from statistical fine-mapping, so causality is probabilistic rather than experimentally confirmed.",
   "The Mendelian set has only 338 positives unevenly spread across traits, so small gaps are not a ranking.",
   "Not blind: variants and controls come from public data that may sit inside model training corpora.",
   "Results are stratified: alignment-based methods lead on Mendelian and complex disease traits, while track-prediction models do better on complex non-disease traits.",
   "The board has had no entries since 2025-03, and AlphaGenome is absent from it, using a different protocol."
  ],
  "sources": [
   {
    "url": "https://doi.org/10.1101/2025.02.11.637758",
    "title": "TraitGym preprint (bioRxiv 10.1101/2025.02.11.637758)",
    "kind": "preprint",
    "date": {
     "date": "2025-02-12",
     "precision": "day",
     "year_hint": null,
     "basis": "bioRxiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://doi.org/10.1101/2025.02.11.637758",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://www.biorxiv.org/content/10.1101/2025.02.11.637758v2",
    "title": "TraitGym preprint v2",
    "kind": "preprint",
    "date": {
     "date": "2025-03-04",
     "precision": "day",
     "year_hint": null,
     "basis": "bioRxiv v2",
     "event_kind": "paper-revision",
     "timestamp_utc": null,
     "source_url": "https://www.biorxiv.org/content/10.1101/2025.02.11.637758v2",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://github.com/songlab-cal/TraitGym",
    "title": "Official repository songlab-cal/TraitGym",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://huggingface.co/datasets/songlab/TraitGym",
    "title": "Hugging Face dataset",
    "kind": "package-registry",
    "date": null
   },
   {
    "url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
    "title": "Hugging Face leaderboard",
    "kind": "leaderboard",
    "date": {
     "date": "2025-03-07",
     "precision": "day",
     "year_hint": null,
     "basis": "leaderboard Space last modified",
     "event_kind": "leaderboard-entry",
     "timestamp_utc": null,
     "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12851941/",
    "title": "AlphaGenome paper (Nature 2026, PMC12851941)",
    "kind": "journal",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "latest_update",
    "was": "2025-07-04",
    "now": "2025-03-25",
    "reason": "Only tag 0.1 (commit date 2025-03-25) was found, with no release object; the Hugging Face dataset and leaderboard were last modified 2025-03-25 and 2025-03-07, and the earlier date has no source.",
    "source_url": "https://github.com/songlab-cal/TraitGym",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update_kind",
    "was": "new-version",
    "now": "code-maintenance-only",
    "reason": "Since then main carries only a CI commit and the revision branches are unmerged, which is not a new version.",
    "source_url": "https://github.com/songlab-cal/TraitGym/branches",
    "date_found": "2026-11-16"
   },
   {
    "field": "status_note",
    "was": "leaderboard continuously running",
    "now": "leaderboard still online but no new entries since 2025-03",
    "reason": "The leaderboard Space was last modified 2025-03-07, while “continuously running” reads as still accepting entries.",
    "source_url": "https://huggingface.co/spaces/songlab/TraitGym-leaderboard",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "ranking.snapshot_date",
   "models_reported[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-37-traitgym/",
   "en": "/bio/benchmark/benchmarks/traitgym/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-38-bend",
  "kind": "benchmark",
  "nn": "38",
  "short_slug": "bend",
  "names": {
   "display": "BEND",
   "full": "Benchmarking DNA Language Models on Biologically Meaningful Tasks",
   "full_null_reason": null,
   "aliases": [
    "BEND",
    "BEND 2.0"
   ],
   "ambiguous_aliases": [
    "bend"
   ]
  },
  "family": null,
  "family_null_reason": "BEND 2.0 is treated as a new version of the same benchmark, not a separate family member.",
  "domain": "genomics-rna",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Fixed datasets with frozen representations and a light downstream head.",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Retrospective; the variant-effect tasks are zero-shot and nothing is blind."
  },
  "publisher": {
   "orgs": [
    "Technical University of Denmark",
    "University of Copenhagen"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Evaluates DNA language models' frozen representations on human-genome functional tasks.",
  "first_release": {
   "date": "2023-11-21",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv 2311.12570 v1",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2311.12570v1",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": null,
   "paper_v1": {
    "date": "2023-11-21",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv 2311.12570 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2311.12570v1",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2024",
    "precision": "year",
    "year_hint": null,
    "basis": "ICLR 2024 conference paper (only the year is sourced)",
    "event_kind": "conference-publication",
    "timestamp_utc": null,
    "source_url": "https://proceedings.iclr.cc/paper_files/paper/2024/hash/429e7b31625a8b7839f9e4d6e2aa9bb9-Abstract-Conference.html",
    "verification": "partial",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-11-02",
   "precision": "day",
   "year_hint": null,
   "basis": "BEND 2.0 preprint (arXiv 2611.03806 v1)",
   "event_kind": "new-version",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2611.03806",
   "verification": "verified",
   "note": null,
   "description": "BEND 2.0 preprint released, reorganized around capabilities needed for DNA design: 8 tasks and 5 experimental datasets; the original repository's last commit was 2024-10-31."
  },
  "next_expected": null,
  "next_expected_null_reason": "The authors have announced no date for a next version.",
  "status": "active",
  "status_note": "Version 2.0 released 2026-11-02; the original and 2.0 use different scopes and must be cited separately.",
  "scale": {
   "summary": "Original: 7 tasks, 285 to 2,005,617 samples, 512 to 100,096 bp. BEND 2.0: 8 tasks over 5 experimental datasets.",
   "items": [
    {
     "label": "Tasks",
     "value": "7",
     "unit": null,
     "version": "BEND v4"
    },
    {
     "label": "Sample-count range",
     "value": "285-2,005,617",
     "unit": null,
     "version": "BEND v4"
    },
    {
     "label": "Sequence-length range",
     "value": "512-100,096",
     "unit": "bp",
     "version": "BEND v4"
    },
    {
     "label": "Tasks",
     "value": "8",
     "unit": null,
     "version": "BEND 2.0"
    },
    {
     "label": "Experimental datasets",
     "value": "5",
     "unit": null,
     "version": "BEND 2.0"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Reuses published genome annotations and functional genomics data; no new experiments.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Freeze pretrained embeddings and train a light head on human-genome functional tasks: gene finding, enhancer annotation, chromatin accessibility, histone modification, methylation and two variant-effect tasks.",
   "truth": "Published genome annotations and functional genomics measurements, with public labels for the variant-effect tasks.",
   "metric_primary": "Per task: MCC, AUPRC or AUROC.",
   "metric_secondary": [
    "BEND 2.0 adds correlation and design-task metrics."
   ],
   "version_traps": "The original and 2.0 define tasks differently and their numbers cannot be mixed; state the embedding chunk length, whether the official head configuration is used, and whether a readout is native or fitted.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Per-task metrics with no single official ranking metric: conditional log loss (bits/base, lower better), macro Spearman correlation, top-1 tissue identification accuracy",
   "eval_set": "BEND 2.0 Table 1 snapshot: 8 endpoints across 5 capability categories, 17 model-interface rows",
   "snapshot_date": {
    "date": "2026-10-22",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://machinelearninglifescience.github.io/BEND2",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://machinelearninglifescience.github.io/BEND2",
   "entries": [
    {
     "rank": 1,
     "model_label": "SpliceAI",
     "model_id": null,
     "site_model": false,
     "score": "0.674",
     "score_unit": null,
     "metric": "Per-task metrics with no single official ranking metric: conditional log loss (bits/base, lower better), macro Spearman correlation, top-1 tissue identification accuracy",
     "eval_set": "BEND 2.0 Table 1 snapshot: 8 endpoints across 5 capability categories, 17 model-interface rows",
     "source_type": "board",
     "date": {
      "date": "2026-10-22",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://machinelearninglifescience.github.io/BEND2",
      "verification": "verified",
      "note": null
     },
     "url": "https://machinelearninglifescience.github.io/BEND2",
     "note": "Not a site model; first on this endpoint, as a specialist model's native output.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "AlphaGenome",
     "model_id": "model-alphagenome",
     "site_model": true,
     "score": "0.650",
     "score_unit": null,
     "metric": "Per-task metrics with no single official ranking metric: conditional log loss (bits/base, lower better), macro Spearman correlation, top-1 tissue identification accuracy",
     "eval_set": "BEND 2.0 Table 1 snapshot: 8 endpoints across 5 capability categories, 17 model-interface rows",
     "source_type": "board",
     "date": {
      "date": "2026-10-22",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://machinelearninglifescience.github.io/BEND2",
      "verification": "verified",
      "note": null
     },
     "url": "https://machinelearninglifescience.github.io/BEND2",
     "note": "Second on this endpoint with a fitted readout and 0.621 native; first on five of the eight endpoints.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "NTv3 650M post",
     "model_id": "model-ntv3",
     "site_model": true,
     "score": "0.564",
     "score_unit": null,
     "metric": "Per-task metrics with no single official ranking metric: conditional log loss (bits/base, lower better), macro Spearman correlation, top-1 tissue identification accuracy",
     "eval_set": "BEND 2.0 Table 1 snapshot: 8 endpoints across 5 capability categories, 17 model-interface rows",
     "source_type": "board",
     "date": {
      "date": "2026-10-22",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://machinelearninglifescience.github.io/BEND2",
      "verification": "verified",
      "note": null
     },
     "url": "https://machinelearninglifescience.github.io/BEND2",
     "note": "Fourth on this endpoint; first on the sequence-prior endpoint at 1.375 bits/base, where lower is better.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 9,
     "model_label": "Evo2 20B",
     "model_id": "model-evo-2",
     "site_model": true,
     "score": "0.367",
     "score_unit": null,
     "metric": "Per-task metrics with no single official ranking metric: conditional log loss (bits/base, lower better), macro Spearman correlation, top-1 tissue identification accuracy",
     "eval_set": "BEND 2.0 Table 1 snapshot: 8 endpoints across 5 capability categories, 17 model-interface rows",
     "source_type": "board",
     "date": {
      "date": "2026-10-22",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://machinelearninglifescience.github.io/BEND2",
      "verification": "verified",
      "note": null
     },
     "url": "https://machinelearninglifescience.github.io/BEND2",
     "note": "Ninth on this endpoint; prior loss 1.388 bits/base (rank 3); last on the design endpoints.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": 1,
   "headline_null_reason": null,
   "caveat": "Values belong to BEND 2.0 (a 2026-11 preprint) and come from its site data file's Table 1 snapshot; there is no official aggregate, ranks hold per endpoint only, and native and fitted readouts are separate rows.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Take data and configs from the official repository, extract frozen embeddings per task and train a small head, which costs far less than fine-tuning. State the embedding chunk length and whether the official head configuration is used; report both mean rank and per-task values rather than a single task; keep the original and 2.0 apart; and report the zero-shot variant tasks alongside TraitGym.",
  "pairs_with": [
   "bench-16-nt_gue",
   "bench-08-enformer",
   "bench-37-traitgym",
   "bench-11-clinvar"
  ],
  "caveats": [
   "Human genome only, and frozen-embedding results are not a fine-tuning ceiling.",
   "The enhancer task is small and sparse and all models do poorly; text and tables disagree on histone modification.",
   "The original baselines are mostly models from around 2023, so newer DNA models are absent from that board.",
   "The two versions differ in scope: 2.0 is organized around design capabilities and its numbers cannot sit beside the original's.",
   "The site models on the 2.0 board are not yet recorded as pairings; their relation type awaits the next round."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2311.12570v1",
    "title": "BEND paper (arXiv 2311.12570 v1)",
    "kind": "preprint",
    "date": {
     "date": "2023-11-21",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2311.12570v1",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://arxiv.org/html/2311.12570v4",
    "title": "BEND paper full text v4",
    "kind": "preprint",
    "date": {
     "date": "2024-04-09",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv v4",
     "event_kind": "paper-revision",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/html/2311.12570v4",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://proceedings.iclr.cc/paper_files/paper/2024/hash/429e7b31625a8b7839f9e4d6e2aa9bb9-Abstract-Conference.html",
    "title": "ICLR 2024 conference paper page",
    "kind": "paper",
    "date": null
   },
   {
    "url": "https://github.com/frederikkemarin/BEND",
    "title": "Official repository frederikkemarin/BEND",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2611.03806",
    "title": "BEND 2.0 preprint (arXiv 2611.03806)",
    "kind": "preprint",
    "date": {
     "date": "2026-11-02",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2611.03806",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://machinelearninglifescience.github.io/BEND2",
    "title": "BEND 2.0 leaderboard page",
    "kind": "leaderboard",
    "date": {
     "date": "2026-10-22",
     "precision": "day",
     "year_hint": null,
     "basis": "Table 1 snapshot date in the site data file",
     "event_kind": "leaderboard-entry",
     "timestamp_utc": null,
     "source_url": "https://machinelearninglifescience.github.io/BEND2",
     "verification": "verified",
     "note": null
    }
   }
  ],
  "corrections": [
   {
    "field": "ranking.eval_set",
    "was": "original BEND standing",
    "now": "BEND 2.0 Table 1 snapshot (2026-10-22), per endpoint",
    "reason": "The original has no live board; all current standings come from the 2.0 site data file and hold per endpoint only.",
    "source_url": "https://machinelearninglifescience.github.io/BEND2",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "release_milestones.official_announcement",
   "ranking.entries"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-38-bend/",
   "en": "/bio/benchmark/benchmarks/bend/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-39-rnagym",
  "kind": "benchmark",
  "nn": "39",
  "short_slug": "rnagym",
  "names": {
   "display": "RNAGym",
   "full": null,
   "full_null_reason": "The authors use only the name RNAGym; no expanded form exists.",
   "aliases": [
    "RNAGym",
    "RNAgym"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "The three tasks belong to one evaluation framework, not a family.",
  "domain": "genomics-rna",
  "domains_secondary": [
   "structure"
  ],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Fixed datasets, with the fitness part scored zero-shot.",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Retrospective and zero-shot, with nothing blind or prospective."
  },
  "publisher": {
   "orgs": [
    "Harvard Medical School"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Zero-shot evaluation framework for RNA fitness and structure.",
  "first_release": {
   "date": "2025-06-17",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv preprint v1 (10.1101/2025.06.16.660049); the Hugging Face dataset creation date is not used as first-release evidence",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://www.biorxiv.org/content/10.1101/2025.06.16.660049",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": null,
    "precision": null,
    "year_hint": null,
    "basis": "The Hugging Face dataset was created 2025-05-15 and the workshop paper has no sourced posting date, so neither is used as first-release evidence.",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://rnagym.org",
    "verification": "unverified",
    "note": null
   },
   "paper_v1": {
    "date": "2025-06-17",
    "precision": "day",
    "year_hint": null,
    "basis": "bioRxiv preprint v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://www.biorxiv.org/content/10.1101/2025.06.16.660049",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": {
    "date": "2025-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "release v0.1.0 in the official repository (repository URL not recorded; see note)",
    "event_kind": "code-release",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "partial",
    "note": null
   },
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-08-14",
   "precision": "day",
   "year_hint": null,
   "basis": "commits to the leaderboard file in the official repository (fitness board update)",
   "event_kind": "leaderboard-entry",
   "timestamp_utc": null,
   "source_url": "https://rnagym.org",
   "verification": "partial",
   "note": null,
   "description": "The fitness board gained two model entries and an outdated mRNA board was removed; a signed-Spearman metric arrived 2026-07-20, and v0.2 branches remain active."
  },
  "next_expected": null,
  "next_expected_null_reason": "The v0.2 branches are active but the authors have announced no release date.",
  "status": "active",
  "status_note": "The fitness board was updated in 2026-08, the project website is noted as not yet synced, and the paper is still a preprint.",
  "scale": {
   "summary": "70 DMS assays (over 1M mutations); 901k chemical-probing reactivity profiles; 215 PDB tertiary structures.",
   "items": [
    {
     "label": "DMS assays",
     "value": "70",
     "unit": null,
     "version": "paper v1"
    },
    {
     "label": "Variants",
     "value": "1,117,295",
     "unit": null,
     "version": "third-party tabulation"
    },
    {
     "label": "Chemical-probing profiles",
     "value": "901,000",
     "unit": null,
     "version": "paper v1"
    },
    {
     "label": "Tertiary structures",
     "value": "215",
     "unit": null,
     "version": "paper v1"
    },
    {
     "label": "Assays on the fitness board",
     "value": "31",
     "unit": null,
     "version": "board, ncRNA only"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Reuses published DMS and chemical-probing experiments; nothing is added.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Three tasks in one framework: zero-shot RNA fitness scoring, secondary-structure reactivity prediction and tertiary-structure prediction.",
   "truth": "Published DMS fitness measurements, chemical-probing reactivity profiles and PDB structures.",
   "metric_primary": "Spearman correlation against measured fitness.",
   "metric_secondary": [
    "AUROC and MCC.",
    "Separate metrics for the structure tasks."
   ],
   "version_traps": "From v0.1.1 the metric is signed Spearman over ncRNA only, so earlier numbers are not comparable; aggregate within RNA category before equal-weight averaging and state whether mRNA assays are included.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Signed Spearman correlation, macro-averaged over three ncRNA categories with equal category weight (from v0.1.1)",
   "eval_set": "RNAGym fitness board, ncRNA only (31 assays); mRNA coding and splicing assays excluded",
   "snapshot_date": {
    "date": "2026-08-14",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://rnagym.org",
    "verification": "partial",
    "note": null
   },
   "board_url": "https://rnagym.org",
   "entries": [
    {
     "rank": 1,
     "model_label": "Evo 2 (40B)",
     "model_id": "model-evo-2",
     "site_model": true,
     "score": "0.2120 (Ribozyme 0.1081 / tRNA 0.4310 / Aptamer 0.0970)",
     "score_unit": null,
     "metric": "Signed Spearman correlation, macro-averaged over three ncRNA categories with equal category weight (from v0.1.1)",
     "eval_set": "RNAGym fitness leaderboard, ncRNA only (31 assays); mRNA coding/splicing assays excluded",
     "source_type": "board",
     "date": {
      "date": "2026-08-14",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://rnagym.org",
      "verification": "partial",
      "note": null
     },
     "url": "https://rnagym.org",
     "note": "First overall on the board and a site model.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "RNA-ERNIE",
     "model_id": null,
     "site_model": false,
     "score": "0.1937",
     "score_unit": null,
     "metric": "Signed Spearman correlation, macro-averaged over three ncRNA categories with equal category weight (from v0.1.1)",
     "eval_set": "RNAGym fitness leaderboard, ncRNA only (31 assays); mRNA coding/splicing assays excluded",
     "source_type": "board",
     "date": {
      "date": "2026-08-14",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://rnagym.org",
      "verification": "partial",
      "note": null
     },
     "url": "https://rnagym.org",
     "note": "Not a site model.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "Evo 2 (7B)",
     "model_id": "model-evo-2",
     "site_model": true,
     "score": "0.1904 (Ribozyme 0.0651 / tRNA 0.3867 / Aptamer 0.1192)",
     "score_unit": null,
     "metric": "Signed Spearman correlation, macro-averaged over three ncRNA categories with equal category weight (from v0.1.1)",
     "eval_set": "RNAGym fitness leaderboard, ncRNA only (31 assays); mRNA coding/splicing assays excluded",
     "source_type": "board",
     "date": {
      "date": "2026-08-14",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://rnagym.org",
      "verification": "partial",
      "note": null
     },
     "url": "https://rnagym.org",
     "note": "A second checkpoint of the same model; two earlier generations rank seventh and eleventh.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "RIBOSPAN (1K-40 checkpoint)",
     "model_id": "model-ribospan",
     "site_model": true,
     "score": "Spearman 0.2524, AUROC 0.6280, MCC 0.1904",
     "score_unit": null,
     "metric": "Spearman, AUROC and MCC under the developer's own aggregation",
     "eval_set": "RNAGym fitness assays under the developer paper's own aggregation (not the current board protocol)",
     "source_type": "self",
     "date": {
      "date": "2026-08-01",
      "precision": "day",
      "year_hint": null,
      "basis": "entry date as given by the ranking source",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/pdf/2608.22849",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/pdf/2608.22849",
     "note": "Not on the official board; developer-reported under the paper's own aggregation with a different metric and assay scope, so no rank is given.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "Values are read from the leaderboard file in the official repository, which the project site had not yet synced; the v0.1.1 metric change breaks comparability with earlier figures; the repository warns the macro-mean over-weights two tiny categories and gaps under about 0.01 are meaningless; only fitness has a board.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-ribospan",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://arxiv.org/pdf/2608.22849",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "The developer self-reports zero-shot results on this set's fitness task under its own aggregation.",
    "resolution": null
   },
   {
    "model_id": "model-evo-2",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/pdf/2608.22849",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "disputed",
    "note": "The site matrix records a board entry while this page's figure is quoted via a third-party paper; the board entry awaits re-checking.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 1
   }
  },
  "how_to_use": "Download the data and baseline scripts and score each assay zero-shot: log-likelihood differences for autoregressive models, masked-marginals for masked ones. Aggregate within RNA category before equal-weight averaging rather than averaging over all assays; report Spearman, AUROC and MCC, and state the sequence-window handling, whether mRNA assays are included and the metric version. Structure tasks need their own pipeline, and a larger nucleic-acid fitness suite makes a useful scale comparison.",
  "pairs_with": [
   "bench-02-proteingym",
   "bench-25-rnapuzzles",
   "bench-26-flip",
   "bench-16-nt_gue"
  ],
  "caveats": [
   "The fitness part is zero-shot only, which an independent team argues is too narrow for fitness prediction.",
   "Data come from published DMS studies that may overlap model pretraining corpora, with no deduplication audit.",
   "Only fitness has a board; the structure tasks have baseline scripts but no board, and the board covers ncRNA only.",
   "Under the macro-mean, the tRNA and aptamer categories hold only single-digit assay counts yet carry full weight.",
   "Naming and protocol follow ProteinGym, but a larger nucleic-acid suite has overtaken its coverage and counts data differently."
  ],
  "sources": [
   {
    "url": "https://www.biorxiv.org/content/10.1101/2025.06.16.660049",
    "title": "RNAGym preprint (bioRxiv 10.1101/2025.06.16.660049)",
    "kind": "preprint",
    "date": {
     "date": "2025-06-17",
     "precision": "day",
     "year_hint": null,
     "basis": "bioRxiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://www.biorxiv.org/content/10.1101/2025.06.16.660049",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://rnagym.org",
    "title": "Project site rnagym.org, which links the official repository",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://rnagym.org",
    "title": "Fitness leaderboard file in the official repository, reached via the project site",
    "kind": "leaderboard",
    "date": {
     "date": "2026-08-14",
     "precision": "day",
     "year_hint": null,
     "basis": "last commit to the leaderboard file",
     "event_kind": "leaderboard-entry",
     "timestamp_utc": null,
     "source_url": "https://rnagym.org",
     "verification": "partial",
     "note": null
    }
   },
   {
    "url": "https://iclr.cc/virtual/2025/32594",
    "title": "ICLR 2025 workshop entry",
    "kind": "official-site",
    "date": null
   },
   {
    "url": "https://arxiv.org/pdf/2608.22849",
    "title": "Third-party use: the RIBOSPAN paper",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2511.02888",
    "title": "Third-party comparison: the NABench paper",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2025-05-15",
    "now": "2025-06-17",
    "reason": "The recorded value was the Hugging Face dataset creation date; repository and Hugging Face creation dates are excluded, so preprint v1 is used.",
    "source_url": "https://www.biorxiv.org/content/10.1101/2025.06.16.660049",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "Hugging Face dataset created (public data release); ICLR 2025 workshop paper same spring; bioRxiv v1 2025-06-17",
    "now": "bioRxiv preprint v1, 2025-06-17; the dataset creation date and the undated workshop paper are notes only",
    "reason": "The basis is realigned with the policy and the earlier value is kept as a note.",
    "source_url": "https://rnagym.org",
    "date_found": "2026-11-16"
   },
   {
    "field": "status_note",
    "was": "no official leaderboard page found",
    "now": "a fitness leaderboard is published in the repository and was updated 2026-08-14",
    "reason": "The site missed the repository's fitness board, release v0.1.0 and the project website.",
    "source_url": "https://rnagym.org",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "release_milestones.first_public_evidence",
   "release_milestones.code_release",
   "latest_update",
   "models_reported[1]",
   "scale.items[1]",
   "ranking.entries",
   "ranking.board_url"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-39-rnagym/",
   "en": "/bio/benchmark/benchmarks/rnagym/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-40-motifbench",
  "kind": "benchmark",
  "nn": "40",
  "short_slug": "motifbench",
  "names": {
   "display": "MotifBench",
   "full": null,
   "full_null_reason": "The name is already the full name; no expanded form is given by the sources.",
   "aliases": [
    "MotifBench"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "One fixed problem set and one leaderboard; no family members.",
  "domain": "protein",
  "domains_secondary": [
   "generation"
  ],
  "inclusion": "supplementary",
  "type": "D",
  "type_note": "Computational-oracle problem set with a submission leaderboard",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Computational self-consistency: ProteinMPNN redesign plus ESMFold refolding"
  },
  "publisher": {
   "orgs": [
    "Multi-lab academic collaboration (MotifBench maintainers)"
   ],
   "org_type": "academic-lab"
  },
  "one_liner": "Thirty fixed motif-scaffolding problems judged by one pipeline, with a submission leaderboard; ground truth stays computational.",
  "first_release": {
   "date": "2025-02-18",
   "precision": "day",
   "year_hint": null,
   "basis": "arXiv 2502.12479 v1 (repository created 2024-06-14, not used as first release)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://arxiv.org/abs/2502.12479v1",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2025-02-18",
    "precision": "day",
    "year_hint": null,
    "basis": "Earliest sourced public evidence is paper v1; the 2024-06-14 repository creation date is a note only, never a first release",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://github.com/blt2114/MotifBench",
    "verification": "verified",
    "note": null
   },
   "paper_v1": {
    "date": "2025-02-18",
    "precision": "day",
    "year_hint": null,
    "basis": "arXiv 2502.12479 v1",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://arxiv.org/abs/2502.12479v1",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-11-13",
   "precision": "day",
   "year_hint": null,
   "basis": "Leaderboard commits in the repository",
   "event_kind": "leaderboard-entry",
   "timestamp_utc": null,
   "source_url": "https://github.com/blt2114/MotifBench/commits/main",
   "verification": "verified",
   "note": null,
   "description": "Leaderboard updated: RFD4-Proteina results added 2026-11-06, reformatted and consolidated into one entry on 2026-11-10/13."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no next version or round.",
  "status": "active",
  "status_note": "Active: the leaderboard has new entries from 2025-02 through 2026-11.",
  "scale": {
   "summary": "Thirty problems in three groups of ten by number of motif segments; 100 scaffolds required per problem.",
   "items": [
    {
     "label": "Problems",
     "value": "30",
     "unit": null,
     "version": "paper v2"
    },
    {
     "label": "Groups",
     "value": "3 x 10",
     "unit": null,
     "version": "paper v2"
    },
    {
     "label": "Scaffolds per problem",
     "value": "100",
     "unit": null,
     "version": "paper v2"
    },
    {
     "label": "Sequences redesigned per scaffold",
     "value": "8",
     "unit": null,
     "version": "paper v2"
    },
    {
     "label": "Leaderboard entries",
     "value": "15",
     "unit": null,
     "version": "2026-11-16"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Purely computational evaluation; reference scaffolds come from experimental structures, but the evaluation itself has no wet lab.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Given a motif, generate fixed-length scaffolds; 100 per problem, judged by sequence redesign and refolding.",
   "truth": "Computational oracle: geometric agreement between the refolded structure and the input motif.",
   "metric_primary": "MotifBench score (0-100)",
   "metric_secondary": [
    "Unique solutions after clustering",
    "Novelty (TM-score against the PDB)",
    "Per-scaffold success rate"
   ],
   "version_traps": "State whether a number comes from the paper or the leaderboard, whether the indexed setting was used, and whether refolding used ESMFold or AlphaFold2.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "MotifBench score (0-100)",
   "eval_set": "30 motif-scaffolding problems, official ProteinMPNN + ESMFold pipeline",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://github.com/blt2114/MotifBench",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://github.com/blt2114/MotifBench",
   "entries": [
    {
     "rank": 1,
     "model_label": "RFD4-Proteina (indexed + medium thinking)",
     "model_id": null,
     "site_model": false,
     "score": "55.06 (± 1.35)",
     "score_unit": null,
     "metric": "MotifBench score (0-100)",
     "eval_set": "30 motif-scaffolding problems, official ProteinMPNN + ESMFold pipeline",
     "source_type": "board",
     "date": {
      "date": "2026-11",
      "precision": "month",
      "year_hint": null,
      "basis": "Entry month as listed in the board table",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/blt2114/MotifBench",
      "verification": "partial",
      "note": "The board gives only the month, not a day."
     },
     "url": "https://github.com/blt2114/MotifBench",
     "note": "Not a site model; overall top-1.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "RFdiffusion3 (indexed)",
     "model_id": "model-rfdiffusion2-3",
     "site_model": true,
     "score": "43.52 (± 1.35)",
     "score_unit": null,
     "metric": "MotifBench score (0-100)",
     "eval_set": "30 motif-scaffolding problems, official ProteinMPNN + ESMFold pipeline",
     "source_type": "board",
     "date": {
      "date": "2026-08",
      "precision": "month",
      "year_hint": null,
      "basis": "Entry month as listed in the board table",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/blt2114/MotifBench",
      "verification": "partial",
      "note": "The board gives only the month, not a day."
     },
     "url": "https://github.com/blt2114/MotifBench",
     "note": "Submitter-run entry; best site model.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 5,
     "model_label": "La-Proteina",
     "model_id": "model-la-proteina",
     "site_model": true,
     "score": "29.75 (± 0.51)",
     "score_unit": null,
     "metric": "MotifBench score (0-100)",
     "eval_set": "30 motif-scaffolding problems, official ProteinMPNN + ESMFold pipeline",
     "source_type": "board",
     "date": {
      "date": "2026-07",
      "precision": "month",
      "year_hint": null,
      "basis": "Entry month as listed in the board table",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/blt2114/MotifBench",
      "verification": "partial",
      "note": "The board gives only the month, not a day."
     },
     "url": "https://github.com/blt2114/MotifBench",
     "note": "Submitted by another generative-model team.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 15,
     "model_label": "ESM3",
     "model_id": "model-esm3",
     "site_model": true,
     "score": "11.38",
     "score_unit": null,
     "metric": "MotifBench score (0-100)",
     "eval_set": "30 motif-scaffolding problems, official ProteinMPNN + ESMFold pipeline",
     "source_type": "board",
     "date": {
      "date": "2025-05",
      "precision": "month",
      "year_hint": null,
      "basis": "Entry month as listed in the board table",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://github.com/blt2114/MotifBench",
      "verification": "partial",
      "note": "The board gives only the month, not a day."
     },
     "url": "https://github.com/blt2114/MotifBench",
     "note": "Last of the 15 entries.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": 1,
   "headline_null_reason": null,
   "caveat": "Ranks follow the board table order (sorted by score, 15 entries); entries are submitter-run, not independently re-run. The top entry is not a site model.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-esm3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/blt2114/MotifBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "Entry on the MotifBench submission board (11.38, 2025-05).",
    "resolution": null
   },
   {
    "model_id": "model-la-proteina",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/blt2114/MotifBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "The board lists this model (29.75, 2026-07); the Chinese model page says there is no score, which is wrong.",
    "resolution": "The board lists La-Proteina (29.75, 2026-07), so the relation is official participation; the model page claiming no score is the error and must be fixed."
   },
   {
    "model_id": "model-rfdiffusion2-3",
    "relation_type": "official-participation",
    "source_type": "board",
    "evidence_url": "https://github.com/blt2114/MotifBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "The board lists RFdiffusion3 (43.52, 2026-08); the Chinese model page says no entry was found, which is wrong.",
    "resolution": "The board and this record agree in listing RFdiffusion3 (43.52, 2026-08) as official participation; the Chinese model page must be fixed."
   }
  ],
  "non_pairings": [
   {
    "model_id": "model-proteina-complexa",
    "note": "The Proteina-Complexa method paper does not mention this problem set; the earlier candidate list was wrong.",
    "source_url": "https://arxiv.org/abs/2603.27950"
   }
  ],
  "n_models_reported": {
   "counted": 3,
   "by_relation": {
    "official-participation": 3,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Take the 30 problems and the official pipeline from the repository, generate 100 fixed-length scaffolds per problem, score with the official scripts and submit to the leaderboard; about 36 GPU hours. Report whether the protocol is the paper's or the board's, whether the indexed setting was used, which refolder was chosen and how many replicates; gaps under about one point support no conclusion. Showing a design can be made still needs your own expression and assays.",
  "pairs_with": [
   "bench-28-adaptyv",
   "bench-31-foldbench",
   "bench-36-aintibody"
  ],
  "caveats": [
   "Scores reflect in-silico self-consistency only, not expression, folding or function.",
   "Replicate standard deviations are about 0.39-0.47, and the authors treat gaps under about one point as unreliable.",
   "Paper and leaderboard give different RFdiffusion baselines (28.05 vs 21.95), so cite the protocol.",
   "Refolding with MSA-free AlphaFold2 instead of ESMFold drops the baseline from 28.1 to 22.5.",
   "Experimental reference scaffolds pass the evaluation in only 20 of 30 problems, which points to a limit of the evaluation itself."
  ],
  "sources": [
   {
    "url": "https://arxiv.org/abs/2502.12479",
    "title": "MotifBench paper, arXiv 2502.12479",
    "kind": "preprint",
    "date": {
     "date": "2025-02-18",
     "precision": "day",
     "year_hint": null,
     "basis": "arXiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://arxiv.org/abs/2502.12479v1",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://arxiv.org/html/2502.12479v2",
    "title": "Paper v2 full text",
    "kind": "preprint",
    "date": null
   },
   {
    "url": "https://github.com/blt2114/MotifBench",
    "title": "Official repository and leaderboard",
    "kind": "leaderboard",
    "date": {
     "date": "2026-11-13",
     "precision": "day",
     "year_hint": null,
     "basis": "Latest leaderboard commit",
     "event_kind": "leaderboard-entry",
     "timestamp_utc": null,
     "source_url": "https://github.com/blt2114/MotifBench/commits/main",
     "verification": "verified",
     "note": null
    }
   }
  ],
  "corrections": [
   {
    "field": "models_reported",
    "was": "RFdiffusion3 and La-Proteina: Chinese model pages state no MotifBench entry was found",
    "now": "both are official-participation entries on the MotifBench board (43.52 and 29.75)",
    "reason": "The board lists both entries explicitly, so the denial on the model pages is wrong (CHANGESET 4.1).",
    "source_url": "https://github.com/blt2114/MotifBench",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported",
    "was": "Proteina-Complexa counted as a MotifBench pairing",
    "now": "recorded as not-a-pairing: the method paper never mentions the benchmark",
    "reason": "CHANGESET 4.1 decision: the method paper does not mention this benchmark.",
    "source_url": "https://arxiv.org/abs/2603.27950",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "ranking.entries[0].date",
   "ranking.entries[1].date",
   "ranking.entries[2].date",
   "ranking.entries[3].date"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-40-motifbench/",
   "en": "/bio/benchmark/benchmarks/motifbench/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-41-bioml-bench",
  "kind": "benchmark",
  "nn": "41",
  "short_slug": "bioml-bench",
  "names": {
   "display": "BioML-bench",
   "full": null,
   "full_null_reason": "The name is the full name; sources give no expanded form.",
   "aliases": [
    "BioML-bench",
    "BioML-Bench",
    "biomlbench"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "One task set with no family members; BioXArena is a separate follow-up benchmark.",
  "domain": "text-agents",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "E",
  "type_note": "End-to-end agent task environment",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": null,
   "qualifier": "Truth hidden from the agent, but data come from already public competitions"
  },
  "publisher": {
   "orgs": [
    "ScienceMachine",
    "University of Cambridge",
    "Shift Bioscience",
    "University of Toronto",
    "Vector Institute"
   ],
   "org_type": "mixed"
  },
  "one_liner": "Agents build and run biomedical ML pipelines end to end, scored against human public-leaderboard results; tested agents average below the human baseline.",
  "first_release": {
   "date": "2025-09-04",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv preprint v1 (the 2025-05-19 first repository commit predates the paper and is a note only)",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.09.01.673319",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2025-09-04",
    "precision": "day",
    "year_hint": null,
    "basis": "Earliest sourced public evidence is preprint v1; repository creation and first commit are not used as a first release",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://github.com/science-machine/biomlbench",
    "verification": "verified",
    "note": null
   },
   "paper_v1": {
    "date": "2025-09-04",
    "precision": "day",
    "year_hint": null,
    "basis": "bioRxiv v1 (DOI 10.1101/2025.09.01.673319)",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.09.01.673319",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2025-09-28",
   "precision": "day",
   "year_hint": null,
   "basis": "bioRxiv version metadata",
   "event_kind": "paper-revision",
   "timestamp_utc": null,
   "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.09.01.673319",
   "verification": "verified",
   "note": null,
   "description": "Preprint v3; last repository commit 2025-09-14 (documentation fixes), maintenance only since."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no next version or new round.",
  "status": "low-frequency",
  "status_note": "Low frequency: no new version after preprint v3; independent follow-up use in 2026.",
  "scale": {
   "summary": "Four domains; tasks taken from public competitions and datasets, with a follow-up report citing 24 tasks.",
   "items": [
    {
     "label": "Domains",
     "value": "4",
     "unit": null,
     "version": "preprint v3"
    },
    {
     "label": "Task sources",
     "value": "PolarisHub, Kaggle, ProteinGym, OpenProblems",
     "unit": null,
     "version": "preprint v3"
    },
    {
     "label": "Tasks (follow-up report)",
     "value": "24",
     "unit": null,
     "version": "third-party report 2026-05"
    },
    {
     "label": "Runs per agent per task",
     "value": "4",
     "unit": null,
     "version": "preprint v3"
    },
    {
     "label": "Failed runs",
     "value": "41",
     "unit": null,
     "version": "preprint v3"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "Produces no new experimental data; tasks and truth come from existing public competitions and datasets.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Given a task description, training data and a submission example, the agent builds a model and submits predictions.",
   "truth": "Hidden test labels from public competitions and datasets; human baselines come from public leaderboards.",
   "metric_primary": "Percentile against human leaderboards, with failed runs penalized",
   "metric_secondary": [
    "Each task's native metric (AUROC, Spearman and others)",
    "Mean rank",
    "Number of failed runs"
   ],
   "version_traps": "State whether failure penalties are applied, which task-set version, which LLM backbone, and the budget and time limits.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "self-reported",
   "metric": "Mean leaderboard percentile against human public-leaderboard baselines, failed runs penalized",
   "eval_set": "BioML-bench tasks across 4 domains (24 tasks in the follow-up report), 4 runs per agent per task",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "Biomni",
     "model_id": "model-biomni",
     "site_model": true,
     "score": "34.45 ± 5.36 (leaderboard percentile)",
     "score_unit": null,
     "metric": "Mean leaderboard percentile against human public-leaderboard baselines, failed runs penalized",
     "eval_set": "BioML-bench tasks across 4 domains (24 tasks in the follow-up report), 4 runs per agent per task",
     "source_type": "self",
     "date": {
      "date": "2025-09",
      "precision": "month",
      "year_hint": null,
      "basis": "Month given by the preprint",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://www.biorxiv.org/content/10.1101/2025.09.01.673319v1.full",
      "verification": "partial",
      "note": "The preprint gives no exact evaluation date."
     },
     "url": "https://www.biorxiv.org/content/10.1101/2025.09.01.673319v1.full",
     "note": "Best of the four agents re-run by the benchmark authors; mean rank 1.88 +/- 0.18, no failed tasks.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "AutoScientists",
     "model_id": null,
     "site_model": false,
     "score": "74.4% (mean leaderboard percentile, 24 tasks)",
     "score_unit": null,
     "metric": "Mean leaderboard percentile against human public-leaderboard baselines, failed runs penalized",
     "eval_set": "BioML-bench tasks across 4 domains (24 tasks in the follow-up report), 4 runs per agent per task",
     "source_type": "self",
     "date": {
      "date": "2026-05-27",
      "precision": "day",
      "year_hint": null,
      "basis": "Preprint date of the follow-up report",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://arxiv.org/abs/2605.28655",
      "verification": "verified",
      "note": null
     },
     "url": "https://arxiv.org/abs/2605.28655",
     "note": "Not a site model; developer-reported with partly different hardware and protocol, and the report does not say whether the protocol matches the row above.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "The source has no rank-1 entry: entries are unranked or protocol-specific, so the caveat is shown as for a not-comparable cell.",
   "caveat": "No official live leaderboard; numbers come from different papers and protocols and are not ranked against each other. Several agent models have no sourced number here.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-biomni",
    "relation_type": "retrospective-dataset-use",
    "source_type": "third",
    "evidence_url": "https://www.biorxiv.org/content/10.1101/2025.09.01.673319",
    "evidence_site_page_only": false,
    "confidence": "medium",
    "verification": "partial",
    "note": "Re-run by the benchmark authors rather than reported by the developer; the Chinese matrix marks it self-reported, which under this round's definition should be third-party and is recorded as an open inconsistency.",
    "resolution": null
   },
   {
    "model_id": "model-co-scientist",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/2502.18864",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report on this benchmark found in official materials.",
    "resolution": null
   },
   {
    "model_id": "model-gpt-rosalind",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found; the only evidence is a site page, so it is shown as unconfirmed.",
    "resolution": null
   },
   {
    "model_id": "model-intern-s",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://github.com/InternLM/Intern-S1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report on this benchmark found in official materials.",
    "resolution": null
   },
   {
    "model_id": "model-quine",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.microsoft.com/en-us/research/blog/introducing-quine-an-ai-research-system-designed-for-the-complexity-of-biology/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report on this benchmark found in official materials.",
    "resolution": null
   },
   {
    "model_id": "model-scireasoner",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "The paper text does not mention this benchmark; the only evidence is a site page.",
    "resolution": null
   },
   {
    "model_id": "model-verid",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found; the only evidence is a site page, so it is shown as unconfirmed.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 1,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 6
   }
  },
  "how_to_use": "Install the official package, prepare data with the per-task scripts, and run each task several times independently (four in the paper). Report both the penalized and unpenalized protocols, the LLM backbone, the budget and the time limit, and do not mix specialised agents with a bare backbone plus generic scaffolding in one table. Because tasks come from public competitions, assess training-data contamination.",
  "pairs_with": [
   "bench-18-labbench",
   "bench-29-bixbench",
   "bench-42-biomysterybench"
  ],
  "caveats": [
   "Results mix agent design with LLM backbone; the authors do not separate them.",
   "Most of the 41 failed runs came from scaffolding problems, not model capability.",
   "Human leaderboard scores mix participant skill, effort and compute, so they are a rough reference only.",
   "Some authors are affiliated with the organization that maintains the evaluation package.",
   "Tasks are taken from public competitions, so training-data contamination is possible."
  ],
  "sources": [
   {
    "url": "https://www.biorxiv.org/content/10.1101/2025.09.01.673319v1.full",
    "title": "BioML-bench preprint, full text",
    "kind": "preprint",
    "date": {
     "date": "2025-09-04",
     "precision": "day",
     "year_hint": null,
     "basis": "bioRxiv v1",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.09.01.673319",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://api.biorxiv.org/details/biorxiv/10.1101/2025.09.01.673319",
    "title": "bioRxiv version metadata",
    "kind": "official-site",
    "date": {
     "date": "2025-09-28",
     "precision": "day",
     "year_hint": null,
     "basis": "preprint v3",
     "event_kind": "paper-revision",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://github.com/science-machine/biomlbench",
    "title": "Official repository",
    "kind": "repository",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2605.28655",
    "title": "Independent follow-up report (24 tasks)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2605.15766",
    "title": "Another follow-up benchmark, described by its authors as complementary",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "ranking.headline",
    "was": "a headline percentile was shown for this benchmark",
    "now": "no headline number; the caveat text is displayed instead",
    "reason": "The source has no rank-1 entry, so per CHANGESET 5.3 no headline number is given.",
    "source_url": "https://www.biorxiv.org/content/10.1101/2025.09.01.673319v1.full",
    "date_found": "2026-11-16"
   },
   {
    "field": "models_reported[model-biomni].relation_type",
    "was": "self-reported (Chinese matrix cell)",
    "now": "benchmark-owner / third-party run, recorded as retrospective dataset use pending the matrix fix",
    "reason": "The re-run was done by the benchmark authors, not the developer (CHANGESET 4.2 open item).",
    "source_url": "https://www.biorxiv.org/content/10.1101/2025.09.01.673319v1.full",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "ranking.entries[0].date",
   "scale.items[2]",
   "models_reported[model-gpt-rosalind]",
   "models_reported[model-scireasoner]",
   "models_reported[model-verid]",
   "models_reported[model-co-scientist]",
   "models_reported[model-intern-s]",
   "models_reported[model-quine]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-41-bioml-bench/",
   "en": "/bio/benchmark/benchmarks/bioml-bench/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-42-biomysterybench",
  "kind": "benchmark",
  "nn": "42",
  "short_slug": "biomysterybench",
  "names": {
   "display": "BioMysteryBench",
   "full": null,
   "full_null_reason": "The name is the full name; sources give no expanded form.",
   "aliases": [
    "BioMysteryBench",
    "BioMysteryBench-preview"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "One question set; the preview is its public subset, not a family.",
  "domain": "text-agents",
  "domains_secondary": [
   "genomics-rna"
  ],
  "inclusion": "supplementary",
  "type": "E",
  "type_note": "Agent question set with objectively verifiable answers",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L2",
   "range": [
    "L2",
    "L4"
   ],
   "qualifier": "Fixed set with objective truth; the full set is gated, close to a hidden-set practice but with no time-blind element"
  },
  "publisher": {
   "orgs": [
    "Anthropic"
   ],
   "org_type": "big-tech-lab"
  },
  "one_liner": "Questions on real sequencing data with metadata stripped, graded on the final answer only; authored by a model vendor, with only five public questions.",
  "first_release": {
   "date": "2026-04-29",
   "precision": "day",
   "year_hint": null,
   "basis": "The publisher's research page, same day as the preview dataset's initial commit (2026-04-29)",
   "event_kind": "official-announcement",
   "timestamp_utc": null,
   "source_url": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2026-04-29",
    "precision": "day",
    "year_hint": null,
    "basis": "Preview dataset initial commit, same day as the research page",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://huggingface.co/api/datasets/Anthropic/BioMysteryBench-preview/commits/main",
    "verification": "verified",
    "note": null
   },
   "paper_v1": null,
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2026-04-29",
    "precision": "day",
    "year_hint": null,
    "basis": "Publisher research page",
    "event_kind": "official-announcement",
    "timestamp_utc": null,
    "source_url": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench",
    "verification": "verified",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-11-09",
   "precision": "day",
   "year_hint": null,
   "basis": "Update date stated on the third-party leaderboard page",
   "event_kind": "leaderboard-entry",
   "timestamp_utc": null,
   "source_url": "https://vals.ai/benchmarks/biomysterybench",
   "verification": "partial",
   "note": "The leaderboard date is carried from an existing record and was not independently re-verified.",
   "description": "Third-party cross-vendor leaderboard updated (31 models, top 83.70%); the latest dataset change is preview v11 on 2026-07-07, swapping two questions."
  },
  "next_expected": null,
  "next_expected_null_reason": "The publisher has announced no next version or new round.",
  "status": "active",
  "status_note": "Active: third-party leaderboard updated 2026-11-09; dataset v11 (2026-07).",
  "scale": {
   "summary": "Research page: 99 questions (76 human-solvable, 23 human-unsolved). Dataset card: 90 in the full set, 5 in the public preview.",
   "items": [
    {
     "label": "Questions (research page)",
     "value": "99",
     "unit": null,
     "version": "2026-04-29"
    },
    {
     "label": "Human-solvable / human-unsolved",
     "value": "76 / 23",
     "unit": null,
     "version": "2026-04-29"
    },
    {
     "label": "Questions (dataset card)",
     "value": "90",
     "unit": null,
     "version": "preview v11, 2026-07-07"
    },
    {
     "label": "Public preview questions",
     "value": "5",
     "unit": null,
     "version": "preview v11, 2026-07-07"
    },
    {
     "label": "Dropped for formatting errors",
     "value": "4",
     "unit": null,
     "version": "2026-04-29"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No new wet lab; questions use existing sequencing data and answers are verified by orthogonal experiments or metadata.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Given sequencing data with metadata stripped, the agent computationally recovers the hidden property; only the final answer is graded.",
   "truth": "Objective properties verifiable from orthogonal experiments or the original metadata.",
   "metric_primary": "Final-answer accuracy",
   "metric_secondary": [
    "Stability of solutions across repeated runs",
    "Separate scores on the human-solvable and human-unsolved subsets"
   ],
   "version_traps": "State whether a number uses the 99-question or 90-question set, the number of runs, the harness, the turn and time limits, and the judge model.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "third-party",
   "metric": "Final-answer accuracy, mean of three full runs, scored by a judge model",
   "eval_set": "Third-party board run: Terminus 2 harness, 200 turns / 4 h per task",
   "snapshot_date": {
    "date": "2026-11-09",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://vals.ai/benchmarks/biomysterybench",
    "verification": "partial",
    "note": "Update date stated on the board page, not independently re-verified."
   },
   "board_url": "https://vals.ai/benchmarks/biomysterybench",
   "entries": [
    {
     "rank": 1,
     "model_label": "Claude Fable 5.2",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "83.70%",
     "score_unit": null,
     "metric": "Final-answer accuracy, mean of three full runs, scored by a judge model",
     "eval_set": "Third-party board run: Terminus 2 harness, 200 turns / 4 h per task",
     "source_type": "third",
     "date": {
      "date": "2026-11-09",
      "precision": "day",
      "year_hint": null,
      "basis": "Update date stated on the board page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://vals.ai/benchmarks/biomysterybench",
      "verification": "partial",
      "note": "The board date was not independently re-verified."
     },
     "url": "https://vals.ai/benchmarks/biomysterybench",
     "note": "Overall top-1.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 2,
     "model_label": "Claude Sonnet 5.5",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "81.11%",
     "score_unit": null,
     "metric": "Final-answer accuracy, mean of three full runs, scored by a judge model",
     "eval_set": "Third-party board run: Terminus 2 harness, 200 turns / 4 h per task",
     "source_type": "third",
     "date": {
      "date": "2026-11-09",
      "precision": "day",
      "year_hint": null,
      "basis": "Update date stated on the board page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://vals.ai/benchmarks/biomysterybench",
      "verification": "partial",
      "note": "The board date was not independently re-verified."
     },
     "url": "https://vals.ai/benchmarks/biomysterybench",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 3,
     "model_label": "GPT-6.1 Sol",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "79.63%",
     "score_unit": null,
     "metric": "Final-answer accuracy, mean of three full runs, scored by a judge model",
     "eval_set": "Third-party board run: Terminus 2 harness, 200 turns / 4 h per task",
     "source_type": "third",
     "date": {
      "date": "2026-11-09",
      "precision": "day",
      "year_hint": null,
      "basis": "Update date stated on the board page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://vals.ai/benchmarks/biomysterybench",
      "verification": "partial",
      "note": "The board date was not independently re-verified."
     },
     "url": "https://vals.ai/benchmarks/biomysterybench",
     "note": null,
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 4,
     "model_label": "GPT-6 Astra",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "79.26%",
     "score_unit": null,
     "metric": "Final-answer accuracy, mean of three full runs, scored by a judge model",
     "eval_set": "Third-party board run: Terminus 2 harness, 200 turns / 4 h per task",
     "source_type": "third",
     "date": {
      "date": "2026-11-09",
      "precision": "day",
      "year_hint": null,
      "basis": "Update date stated on the board page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://vals.ai/benchmarks/biomysterybench",
      "verification": "partial",
      "note": "The board date was not independently re-verified."
     },
     "url": "https://vals.ai/benchmarks/biomysterybench",
     "note": "Tied at 79.26% with two other entries.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 7,
     "model_label": "Gemini 4 Argon",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "76.30%",
     "score_unit": null,
     "metric": "Final-answer accuracy, mean of three full runs, scored by a judge model",
     "eval_set": "Third-party board run: Terminus 2 harness, 200 turns / 4 h per task",
     "source_type": "third",
     "date": {
      "date": "2026-11-09",
      "precision": "day",
      "year_hint": null,
      "basis": "Update date stated on the board page",
      "event_kind": "leaderboard-entry",
      "timestamp_utc": null,
      "source_url": "https://vals.ai/benchmarks/biomysterybench",
      "verification": "partial",
      "note": "The board date was not independently re-verified."
     },
     "url": "https://vals.ai/benchmarks/biomysterybench",
     "note": "That vendor's highest entry.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": null,
     "model_label": "Claude Opus 4.6",
     "model_id": "model-frontier-llms",
     "site_model": true,
     "score": "77.4% (human-solvable) / 23.5% (human-unsolved)",
     "score_unit": null,
     "metric": "Final-answer accuracy, mean of five runs per question, vendor protocol",
     "eval_set": "Publisher research page: 5 runs per question; human-solvable and human-unsolved subsets reported separately",
     "source_type": "self",
     "date": {
      "date": "2026-04-29",
      "precision": "day",
      "year_hint": null,
      "basis": "Research page publication date",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench",
      "verification": "verified",
      "note": null
     },
     "url": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench",
     "note": "Vendor self-report with run settings different from the board; not ranked.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "The board is third-party run (31 models, 2026-11-09); the benchmark is vendor-built, the full set is gated, and vendor self-reported numbers are not comparable with the board.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [
   {
    "model_id": "model-frontier-llms",
    "relation_type": "self-reported-official-protocol",
    "source_type": "self",
    "evidence_url": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "verified",
    "note": "The publisher runs the benchmark itself and reports scores; the cross-vendor board is run by a third party.",
    "resolution": null
   },
   {
    "model_id": "model-co-scientist",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://arxiv.org/abs/2502.18864",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report on this benchmark found in official materials.",
    "resolution": null
   },
   {
    "model_id": "model-gpt-rosalind",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found; the only evidence is a site page, so it is shown as unconfirmed.",
    "resolution": null
   },
   {
    "model_id": "model-intern-s",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://github.com/InternLM/Intern-S1",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report on this benchmark found in official materials.",
    "resolution": null
   },
   {
    "model_id": "model-quine",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": "https://www.microsoft.com/en-us/research/blog/introducing-quine-an-ai-research-system-designed-for-the-complexity-of-biology/",
    "evidence_site_page_only": false,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report on this benchmark found in official materials.",
    "resolution": null
   },
   {
    "model_id": "model-scireasoner",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "The paper text does not mention this benchmark; the only evidence is a site page.",
    "resolution": null
   },
   {
    "model_id": "model-verid",
    "relation_type": "disputed-or-unconfirmed",
    "source_type": "unknown",
    "evidence_url": null,
    "evidence_site_page_only": true,
    "confidence": "high",
    "verification": "disputed",
    "note": "No report found; the only evidence is a site page, so it is shown as unconfirmed.",
    "resolution": null
   }
  ],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 1,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 1,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 6
   }
  },
  "how_to_use": "The five public preview questions support only a smoke test; the full set requires a request to the publisher under evaluation-only terms, or submission to the third-party board. Run each question several times and average, state the harness, turn and time limits and judge model, and report the human-solvable and human-unsolved subsets separately. Looking up source data by accession is prohibited. The sample is small, so gaps of a few points are not a ranking.",
  "pairs_with": [
   "bench-18-labbench",
   "bench-29-bixbench",
   "bench-41-bioml-bench"
  ],
  "caveats": [
   "The questions were written by a model vendor that also develops models under test, which is a conflict of interest.",
   "The full set is gated and the public preview has only five questions, so outsiders cannot fully reproduce results.",
   "Question counts differ (99 vs 90) with no full public explanation, so cite the source.",
   "The sample is small with wide intervals, and success on hard questions is unstable, with marginal solves clearly more frequent.",
   "The vendor self-report and the third-party board use different run settings, so their numbers are not directly comparable."
  ],
  "sources": [
   {
    "url": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench",
    "title": "Publisher research page",
    "kind": "company-blog",
    "date": {
     "date": "2026-04-29",
     "precision": "day",
     "year_hint": null,
     "basis": "Research page publication date",
     "event_kind": "official-announcement",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://huggingface.co/datasets/Anthropic/BioMysteryBench-preview",
    "title": "Preview dataset card",
    "kind": "official-site",
    "date": {
     "date": "2026-07-07",
     "precision": "day",
     "year_hint": null,
     "basis": "preview v11 commit",
     "event_kind": "data-release",
     "timestamp_utc": null,
     "source_url": "https://huggingface.co/api/datasets/Anthropic/BioMysteryBench-preview/commits/main",
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://vals.ai/benchmarks/biomysterybench",
    "title": "Third-party cross-vendor leaderboard",
    "kind": "leaderboard",
    "date": {
     "date": "2026-11-09",
     "precision": "day",
     "year_hint": null,
     "basis": "Update date stated on the board page (not independently re-verified)",
     "event_kind": "leaderboard-entry",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   }
  ],
  "corrections": [
   {
    "field": "scale",
    "was": "99 expert questions presented as the set size",
    "now": "research page 99 (76 human-solvable, 23 human-unsolved) and dataset card 90 recorded side by side, with the public preview at 5",
    "reason": "The two counts disagree and the reason is not fully public, so both are recorded side by side.",
    "source_url": "https://huggingface.co/datasets/Anthropic/BioMysteryBench-preview",
    "date_found": "2026-11-16"
   },
   {
    "field": "latest_update.verification",
    "was": "treated as fully verified",
    "now": "partial: the 2026-11-09 board date is carried from an existing record and was not independently re-fetched",
    "reason": "CHANGESET section 6 requires dates that depend on a dynamic page to be marked unverified.",
    "source_url": "https://vals.ai/benchmarks/biomysterybench",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "ranking.snapshot_date",
   "scale.items[2]",
   "models_reported[model-gpt-rosalind]",
   "models_reported[model-scireasoner]",
   "models_reported[model-verid]",
   "models_reported[model-co-scientist]",
   "models_reported[model-intern-s]",
   "models_reported[model-quine]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-42-biomysterybench/",
   "en": "/bio/benchmark/benchmarks/biomysterybench/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-43-gdpa1",
  "kind": "benchmark",
  "nn": "43",
  "short_slug": "gdpa1",
  "names": {
   "display": "Ginkgo GDPa1",
   "full": "GDPa1 antibody developability dataset and the AbDev competition",
   "full_null_reason": null,
   "aliases": [
    "GDPa1",
    "Ginkgo GDPa1",
    "AbDev competition"
   ],
   "ambiguous_aliases": [
    "AbDev"
   ]
  },
  "family": null,
  "family_null_reason": "One dataset plus a one-off competition; no family members.",
  "domain": "antibody",
  "domains_secondary": [
   "protein"
  ],
  "inclusion": "supplementary",
  "type": "A+B",
  "type_note": "Public dataset plus a private held-out blind test",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L4",
   "range": null,
   "qualifier": "Blind test with hidden truth, but the held-out antibodies are existing clinical antibodies, not prospectively synthesised"
  },
  "publisher": {
   "orgs": [
    "Ginkgo Datapoints",
    "Ginkgo Bioworks",
    "Hugging Face"
   ],
   "org_type": "platform-company"
  },
  "one_liner": "Uniform-assay developability data on 246 clinical antibodies plus a one-off blind test on 80 private held-out antibodies.",
  "first_release": {
   "date": "2025-09-08",
   "precision": "day",
   "year_hint": null,
   "basis": "Official announcement of the public dataset and the competition (2025-09-08); the 2025-06-13 dataset-object creation date is a gated object and is not used as a first release",
   "event_kind": "official-announcement",
   "timestamp_utc": null,
   "source_url": "https://huggingface.co/blog/ginkgo-datapoints/2025-abdev-competition",
   "verification": "partial",
   "note": "Day precision still rests on the announcement page and stays flagged."
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2025-06-13",
    "precision": "day",
    "year_hint": null,
    "basis": "Dataset object creation date (gated); a note only, never displayed as a release",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://huggingface.co/api/datasets/ginkgo-datapoints/GDPa1",
    "verification": "partial",
    "note": null
   },
   "paper_v1": null,
   "journal_online": {
    "date": "2026-02-22",
    "precision": "day",
    "year_hint": null,
    "basis": "Results paper online in a journal (mAbs 18(1))",
    "event_kind": "journal-online",
    "timestamp_utc": null,
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
    "verification": "verified",
    "note": null
   },
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": {
    "date": "2025-09-08",
    "precision": "day",
    "year_hint": null,
    "basis": "Competition and public dataset announcement",
    "event_kind": "official-announcement",
    "timestamp_utc": null,
    "source_url": "https://huggingface.co/blog/ginkgo-datapoints/2025-abdev-competition",
    "verification": "partial",
    "note": null
   }
  },
  "latest_update": {
   "date": "2026-11-03",
   "precision": "day",
   "year_hint": null,
   "basis": "Dataset card changelog and the platform's last-modified time",
   "event_kind": "data-release",
   "timestamp_utc": null,
   "source_url": "https://huggingface.co/datasets/ginkgo-datapoints/GDPa1",
   "verification": "partial",
   "note": "The dataset is gated and the changelog unreadable, so the v1.4 label is carried from an existing record.",
   "description": "Dataset card updated to v1.4; before that: results paper 2026-02-22, competition closed 2025-11-18."
  },
  "next_expected": null,
  "next_expected_null_reason": "The organizers have announced no date for a next round.",
  "status": "low-frequency",
  "status_note": "Low frequency: the competition closed (2025-09-08 to 2025-11-18); the public leaderboard still runs and the dataset is at v1.4.",
  "scale": {
   "summary": "246 public clinical antibodies with 10 assays; 80 private held-out antibodies; 5 properties predicted; 113 teams.",
   "items": [
    {
     "label": "Public antibodies",
     "value": "246",
     "unit": null,
     "version": "dataset card"
    },
    {
     "label": "Assays",
     "value": "10",
     "unit": null,
     "version": "dataset card"
    },
    {
     "label": "Private held-out antibodies",
     "value": "80",
     "unit": null,
     "version": "results paper abstract"
    },
    {
     "label": "Properties predicted",
     "value": "5",
     "unit": null,
     "version": "competition 2025"
    },
    {
     "label": "Teams",
     "value": "113",
     "unit": null,
     "version": "results paper abstract"
    }
   ]
  },
  "wetlab_refresh": "new",
  "wetlab_note": "The 80 held-out antibodies were measured specifically for the competition, once only; not a standing round.",
  "needs_own_wetlab": "no",
  "what_it_measures": {
   "task": "Predict five developability properties from sequence, scored on 80 private held-out antibodies.",
   "truth": "Experimental measurements from one laboratory under uniform assays: hydrophobicity, polyreactivity, self-association, thermostability and titer.",
   "metric_primary": "Spearman correlation per property",
   "metric_secondary": [
    "Per-property prize standings"
   ],
   "version_traps": "Distinguish GDPa1 cross-validation from the competition's private held-out protocol; the paper shows the two differ systematically.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "board",
   "metric": "Spearman correlation per property on the private held-out set",
   "eval_set": "AbDev competition 2025, 80 private held-out antibodies, 5 properties",
   "snapshot_date": {
    "date": "2026-02-22",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": "https://huggingface.co/spaces/ginkgo-datapoints/abdev-leaderboard",
    "verification": "verified",
    "note": null
   },
   "board_url": "https://huggingface.co/spaces/ginkgo-datapoints/abdev-leaderboard",
   "entries": [
    {
     "rank": 1,
     "model_label": "Anonymous",
     "model_id": null,
     "site_model": false,
     "score": "0.708",
     "score_unit": null,
     "metric": "Spearman correlation per property on the private held-out set",
     "eval_set": "AbDev competition 2025, 80 private held-out antibodies, 5 properties",
     "source_type": "board",
     "date": {
      "date": "2026-02-22",
      "precision": "day",
      "year_hint": null,
      "basis": "Journal-online date of the results paper",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
     "note": "Not a site model; top-1 on hydrophobicity (HIC).",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 1,
     "model_label": "microcrisprtm2",
     "model_id": null,
     "site_model": false,
     "score": "0.392",
     "score_unit": null,
     "metric": "Spearman correlation per property on the private held-out set",
     "eval_set": "AbDev competition 2025, 80 private held-out antibodies, 5 properties",
     "source_type": "board",
     "date": {
      "date": "2026-02-22",
      "precision": "day",
      "year_hint": null,
      "basis": "Journal-online date of the results paper",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
     "note": "Not a site model; top-1 on thermostability (Tm2).",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 1,
     "model_label": "PR_CHO_NOV",
     "model_id": null,
     "site_model": false,
     "score": "0.356",
     "score_unit": null,
     "metric": "Spearman correlation per property on the private held-out set",
     "eval_set": "AbDev competition 2025, 80 private held-out antibodies, 5 properties",
     "source_type": "board",
     "date": {
      "date": "2026-02-22",
      "precision": "day",
      "year_hint": null,
      "basis": "Journal-online date of the results paper",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
     "note": "Not a site model; top-1 on polyreactivity (abstract 0.356, Results 0.337; swapped with self-association between the two).",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 1,
     "model_label": "AbDevelop",
     "model_id": null,
     "site_model": false,
     "score": "0.337",
     "score_unit": null,
     "metric": "Spearman correlation per property on the private held-out set",
     "eval_set": "AbDev competition 2025, 80 private held-out antibodies, 5 properties",
     "source_type": "board",
     "date": {
      "date": "2026-02-22",
      "precision": "day",
      "year_hint": null,
      "basis": "Journal-online date of the results paper",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
     "note": "Not a site model; top-1 on self-association (abstract 0.337, Results 0.356).",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    },
    {
     "rank": 1,
     "model_label": "meta2",
     "model_id": null,
     "site_model": false,
     "score": "0.310",
     "score_unit": null,
     "metric": "Spearman correlation per property on the private held-out set",
     "eval_set": "AbDev competition 2025, 80 private held-out antibodies, 5 properties",
     "source_type": "board",
     "date": {
      "date": "2026-02-22",
      "precision": "day",
      "year_hint": null,
      "basis": "Journal-online date of the results paper",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
      "verification": "verified",
      "note": null
     },
     "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
     "note": "Not a site model; top-1 on titer.",
     "second_hand": false,
     "withheld": false,
     "withheld_reason": null
    }
   ],
   "headline": 0,
   "best_site_model": null,
   "headline_null_reason": null,
   "caveat": "The competition closed 2025-11-18 and standings come from the results paper; the leaderboard Space errored on 2026-11-16 and was not loaded; no site model appears among the winners.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Request GDPa1 from the data platform (terms must be accepted), use the recommended stratified cross-validation split, and report Spearman per property for all five. The private held-out set was consumed by the competition and cannot be submitted to again; a real blind test needs new antibodies measured under the same assays, or joining the related consortium. State whether a number is cross-validation or the competition held-out set.",
  "pairs_with": [
   "bench-34-abbibench",
   "bench-35-flab",
   "bench-36-aintibody",
   "bench-27-polaris"
  ],
  "caveats": [
   "The key negative result of the blind test: cross-validation correlations are systematically higher than held-out correlations.",
   "Only 246 training and 80 test antibodies, all clinical-stage, so extrapolation to early-stage or designed antibodies is untested.",
   "A one-off competition: the held-out set is exposed once it closes, so later work can only compare cross-validation scores.",
   "Winning methods differ by property; no single architecture wins across the board.",
   "Not a prospective synthesis test; the held-out antibodies are existing clinical antibodies."
  ],
  "sources": [
   {
    "url": "https://huggingface.co/blog/ginkgo-datapoints/2025-abdev-competition",
    "title": "Competition and dataset announcement",
    "kind": "official-site",
    "date": {
     "date": "2025-09-08",
     "precision": "day",
     "year_hint": null,
     "basis": "Announcement date",
     "event_kind": "official-announcement",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
    "title": "Competition results paper (mAbs 18(1))",
    "kind": "journal",
    "date": {
     "date": "2026-02-22",
     "precision": "day",
     "year_hint": null,
     "basis": "Journal online",
     "event_kind": "journal-online",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://huggingface.co/datasets/ginkgo-datapoints/GDPa1",
    "title": "GDPa1 dataset card (gated)",
    "kind": "official-site",
    "date": {
     "date": "2026-11-03",
     "precision": "day",
     "year_hint": null,
     "basis": "Last modified (card changelog v1.4, not independently verified)",
     "event_kind": "data-release",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://huggingface.co/spaces/ginkgo-datapoints/abdev-leaderboard",
    "title": "Public leaderboard",
    "kind": "leaderboard",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2025-06-13 (dataset on HF); competition announced 2025-09-08",
    "now": "2025-09-08",
    "reason": "2025-06-13 is only the creation date of a gated dataset object and cannot support day precision for a public release; the sourced public event is the competition and data announcement.",
    "source_url": "https://huggingface.co/datasets/ginkgo-datapoints/GDPa1",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "HF blog/competition launch with public GDPa1 data (HF repo object created 2025-06-13, likely private/gated earlier)",
    "now": "official announcement of the public GDPa1 dataset and AbDev competition, 2025-09-08. HF dataset object created 2025-06-13 (creation date, gated, not displayed as release).",
    "reason": "Basis realigned; the repository creation date is demoted to a note.",
    "source_url": "https://huggingface.co/datasets/ginkgo-datapoints/GDPa1",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release",
    "was": "2025-09-08",
    "now": "2025-06-13 (dataset on HF); competition announced 2025-09-08",
    "reason": "This was the previous round's change, moving the first release to the dataset object creation date; this round reverses it, as a creation date may not be a first release.",
    "source_url": "https://huggingface.co/api/datasets/ginkgo-datapoints/GDPa1",
    "date_found": "2026-11-16"
   },
   {
    "field": "scale",
    "was": "246 public antibodies, 10 assays; 80 private holdout; 113 teams (presented as fully verified)",
    "now": "same figures, but 246/10 are verified on the dataset card while 80 and 113 come from the results paper abstract",
    "reason": "The wet-lab check returned partial, so each figure's source must be stated.",
    "source_url": "https://huggingface.co/datasets/ginkgo-datapoints/GDPa1",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking.entries",
    "was": "polyreactivity 0.337 and self-association 0.356",
    "now": "polyreactivity 0.356 and self-association 0.337 per the abstract; the Results section swaps the two",
    "reason": "The paper's abstract and Results section swap these two properties; the abstract is recorded and the discrepancy noted.",
    "source_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12928636/",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "first_release",
   "latest_update",
   "release_milestones.first_public_evidence",
   "scale.items[2]",
   "scale.items[4]",
   "ranking.entries[2]",
   "ranking.entries[3]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-43-gdpa1/",
   "en": "/bio/benchmark/benchmarks/gdpa1/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 },
 {
  "id": "bench-44-fsmol",
  "kind": "benchmark",
  "nn": "44",
  "short_slug": "fsmol",
  "names": {
   "display": "FS-Mol",
   "full": "FS-Mol: a few-shot molecular activity prediction benchmark",
   "full_null_reason": null,
   "aliases": [
    "FS-Mol",
    "fsmol-0.1"
   ],
   "ambiguous_aliases": []
  },
  "family": null,
  "family_null_reason": "Only one data version, fsmol-0.1; the repository and the paper are milestones of one entry, not a family.",
  "domain": "small-molecule",
  "domains_secondary": [],
  "inclusion": "supplementary",
  "type": "B",
  "type_note": "Fixed task split plus an evaluation protocol",
  "type_note_null_reason": null,
  "evidence_level": {
   "primary": "L3",
   "range": null,
   "qualifier": "Cross-assay transfer split by target assay, with no blind test"
  },
  "publisher": {
   "orgs": [
    "Microsoft Research",
    "Novartis"
   ],
   "org_type": "mixed"
  },
  "one_liner": "With only 16-256 labels on a brand-new assay, predict the rest; it fills the few-shot adaptation gap.",
  "first_release": {
   "date": "2021-12",
   "precision": "month",
   "year_hint": null,
   "basis": "NeurIPS 2021 Datasets and Benchmarks paper, 2021-12 (month precision; no arXiv version found). The 2021-08-13 first repository commit is a note only.",
   "event_kind": "paper-v1",
   "timestamp_utc": null,
   "source_url": "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/8d3bba7425e7c98c50f52ca1b52d3735-Paper-round2.pdf",
   "verification": "verified",
   "note": null
  },
  "release_milestones": {
   "first_public_evidence": {
    "date": "2021-08",
    "precision": "month",
    "year_hint": null,
    "basis": "First repository commit 2021-08-13 with data and code; by policy not a first release, recorded only as earlier public evidence",
    "event_kind": "first-public-evidence",
    "timestamp_utc": null,
    "source_url": "https://github.com/microsoft/FS-Mol",
    "verification": "partial",
    "note": null
   },
   "paper_v1": {
    "date": "2021-12",
    "precision": "month",
    "year_hint": null,
    "basis": "NeurIPS 2021 Datasets and Benchmarks proceedings",
    "event_kind": "paper-v1",
    "timestamp_utc": null,
    "source_url": "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/8d3bba7425e7c98c50f52ca1b52d3735-Paper-round2.pdf",
    "verification": "verified",
    "note": null
   },
   "journal_online": null,
   "code_release": null,
   "weights_release": null,
   "challenge_open": null,
   "official_announcement": null
  },
  "latest_update": {
   "date": "2026-10-21",
   "precision": "day",
   "year_hint": null,
   "basis": "Repository archive status (the archive date comes from an existing record; the API was rate-limited and it was not re-verified)",
   "event_kind": "repository-archived",
   "timestamp_utc": null,
   "source_url": "https://github.com/microsoft/FS-Mol",
   "verification": "partial",
   "note": "The archive date was not independently re-verified.",
   "description": "Repository archived read-only; last code commit 2021-12-08 (a typing fix), maintenance only since."
  },
  "next_expected": null,
  "next_expected_null_reason": "The maintainers have announced no further version and the repository is archived.",
  "status": "frozen",
  "status_note": "Frozen: the repository has been archived read-only since 2026-10-21; method papers still cite it.",
  "scale": {
   "summary": "5,120 assays, 233,786 compounds, 489,133 measurements; 4,938 training, 40 validation and 157 test tasks.",
   "items": [
    {
     "label": "Assays",
     "value": "5,120",
     "unit": null,
     "version": "fsmol-0.1"
    },
    {
     "label": "Compounds",
     "value": "233,786",
     "unit": null,
     "version": "fsmol-0.1"
    },
    {
     "label": "Measurements",
     "value": "489,133",
     "unit": null,
     "version": "fsmol-0.1"
    },
    {
     "label": "Training / validation / test tasks",
     "value": "4,938 / 40 / 157",
     "unit": null,
     "version": "fsmol-0.1"
    },
    {
     "label": "Kinase tasks among the test tasks",
     "value": "125",
     "unit": null,
     "version": "fsmol-0.1"
    }
   ]
  },
  "wetlab_refresh": "static",
  "wetlab_note": "No new experimental data; built from ChEMBL27, with no later release by the maintainers.",
  "needs_own_wetlab": "yes-for-real-generalization",
  "what_it_measures": {
   "task": "Pre-train on training tasks, then use 16-256 labelled molecules from a new assay to predict the query set's activity.",
   "truth": "Activity measurements per assay in ChEMBL27, binarized at that assay's median.",
   "metric_primary": "Delta AUPRC: query AUPRC minus the positive rate",
   "metric_secondary": [
    "Mean rank",
    "Scores stratified by target class"
   ],
   "version_traps": "State the support-set size, whether ECFP or physicochemical features were added, how many tasks qualified, and any extra pre-training data.",
   "version_traps_null_reason": null
  },
  "ranking": {
   "status": "not-comparable",
   "metric": "Delta AUPRC (support set 16-256)",
   "eval_set": "157 test tasks (fsmol-0.1, ChEMBL27)",
   "snapshot_date": {
    "date": "2026-11-16",
    "precision": "day",
    "year_hint": null,
    "basis": "board snapshot date, or the newest entry date for self/third-party standings",
    "event_kind": "leaderboard-entry",
    "timestamp_utc": null,
    "source_url": null,
    "verification": "verified",
    "note": null
   },
   "board_url": null,
   "entries": [
    {
     "rank": null,
     "model_label": "PN (prototypical network)",
     "model_id": null,
     "site_model": false,
     "score": null,
     "score_unit": null,
     "metric": "Delta AUPRC (support set 16-256)",
     "eval_set": "157 test tasks (fsmol-0.1, ChEMBL27)",
     "source_type": "self",
     "date": {
      "date": "2021",
      "precision": "year",
      "year_hint": null,
      "basis": "Publication year of the paper (the proceedings give no day)",
      "event_kind": "results-published",
      "timestamp_utc": null,
      "source_url": "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/8d3bba7425e7c98c50f52ca1b52d3735-Paper-round2.pdf",
      "verification": "verified",
      "note": null
     },
     "url": "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/8d3bba7425e7c98c50f52ca1b52d3735-Paper-round2.pdf",
     "note": "Not a site model; the best baseline in the original paper's table at support size 16.",
     "second_hand": false,
     "withheld": true,
     "withheld_reason": "Not comparable: protocols and subsets differ, so no score is shown in the ranking row; per-protocol values stay on the entity page."
    }
   ],
   "headline": null,
   "best_site_model": null,
   "headline_null_reason": "Not comparable: support sizes and features differ across sources, so no headline number is given and the caveat text is shown instead.",
   "caveat": "No live leaderboard and the repository is archived; later method papers self-report under different support sizes and features, which are not comparable; no site model reports this benchmark.",
   "retrieved": "2026-11-16"
  },
  "models_reported": [],
  "non_pairings": [],
  "n_models_reported": {
   "counted": 0,
   "by_relation": {
    "official-participation": 0,
    "self-reported-official-protocol": 0,
    "self-reported-modified-protocol": 0,
    "third-party-evaluation": 0,
    "retrospective-dataset-use": 0,
    "cited-as-comparison-only": 0,
    "disputed-or-unconfirmed": 0
   }
  },
  "how_to_use": "Take the data and evaluation code from the official repository, run the official evaluation on the 157 test tasks and tune on the 40 validation tasks. Report delta AUPRC separately for support sizes 16 to 256 and include a random-forest baseline; the number of qualifying tasks varies with support size, so state task coverage. Declare any extra pre-training data and its possible overlap with the test assays. Stronger generalization claims need strict target-family or scaffold splits.",
  "pairs_with": [
   "bench-03-moleculenet",
   "bench-04-tdc",
   "bench-27-polaris"
  ],
  "caveats": [
   "Test tasks are enzyme targets only: 125 of 157 are kinases, with 1-20 tasks per other class.",
   "Split by target, yet protein families and chemical series are not fully separated, so it is closer to neighbourhood interpolation than true out-of-distribution.",
   "Labels are median-binarized and the positive rate is constrained to 0.30-0.70, unlike the rare positives of real screening.",
   "Papers use different support-set sizes and features, so numbers are often not directly comparable.",
   "The official repository is archived read-only and the data do not follow newer ChEMBL releases."
  ],
  "sources": [
   {
    "url": "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/8d3bba7425e7c98c50f52ca1b52d3735-Paper-round2.pdf",
    "title": "FS-Mol paper (NeurIPS 2021 Datasets and Benchmarks)",
    "kind": "paper",
    "date": {
     "date": "2021-12",
     "precision": "month",
     "year_hint": null,
     "basis": "Proceedings",
     "event_kind": "paper-v1",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://github.com/microsoft/FS-Mol",
    "title": "Official repository (archived read-only)",
    "kind": "repository",
    "date": {
     "date": "2026-10-21",
     "precision": "day",
     "year_hint": null,
     "basis": "Archive date (not independently re-verified)",
     "event_kind": "repository-archived",
     "timestamp_utc": null,
     "source_url": null,
     "verification": "verified",
     "note": null
    }
   },
   {
    "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC12076497/",
    "title": "Few-shot method paper (journal version, 2025)",
    "kind": "third-party-paper",
    "date": null
   },
   {
    "url": "https://arxiv.org/abs/2502.12453",
    "title": "Method paper reporting on this benchmark and on molecular property sets",
    "kind": "third-party-paper",
    "date": null
   }
  ],
  "corrections": [
   {
    "field": "first_release",
    "was": "2021-08-13",
    "now": "2021-12",
    "reason": "2021-08-13 is the repository's initial commit, which the policy excludes as first-release evidence; the sourced public milestone is the NeurIPS 2021 Datasets and Benchmarks paper, at month precision.",
    "source_url": "https://github.com/microsoft/FS-Mol",
    "date_found": "2026-11-16"
   },
   {
    "field": "first_release_basis",
    "was": "GitHub repo microsoft/FS-Mol initial commit (data + code); NeurIPS 2021 D&B paper Dec 2021; no arXiv version found",
    "now": "NeurIPS 2021 Datasets & Benchmarks paper, 2021-12 (month precision; no arXiv version found). Repo initial commit 2021-08-13 kept as a note only.",
    "reason": "Basis aligned with the policy; the first repository commit is demoted to a note.",
    "source_url": "https://github.com/microsoft/FS-Mol",
    "date_found": "2026-11-16"
   },
   {
    "field": "ranking",
    "was": "baseline delta AUPRC values shown as a ranking",
    "now": "status not-comparable: no score or rank is displayed in the ranking row",
    "reason": "Under the ranking file's display rules, not-comparable entries carry no score and no rank.",
    "source_url": "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/8d3bba7425e7c98c50f52ca1b52d3735-Paper-round2.pdf",
    "date_found": "2026-11-16"
   }
  ],
  "unverified_fields": [
   "latest_update",
   "release_milestones.first_public_evidence",
   "ranking.entries[0]"
  ],
  "as_of": "2026-11-16",
  "site_urls": {
   "zh": "/AI4S/bio/benchmarks/bench-44-fsmol/",
   "en": "/bio/benchmark/benchmarks/fsmol/"
  },
  "global_entity": null,
  "global_entity_null_reason": "No site-wide /entities/ page exists for this benchmark."
 }
]
