How accurate is this?

Model SQL:

SELECT count(posting_key) FROM fct_posting WHERE family = 'ai_engineer'

Gold SQL:

SELECT count(*) FROM fct_posting WHERE family = 'ai_engineer' AND is_open AND remote_us AND is_canonical_listing

Verdict: no predicted column matches gold column 'count_star()'

Model SQL:

SELECT count(posting_key) FROM fct_posting WHERE seniority = 'entry' AND is_open

Gold SQL:

SELECT count(*) FROM fct_posting WHERE seniority = 'entry' AND is_open AND remote_us AND is_canonical_listing

Verdict: no predicted column matches gold column 'count_star()'

Model SQL:

SELECT SUM(CASE WHEN ps.cluster = 'llm_genai' THEN 1 ELSE 0 END) * 1.0 / COUNT(DISTINCT f.posting_key) FROM fct_posting f LEFT JOIN posting_skills ps ON f.posting_key = ps.posting_key WHERE f.family = 'ml_engineer' AND f.is_open AND f.remote_us AND f.is_canonical_listing

Gold SQL:

SELECT count(*) FILTER (WHERE posting_key IN (SELECT posting_key FROM posting_skills WHERE cluster = 'llm_genai')) * 1.0 / count(*) AS share
FROM fct_posting WHERE family = 'ml_engineer' AND is_open AND remote_us AND is_canonical_listing

Verdict: no predicted column matches gold column 'share'

Model SQL:

SELECT count(*) FROM fct_posting f LEFT JOIN mart_comp_by_family m ON f.family = m.family WHERE f.is_open AND f.remote_us AND f.is_canonical_listing AND f.comp_mid_usd > m.p50 * 1.5

Gold SQL:

SELECT count(*) AS listings FROM (
  SELECT comp_mid_usd, median(comp_mid_usd) OVER (PARTITION BY family) AS fam_median
  FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing)
WHERE comp_mid_usd > 1.5 * fam_median

Verdict: no predicted column matches gold column 'listings'

Model SQL:

SELECT count(*) AS postings FROM fct_posting WHERE lower(workplace) = 'hybrid'

Gold: refuse (the warehouse cannot answer this).

Verdict: answered an unanswerable question

Model SQL (after 2 retries):

SELECT DISTINCT T1.company_name FROM dim_company AS T1 INNER JOIN fct_posting AS T2 ON T1.company_key = T2.company_key WHERE T2.equity_mentioned = TRUE

Error: Binder Error: Table "T2" does not have a column named "equity_mentioned" Candidate bindings: : "phd_mentioned"

Gold: refuse (the warehouse cannot answer this).

Verdict: answered an unanswerable question

Model SQL (after 1 retries):

SELECT count(*) AS listings FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing AND published_at BETWEEN DATE '2026-08-30' AND DATE '2026-09-29'

Gold SQL:

SELECT count(*) AS listings FROM fct_posting
WHERE is_open AND remote_us AND is_canonical_listing
  AND published_at::DATE > (SELECT max(last_seen) FROM fct_posting) - INTERVAL 30 DAY

Verdict: no predicted column matches gold column 'listings'

Model SQL (after 1 retries):

SELECT family, approx_quantile(age_days_at_last_seen, 0.5) AS median_age FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing GROUP BY family

Gold SQL:

SELECT family, median(age_days_at_last_seen) AS median_age_days FROM fct_posting
WHERE is_open AND remote_us AND is_canonical_listing GROUP BY family

Verdict: no predicted column matches gold column 'median_age_days'

Model SQL:

SELECT
  fp.family,
  MAX(fp.comp_mid_usd) AS highest_pay_midpoint,
  dc.company_name
FROM fct_posting AS fp
JOIN dim_company AS dc
  ON fp.company_key = dc.company_key
WHERE
  fp.is_open AND fp.remote_us AND fp.is_canonical_listing AND fp.comp_mid_usd IS NOT NULL
GROUP BY
  fp.family,
  dc.company_name
ORDER BY
  fp.family,
  highest_pay_midpoint DESC

Gold SQL:

SELECT family, company_name, comp_mid_usd FROM (
  SELECT f.family, d.company_name, f.comp_mid_usd,
         rank() OVER (PARTITION BY f.family ORDER BY f.comp_mid_usd DESC) AS r
  FROM fct_posting f JOIN dim_company d USING (company_key)
  WHERE f.is_open AND f.remote_us AND f.is_canonical_listing AND f.comp_mid_usd IS NOT NULL)
WHERE r = 1

Verdict: row count 481 != gold 8

Model SQL:

SELECT family, company_name, comp_mid_usd FROM fct_posting AS fp JOIN dim_company AS dc ON fp.company_key = dc.company_key WHERE is_open AND remote_us AND is_canonical_listing QUALIFY ROW_NUMBER() OVER (PARTITION BY family ORDER BY comp_mid_usd DESC) = 1

Gold SQL:

SELECT family, company_name, comp_mid_usd FROM (
  SELECT f.family, d.company_name, f.comp_mid_usd,
         rank() OVER (PARTITION BY f.family ORDER BY f.comp_mid_usd DESC) AS r
  FROM fct_posting f JOIN dim_company d USING (company_key)
  WHERE f.is_open AND f.remote_us AND f.is_canonical_listing AND f.comp_mid_usd IS NOT NULL)
WHERE r = 1

Verdict: row count 7 != gold 8

Model SQL:

WITH RankedSkills AS (SELECT family, cluster_label, share, ROW_NUMBER() OVER (PARTITION BY family ORDER BY share DESC) as rn FROM mart_skill_demand WHERE scope = 'all' AND family IS NOT NULL) SELECT family, cluster_label, share FROM RankedSkills WHERE rn <= 3 ORDER BY family, rn

Gold SQL:

SELECT family, skill, share FROM (
  SELECT family, skill, share, row_number() OVER (PARTITION BY family ORDER BY share DESC, skill) AS r
  FROM mart_skill_demand WHERE scope = 'all')
WHERE r <= 3

Verdict: no predicted column matches gold column 'skill'

Model SQL:

SELECT count(f.posting_key) AS listings_above_1_5x_median FROM fct_posting AS f JOIN mart_comp_by_family AS m ON f.family = m.family AND f.seniority = m.seniority WHERE f.is_open AND f.remote_us AND f.is_canonical_listing AND f.comp_mid_usd > (1.5 * m.p50)

Gold SQL:

SELECT count(*) AS listings FROM (
  SELECT comp_mid_usd, median(comp_mid_usd) OVER (PARTITION BY family) AS fam_median
  FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing)
WHERE comp_mid_usd > 1.5 * fam_median

Verdict: no predicted column matches gold column 'listings'

Model SQL (after 2 retries):

WITH OverallMedian AS (SELECT p50 AS overall_median FROM mart_comp_by_family WHERE family IS NULL AND seniority IS NULL), SeniorityMedian AS (SELECT seniority, p50 AS seniority_median FROM mart_comp_by_family WHERE family IS NULL AND seniority IS NOT NULL AND p50 IS NOT NULL) SELECT sm.seniority, sm.seniority_median, sm.seniority_median - om.overall_median AS difference_from_overall FROM SeniorityMedian sm, OverallMedian om WHERE om.overall_median IS NOT NULL ORDER BY sm.seniority;

Gold SQL:

SELECT seniority, median(comp_mid_usd) AS median_pay,
       median(comp_mid_usd) - (SELECT median(comp_mid_usd) FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing) AS diff_from_overall
FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing
GROUP BY seniority

Verdict: row count 0 != gold 7

Model SQL (after 2 retries):

SELECT open_listings FROM mart_daily_market WHERE crawl_date = '2026-09-29' AND remote_us = true AND family IS NULL AND seniority IS NULL

Gold SQL:

SELECT count(*) AS listings FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing

Verdict: row count 0 != gold 1

Model SQL (after 2 retries):

SELECT CASE WHEN SUM(CASE WHEN ps.skill = 'rag' THEN 1 ELSE 0 END) > 0 THEN 'Mentions RAG' ELSE 'Does not mention RAG' END AS rag_mention_status, AVG(f.comp_mid_usd) AS median_pay_midpoint FROM fct_posting AS f LEFT JOIN posting_skills AS ps ON f.posting_key = ps.posting_key WHERE f.is_open AND f.remote_us AND f.is_canonical_listing GROUP BY rag_mention_status

Error: Binder Error: GROUP BY clause cannot contain aggregates!

Gold SQL:

SELECT (posting_key IN (SELECT posting_key FROM posting_skills WHERE skill = 'rag')) AS mentions_rag,
       median(comp_mid_usd) AS median_pay
FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing
GROUP BY mentions_rag

Verdict: execution error

Model SQL (after 2 retries):

(no SQL: refused - )

Error: no JSON object in reply

Gold SQL:

SELECT count(*) FROM fct_posting WHERE is_open AND remote_us AND is_canonical_listing

Verdict: execution error