[ { "id": "Kestin2025", "tier": "Tier 1", "full_citation": "Kestin, G., Miller, K., Klales, A., Milbourne, T., \u0026 Ponti, G. (2025). AI tutoring outperforms in-class active learning: an RCT introducing a novel research-based design in an authentic educational setting. Scientific Reports, 15, 17458.", "link": "https://doi.org/10.1038/s41598-025-97652-6", "peer_reviewed": "Yes", "population": "Introductory physics for life sciences; 194 eligible undergraduates; Harvard University, USA.", "design": "Cluster-randomized crossover RCT; two topics over two consecutive weeks; immediate pre/post-tests.", "finding_verbatim": "Students learn significantly more in less time when using the AI tutor.", "effect_size": "Linear-regression estimate 0.63 SD; quantile-regression estimate 0.73–1.3 SD; median post-test 4.5 vs 3.5; p\u003c10^-8.", "conditions_and_limits": "GPT-4; expert-crafted question-specific prompts; pre-written solutions; instructional videos; identical worksheets; understanding/applying/analyzing outcomes; immediate tests only.", "criticism": "Single selective institution and course; two lessons; no delayed retention; attrition from 233 enrolled to 194 eligible; ceiling-effect adjustment produces a wide estimate; no generic-AI arm; authors designed/taught the intervention.", "which_workflow": "AI teaching assistant—direct; case/worked-example scaffolding—indirect.", "claimable_sentence": "In one randomized Harvard physics study, an instructor-authored, carefully scaffolded AI tutor produced higher immediate post-test performance than in-class active learning." }, { "id": "Bastani2025", "tier": "Tier 1", "full_citation": "Bastani, H., Bastani, O., Sungu, A., Ge, H., Kabakcı, Ö., \u0026 Mariman, R. (2025). Generative AI without guardrails can harm learning: Evidence from high school mathematics. Proceedings of the National Academy of Sciences, 122(26), e2422633122.", "link": "https://doi.org/10.1073/pnas.2422633122", "peer_reviewed": "Yes", "population": "Nearly 1,000 students in grades 9–11; mathematics; one private high school in Turkey.", "design": "Preregistered cluster RCT across four sessions; control vs generic GPT-4 vs teacher-grounded guarded GPT-4; assisted practice followed by unassisted exams.", "finding_verbatim": "When access is subsequently taken away, students actually perform worse than those who never had access.", "effect_size": "Practice: GPT Base +0.137/1 (48%) and GPT Tutor +0.361/1 (127%) vs control. Unassisted exam: GPT Base −0.054/1 (−17%); GPT Tutor −0.004, not significant.", "conditions_and_limits": "GPT Tutor prompt contained teacher-designed hints and correct solutions and withheld direct answers; outcomes were session-level math practice/exams.", "criticism": "K–12, not higher education; one school; guarded tutor prevented harm but did not improve unassisted learning; short duration; classroom-level randomization.", "which_workflow": "AI teaching assistant—direct design contrast; Socratic withholding—direct.", "claimable_sentence": "A large classroom experiment found that generic AI reduced subsequent unassisted performance, while teacher-designed hints and guardrails eliminated that harm." }, { "id": "KulikFletcher2016", "tier": "Tier 1", "full_citation": "Kulik, J. A., \u0026 Fletcher, J. D. (2016). Effectiveness of intelligent tutoring systems: A meta-analytic review. Review of Educational Research, 86(1), 42–78.", "link": "https://doi.org/10.3102/0034654315581420", "peer_reviewed": "Yes", "population": "50 controlled evaluations across educational levels, subjects, and intelligent tutoring systems.", "design": "Meta-analysis of controlled evaluations.", "finding_verbatim": "The median effect of intelligent tutoring in the 50 evaluations was to raise test scores 0.66 standard deviations.", "effect_size": "Median ES=0.66 SD; authors equate this with a move from the 50th to 75th percentile.", "conditions_and_limits": "Effects depended strongly on alignment: locally developed tests yielded larger effects than standardized tests; adequate implementation and conventional comparison groups mattered.", "criticism": "Predates generative AI; heterogeneous populations and systems; not specific to higher education or instructor-configured LLMs; median is not a pooled causal effect for a new product.", "which_workflow": "AI teaching assistant—direct mechanism, indirect technology.", "claimable_sentence": "Across 50 controlled evaluations, intelligent tutoring systems improved test performance relative to conventional instruction, with results strongly dependent on test alignment and implementation." }, { "id": "VanLehn2011", "tier": "Tier 1", "full_citation": "VanLehn, K. (2011). The relative effectiveness of human tutoring, intelligent tutoring systems, and other tutoring systems. Educational Psychologist, 46(4), 197–221.", "link": "https://doi.org/10.1080/00461520.2011.611369", "peer_reviewed": "Yes", "population": "Experimental comparisons of human tutoring, computer tutoring, and matched no-tutoring instruction; mixed educational settings.", "design": "Quantitative research review.", "finding_verbatim": "This review did not confirm these beliefs.", "effect_size": "Human tutoring d=0.79; intelligent tutoring systems d=0.76; answer-based systems approximately d=0.31 vs no tutoring.", "conditions_and_limits": "Comparisons attempted to hold content and instructional time constant; effect varied with interaction granularity.", "criticism": "Not a formal modern meta-analysis; small sets for some comparisons; heterogeneous tutors and outcomes; predates LLMs; does not establish equivalence in every context.", "which_workflow": "AI teaching assistant—direct mechanism; Bloom critique.", "claimable_sentence": "A quantitative review found substantially smaller tutoring effects than Bloom’s 2σ benchmark and similar average effects for human and step-based computer tutoring." }, { "id": "Wisniewski2020", "tier": "Tier 2", "full_citation": "Wisniewski, B., Zierer, K., \u0026 Hattie, J. (2020). The power of feedback revisited: A meta-analysis of educational feedback research. Frontiers in Psychology, 10, 3087.", "link": "https://doi.org/10.3389/fpsyg.2019.03087", "peer_reviewed": "Yes", "population": "435 studies; 994 effects; more than 61,000 learners across educational levels and domains.", "design": "Random-effects meta-analysis.", "finding_verbatim": "Feedback cannot be understood as a single consistent form of treatment.", "effect_size": "Overall d=0.48; significant heterogeneity.", "conditions_and_limits": "Effects were larger for cognitive and motor outcomes and depended substantially on the information content of feedback.", "criticism": "Broad mixture of populations, feedback types, designs, and outcomes; includes Hattie as coauthor but uses study-level meta-analysis; not specific to AI or timely feedback.", "which_workflow": "AI teaching assistant—timely formative feedback; case practice—expert feedback.", "claimable_sentence": "Information-rich feedback can improve learning, but its impact varies substantially with what the feedback communicates." }, { "id": "Springer1999", "tier": "Tier 2", "full_citation": "Springer, L., Stanne, M. E., \u0026 Donovan, S. S. (1999). Effects of small-group learning on undergraduates in science, mathematics, engineering, and technology: A meta-analysis. Review of Educational Research, 69(1), 21–51.", "link": "https://doi.org/10.3102/00346543069001021", "peer_reviewed": "Yes", "population": "Undergraduate STEM; 39 included studies; achievement N=3,472, persistence N=2,014, attitudes N=1,293.", "design": "Meta-analysis of controlled and pre/post field studies from 1980 onward.", "finding_verbatim": "Students who learned in small groups demonstrated greater achievement.", "effect_size": "Achievement d=0.51; persistence d=0.46; attitudes d=0.55; all weighted main effects significant.", "conditions_and_limits": "Cooperative, collaborative, and mixed small-group formats; postsecondary STEM; effects varied by design and setting.", "criticism": "Older literature; achievement and attitude effects heterogeneous; instructor-investigator studies had larger effects; sparse descriptions prevented analysis of detailed procedures; not AI-mediated.", "which_workflow": "Group work/shared team model—direct mechanism.", "claimable_sentence": "Across undergraduate STEM studies, structured small-group learning was associated with higher achievement, persistence, and more favorable attitudes than non-group instruction." }, { "id": "Biesma2019", "tier": "Tier 2", "full_citation": "Biesma, R., Kennedy, M.-C., Pawlikowska, T., Brugha, R., Conroy, R., \u0026 Doyle, F. (2019). Peer assessment to improve medical student’s contributions to team-based projects: randomised controlled trial and qualitative follow-up. BMC Medical Education, 19, 371.", "link": "https://doi.org/10.1186/s12909-019-1783-8", "peer_reviewed": "Yes", "population": "223 second-year medical students in 37 teams; Royal College of Surgeons in Ireland.", "design": "Cluster RCT with qualitative focus-group follow-up; 19 intervention and 18 control teams.", "finding_verbatim": "There was no difference in team contribution, and other forms of team effectiveness.", "effect_size": "Primary outcome β=0.76, 95% CI [−0.58, 2.09], p=.26; all CATME secondary outcomes nonsignificant.", "conditions_and_limits": "Transparent consensus-based peer marking with fixed points; students had no prior experience; 10-week team project.", "criticism": "Poor implementation was part of the causal result: students avoided differentiating peers because of relationship concerns; per-protocol analysis; one medical school.", "which_workflow": "Group work/accountability—direct negative evidence.", "claimable_sentence": "A cluster trial found that a transparent peer-marking system did not improve team contribution because students largely resisted implementing it." }, { "id": "Dochy2003", "tier": "Tier 2", "full_citation": "Dochy, F., Segers, M., Van den Bossche, P., \u0026 Gijbels, D. (2003). Effects of problem-based learning: A meta-analysis. Learning and Instruction, 13(5), 533–568.", "link": "https://doi.org/10.1016/S0959-4752(02)00025-7", "peer_reviewed": "Yes", "population": "43 empirical studies of problem-based learning in tertiary education in real classrooms; most medical/health contexts.", "design": "Meta-analysis; included studies were quasi-experimental.", "finding_verbatim": "There is a robust positive effect from PBL on the skills of students.", "effect_size": "Skills ES=+0.460 ±0.058; knowledge ES=−0.223 ±0.058, with the knowledge result driven by two outliers.", "conditions_and_limits": "Effects depended on expertise, implementation scope, retention interval, assessment method, and research design.", "criticism": "No randomized included studies; substantial heterogeneity; knowledge estimate negative and non-robust; many medical curricula; PBL is not equivalent to an AI case cast.", "which_workflow": "Case study/AI expert cast—direct for PBL, indirect for interface.", "claimable_sentence": "In tertiary education, problem-based learning showed a moderate positive effect on applying knowledge, while immediate factual-knowledge results were mixed." }, { "id": "Atkinson2000", "tier": "Tier 2", "full_citation": "Atkinson, R. K., Derry, S. J., Renkl, A., \u0026 Wortham, D. (2000). Learning from examples: Instructional principles from the worked examples research. Review of Educational Research, 70(2), 181–214.", "link": "https://doi.org/10.3102/00346543070002181", "peer_reviewed": "Yes", "population": "Experimental worked-example literature across mathematics and other problem-solving domains; learners largely in early skill acquisition.", "design": "Narrative research review and design-principle synthesis.", "finding_verbatim": "Worked examples are associated with early stages of skill development.", "effect_size": "No single pooled effect size reported.", "conditions_and_limits": "Benefits depend on integrated components, multiple examples, signaling deep structure, self-explanation, and learner expertise; support should fade as expertise grows.", "criticism": "Narrative rather than meta-analytic; much laboratory and school research; expertise-reversal limits; expert modeling by an AI persona was not tested.", "which_workflow": "Case study/AI expert cast—expert modeling and worked examples, indirect.", "claimable_sentence": "Worked examples can support early skill development when they expose conceptual structure and are adapted as learners gain expertise." }, { "id": "Freeman2014", "tier": "Tier 2", "full_citation": "Freeman, S., Eddy, S. L., McDonough, M., Smith, M. K., Okoroafor, N., Jordt, H., \u0026 Wenderoth, M. P. (2014). Active learning increases student performance in science, engineering, and mathematics. Proceedings of the National Academy of Sciences, 111(23), 8410–8415.", "link": "https://doi.org/10.1073/pnas.1319030111", "peer_reviewed": "Yes", "population": "225 studies in undergraduate STEM; 158 studies for exam/concept-inventory effects and 67 for failure rates.", "design": "Meta-analysis comparing active learning with traditional lecturing.", "finding_verbatim": "Student performance on examinations and concept inventories increased by 0.47 SDs under active learning.", "effect_size": "0.47 SD; traditional lecture failure odds ratio 1.95; average exam scores about 6% higher under active learning.", "conditions_and_limits": "Undergraduate STEM; active learning broadly defined; section-level instructional comparisons.", "criticism": "Interventions heterogeneous; study quality varied and many were not randomized; possible publication bias; does not isolate cases, group work, AI, or tutoring.", "which_workflow": "Group work and case study—broad indirect mechanism.", "claimable_sentence": "Across undergraduate STEM, active-learning sections produced higher assessment scores and lower failure rates than traditional lectures." }, { "id": "Deslauriers2019", "tier": "Tier 2", "full_citation": "Deslauriers, L., McCarty, L. S., Miller, K., Callaghan, K., \u0026 Kestin, G. (2019). Measuring actual learning versus feeling of learning in response to being actively engaged in the classroom. Proceedings of the National Academy of Sciences, 116(39), 19251–19257.", "link": "https://doi.org/10.1073/pnas.1821936116", "peer_reviewed": "Yes", "population": "149 students across introductory physics courses at Harvard University, USA.", "design": "Randomized crossover classroom experiment over two class meetings.", "finding_verbatim": "Students in the active classroom learn more, but they feel like they learn less.", "effect_size": "Learning effect reported as 0.46 SD in the primary experiment; feeling-of-learning favored passive lecture; exact estimates vary by course/analysis.", "conditions_and_limits": "Identical content; highly experienced instructors; immediate custom multiple-choice tests; active vs fluent lecture delivery.", "criticism": "One institution and discipline; two sessions; immediate researcher-built outcomes; instructor and treatment could not be blinded; useful for adoption, not product efficacy.", "which_workflow": "All workflows—faculty/student adoption objection.", "claimable_sentence": "Students may underestimate active learning because the effort that improves performance can make the experience feel less fluent." }, { "id": "Bisra2018", "tier": "Tier 2", "full_citation": "Bisra, K., Liu, Q., Nesbit, J. C., Salimi, F., \u0026 Winne, P. H. (2018). Inducing self-explanation: A meta-analysis. Educational Psychology Review, 30(3), 703–725.", "link": "https://doi.org/10.1007/s10648-018-9434-x", "peer_reviewed": "Yes", "population": "64 research reports; 69 effects; approximately 5,917 learners across school, undergraduate, and adult settings and multiple subjects.", "design": "Random-effects meta-analysis.", "finding_verbatim": "Self-explanation prompts are a potentially powerful intervention across a range of instructional conditions.", "effect_size": "Overall Hedges g=0.55, 95% CI approximately [0.45, 0.65].", "conditions_and_limits": "Prompts induced explanations while studying or problem solving; effects varied with prompt format, knowledge type, task, and educational level.", "criticism": "Mixed age groups and tasks; not specifically oral or AI-mediated; possible publication dependence and heterogeneity; explaining aloud was not isolated from written self-explanation.", "which_workflow": "Voice assessment—self-explanation; case practice—direct mechanism.", "claimable_sentence": "Prompting learners to explain relationships and reasoning improves learning across a range of study and problem-solving conditions." }, { "id": "RoedigerKarpicke2006", "tier": "Tier 2", "full_citation": "Roediger, H. L., III, \u0026 Karpicke, J. D. (2006). Test-enhanced learning: Taking memory tests improves long-term retention. Psychological Science, 17(3), 249–255.", "link": "https://doi.org/10.1111/j.1467-9280.2006.01693.x", "peer_reviewed": "Yes", "population": "Undergraduate psychology participant pools; two experiments using educational prose passages in the USA.", "design": "Randomized laboratory experiments comparing repeated study with one or repeated free-recall tests without feedback.", "finding_verbatim": "On the delayed tests, prior testing produced substantially greater retention than studying.", "effect_size": "Experiment 2 at one week: repeated testing about 61% recall vs repeated study about 40%; authors did not report a standardized effect in the abstract.", "conditions_and_limits": "Free recall without feedback; 5-minute, 2-day, and 1-week delays; prose learning rather than course grades.", "criticism": "Laboratory-style, narrow materials, short retention window, and written recall; does not establish validity or security of oral assessment or automated scoring.", "which_workflow": "Voice assessment—retrieval practice, direct mechanism.", "claimable_sentence": "Retrieving learned material can strengthen delayed retention more than spending the same opportunities restudying it." }, { "id": "Nallaya2024", "tier": "Tier 2", "full_citation": "Nallaya, S., Gentili, S., Weeks, S., \u0026 Baldock, K. (2024). The validity, reliability, academic integrity and integration of oral assessments in higher education: A systematic review. Issues in Educational Research, 34(2), 629–646.", "link": "https://www.iier.org.au/iier34/nallaya.pdf", "peer_reviewed": "Yes", "population": "17 peer-reviewed higher-education studies published 2010–2021 across disciplines and countries.", "design": "PRISMA systematic review with narrative synthesis.", "finding_verbatim": "Oral assessments can be both valid and reliable.", "effect_size": "No pooled effect size; review reports study-level validity/reliability evidence.", "conditions_and_limits": "Validity and reliability require clear criteria, authentic alignment, assessor training, moderation, practice opportunities, and inclusive design.", "criticism": "Only 17 heterogeneous studies; English-language and date restrictions; narrative synthesis; academic-integrity claims often perceptual or case-based; no validation of LLM scoring.", "which_workflow": "Voice assessment—oral validity/integrity, direct mechanism.", "claimable_sentence": "Structured oral assessment can validly assess reasoning when criteria, assessor training, moderation, practice, and accessibility are deliberately designed." }, { "id": "Huxham2012", "tier": "Tier 2", "full_citation": "Huxham, M., Campbell, F., \u0026 Westwood, J. (2012). Oral versus written assessments: A test of student performance and attitudes. Assessment \u0026 Evaluation in Higher Education, 37(1), 125–136.", "link": "https://doi.org/10.1080/02602938.2010.515012", "peer_reviewed": "Yes", "population": "Two biology cohorts at Edinburgh Napier University, UK: N=99 randomized between modes and N=29 completing both modes.", "design": "Randomized mode comparison plus within-subject comparison and student-attitude study.", "finding_verbatim": "Both cohorts showed highly significant differences in the mean marks achieved, with better performance in the oral assessment.", "effect_size": "Higher oral marks; exact standardized effect not reported in the abstract/source record.", "conditions_and_limits": "Same biology questions delivered orally or in writing; small cohorts; attitudes also collected.", "criticism": "Higher marks do not prove more learning or validity; possible mode, interaction, and examiner effects; small single-institution study; oral anxiety remains relevant.", "which_workflow": "Voice assessment—oral performance, direct but limited.", "claimable_sentence": "In two university biology cohorts, students earned higher marks on oral than written versions of comparable questions." }, { "id": "Scarfe2024", "tier": "Tier 3", "full_citation": "Scarfe, P., Watcham, K., Clarke, A., \u0026 Roesch, E. (2024). A real-world test of artificial intelligence infiltration of a university examinations system: A ‘Turing Test’ case study. PLOS ONE, 19(6), e0305354.", "link": "https://doi.org/10.1371/journal.pone.0305354", "peer_reviewed": "Yes", "population": "Five undergraduate psychology modules across three years; one UK university; 63 AI submissions and 1,134 real submissions.", "design": "Blind real-world observational infiltration study in live online take-home examinations.", "finding_verbatim": "We found that 94% of our AI submissions were undetected.", "effect_size": "94% unflagged by markers; AI answers averaged half a grade boundary higher; 83.4% probability the AI set outperformed an equal-size random real-student set.", "conditions_and_limits": "100% GPT-4-written answers inserted into 2022–23 exams; approximately 5% AI prevalence; ordinary marking system; short-answer and essay exams.", "criticism": "One institution/discipline/year; only 63 AI submissions; model and assessment conditions are time-sensitive; ‘detection’ was marker concern, not a controlled detector test; three finalist submissions.", "which_workflow": "Voice assessment—why now, indirect; assessment integrity.", "claimable_sentence": "In one blind study of five UK psychology modules, markers did not flag 94% of wholly AI-written exam submissions." }, { "id": "WeberWulff2023", "tier": "Tier 3", "full_citation": "Weber-Wulff, D., Anohina-Naumeca, A., Bjelobaba, S., Foltýnek, T., Guerrero-Dib, J., Popoola, O., Šigut, P., \u0026 Waddington, L. (2023). Testing of detection tools for AI-generated text. International Journal for Educational Integrity, 19, 26.", "link": "https://doi.org/10.1007/s40979-023-00146-z", "peer_reviewed": "Yes", "population": "14 AI-text detectors: 12 public tools plus Turnitin and PlagiarismCheck; human, machine-translated, AI, edited, paraphrased, and obfuscated texts.", "design": "Comparative diagnostic-accuracy evaluation.", "finding_verbatim": "The available detection tools are neither accurate nor reliable.", "effect_size": "Highest tool accuracies were below 80% in the authors’ inclusive binary analysis; performance varied by text condition.", "conditions_and_limits": "Tools and models available in 2023; original document set; multiple transformation conditions; accuracy/error analysis.", "criticism": "Rapidly obsolete tool versions; small purpose-built corpus; accuracy depends on threshold, prevalence, language, genre, and scoring rule; not a live disciplinary process.", "which_workflow": "Voice assessment—why now; detection unreliability.", "claimable_sentence": "AI-text detector output should not be treated as proof of authorship because accuracy varies and obfuscation can sharply reduce performance." }, { "id": "Liang2023", "tier": "Tier 3", "full_citation": "Liang, W., Yuksekgonul, M., Mao, Y., Wu, E., \u0026 Zou, J. (2023). GPT detectors are biased against non-native English writers. Patterns, 4(7), 100779.", "link": "https://doi.org/10.1016/j.patter.2023.100779", "peer_reviewed": "Yes", "population": "91 human-written TOEFL essays by non-native English writers and 88 US eighth-grade essays; seven detectors.", "design": "Comparative diagnostic and perturbation study.", "finding_verbatim": "They incorrectly labeled more than half of the TOEFL essays as ‘AI-generated.’", "effect_size": "Average false-positive rate 61.3% on TOEFL essays; all seven flagged 19.8% unanimously; at least one flagged 97.8%.", "conditions_and_limits": "Specific 2023 detectors and short essay corpora; comparison of native and non-native writing; vocabulary perturbations.", "criticism": "Convenience samples differed in task, age, and provenance; rapidly changing tools; not a prevalence-representative campus deployment; later studies may obtain different rates.", "which_workflow": "Voice assessment—why now; fairness of text detection.", "claimable_sentence": "In a 2023 test, seven detectors falsely labeled human-written non-native-English essays as AI-generated at a high average rate." }, { "id": "Bloom1984", "tier": "Contested", "full_citation": "Bloom, B. S. (1984). The 2 sigma problem: The search for methods of group instruction as effective as one-to-one tutoring. Educational Researcher, 13(6), 4–16.", "link": "https://doi.org/10.3102/0013189X013006004", "peer_reviewed": "Yes", "population": "Selected mastery-learning and tutoring studies, including small studies conducted by Bloom’s students.", "design": "Narrative synthesis and problem statement; not a modern meta-analysis.", "finding_verbatim": "The average tutored student was above 98% of the students in the control class.", "effect_size": "Claimed approximately 2 SD for mastery-based one-to-one tutoring vs conventional instruction in selected studies.", "conditions_and_limits": "Intensive one-to-one tutoring combined with mastery learning and corrective feedback; historical, selected comparisons.", "criticism": "The 2σ figure is not a replicable general tutoring estimate; selection, small samples, bundled treatment, and historical controls limit inference. VanLehn found human tutoring d=0.79; other syntheses are smaller.", "which_workflow": "AI teaching assistant—historical context only.", "claimable_sentence": "Bloom framed an influential challenge about scaling mastery tutoring, but later reviews report much smaller typical effects." } ]