Interactive: the p-value is the tail of the null distribution - Introduction to Statistical Concepts and Methods (MATH 2200)

Drag the observed-statistic slider below. The blue histogram is a randomization null distribution: 10,000 differences in group means, each from one random shuffle of the group labels of a simulated two-group experiment. Every shuffle is a world where the null hypothesis is true, so the histogram shows the wobble chance alone produces (centered on the green dotted no-effect line at 0).

The slider sets a hypothetical observed test statistic $|D|$ . The figure shades the two-sided tail beyond $\pm|D|$ (vermillion, hatched) and reports the simulated p-value -- the proportion of shuffles at least as extreme as the observed value:

\text{p-value} = \frac{\#\{\,i : |D^{*}_i| \ge |D_{\text{obs}}|\,\}}{M}.

Slide right and watch the shaded tail -- and the p-value -- shrink. Somewhere around $|D| \approx 4$ the tail proportion drops past 0.05: that is the $\alpha = 0.05$ line made visible.

import numpy as np
import plotly.graph_objects as go
from IPython.display import HTML

# Okabe-Ito colorblind-safe palette (CLAUDE.md Part 1 accessibility rule)
OI_BLUE       = "#0072B2"   # the null distribution (typical, "chance" outcomes)
OI_VERMILLION = "#D55E00"   # the two-sided tail beyond the observed statistic
OI_GREEN      = "#009E73"   # the zero / no-effect reference line

# -- A simulated randomized two-group experiment -------------------------------
# 120 students, 60 per arm. We draw all 120 "measurements" from ONE common
# distribution, then build the NULL distribution the honest way: by repeatedly
# shuffling the 120 group labels and recomputing the difference in group means.
# Each shuffle is a world where the null hypothesis is literally true (the labels
# are meaningless), so the spread of these differences is exactly the wobble that
# chance alone produces. SIMULATED data, fixed seed -> identical every build.
# No real dataset value is claimed here.
rng = np.random.default_rng(2200)
N, n1 = 120, 60
values = rng.normal(loc=50.0, scale=11.0, size=N)   # pooled measurements (sim)

M = 10_000                                          # number of label shuffles
perm = np.argsort(rng.random((M, N)), axis=1)       # M random permutations
g1, g2 = perm[:, :n1], perm[:, n1:]                 # split into two fake groups
null_diffs = values[g1].mean(axis=1) - values[g2].mean(axis=1)   # D* per shuffle
null_sd = null_diffs.std()

# Fixed, symmetric binning so the bars never jump as the slider moves.
edges   = np.linspace(-7, 7, 42)                    # 41 bins
centers = (edges[:-1] + edges[1:]) / 2
binw    = edges[1] - edges[0]
counts, _ = np.histogram(null_diffs, bins=edges)

# Slider stops: a grid of hypothetical OBSERVED test statistics |D_obs|.
obs_grid = np.round(np.arange(0.5, 6.01, 0.5), 1)   # 0.5, 1.0, ... , 6.0
start_k  = int(np.where(np.isclose(obs_grid, 4.0))[0][0])  # open near p ~ 0.05

def pval(t):
    """Two-sided SIMULATED p-value: proportion of shuffles at least as extreme."""
    return float(np.mean(np.abs(null_diffs) >= t))

# Trace 0: the full null distribution (always visible).
fig = go.Figure()
fig.add_trace(go.Bar(
    x=centers, y=counts, width=binw, name="null distribution",
    marker=dict(color=OI_BLUE, line=dict(color="white", width=0.5)),
    hovertemplate="difference: %{x:.2f}<br>shuffles: %{y}<extra></extra>",
))

# Traces 1..K: the two-sided tail overlay for each observed value (hatched
# vermillion). Bars are zero outside the tail, so only the extreme region is
# shaded; the diagonal pattern means the tail is NOT signalled by colour alone.
for t in obs_grid:
    tail_y = np.where(np.abs(centers) >= t, counts, 0)
    fig.add_trace(go.Bar(
        x=centers, y=tail_y, width=binw, visible=False,
        name="tail (>= observed)",
        marker=dict(color=OI_VERMILLION, pattern=dict(shape="/", size=6),
                    line=dict(color="white", width=0.5)),
        hovertemplate="difference: %{x:.2f}<br>shuffles in tail: %{y}<extra></extra>",
    ))

def layout_for(t):
    """The per-step layout patch: title, the +-|D| boundary lines, p-value box."""
    p = pval(t)
    n_extreme = int(round(p * M))
    p_txt = "< 0.0001" if p == 0 else f"{p:.4f}"
    shapes = [
        # zero / no-effect reference (green dashed, full height)
        dict(type="line", xref="x", yref="paper", x0=0, x1=0, y0=0, y1=1,
             line=dict(color=OI_GREEN, width=2, dash="dot")),
        # observed boundaries at +-|D| (vermillion dashed, full height)
        dict(type="line", xref="x", yref="paper", x0=t, x1=t, y0=0, y1=1,
             line=dict(color=OI_VERMILLION, width=2.5, dash="dash")),
        dict(type="line", xref="x", yref="paper", x0=-t, x1=-t, y0=0, y1=1,
             line=dict(color=OI_VERMILLION, width=2.5, dash="dash")),
    ]
    annotations = [
        dict(xref="x", yref="paper", x=t, y=1.0, yanchor="bottom",
             showarrow=False, font=dict(color=OI_VERMILLION, size=12),
             text=f"+observed = +{t:.1f}"),
        dict(xref="x", yref="paper", x=-t, y=1.0, yanchor="bottom",
             showarrow=False, font=dict(color=OI_VERMILLION, size=12),
             text=f"-observed = -{t:.1f}"),
        dict(xref="paper", yref="paper", x=0.015, y=0.97,
             xanchor="left", yanchor="top", align="left", showarrow=False,
             bordercolor=OI_VERMILLION, borderwidth=1.5, borderpad=6,
             bgcolor="rgba(255,255,255,0.85)", font=dict(size=12),
             text=("<b>two-sided p-value = tail share</b><br>"
                   f"|D*| &#8805; {t:.1f} in {n_extreme} of {M:,} shuffles<br>"
                   f"<b>p = {p_txt}</b>")),
    ]
    title = (f"Randomization null distribution (simulated)  -  observed |D| = {t:.1f},  "
             f"two-sided p = {p_txt}")
    return shapes, annotations, title

# Build slider steps: toggle the matching tail trace + patch the layout.
steps = []
for k, t in enumerate(obs_grid):
    vis = [True] + [j == k for j in range(len(obs_grid))]   # trace 0 always on
    shapes, annotations, title = layout_for(t)
    steps.append(dict(
        method="update", label=f"{t:.1f}",
        args=[{"visible": vis},
              {"shapes": shapes, "annotations": annotations, "title.text": title}]))

# Make the starting step visible and seed the initial layout from it.
fig.data[1 + start_k].visible = True
shapes0, annotations0, title0 = layout_for(obs_grid[start_k])

fig.update_layout(
    template="simple_white", barmode="overlay", bargap=0,
    showlegend=False,
    xaxis_title="Difference in group means under the null (treatment - control)",
    yaxis_title="Number of shuffles (out of 10,000)",
    xaxis_range=[-7, 7],
    height=480, margin=dict(t=80, r=20, b=110, l=70),
    shapes=shapes0, annotations=annotations0, title=title0,
    sliders=[dict(active=start_k, pad={"t": 50},
                  currentvalue={"prefix": "Observed test statistic  |D| = "},
                  steps=steps)],
)

# THE GOTCHA FIX: emit text/html ourselves (embeddable div + CDN plotly.js).
HTML(fig.to_html(full_html=False, include_plotlyjs="cdn"))