A/B testing & experimentation

import numpy as np
from scipy import stats

# Simulated A/B test data
n_A = 1000
n_B = 1000
conversions_A = 120
conversions_B = 140

# Prior: Beta(1, 1) uniform
alpha_prior, beta_prior = 1, 1

# Posterior parameters
alpha_A = alpha_prior + conversions_A
beta_A = beta_prior + n_A - conversions_A
alpha_B = alpha_prior +…

import numpy as np

def benjamini_hochberg(p_values, alpha=0.05):
    p_values = np.array(p_values)
    n = len(p_values)
    sorted_idx = np.argsort(p_values)
    sorted_p = p_values[sorted_idx]
    
    thresholds = (np.arange(1, n + 1) / n) * alpha
    significant = sorted_p <= thresholds
    
    if not significan…

import random


def bootstrap_ci(data, statistic, n_bootstraps=1000, ci_level=0.95, seed=42):
    random.seed(seed)
    n = len(data)
    boot_stats = []

    for _ in range(n_bootstraps):
        sample = [random.choice(data) for _ in range(n)]
        boot_stats.append(statistic(sample))

    boot_stats.sort()
    l…

import random


def sample_ratio_mismatch(population_size: int, sample_size: int, p: float) -> float:
    """
    Estimate the probability that a simple random sample's proportion
    differs from the population proportion by more than 10%.
    """
    total_counts = [0, 0]
    for _ in range(10000):
        sample = …

import math
from collections import Counter

def chi_square_statistic(observed):
    """
    Compute chi-square statistic for a mock conversion test.
    observed: dict mapping outcomes to observed frequencies.
    """
    observed = Counter(observed)
    n = sum(observed.values())
    expected = n / len(observed) if …

import numpy as np
from scipy.stats import norm


def delta_method_ratio_delta(control: np.ndarray, treatment: np.ndarray, confidence: float = 0.95):
    """Estimate confidence interval for ratio metric using delta method.

    Args:
        control: numerator/denominator pairs from control group (n x 2 array)
       …

import numpy as np
from scipy import stats

def mann_whitney_u_mock(sample_a, sample_b):
    """Compute Mann-Whitney U and p-value manually."""
    # Combine and rank
    combined = sample_a + sample_b
    n_a, n_b = len(sample_a), len(sample_b)
    n_total = n_a + n_b
    
    # Rank with ties handling (average ranks…

import numpy as np

def two_sample_t_test(sample1, sample2):
    """Perform Welch's t-test for two independent samples."""
    n1, n2 = len(sample1), len(sample2)
    mean1, mean2 = np.mean(sample1), np.mean(sample2)
    var1, var2 = np.var(sample1, ddof=1), np.var(sample2, ddof=1)

    # Standard error of difference
…

import pandas as pd
import numpy as np


def intent_to_treat_analysis(data):
    """Perform intent-to-treat (ITT) analysis.

    ITT compares outcomes based on initial treatment assignment,
    regardless of whether participants actually received the treatment.
    """
    # Create a copy to avoid mutating the origina…

import math
from statistics import mean, variance


def welch_t_test(sample1, sample2):
    n1, n2 = len(sample1), len(sample2)
    mean1, mean2 = mean(sample1), mean(sample2)
    var1, var2 = variance(sample1), variance(sample2)

    # Welch's t statistic
    t_stat = (mean1 - mean2) / math.sqrt(var1 / n1 + var2 / n2…

from math import comb, factorial
from itertools import combinations


def hypergeometric_probability(a, b, c, d):
    """Probability of observing table [[a, b], [c, d]] under the null."""
    row1 = a + b
    row2 = c + d
    col1 = a + c
    col2 = b + d
    total = row1 + row2
    return (comb(row1, a) * comb(row2, …

import random
import statistics

def permutation_test(group_a, group_b, n_permutations=10000, seed=42):
    random.seed(seed)
    combined = group_a + group_b
    observed_diff = abs(statistics.mean(group_a) - statistics.mean(group_b))
    
    count = 0
    n = len(group_a)
    for _ in range(n_permutations):
       …