Hypothesis tests

One numerical variable (mean)

Observed stat

( x_bar <- fli_small %>%
  specify(response = dep_delay) %>%
  calculate(stat = "mean") )

stat
11.49

null_distn <- fli_small %>%
  specify(response = dep_delay) %>%
  hypothesize(null = "point", mu = 10) %>%
  generate(reps = 1000) %>%
  calculate(stat = "mean")

## Setting `type = "bootstrap"` in `generate()`.

visualize(null_distn) +
  shade_p_value(obs_stat = x_bar, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = x_bar, direction = "two_sided")

p_value
0.356

One numerical variable (standardized mean \(t\))

Observed stat

( t_bar <- fli_small %>%
  specify(response = dep_delay) %>%
  calculate(stat = "t") )

stat
6.827

null_distn <- fli_small %>%
  specify(response = dep_delay) %>%
  hypothesize(null = "point", mu = 8) %>%
  generate(reps = 1000) %>%
  calculate(stat = "t")

## Setting `type = "bootstrap"` in `generate()`.

visualize(null_distn) +
  shade_p_value(obs_stat = t_bar, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = t_bar, direction = "two_sided")

p_value
0

One numerical variable (median)

Observed stat

( x_tilde <- fli_small %>%
  specify(response = dep_delay) %>%
  calculate(stat = "median") )

stat
-2

null_distn <- fli_small %>%
  specify(response = dep_delay) %>%
  hypothesize(null = "point", med = -1) %>% 
  generate(reps = 1000) %>% 
  calculate(stat = "median")

## Setting `type = "bootstrap"` in `generate()`.

visualize(null_distn) +
  shade_p_value(obs_stat = x_tilde, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = x_tilde, direction = "two_sided")

p_value
0.018

One categorical (one proportion)

Observed stat

( p_hat <- fli_small %>%
  specify(response = day_hour, success = "morning") %>%
  calculate(stat = "prop") )

stat
0.452

null_distn <- fli_small %>%
  specify(response = day_hour, success = "morning") %>%
  hypothesize(null = "point", p = .5) %>%
  generate(reps = 1000) %>%
  calculate(stat = "prop")

## Setting `type = "simulate"` in `generate()`.

visualize(null_distn) +
  shade_p_value(obs_stat = p_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = p_hat, direction = "two_sided")

p_value
0.036

Logical variables will be coerced to factors:

null_distn <- fli_small %>%
  mutate(day_hour_logical = (day_hour == "morning")) %>%
  specify(response = day_hour_logical, success = "TRUE") %>%
  hypothesize(null = "point", p = .5) %>%
  generate(reps = 1000) %>%
  calculate(stat = "prop")

## Setting `type = "simulate"` in `generate()`.

One categorical variable (standardized proportion \(z\))

Not yet implemented.

Two categorical (2 level) variables

Observed stat

( d_hat <- fli_small %>% 
  specify(day_hour ~ season, success = "morning") %>%
  calculate(stat = "diff in props", order = c("winter", "summer")) )

stat
0.0044

null_distn <- fli_small %>%
  specify(day_hour ~ season, success = "morning") %>%
  hypothesize(null = "independence") %>% 
  generate(reps = 1000) %>% 
  calculate(stat = "diff in props", order = c("winter", "summer"))

## Setting `type = "permute"` in `generate()`.

visualize(null_distn) +
  shade_p_value(obs_stat = d_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = d_hat, direction = "two_sided")

p_value
0.954

Two categorical (2 level) variables (z)

Standardized observed stat

( z_hat <- fli_small %>% 
  specify(day_hour ~ season, success = "morning") %>%
  calculate(stat = "z", order = c("winter", "summer")) )

stat
0.0985

null_distn <- fli_small %>%
  specify(day_hour ~ season, success = "morning") %>%
  hypothesize(null = "independence") %>% 
  generate(reps = 1000) %>% 
  calculate(stat = "z", order = c("winter", "summer"))

## Setting `type = "permute"` in `generate()`.

visualize(null_distn) +
  shade_p_value(obs_stat = z_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = z_hat, direction = "two_sided")

p_value
0.95

Note the similarities in this plot and the previous one.

One categorical (>2 level) - GoF

Observed stat

Note the need to add in the hypothesized values here to compute the observed statistic.

( Chisq_hat <- fli_small %>%
  specify(response = origin) %>%
  hypothesize(null = "point", 
              p = c("EWR" = .33, "JFK" = .33, "LGA" = .34)) %>% 
  calculate(stat = "Chisq") )

stat
7.009

null_distn <- fli_small %>%
  specify(response = origin) %>%
  hypothesize(null = "point", 
              p = c("EWR" = .33, "JFK" = .33, "LGA" = .34)) %>% 
  generate(reps = 1000, type = "simulate") %>% 
  calculate(stat = "Chisq")

visualize(null_distn) +
  shade_p_value(obs_stat = Chisq_hat, direction = "greater")

null_distn %>%
  get_p_value(obs_stat = Chisq_hat, direction = "greater")

p_value
0.037

Two categorical (>2 level) variables

Observed stat

( Chisq_hat <- fli_small %>%
  specify(formula = day_hour ~ origin) %>% 
  calculate(stat = "Chisq") )

stat
0.5284

null_distn <- fli_small %>%
  specify(day_hour ~ origin) %>%
  hypothesize(null = "independence") %>% 
  generate(reps = 1000, type = "permute") %>% 
  calculate(stat = "Chisq")

visualize(null_distn) +
  shade_p_value(obs_stat = Chisq_hat, direction = "greater")

null_distn %>%
  get_p_value(obs_stat = Chisq_hat, direction = "greater")

p_value
0.77

One numerical variable, one categorical (2 levels) (diff in means)

Observed stat

( d_hat <- fli_small %>% 
  specify(dep_delay ~ season) %>% 
  calculate(stat = "diff in means", order = c("summer", "winter")) )

stat
3

null_distn <- fli_small %>%
  specify(dep_delay ~ season) %>%
  hypothesize(null = "independence") %>%
  generate(reps = 1000, type = "permute") %>%
  calculate(stat = "diff in means", order = c("summer", "winter"))

visualize(null_distn) +
  shade_p_value(obs_stat = d_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = d_hat, direction = "two_sided")

p_value
0.338

One numerical variable, one categorical (2 levels) (t)

Standardized observed stat

( t_hat <- fli_small %>% 
  specify(dep_delay ~ season) %>% 
  calculate(stat = "t", order = c("summer", "winter")) )

stat
0.8909

null_distn <- fli_small %>%
  specify(dep_delay ~ season) %>%
  hypothesize(null = "independence") %>%
  generate(reps = 1000, type = "permute") %>%
  calculate(stat = "t", order = c("summer", "winter"))

visualize(null_distn) +
  shade_p_value(obs_stat = t_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = t_hat, direction = "two_sided")

p_value
0.4

Note the similarities in this plot and the previous one.

One numerical variable, one categorical (2 levels) (diff in medians)

Observed stat

( d_hat <- fli_small %>% 
  specify(dep_delay ~ season) %>% 
  calculate(stat = "diff in medians", order = c("summer", "winter")) )

stat
1

null_distn <- fli_small %>%
  specify(dep_delay ~ season) %>% # alt: response = dep_delay, 
  # explanatory = season
  hypothesize(null = "independence") %>%
  generate(reps = 1000, type = "permute") %>%
  calculate(stat = "diff in medians", order = c("summer", "winter"))

visualize(null_distn) +
  shade_p_value(obs_stat = d_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = d_hat, direction = "two_sided")

p_value
0.64

One numerical, one categorical (>2 levels) - ANOVA

Observed stat

( F_hat <- fli_small %>% 
  specify(arr_delay ~ origin) %>%
  calculate(stat = "F") )

stat
0.6858

null_distn <- fli_small %>%
   specify(arr_delay ~ origin) %>%
   hypothesize(null = "independence") %>%
   generate(reps = 1000, type = "permute") %>%
   calculate(stat = "F")

visualize(null_distn) +
  shade_p_value(obs_stat = F_hat, direction = "greater")

null_distn %>%
  get_p_value(obs_stat = F_hat, direction = "greater")

p_value
0.529

Two numerical vars - SLR

Observed stat

( slope_hat <- fli_small %>% 
  specify(arr_delay ~ dep_delay) %>% 
  calculate(stat = "slope") )

stat
0.9916

null_distn <- fli_small %>%
   specify(arr_delay ~ dep_delay) %>% 
   hypothesize(null = "independence") %>%
   generate(reps = 1000, type = "permute") %>%
   calculate(stat = "slope")

visualize(null_distn) +
  shade_p_value(obs_stat = slope_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = slope_hat, direction = "two_sided")

p_value
0

Two numerical vars - correlation

Observed stat

( correlation_hat <- fli_small %>% 
  specify(arr_delay ~ dep_delay) %>% 
  calculate(stat = "correlation") )

stat
0.8951

null_distn <- fli_small %>%
   specify(arr_delay ~ dep_delay) %>% 
   hypothesize(null = "independence") %>%
   generate(reps = 1000, type = "permute") %>%
   calculate(stat = "correlation")

visualize(null_distn) +
  shade_p_value(obs_stat = correlation_hat, direction = "two_sided")

null_distn %>%
  get_p_value(obs_stat = correlation_hat, direction = "two_sided")

p_value
0

Two numerical vars - SLR (t)

Not currently implemented since \(t\) could refer to standardized slope or standardized correlation.

Confidence intervals

One numerical (one mean)

Point estimate

( x_bar <- fli_small %>% 
  specify(response = arr_delay) %>%
  calculate(stat = "mean") )

stat
6.154

boot <- fli_small %>%
   specify(response = arr_delay) %>%
   generate(reps = 1000, type = "bootstrap") %>%
   calculate(stat = "mean")
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
2.606	9.602

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = x_bar) )

lower	upper
2.609	9.699

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

One numerical (one mean - standardized)

Point estimate

( t_hat <- fli_small %>% 
  specify(response = arr_delay) %>%
  calculate(stat = "t") )

stat
3.3

boot <- fli_small %>%
   specify(response = arr_delay) %>%
   generate(reps = 1000, type = "bootstrap") %>%
   calculate(stat = "t")
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
1.625	4.879

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = t_hat) )

lower	upper
1.7	4.9

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

One categorical (one proportion)

Point estimate

( p_hat <- fli_small %>% 
   specify(response = day_hour, success = "morning") %>%
   calculate(stat = "prop") )

stat
0.452

boot <- fli_small %>%
 specify(response = day_hour, success = "morning") %>%
 generate(reps = 1000, type = "bootstrap") %>%
 calculate(stat = "prop")
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
0.406	0.496

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = p_hat) )

lower	upper
0.4079	0.4961

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

One categorical variable (standardized proportion \(z\))

Not yet implemented.

One numerical variable, one categorical (2 levels) (diff in means)

Point estimate

( d_hat <- fli_small %>%
  specify(arr_delay ~ season) %>%
  calculate(stat = "diff in means", order = c("summer", "winter")) )

stat
5.629

boot <- fli_small %>%
   specify(arr_delay ~ season) %>%
   generate(reps = 1000, type = "bootstrap") %>%
   calculate(stat = "diff in means", order = c("summer", "winter"))
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
-2.025	12.54

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = d_hat) )

lower	upper
-1.605	12.86

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

One numerical variable, one categorical (2 levels) (t)

Standardized point estimate

( t_hat <- fli_small %>%
  specify(arr_delay ~ season) %>%
  calculate(stat = "t", order = c("summer", "winter")) )

stat
1.511

boot <- fli_small %>%
   specify(arr_delay ~ season) %>%
   generate(reps = 1000, type = "bootstrap") %>%
   calculate(stat = "t", order = c("summer", "winter"))
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
-0.3589	3.736

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = t_hat) )

lower	upper
-0.5783	3.601

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

Two categorical variables (diff in proportions)

Point estimate

( d_hat <- fli_small %>% 
  specify(day_hour ~ season, success = "morning") %>%
  calculate(stat = "diff in props", order = c("summer", "winter")) )

stat
-0.0044

boot <- fli_small %>%
  specify(day_hour ~ season, success = "morning") %>%
  generate(reps = 1000, type = "bootstrap") %>% 
  calculate(stat = "diff in props", order = c("summer", "winter"))
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
-0.0957	0.0818

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = d_hat) )

lower	upper
-0.0914	0.0826

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

Two categorical variables (z)

Standardized point estimate

( z_hat <- fli_small %>% 
  specify(day_hour ~ season, success = "morning") %>%
  calculate(stat = "z", order = c("summer", "winter")) )

stat
-0.0985

boot <- fli_small %>%
  specify(day_hour ~ season, success = "morning") %>%
  generate(reps = 1000, type = "bootstrap") %>% 
  calculate(stat = "z", order = c("summer", "winter"))
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
-1.962	1.788

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = z_hat) )

lower	upper
-2.042	1.845

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

Two numerical vars - SLR

Point estimate

( slope_hat <- fli_small %>% 
  specify(arr_delay ~ dep_delay) %>%
  calculate(stat = "slope") )

stat
0.9916

boot <- fli_small %>%
   specify(arr_delay ~ dep_delay) %>% 
   generate(reps = 1000, type = "bootstrap") %>%
   calculate(stat = "slope")
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
0.9463	1.032

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", point_estimate = slope_hat) )

lower	upper
0.9468	1.036

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

Two numerical vars - correlation

Point estimate

( correlation_hat <- fli_small %>% 
  specify(arr_delay ~ dep_delay) %>%
  calculate(stat = "correlation") )

stat
0.8951

boot <- fli_small %>%
   specify(arr_delay ~ dep_delay) %>% 
   generate(reps = 1000, type = "bootstrap") %>%
   calculate(stat = "correlation")
( percentile_ci <- get_ci(boot) )

2.5%	97.5%
0.827	0.9332

visualize(boot) +
  shade_confidence_interval(endpoints = percentile_ci)

( standard_error_ci <- get_ci(boot, type = "se", 
                            point_estimate = correlation_hat) )

lower	upper
0.8418	0.9485

visualize(boot) +
  shade_confidence_interval(endpoints = standard_error_ci)

Two numerical vars - t

Not currently implemented since \(t\) could refer to standardized slope or standardized correlation.

Full infer pipeline examples using `nycflights13` `flights` data

Chester Ismay

Updated on 2018-06-14

Data preparation

Hypothesis tests

One numerical variable (mean)

One numerical variable (standardized mean \(t\))

One numerical variable (median)

One categorical (one proportion)

One categorical variable (standardized proportion \(z\))

Two categorical (2 level) variables

Two categorical (2 level) variables (z)

One categorical (>2 level) - GoF

Two categorical (>2 level) variables

One numerical variable, one categorical (2 levels) (diff in means)

One numerical variable, one categorical (2 levels) (t)

One numerical variable, one categorical (2 levels) (diff in medians)

One numerical, one categorical (>2 levels) - ANOVA

Two numerical vars - SLR

Two numerical vars - correlation

Two numerical vars - SLR (t)

Confidence intervals

One numerical (one mean)

One numerical (one mean - standardized)

One categorical (one proportion)

One categorical variable (standardized proportion \(z\))

One numerical variable, one categorical (2 levels) (diff in means)

One numerical variable, one categorical (2 levels) (t)

Two categorical variables (diff in proportions)

Two categorical variables (z)

Two numerical vars - SLR

Two numerical vars - correlation

Two numerical vars - t

Full infer pipeline examples using nycflights13 flights data

Chester Ismay

Updated on 2018-06-14

Data preparation

Hypothesis tests

One numerical variable (mean)

One numerical variable (standardized mean \(t\))

One numerical variable (median)

One categorical (one proportion)

One categorical variable (standardized proportion \(z\))

Two categorical (2 level) variables

Two categorical (2 level) variables (z)

One categorical (>2 level) - GoF

Two categorical (>2 level) variables

One numerical variable, one categorical (2 levels) (diff in means)

One numerical variable, one categorical (2 levels) (t)

One numerical variable, one categorical (2 levels) (diff in medians)

One numerical, one categorical (>2 levels) - ANOVA

Two numerical vars - SLR

Two numerical vars - correlation

Two numerical vars - SLR (t)

Confidence intervals

One numerical (one mean)

One numerical (one mean - standardized)

One categorical (one proportion)

One categorical variable (standardized proportion \(z\))

One numerical variable, one categorical (2 levels) (diff in means)

One numerical variable, one categorical (2 levels) (t)

Two categorical variables (diff in proportions)

Two categorical variables (z)

Two numerical vars - SLR

Two numerical vars - correlation

Two numerical vars - t

Full infer pipeline examples using `nycflights13` `flights` data