# HSCI 410 Lesson 2, Activity 2.2: clean the course data and record every change
# Run-along script for the narrated walkthrough. It reads the course data
# directly from the course website, so there is nothing to set up: open it
# in RStudio and run each part with Ctrl+Enter when the narration asks.

# 1. Read the raw data and check the type of age and bmi
phaa <- read.csv("https://heal-learn.ca/r-activities/data/phaa_survey.csv",   # course data, read from the website
                 stringsAsFactors = FALSE, na.strings = c("", "NA"))
summary(phaa$age)                  # before the conversion
phaa$age <- as.numeric(phaa$age)   # text such as "unknown" would become NA
phaa$bmi <- as.numeric(phaa$bmi)
summary(phaa$age)                  # after the conversion

# 2. Outlier rules flag values for checking; they change nothing
x <- c(96, 102, 106, 110, 112, 115, 118, 121, 125, 131, 184)
quantile(x, probs = c(0.25, 0.75))   # Q1 and Q3
IQR(x)                               # Q3 minus Q1
boxplot.stats(x)$out                 # values beyond the fences
q <- quantile(phaa$bmi, probs = c(0.25, 0.75), na.rm = TRUE)
fences <- c(q[1] - 1.5 * IQR(phaa$bmi), q[2] + 1.5 * IQR(phaa$bmi))
fences
phaa[which(phaa$bmi < fences[1] | phaa$bmi > fences[2]), c("id", "bmi")]
z <- as.numeric(scale(phaa$bmi))     # z = (x - mean) / SD
phaa$id[which(abs(z) > 3)]           # IDs with |z| above 3
round(z[which(phaa$bmi == 31.6)], 2)
bmi_ok <- phaa$bmi[phaa$bmi < 60]    # without the impossible 95
round(c(sd_all = sd(phaa$bmi), sd_ok = sd(bmi_ok)), 2)
round((c(30.6, 30.8, 31.6) - mean(bmi_ok)) / sd(bmi_ok), 2)
sum(phaa$systolic_bp < 90 | phaa$systolic_bp > 180)   # soft limits
sum(phaa$bmi < 16 | phaa$bmi > 50)

# 3. Hard limits: list the impossible values, then set them to NA
phaa[which(phaa$age < 18 | phaa$age > 100), c("id", "age")]
phaa[which(phaa$systolic_bp < 80 | phaa$systolic_bp > 220), c("id", "systolic_bp")]
phaa[which(phaa$bmi < 13 | phaa$bmi > 60), c("id", "bmi")]
phaa$age[phaa$age < 18 | phaa$age > 100] <- NA
phaa$systolic_bp[phaa$systolic_bp < 80 | phaa$systolic_bp > 220] <- NA
phaa$bmi[phaa$bmi < 13 | phaa$bmi > 60] <- NA
summary(phaa$age)                  # 2 NAs, and every row is kept
phaa$education <- factor(phaa$education, levels = c("Less than high school",
  "High school", "Some college", "Bachelor's", "Graduate degree"), ordered = TRUE)
levels(phaa$education)             # lowest to highest

# 4. Record each change in a cleaning log
cleaning_log <- data.frame(date = "2026-09-30", analyst = "AB",
  id       = c("P0033", "P0612", "P0017", "P0245", "P0078"),
  variable = c("age", "age", "systolic_bp", "systolic_bp", "bmi"),
  old      = c(-3, 220, 300, 60, 95), new = NA,
  reason   = c("below 18: impossible", "above 100: impossible",
               "above 220: implausible", "below 80: implausible",
               "above 60: implausible"))
cleaning_log

# 5. Transform and recode into new variables; never overwrite
crp <- c(0.4, 0.8, 1.1, 1.5, 2.3, 3.0, 4.8, 9.5, 21.0)   # CRP, mg/L
c(mean = mean(crp), median = median(crp))
mean(log(crp))                     # the mean on the log scale
exp(mean(log(crp)))                # back-transformed: geometric mean
visits <- c(0, 0, 1, 2, 5, 12)
log(visits)                        # log(0) is -Inf
log1p(visits)                      # log(x + 1), so 0 stays 0
expm1(log1p(visits))               # expm1() undoes log1p()
1 / c(2, 5, 10)                    # the reciprocal reverses the order
-1 / c(2, 5, 10)                   # a minus sign restores it
smoker01 <- ifelse(phaa$smoker == "Yes", 1, 0)   # 1 = smoker
table(phaa$smoker, smoker01)       # check every recode
bmi_group <- cut(phaa$bmi, breaks = c(0, 18.5, 25, 30, Inf), right = FALSE,
  labels = c("Under 18.5", "18.5 to 24.9", "25 to 29.9", "30 or more"))
table(bmi_group, useNA = "ifany")

# 6. Clean text and check for duplicate IDs
city <- c("Vancouver", "vancouver ", "VANCOUVER", "Vancoouver", " Burnaby")
city <- trimws(tolower(city))      # remove stray spaces, lower case
city
city[grepl("^vanc", city)] <- "vancouver"   # starts with "vanc"
table(city)
sum(duplicated(phaa$id))           # IDs present more than once

# 7. Missing values: codes, how many, and who is missing
age_raw <- c(34, -9, 51, 999, 47)  # -9 = refused, 999 = not asked
mean(age_raw)                      # wrong: the codes count as ages
age_raw[age_raw %in% c(-9, 999)] <- NA
mean(age_raw, na.rm = TRUE)
n_miss <- colSums(is.na(phaa))
n_miss[n_miss > 0]                 # only the columns with gaps
round(100 * n_miss[n_miss > 0] / nrow(phaa), 1)
sum(complete.cases(phaa))          # rows with no gap in any column
nrow(na.omit(phaa))                # na.omit() keeps only those rows
inc_miss <- is.na(phaa$income)     # TRUE = income missing
table(inc_miss)
round(tapply(phaa$age, inc_miss, mean, na.rm = TRUE), 1)
round(tapply(phaa$systolic_bp, inc_miss, mean, na.rm = TRUE), 1)
round(prop.table(table(inc_miss, phaa$smoker), margin = 1) * 100, 1)

# 8. A simulation: MCAR and MNAR and the complete-case mean
set.seed(410)
income <- round(rlnorm(10000, meanlog = log(55), sdlog = 0.6))   # $1000s
mean(income)                       # the true mean
inc_mcar <- income
inc_mcar[runif(10000) < 0.30] <- NA          # MCAR: 30% for everyone
mean(inc_mcar, na.rm = TRUE)
p_miss <- ifelse(income > quantile(income, 0.75), 0.70, 0.15)
inc_mnar <- income
inc_mnar[runif(10000) < p_miss] <- NA        # MNAR: top earners 70%
mean(inc_mnar, na.rm = TRUE)
sd(income) / sqrt(10000)                     # SE of the mean, everyone
sd(inc_mcar, na.rm = TRUE) / sqrt(sum(!is.na(inc_mcar)))   # SE, MCAR
