# =============================================================================
# HSCI 841 Qualitative Research Methods and Analysis in Public Health  -  Lesson 7:
#   Comparing Variables and Grounded Theory
# Answer key for the in-lesson R activities
# Data file(s): the 20 transcripts in term projects/HSCI_841/transcripts/,
#   term projects/HSCI_841/exports/taguette_loneliness_export.csv,
#   term projects/HSCI_841/codebook_week7.csv, term projects/HSCI_841/participant_metadata.csv
#   (all of them are in HSCI_841_loneliness_data.zip)
# Packages: tidyverse, readtext, ggplot2  (install once with
#   install.packages(c("tidyverse", "readtext")))
# Reproduces every code block in the lesson.
#
# Paths are relative to the course repository root. Set your working directory there
# (Session > Set Working Directory > Choose Directory, or open the course .Rproj) before running.
# =============================================================================

# NOTE (answer key): the transcripts contain typographic dashes and curly quotation marks. If the R
# session is not running in a UTF-8 locale (plain Rscript on a server, for example), those characters
# are escaped rather than read. RStudio is already UTF-8; this line only matters for a bare Rscript run.
if (!isTRUE(l10n_info()[["UTF-8"]])) invisible(suppressWarnings(Sys.setlocale("LC_CTYPE", "C.UTF-8")))

# ==== Section 4.1: Step 1 / Read transcripts and a Taguette export into R ====
library(tidyverse)
library(readtext)

# Ingest all 20 transcripts
# NOTE (answer key): path corrected from "../term projects/..." to the repository-root convention.
transcripts <- readtext("term projects/HSCI_841/transcripts/*.txt") %>%
  as_tibble() %>%
  mutate(participant = str_extract(doc_id, "^P[0-9]+_[A-Za-z]+"))

glimpse(transcripts)

# Ingest the Taguette export (your coded extracts CSV)
# NOTE (answer key): Taguette names each document by its filename ("P01_Maya.txt"), so the
# participant label is taken from the filename with str_extract(), the same way it is for the
# transcripts above. Without this the two objects cannot be compared.
codes <- read_csv("term projects/HSCI_841/exports/taguette_loneliness_export.csv",
                  show_col_types = FALSE) %>%
  rename(participant = document, code = tag, extract = content) %>%
  mutate(participant = str_extract(participant, "^P[0-9]+_[A-Za-z]+"))

glimpse(codes)

cat("Transcripts:", nrow(transcripts), " coded extracts:", n_distinct(codes$id),
    " code applications:", nrow(codes), " codes:", n_distinct(codes$code), "\n")
print(codes %>% count(code, sort = TRUE))

# The codebook these codes come from (code, focal, definition, inclusion, exclusion, example)
codebook <- read_csv("term projects/HSCI_841/codebook_week7.csv", show_col_types = FALSE)
print(codebook %>% select(code, focal))

# ==== Section 4.2: Step 2 / Build a code-by-case matrix (rows = codes, columns = participants) ====
focal_codes <- c("trigger", "embodied_feature", "coping_move",
                 "surprising_help", "structural_critique", "identity_stake")

focal_participants <- c("P01_Maya", "P05_Linda", "P06_Aarav",
                        "P11_Helen", "P15_Amira", "P16_Elena",
                        "P18_Chen", "P20_Frank")

matrix_long <- codes %>%
  filter(code %in% focal_codes, participant %in% focal_participants) %>%
  count(code, participant, name = "n_extracts")

matrix_wide <- matrix_long %>%
  pivot_wider(names_from = participant, values_from = n_extracts, values_fill = 0)

print(matrix_wide)

# ---- Reading the matrix ----
# Rows first: how one code varies across the eight cases.
print(matrix_long %>% group_by(code) %>%
        summarise(cases = n(), extracts = sum(n_extracts)) %>% arrange(desc(extracts)))
# Then columns: how many of the six codes each case carries.
print(matrix_long %>% group_by(participant) %>%
        summarise(codes_present = n(), extracts = sum(n_extracts)) %>% arrange(participant))
# Then the empty cells. Complete the grid so absences are visible instead of missing.
grid <- expand_grid(code = focal_codes, participant = focal_participants) %>%
  left_join(matrix_long, by = c("code", "participant")) %>%
  mutate(n_extracts = replace_na(n_extracts, 0))
empty <- grid %>% filter(n_extracts == 0)
cat("Empty cells:", nrow(empty), "\n")
print(empty)
# P06 Aarav has no identity_stake extract because the state-versus-trait question
# ("do you think of yourself as a lonely person?") was not put to him. That is an elicitation
# absence, which belongs in the limitations section, not a substantive one. Compare P01 Maya, who
# was not asked it either and volunteered the material anyway ("failing at being 22"): the same
# missing question produced a filled cell in one case and an empty cell in the other, which is
# exactly why an empty cell has to be interrogated before it is interpreted.

# ==== Section 4.3: Step 3 / Visualize the matrix as a tile heatmap ====
library(ggplot2)

ggplot(matrix_long, aes(x = participant, y = code, fill = n_extracts)) +
  geom_tile(color = "white", linewidth = 0.6) +
  geom_text(aes(label = n_extracts), color = "#03241F", size = 3.5) +
  scale_fill_gradient(low = "#E6F3F0", high = "#0B7B6B") +
  labs(
    title = "Loneliness corpus: 6 codes x 8 participants",
    subtitle = "Cell value = number of extracts coded; empty cells indicate informative absence or under-elicitation",
    x = NULL, y = NULL, fill = "# extracts"
  ) +
  theme_minimal(base_size = 11) +
  theme(axis.text.x = element_text(angle = 35, hjust = 1))

# NOTE (answer key): create the output folder first, or ggsave() stops with "cannot open file".
dir.create("capstone", showWarnings = FALSE)
ggsave("capstone/wk07_matrix_loneliness.png", width = 9, height = 5, dpi = 150)
cat("Wrote capstone/wk07_matrix_loneliness.png\n")

# ==== Section 4.4: Step 4 (optional) / Code co-occurrence within participants ====
co_occur <- codes %>%
  filter(code %in% focal_codes) %>%
  distinct(participant, code) %>%
  mutate(present = 1L) %>%
  pivot_wider(names_from = code, values_from = present, values_fill = 0L) %>%
  select(-participant) %>%
  as.matrix()

co_occur_mat <- t(co_occur) %*% co_occur  # symmetric code x code matrix

print(co_occur_mat)

# ---- Reading the co-occurrence matrix ----
# The diagonal is the number of transcripts carrying each code; the off-diagonal cells are the
# number of transcripts carrying both. Because all six codes follow the interview guide, they
# co-occur almost everywhere, which is the artefactual counting problem section 1.5 warns about:
# a directly elicited code is weak evidence of commonness.
diag_n <- diag(co_occur_mat)
cat("Transcripts per focal code:\n"); print(diag_n)
pairs <- as.data.frame(as.table(co_occur_mat)) %>%
  filter(as.character(Var1) < as.character(Var2)) %>%
  arrange(desc(Freq))
print(head(pairs, 5))

# A comparison that is not artefactual: subgroup contrasts on the open codes, which were not
# elicited by a standard question. This is the Lesson 7 move from matrix to comparison.
participants <- read_csv("term projects/HSCI_841/participant_metadata.csv", show_col_types = FALSE) %>%
  mutate(participant = paste(participant_id, pseudonym, sep = "_"))
codes_meta <- codes %>% left_join(participants, by = "participant")
print(codes_meta %>%
        filter(code %in% c("failed_strategy", "help_seeking", "change_over_time")) %>%
        count(life_stage, code) %>%
        pivot_wider(names_from = code, values_from = n, values_fill = 0))
print(codes_meta %>% distinct(participant, code, immigrant) %>%
        count(immigrant, code) %>%
        pivot_wider(names_from = immigrant, values_from = n, values_fill = 0))