167 lines
7.3 KiB
R
167 lines
7.3 KiB
R
library(tibble)
|
|
library(dplyr)
|
|
library(tidyr)
|
|
library(purrr)
|
|
library(yaml)
|
|
library(glue)
|
|
library(lubridate)
|
|
options(scipen = 999)
|
|
|
|
buils_tz_intervals <- function(tz_codes, device_type){
|
|
tz_codes <- tz_codes %>%
|
|
group_by(device_id) %>%
|
|
arrange(timestamp)
|
|
|
|
if(device_type == "fitbit" )
|
|
tz_codes <- tz_codes %>%
|
|
mutate(end_timestamp = lead(timestamp), end_local_date_time = lead(local_date_time)) %>%
|
|
ungroup() %>%
|
|
replace_na(list(end_timestamp = as.numeric(Sys.time())*1000, end_local_date_time = format(Sys.time(), format="%Y-%m-%d %H:%M:%S") )) %>%
|
|
mutate(local_date_time = lubridate::ymd_hms(local_date_time), end_local_date_time = lubridate::ymd_hms(end_local_date_time))
|
|
else
|
|
tz_codes <- tz_codes %>%
|
|
mutate(end_timestamp = lead(timestamp)) %>%
|
|
ungroup() %>%
|
|
replace_na(list(end_timestamp = as.numeric(Sys.time())*1000 ))
|
|
return(tz_codes)
|
|
}
|
|
|
|
|
|
assign_tz_code <- function(data, device_id, tz_codes, device_type){
|
|
tz_codes <- tz_codes %>% filter(device_id == !!device_id) %>% select(-device_id)
|
|
|
|
if(device_type == "fitbit" && all(data$timestamp == 0) && "local_date_time" %in% colnames(data)){
|
|
# Only do this for Fitbit raw data, other devices and Fitbit sleep episodes can use timestamps
|
|
column <- "local_date_time"
|
|
data <- data %>% mutate(local_date_time=lubridate::ymd_hms(local_date_time))
|
|
} else
|
|
column <- "timestamp"
|
|
|
|
for(i in 1:nrow(tz_codes)) {
|
|
start <- tz_codes[[i, column]]
|
|
end <- tz_codes[[i, paste0("end_", column)]]
|
|
time_zone <- trimws(tz_codes[[i, "tzcode"]], which="both")
|
|
|
|
data$local_timezone <- if_else(start <= data[[column]] & data[[column]] < end, time_zone, data$local_timezone)
|
|
}
|
|
|
|
if(column == "local_date_time")
|
|
data$local_date_time <- format(data$local_date_time, format="%Y-%m-%d %H:%M:%S")
|
|
|
|
return(data %>% filter(!is.na(local_timezone)))
|
|
|
|
}
|
|
|
|
validate_single_tz_per_fitbit_device <- function(tz_codes, INFER_FROM_SMARTPHONE_TZ){
|
|
|
|
if(INFER_FROM_SMARTPHONE_TZ)
|
|
stop("If [TIMEZONE][MULTIPLE][FITBIT][INFER_FROM_SMARTPHONE_TZ] is True (you want to infer Fitbit time zones with smartphone data), you need to set ALLOW_MULTIPLE_TZ_PER_DEVICE to True. However, read the docs to understand why this can be innacurate")
|
|
|
|
tz_per_device <- tz_codes %>% group_by(device_id) %>% summarise(n = n(), .groups = "drop_last") %>% filter(n > 1)
|
|
if(nrow(tz_per_device) > 0)
|
|
stop("If [TIMEZONE][MULTIPLE][FITBIT][ALLOW_MULTIPLE_TZ_PER_DEVICE] is False, every fitbit device id in [MULTIPLE][TZCODES_FILE] must have only one timezone with a timestamp equal to 0. The following device ids do not comply:", paste(tz_per_device %>% pull(device_id), collapse = ","))
|
|
|
|
zero_ts <- tz_codes %>% filter(timestamp > 0)
|
|
if(nrow(zero_ts) > 0)
|
|
stop("If [TIMEZONE][MULTIPLE][FITBIT][ALLOW_MULTIPLE_TZ_PER_DEVICE] is False, every fitbit device id in [MULTIPLE][TZCODES_FILE] must have only one timezone with a timestamp equal to 0. The following device ids do not comply:", paste(zero_ts %>% pull(device_id), collapse = ","))
|
|
|
|
}
|
|
|
|
infer_tz_codes_from_phones <- function(data_device_ids, tz_codes, participant_file){
|
|
participant_data <- read_yaml(participant_file)
|
|
phone_device_ids <- participant_data$PHONE$DEVICE_IDS
|
|
phone_tz_codes <- tz_codes %>% filter(device_id %in% phone_device_ids)
|
|
|
|
if(nrow(phone_tz_codes) == 0)
|
|
warning("The PHONE device ids that we were supposed to use to infer fitbit devices timezones do not have timezone data in [MULTIPLE][TZCODES_FILE]. ",
|
|
" Problematic phone devices ids: ", paste0(phone_device_ids, collapse = ", "))
|
|
|
|
data_tz_codes <- NULL
|
|
for(data_device_id in data_device_ids)
|
|
data_tz_codes <- bind_rows(data_tz_codes, phone_tz_codes %>% mutate(device_id = data_device_id) %>% arrange(timestamp))
|
|
data_tz_codes
|
|
}
|
|
|
|
get_devices_ids <- function(participant_data){
|
|
devices_ids = c()
|
|
for(device in participant_data)
|
|
for(attribute in names(device))
|
|
if(attribute == "DEVICE_IDS")
|
|
devices_ids <- c(devices_ids, device[[attribute]])
|
|
return(devices_ids)
|
|
}
|
|
|
|
get_participant_most_common_tz <- function(tz_codes_file, participant_file){
|
|
tz_codes <- read.csv(tz_codes_file)
|
|
participant_device_ids <- get_devices_ids(read_yaml(participant_file))
|
|
|
|
participant_tz_codes <- tz_codes %>% filter(device_id %in% participant_device_ids)
|
|
most_common_tz <- buils_tz_intervals(participant_tz_codes, "all") %>%
|
|
mutate(duration = end_timestamp - timestamp) %>%
|
|
filter(duration == max(duration)) %>%
|
|
head(1) %>%
|
|
pull(tzcode)
|
|
|
|
if(length(most_common_tz)==0)
|
|
most_common_tz <- "UTC"
|
|
return(most_common_tz)
|
|
}
|
|
|
|
multiple_time_zone_assignment <- function(sensor_data, timezone_parameters, device_type, pid, participant_file){
|
|
if(nrow(sensor_data) == 0)
|
|
return(sensor_data %>% mutate(local_timezone = NA_character_))
|
|
|
|
tz_codes <- read.csv(timezone_parameters$MULTIPLE$TZCODES_FILE)
|
|
default <- timezone_parameters$MULTIPLE$DEFAULT_TZCODE
|
|
IF_MISSING_TZCODE <- timezone_parameters$MULTIPLE$IF_MISSING_TZCODE
|
|
ALLOW_MULTIPLE_TZ_PER_DEVICE <- timezone_parameters$MULTIPLE$FITBIT$ALLOW_MULTIPLE_TZ_PER_DEVICE
|
|
INFER_FROM_SMARTPHONE_TZ <- timezone_parameters$MULTIPLE$FITBIT$INFER_FROM_SMARTPHONE_TZ
|
|
data_device_ids <- sensor_data %>% distinct(device_id) %>% pull(device_id)
|
|
|
|
if(INFER_FROM_SMARTPHONE_TZ && device_type == "fitbit")
|
|
data_tz_codes <- infer_tz_codes_from_phones(data_device_ids, tz_codes, participant_file)
|
|
else
|
|
data_tz_codes <- tz_codes %>% filter(device_id %in% data_device_ids)
|
|
|
|
# Check if we have timezones for all device ids
|
|
if(length(unique(data_tz_codes$device_id)) < length(data_device_ids)){
|
|
if(IF_MISSING_TZCODE == "STOP")
|
|
stop(glue("One or more device ids do not have any time zone codes in your [MULTIPLE][TZCODES_FILE].",
|
|
"You can add one or set [MULTIPLE][IF_MISSING_TZCODE] to 'USE_DEFAULT'. The missing device ids are [{ids}]",
|
|
ids=paste0(setdiff(data_device_ids, data_tz_codes %>% pull(device_id)), collapse = ",")))
|
|
else if(IF_MISSING_TZCODE == "USE_DEFAULT"){
|
|
warning("Using DEFAULT time zone for ", paste0(setdiff(data_device_ids, data_tz_codes %>% pull(device_id)), collapse = ","))
|
|
default_tz_codes <- data.frame(timestamp = rep_along(c(data_device_ids),0), tzcode = rep_along(c(data_device_ids),default), device_id=data_device_ids) %>%
|
|
filter(!device_id %in% data_tz_codes$device_id)
|
|
data_tz_codes <- bind_rows(data_tz_codes, default_tz_codes)
|
|
}
|
|
}
|
|
|
|
if(device_type == "fitbit"){
|
|
if(!ALLOW_MULTIPLE_TZ_PER_DEVICE)
|
|
validate_single_tz_per_fitbit_device(data_tz_codes, INFER_FROM_SMARTPHONE_TZ)
|
|
# We only use datetimes for raw Fitbit data
|
|
data_tz_codes <- data_tz_codes %>%
|
|
group_by(tzcode) %>%
|
|
nest() %>%
|
|
mutate(data = map2(data, tzcode, function(nested_data, tz){
|
|
nested_data %>% mutate(local_date_time = format(as_datetime(timestamp / 1000, tz=tz), format="%Y-%m-%d %H:%M:%S"))
|
|
})) %>%
|
|
unnest(cols=everything()) %>%
|
|
ungroup()
|
|
}
|
|
|
|
tz_intervals <- buils_tz_intervals(data_tz_codes, device_type)
|
|
sensor_data <- sensor_data %>% mutate(local_timezone = NA_character_)
|
|
|
|
if(nrow(sensor_data) > 0){
|
|
sensor_data <- sensor_data %>%
|
|
group_by(device_id) %>%
|
|
nest() %>%
|
|
mutate(data = map2(data, device_id, assign_tz_code, tz_intervals, device_type)) %>%
|
|
unnest(cols = data) %>%
|
|
ungroup()
|
|
}
|
|
|
|
return(sensor_data)
|
|
} |