diff --git a/rules/features.smk b/rules/features.smk index 911e7cff..55703574 100644 --- a/rules/features.smk +++ b/rules/features.smk @@ -64,7 +64,7 @@ rule ios_activity_recognition_deltas: rule locations_python_features: input: - location_data = expand("data/raw/{{pid}}/{sensor}_processed_{locations_to_use}.csv", sensor=config["LOCATIONS"]["DB_TABLE"], locations_to_use=config["LOCATIONS"]["LOCATIONS_TO_USE"]), + sensor_data = expand("data/raw/{{pid}}/{sensor}_processed_{locations_to_use}.csv", sensor=config["LOCATIONS"]["DB_TABLE"], locations_to_use=config["LOCATIONS"]["LOCATIONS_TO_USE"]), day_segments_labels = "data/interim/day_segments_labels.csv" params: provider = lambda wildcards: config["LOCATIONS"]["PROVIDERS"][wildcards.provider_key], @@ -72,11 +72,11 @@ rule locations_python_features: output: "data/interim/{pid}/locations_features/locations_python_{provider_key}.csv" script: - "../src/features/location/locations_entry.py" + "../src/features/locations/locations_entry.py" rule locations_r_features: input: - location_data = expand("data/raw/{{pid}}/{sensor}_processed_{locations_to_use}.csv", sensor=config["LOCATIONS"]["DB_TABLE"], locations_to_use=config["LOCATIONS"]["LOCATIONS_TO_USE"]), + sensor_data = expand("data/raw/{{pid}}/{sensor}_processed_{locations_to_use}.csv", sensor=config["LOCATIONS"]["DB_TABLE"], locations_to_use=config["LOCATIONS"]["LOCATIONS_TO_USE"]), day_segments_labels = "data/interim/day_segments_labels.csv" params: provider = lambda wildcards: config["LOCATIONS"]["PROVIDERS"][wildcards.provider_key], @@ -84,7 +84,7 @@ rule locations_r_features: output: "data/interim/{pid}/locations_features/locations_r_{provider_key}.csv" script: - "../src/features/location/locations_entry.R" + "../src/features/locations/locations_entry.R" rule bluetooth_features: input: diff --git a/src/features/call/call_base.R b/src/features/call/call_base.R deleted file mode 100644 index 9c20e81d..00000000 --- a/src/features/call/call_base.R +++ /dev/null @@ -1,78 +0,0 @@ -library('tidyr') -library('stringr') -library('entropy') - -Mode <- function(v) { - uniqv <- unique(v) - uniqv[which.max(tabulate(match(v, uniqv)))] -} - -base_call_features <- function(calls, call_type, day_segment, requested_features){ - # Output dataframe - features = data.frame(local_segment = character(), stringsAsFactors = FALSE) - - # The name of the features this function can compute - base_features_names <- c("count", "distinctcontacts", "meanduration", "sumduration", "minduration", "maxduration", "stdduration", "modeduration", "entropyduration", "timefirstcall", "timelastcall", "countmostfrequentcontact") - - # The subset of requested features this function can compute - features_to_compute <- intersect(base_features_names, requested_features) - - # Filter rows that belong to the calls type and day segment of interest - call_type_label = ifelse(call_type == "incoming", "1", ifelse(call_type == "outgoing", "2", ifelse(call_type == "missed", "3", NA))) - if(is.na(call_type_label)) - stop(paste("Call type can online be incoming, outgoing or missed but instead you typed: ", call_type)) - - # Filter the rows that belong to day_segment, and put the segment full name in a new column for grouping - date_regex = "[0-9]{4}[\\-|\\/][0-9]{2}[\\-|\\/][0-9]{2}" - hour_regex = "[0-9]{2}:[0-9]{2}:[0-9]{2}" - calls <- calls %>% - filter(call_type == call_type_label) %>% - filter(grepl(paste0("\\[", day_segment, "#"),assigned_segments)) %>% - mutate(local_segment = str_extract(assigned_segments, paste0("\\[", day_segment, "#", date_regex, "#", hour_regex, "#", date_regex, "#", hour_regex, "\\]")), - local_segment = str_sub(local_segment, 2, -2)) # get rid of first and last character([]) - - # If there are not features or data to work with, return an empty df with appropiate columns names - if(length(features_to_compute) == 0) - return(features) - if(nrow(calls) < 1) - return(cbind(features, read.csv(text = paste(paste("call", call_type, features_to_compute, sep = "_"), collapse = ","), stringsAsFactors = FALSE))) - - for(feature_name in features_to_compute){ - if(feature_name == "countmostfrequentcontact"){ - # Get the number of messages for the most frequent contact throughout the study - mostfrequentcontact <- calls %>% - group_by(trace) %>% - mutate(N=n()) %>% - ungroup() %>% - filter(N == max(N)) %>% - head(1) %>% # if there are multiple contacts with the same amount of messages pick the first one only - pull(trace) - feature <- calls %>% - filter(trace == mostfrequentcontact) %>% - group_by(local_segment) %>% - summarise(!!paste("call", call_type, feature_name, sep = "_") := n()) %>% - replace(is.na(.), 0) - features <- merge(features, feature, by="local_segment", all = TRUE) - } else { - feature <- calls %>% - group_by(local_segment) - - feature <- switch(feature_name, - "count" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := n()), - "distinctcontacts" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := n_distinct(trace)), - "meanduration" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := mean(call_duration)), - "sumduration" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := sum(call_duration)), - "minduration" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := min(call_duration)), - "maxduration" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := max(call_duration)), - "stdduration" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := sd(call_duration)), - "modeduration" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := Mode(call_duration)), - "entropyduration" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := entropy.MillerMadow(call_duration)), - "timefirstcall" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := first(local_hour) * 60 + first(local_minute)), - "timelastcall" = feature %>% summarise(!!paste("call", call_type, feature_name, sep = "_") := last(local_hour) * 60 + last(local_minute))) - - features <- merge(features, feature, by="local_segment", all = TRUE) - } - } - features <- features %>% mutate_at(vars(contains("countmostfrequentcontact")), list( ~ replace_na(., 0))) - return(features) -} \ No newline at end of file diff --git a/src/features/location/locations_entry.R b/src/features/location/locations_entry.R deleted file mode 100644 index b2349a78..00000000 --- a/src/features/location/locations_entry.R +++ /dev/null @@ -1,44 +0,0 @@ -source("renv/activate.R") -source("src/features/utils/utils.R") -library("dplyr") -library("stringr") -library("tidyr") - -location_data <- read.csv(snakemake@input[["location_data"]], stringsAsFactors = FALSE) -day_segments_labels <- read.csv(snakemake@input[["day_segments_labels"]], stringsAsFactors = FALSE) -provider <- snakemake@params["provider"][["provider"]] -provider_key <- snakemake@params["provider_key"] - -location_features <- data.frame(local_segment = character(), stringsAsFactors = FALSE) - -if(!"FEATURES" %in% names(provider)) - stop(paste0("Provider config[LOCATION][PROVIDERS][", provider_key,"] is missing a FEATURES attribute in config.yaml")) - -if(provider[["COMPUTE"]] == TRUE){ - code_path <- paste0("src/features/location/", provider[["SRC_FOLDER"]], "/main.R") - source(code_path) - features_function <- match.fun(paste0(provider[["SRC_FOLDER"]], "_location_features")) - day_segments <- day_segments_labels %>% pull(label) - for (day_segment in day_segments){ - print(paste(rapids_log_tag,"Processing", provider_key, day_segment)) - - features <- features_function(location_data, day_segment, provider) - - # Check all features names contain the provider key so they are unique - features_names <- colnames(features %>% select(-local_segment)) - if(any(!grepl(paste0(".*(",str_to_lower(provider_key),").*"), features_names))) - stop(paste("The name of all location features of", provider_key," must contain its name in lower case but the following don't [", paste(features_names[!grepl(paste0(".*(",str_to_lower(provider_key),").*"), features_names)], collapse = ", "), "]")) - - location_features <- merge(location_features, features, all = TRUE) - } -} else { - for(feature in provider[["FEATURES"]]) - location_features[,feature] <- NA -} - -location_features <- location_features %>% separate(col = local_segment, - into = c("local_segment_label", "local_start_date", "local_start_time", "local_end_date", "local_end_time"), - sep = "#", - remove = FALSE) - -write.csv(location_features, snakemake@output[[1]], row.names = FALSE) diff --git a/src/features/location/locations_entry.py b/src/features/location/locations_entry.py deleted file mode 100644 index c47fdf5a..00000000 --- a/src/features/location/locations_entry.py +++ /dev/null @@ -1,39 +0,0 @@ -import pandas as pd -from importlib import import_module, util -from pathlib import Path - -# import filter_data_by_segment from src/features/utils/utils.py -spec = util.spec_from_file_location("util", str(Path(snakemake.scriptdir).parent / "utils" / "utils.py")) -mod = util.module_from_spec(spec) -spec.loader.exec_module(mod) -filter_data_by_segment = getattr(mod, "filter_data_by_segment") -rapids_log_tag = getattr(mod, "rapids_log_tag") - -location_data = pd.read_csv(snakemake.input["location_data"][0]) -day_segments_labels = pd.read_csv(snakemake.input["day_segments_labels"], header=0) -mypath = snakemake.params["mypath"] -provider = snakemake.params["provider"] -provider_key = snakemake.params["provider_key"] -location_features = pd.DataFrame(columns=["local_segment"]) - -if "FEATURES" not in provider: - raise ValueError("Provider config[LOCATION][PROVIDERS][{}] is missing a FEATURES attribute in config.yaml".format(provider_key)) - -if provider["COMPUTE"] == True: - code_path = provider["SRC_FOLDER"] + ".main" - feature_module = import_module(code_path) - feature_function = getattr(feature_module, provider["SRC_FOLDER"] + "_location_features") - - for day_segment in day_segments_labels["label"]: - print("{} Processing {} {}".format(rapids_log_tag, provider_key, day_segment)) - features = feature_function(location_data, day_segment, provider, filter_data_by_segment=filter_data_by_segment) - location_features = location_features.merge(features, how="outer") -else: - for feature in provider["FEATURES"]: - location_features[feature] = None - -segment_colums = pd.DataFrame() -segment_colums[["local_segment_label", "local_start_date", "local_start_time", "local_end_date", "local_end_time"]] = location_features["local_segment"].str.split(pat="#", expand=True) -for i in range(segment_colums.shape[1]): - location_features.insert(1 + i, segment_colums.columns[i], segment_colums[segment_colums.columns[i]]) -location_features.to_csv(snakemake.output[0], index=False) \ No newline at end of file diff --git a/src/features/location/barnett/library/AvgFlightDur.R b/src/features/locations/barnett/library/AvgFlightDur.R similarity index 100% rename from src/features/location/barnett/library/AvgFlightDur.R rename to src/features/locations/barnett/library/AvgFlightDur.R diff --git a/src/features/location/barnett/library/AvgFlightLen.R b/src/features/locations/barnett/library/AvgFlightLen.R similarity index 100% rename from src/features/location/barnett/library/AvgFlightLen.R rename to src/features/locations/barnett/library/AvgFlightLen.R diff --git a/src/features/location/barnett/library/Collapse2Pause.R b/src/features/locations/barnett/library/Collapse2Pause.R similarity index 100% rename from src/features/location/barnett/library/Collapse2Pause.R rename to src/features/locations/barnett/library/Collapse2Pause.R diff --git a/src/features/location/barnett/library/DailyMobilityPlots.R b/src/features/locations/barnett/library/DailyMobilityPlots.R similarity index 100% rename from src/features/location/barnett/library/DailyMobilityPlots.R rename to src/features/locations/barnett/library/DailyMobilityPlots.R diff --git a/src/features/location/barnett/library/DailyRoutineIndex.R b/src/features/locations/barnett/library/DailyRoutineIndex.R similarity index 100% rename from src/features/location/barnett/library/DailyRoutineIndex.R rename to src/features/locations/barnett/library/DailyRoutineIndex.R diff --git a/src/features/location/barnett/library/DayDist.R b/src/features/locations/barnett/library/DayDist.R similarity index 100% rename from src/features/location/barnett/library/DayDist.R rename to src/features/locations/barnett/library/DayDist.R diff --git a/src/features/location/barnett/library/DistanceTravelled.R b/src/features/locations/barnett/library/DistanceTravelled.R similarity index 100% rename from src/features/location/barnett/library/DistanceTravelled.R rename to src/features/locations/barnett/library/DistanceTravelled.R diff --git a/src/features/location/barnett/library/ExtractFlights.R b/src/features/locations/barnett/library/ExtractFlights.R similarity index 100% rename from src/features/location/barnett/library/ExtractFlights.R rename to src/features/locations/barnett/library/ExtractFlights.R diff --git a/src/features/location/barnett/library/ExtractTimePeriod.R b/src/features/locations/barnett/library/ExtractTimePeriod.R similarity index 100% rename from src/features/location/barnett/library/ExtractTimePeriod.R rename to src/features/locations/barnett/library/ExtractTimePeriod.R diff --git a/src/features/location/barnett/library/GPS2MobMat.R b/src/features/locations/barnett/library/GPS2MobMat.R similarity index 100% rename from src/features/location/barnett/library/GPS2MobMat.R rename to src/features/locations/barnett/library/GPS2MobMat.R diff --git a/src/features/location/barnett/library/GPSmobility-internal.R b/src/features/locations/barnett/library/GPSmobility-internal.R similarity index 100% rename from src/features/location/barnett/library/GPSmobility-internal.R rename to src/features/locations/barnett/library/GPSmobility-internal.R diff --git a/src/features/location/barnett/library/GetMobilityFeaturesMat.R b/src/features/locations/barnett/library/GetMobilityFeaturesMat.R similarity index 100% rename from src/features/location/barnett/library/GetMobilityFeaturesMat.R rename to src/features/locations/barnett/library/GetMobilityFeaturesMat.R diff --git a/src/features/location/barnett/library/GuessPause.R b/src/features/locations/barnett/library/GuessPause.R similarity index 100% rename from src/features/location/barnett/library/GuessPause.R rename to src/features/locations/barnett/library/GuessPause.R diff --git a/src/features/location/barnett/library/Hometime.R b/src/features/locations/barnett/library/Hometime.R similarity index 100% rename from src/features/location/barnett/library/Hometime.R rename to src/features/locations/barnett/library/Hometime.R diff --git a/src/features/location/barnett/library/InitializeParams.R b/src/features/locations/barnett/library/InitializeParams.R similarity index 100% rename from src/features/location/barnett/library/InitializeParams.R rename to src/features/locations/barnett/library/InitializeParams.R diff --git a/src/features/location/barnett/library/IsFlight.R b/src/features/locations/barnett/library/IsFlight.R similarity index 100% rename from src/features/location/barnett/library/IsFlight.R rename to src/features/locations/barnett/library/IsFlight.R diff --git a/src/features/location/barnett/library/LatLong2XY.R b/src/features/locations/barnett/library/LatLong2XY.R similarity index 100% rename from src/features/location/barnett/library/LatLong2XY.R rename to src/features/locations/barnett/library/LatLong2XY.R diff --git a/src/features/location/barnett/library/LocationAt.R b/src/features/locations/barnett/library/LocationAt.R similarity index 100% rename from src/features/location/barnett/library/LocationAt.R rename to src/features/locations/barnett/library/LocationAt.R diff --git a/src/features/location/barnett/library/MaxDiam.R b/src/features/locations/barnett/library/MaxDiam.R similarity index 100% rename from src/features/location/barnett/library/MaxDiam.R rename to src/features/locations/barnett/library/MaxDiam.R diff --git a/src/features/location/barnett/library/MaxDistBetweenTrajectories.R b/src/features/locations/barnett/library/MaxDistBetweenTrajectories.R similarity index 100% rename from src/features/location/barnett/library/MaxDistBetweenTrajectories.R rename to src/features/locations/barnett/library/MaxDistBetweenTrajectories.R diff --git a/src/features/location/barnett/library/MaxHomeDist.R b/src/features/locations/barnett/library/MaxHomeDist.R similarity index 100% rename from src/features/location/barnett/library/MaxHomeDist.R rename to src/features/locations/barnett/library/MaxHomeDist.R diff --git a/src/features/location/barnett/library/MaxRadius.R b/src/features/locations/barnett/library/MaxRadius.R similarity index 100% rename from src/features/location/barnett/library/MaxRadius.R rename to src/features/locations/barnett/library/MaxRadius.R diff --git a/src/features/location/barnett/library/MinsMissing.R b/src/features/locations/barnett/library/MinsMissing.R similarity index 100% rename from src/features/location/barnett/library/MinsMissing.R rename to src/features/locations/barnett/library/MinsMissing.R diff --git a/src/features/location/barnett/library/MobilityFeatures.R b/src/features/locations/barnett/library/MobilityFeatures.R similarity index 100% rename from src/features/location/barnett/library/MobilityFeatures.R rename to src/features/locations/barnett/library/MobilityFeatures.R diff --git a/src/features/location/barnett/library/MobmatQualityOK.R b/src/features/locations/barnett/library/MobmatQualityOK.R similarity index 100% rename from src/features/location/barnett/library/MobmatQualityOK.R rename to src/features/locations/barnett/library/MobmatQualityOK.R diff --git a/src/features/location/barnett/library/ProbPause.R b/src/features/locations/barnett/library/ProbPause.R similarity index 100% rename from src/features/location/barnett/library/ProbPause.R rename to src/features/locations/barnett/library/ProbPause.R diff --git a/src/features/location/barnett/library/ProgressBar.R b/src/features/locations/barnett/library/ProgressBar.R similarity index 100% rename from src/features/location/barnett/library/ProgressBar.R rename to src/features/locations/barnett/library/ProgressBar.R diff --git a/src/features/location/barnett/library/RadiusOfGyration.R b/src/features/locations/barnett/library/RadiusOfGyration.R similarity index 100% rename from src/features/location/barnett/library/RadiusOfGyration.R rename to src/features/locations/barnett/library/RadiusOfGyration.R diff --git a/src/features/location/barnett/library/RandomBridge.R b/src/features/locations/barnett/library/RandomBridge.R similarity index 100% rename from src/features/location/barnett/library/RandomBridge.R rename to src/features/locations/barnett/library/RandomBridge.R diff --git a/src/features/location/barnett/library/SigLocEntropy.R b/src/features/locations/barnett/library/SigLocEntropy.R similarity index 100% rename from src/features/location/barnett/library/SigLocEntropy.R rename to src/features/locations/barnett/library/SigLocEntropy.R diff --git a/src/features/location/barnett/library/SigLocs.R b/src/features/locations/barnett/library/SigLocs.R similarity index 100% rename from src/features/location/barnett/library/SigLocs.R rename to src/features/locations/barnett/library/SigLocs.R diff --git a/src/features/location/barnett/library/SigLocsVisited.R b/src/features/locations/barnett/library/SigLocsVisited.R similarity index 100% rename from src/features/location/barnett/library/SigLocsVisited.R rename to src/features/locations/barnett/library/SigLocsVisited.R diff --git a/src/features/location/barnett/library/SimulateMobilityGaps.R b/src/features/locations/barnett/library/SimulateMobilityGaps.R similarity index 100% rename from src/features/location/barnett/library/SimulateMobilityGaps.R rename to src/features/locations/barnett/library/SimulateMobilityGaps.R diff --git a/src/features/location/barnett/library/StdFlightDur.R b/src/features/locations/barnett/library/StdFlightDur.R similarity index 100% rename from src/features/location/barnett/library/StdFlightDur.R rename to src/features/locations/barnett/library/StdFlightDur.R diff --git a/src/features/location/barnett/library/StdFlightLen.R b/src/features/locations/barnett/library/StdFlightLen.R similarity index 100% rename from src/features/location/barnett/library/StdFlightLen.R rename to src/features/locations/barnett/library/StdFlightLen.R diff --git a/src/features/location/barnett/library/WriteSurveyAnswers2File.R b/src/features/locations/barnett/library/WriteSurveyAnswers2File.R similarity index 100% rename from src/features/location/barnett/library/WriteSurveyAnswers2File.R rename to src/features/locations/barnett/library/WriteSurveyAnswers2File.R diff --git a/src/features/location/barnett/library/plot.flights.R b/src/features/locations/barnett/library/plot.flights.R similarity index 100% rename from src/features/location/barnett/library/plot.flights.R rename to src/features/locations/barnett/library/plot.flights.R diff --git a/src/features/location/barnett/library/plotlimits.R b/src/features/locations/barnett/library/plotlimits.R similarity index 100% rename from src/features/location/barnett/library/plotlimits.R rename to src/features/locations/barnett/library/plotlimits.R diff --git a/src/features/location/barnett/main.R b/src/features/locations/barnett/main.R similarity index 96% rename from src/features/location/barnett/main.R rename to src/features/locations/barnett/main.R index 54dc9628..9aafe523 100644 --- a/src/features/location/barnett/main.R +++ b/src/features/locations/barnett/main.R @@ -3,7 +3,7 @@ library("dplyr") library("stringr") # Load Ian Barnett's code. Taken from https://scholar.harvard.edu/ibarnett/software/gpsmobility -file.sources = list.files(c("src/features/location/barnett/library"), pattern="*.R$", full.names=TRUE, ignore.case=TRUE) +file.sources = list.files(c("src/features/locations/barnett/library"), pattern="*.R$", full.names=TRUE, ignore.case=TRUE) sapply(file.sources,source,.GlobalEnv) create_empty_file <- function(requested_features){ @@ -27,7 +27,7 @@ create_empty_file <- function(requested_features){ ) %>% select(all_of(requested_features))) } -barnett_location_features <- function(location_data, day_segment, params){ +barnett_features <- function(location_data, day_segment, params){ location_features <- NULL location <- location_data accuracy_limit <- params[["ACCURACY_LIMIT"]] diff --git a/src/features/location/doryab/main.py b/src/features/locations/doryab/main.py similarity index 99% rename from src/features/location/doryab/main.py rename to src/features/locations/doryab/main.py index cffb2597..ed7b614a 100644 --- a/src/features/location/doryab/main.py +++ b/src/features/locations/doryab/main.py @@ -4,7 +4,7 @@ from astropy.timeseries import LombScargle from sklearn.cluster import DBSCAN from math import radians, cos, sin, asin, sqrt -def doryab_location_features(location_data, day_segment, params, filter_data_by_segment, *args, **kwargs): +def doryab_features(location_data, day_segment, params, filter_data_by_segment, *args, **kwargs): requested_features = params["FEATURES"] dbscan_eps = params["DBSCAN_EPS"] dbscan_minsamples = params["DBSCAN_MINSAMPLES"] diff --git a/src/features/locations/locations_entry.R b/src/features/locations/locations_entry.R new file mode 100644 index 00000000..f8786432 --- /dev/null +++ b/src/features/locations/locations_entry.R @@ -0,0 +1,13 @@ +source("renv/activate.R") +source("src/features/utils/utils.R") +library("dplyr") +library("tidyr") + +sensor_data_file <- snakemake@input[["sensor_data"]] +day_segments_file <- snakemake@input[["day_segments_labels"]] +provider <- snakemake@params["provider"][["provider"]] +provider_key <- snakemake@params["provider_key"] + +sensor_features <- fetch_provider_features(provider, provider_key, "locations", sensor_data_file, day_segments_file) + +write.csv(sensor_features, snakemake@output[[1]], row.names = FALSE) diff --git a/src/features/locations/locations_entry.py b/src/features/locations/locations_entry.py new file mode 100644 index 00000000..9ff0470f --- /dev/null +++ b/src/features/locations/locations_entry.py @@ -0,0 +1,18 @@ +import pandas as pd +from importlib import import_module, util +from pathlib import Path + +# import fetch_provider_features from src/features/utils/utils.py +spec = util.spec_from_file_location("util", str(Path(snakemake.scriptdir).parent / "utils" / "utils.py")) +mod = util.module_from_spec(spec) +spec.loader.exec_module(mod) +fetch_provider_features = getattr(mod, "fetch_provider_features") + +sensor_data_file = snakemake.input["sensor_data"][0] +day_segments_file = snakemake.input["day_segments_labels"] +provider = snakemake.params["provider"] +provider_key = snakemake.params["provider_key"] + +sensor_features = fetch_provider_features(provider, provider_key, "locations", sensor_data_file, day_segments_file) + +sensor_features.to_csv(snakemake.output[0], index=False) \ No newline at end of file diff --git a/src/features/utils/utils.R b/src/features/utils/utils.R index 544a0655..645ef78c 100644 --- a/src/features/utils/utils.R +++ b/src/features/utils/utils.R @@ -1,4 +1,7 @@ library("stringr") + +rapids_log_tag <- "RAPIDS:" + filter_data_by_segment <- function(data, day_segment){ # Filter the rows that belong to day_segment, and put the segment full name in a new column for grouping date_regex = "[0-9]{4}[\\-|\\/][0-9]{2}[\\-|\\/][0-9]{2}" @@ -9,4 +12,41 @@ filter_data_by_segment <- function(data, day_segment){ local_segment = str_sub(local_segment, 2, -2)) # get rid of first and last character([]) return(data) } -rapids_log_tag <- "RAPIDS:" \ No newline at end of file + +fetch_provider_features <- function(provider, provider_key, config_key, sensor_data_file, day_segments_file){ + sensor_features <- data.frame(local_segment = character(), stringsAsFactors = FALSE) + + sensor_data <- read.csv(sensor_data_file, stringsAsFactors = FALSE) + day_segments_labels <- read.csv(day_segments_file, stringsAsFactors = FALSE) + + if(!"FEATURES" %in% names(provider)) + stop(paste0("Provider config[CALLS][PROVIDERS][", provider_key,"] is missing a FEATURES attribute in config.yaml")) + + if(provider[["COMPUTE"]] == TRUE){ + code_path <- paste0("src/features/", config_key,"/", provider[["SRC_FOLDER"]], "/main.R") + source(code_path) + features_function <- match.fun(paste0(provider[["SRC_FOLDER"]], "_features")) + day_segments <- day_segments_labels %>% pull(label) + for (day_segment in day_segments){ + print(paste(rapids_log_tag,"Processing", config_key, provider_key, day_segment)) + + features <- features_function(sensor_data, day_segment, provider) + + # Check all features names contain the provider key so they are unique + features_names <- colnames(features %>% select(-local_segment)) + if(any(!grepl(paste0(".*(",str_to_lower(provider_key),").*"), features_names))) + stop(paste("The name of all calls features of", provider_key," must contain its name in lower case but the following don't [", paste(features_names[!grepl(paste0(".*(",str_to_lower(provider_key),").*"), features_names)], collapse = ", "), "]")) + + sensor_features <- merge(sensor_features, features, all = TRUE) + } + } else { + for(feature in provider[["FEATURES"]]) + sensor_features[,feature] <- NA + } + + sensor_features <- sensor_features %>% separate(col = local_segment, + into = c("local_segment_label", "local_start_date", "local_start_time", "local_end_date", "local_end_time"), + sep = "#", + remove = FALSE) + return(sensor_features) +} \ No newline at end of file diff --git a/src/features/utils/utils.py b/src/features/utils/utils.py index a0736a78..a51fd731 100644 --- a/src/features/utils/utils.py +++ b/src/features/utils/utils.py @@ -1,3 +1,4 @@ +rapids_log_tag = "RAPIDS:" def filter_data_by_segment(data, day_segment): date_regex = "[0-9]{4}[\-|\/][0-9]{2}[\-|\/][0-9]{2}" @@ -6,4 +7,36 @@ def filter_data_by_segment(data, day_segment): data["local_segment"] = data["assigned_segments"].str.extract(segment_regex, expand=True) return(data.dropna(subset = ["local_segment"])) -rapids_log_tag = "RAPIDS:" \ No newline at end of file +def fetch_provider_features(provider, provider_key, config_key, sensor_data_file, day_segments_file): + import pandas as pd + from importlib import import_module, util + + sensor_features = pd.DataFrame(columns=["local_segment"]) + sensor_data = pd.read_csv(sensor_data_file) + day_segments_labels = pd.read_csv(day_segments_file, header=0) + if "FEATURES" not in provider: + raise ValueError("Provider config[{}][PROVIDERS][{}] is missing a FEATURES attribute in config.yaml".format(config_key.upper(), provider_key)) + + if provider["COMPUTE"] == True: + code_path = provider["SRC_FOLDER"] + ".main" + feature_module = import_module(code_path) + feature_function = getattr(feature_module, provider["SRC_FOLDER"] + "_features") + + for day_segment in day_segments_labels["label"]: + print("{} Processing {} {} {}".format(rapids_log_tag, config_key, provider_key, day_segment)) + print("---") + features = feature_function(sensor_data, day_segment, provider, filter_data_by_segment=filter_data_by_segment) + print("2") + sensor_features = sensor_features.merge(features, how="outer") + else: + for feature in provider["FEATURES"]: + sensor_features[feature] = None + print("3") + segment_colums = pd.DataFrame() + split_segemnt_columns = sensor_features["local_segment"].str.split(pat="#", expand=True) + new_segment_columns = split_segemnt_columns if split_segemnt_columns.shape[1] == 5 else pd.DataFrame(columns=["local_segment_label", "local_start_date", "local_start_time", "local_end_date", "local_end_time"]) + segment_colums[["local_segment_label", "local_start_date", "local_start_time", "local_end_date", "local_end_time"]] = new_segment_columns + for i in range(segment_colums.shape[1]): + sensor_features.insert(1 + i, segment_colums.columns[i], segment_colums[segment_colums.columns[i]]) + + return sensor_features \ No newline at end of file