From d1c38016dea82f24fbf1d7ca91ff3fe28279143e Mon Sep 17 00:00:00 2001 From: Mingze Cao <29229557+Martinze@users.noreply.github.com> Date: Wed, 8 Apr 2020 10:51:18 -0500 Subject: [PATCH] Refactor call features: replace "metrics" with "features" Co-authored-by: Meng Li --- config.yaml | 4 +- docs/features/extracted.rst | 36 ++++++++-------- rules/features.snakefile | 6 +-- .../{call_metrics.R => call_features.R} | 42 +++++++++---------- 4 files changed, 44 insertions(+), 44 deletions(-) rename src/features/{call_metrics.R => call_features.R} (59%) diff --git a/config.yaml b/config.yaml index c69e14e0..ba3a4e07 100644 --- a/config.yaml +++ b/config.yaml @@ -42,10 +42,10 @@ SMS: sent: [count, distinctcontacts, timefirstsms, timelastsms, countmostfrequentcontact] DAY_SEGMENTS: *day_segments -# Communication call features config, TYPES and METRICS keys need to match +# Communication call features config, TYPES and FEATURES keys need to match CALLS: TYPES: [missed, incoming, outgoing] - METRICS: + FEATURES: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, hubermduration, varqnduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, hubermduration, varqnduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] diff --git a/docs/features/extracted.rst b/docs/features/extracted.rst index 05f5497b..99ba1a62 100644 --- a/docs/features/extracted.rst +++ b/docs/features/extracted.rst @@ -176,7 +176,7 @@ See `Call Config Code`_ .. - Apply readable datetime to Calls dataset: ``expand("data/raw/{pid}/{sensor}_with_datetime.csv", pid=config["PIDS"], sensor=config["SENSORS"]),`` -- Extract Calls Metrics +- Extract Calls Features | ``expand("data/processed/{pid}/call_{call_type}_{segment}.csv",`` | ``pid=config["PIDS"],`` @@ -193,9 +193,9 @@ See `Call Config Code`_ - **Script:** ``src/data/readable_datetime.R`` - See the readable_datetime.R_ script. -- **Rule:** ``rules/features.snakefile/call_metrics`` - See the call_metrics_ rule. +- **Rule:** ``rules/features.snakefile/call_features`` - See the call_features_ rule. - - **Script:** ``src/features/call_metrics.R`` - See the call_metrics.R_ script. + - **Script:** ``src/features/call_features.R`` - See the call_features.R_ script. .. _calls-parameters: @@ -207,14 +207,14 @@ Name Description ============ =================== call_type The particular ``call_type`` that will be analyzed. The options for this parameter are ``incoming``, ``outgoing`` or ``missed``. day_segment The particular ``day_segments`` that will be analyzed. The available options are ``daily``, ``morning``, ``afternoon``, ``evening``, ``night`` -metrics The different measures that can be retrieved from the calls dataset. Note that the same metrics are available for both ``incoming`` and ``outgoing`` calls, while ``missed`` calls has its own set of metrics. See :ref:`Available Incoming and Outgoing Call Metrics ` Table and :ref:`Available Missed Call Metrics ` Table below. +features The different measures that can be retrieved from the calls dataset. Note that the same features are available for both ``incoming`` and ``outgoing`` calls, while ``missed`` calls has its own set of features. See :ref:`Available Incoming and Outgoing Call Features ` Table and :ref:`Available Missed Call Features ` Table below. ============ =================== -.. _available-in-and-out-call-metrics: +.. _available-in-and-out-call-features: -**Available Incoming and Outgoing Call Metrics** +**Available Incoming and Outgoing Call Features** -The following table shows a list of the available metrics for ``incoming`` and ``outgoing`` calls. +The following table shows a list of the available features for ``incoming`` and ``outgoing`` calls. ========================= ========= ============= Name Units Description @@ -235,11 +235,11 @@ timelastcall minutes The time in minutes from 12:00am (Midn countmostfrequentcontact calls The count of the number of calls of a particular ``call_type`` and ``day_segment`` for the most contacted contact. ========================= ========= ============= -.. _available-missed-call-metrics: +.. _available-missed-call-features: -**Available Missed Call Metrics** +**Available Missed Call Features** -The following table shows a list of the available metrics for ``missed`` calls. +The following table shows a list of the available features for ``missed`` calls. ========================= ========= ============= Name Units Description @@ -248,19 +248,19 @@ count calls A count of the number of times a ``mis distinctcontacts contacts A count of distinct contacts whose calls were ``missed``. timefirstcall minutes The time in minutes from 12:00am (Midnight) that the first ``missed`` call occurred. timelastcall minutes The time in minutes from 12:00am (Midnight) that the last ``missed`` call occurred. -countmostfrequentcontact SMS The count of the number of ``missed`` calls for the contact with the most ``missed`` calls. +countmostfrequentcontact CALLS The count of the number of ``missed`` calls for the contact with the most ``missed`` calls. ========================= ========= ============= **Assumptions/Observations:** - #. ``TYPES`` and ``METRICS`` keys need to match. From example:: + #. ``TYPES`` and ``FEATURES`` keys need to match. From example:: - SMS: + CALLS: TYPES: [missed] - METRICS: - missed: [count, distinctcontacts, timefirstsms, timelastsms, countmostfrequentcontact] + FEATURES: + missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] -In the above config setting code the ``TYPE`` ``missed`` matches the ``METRICS`` key ``missed``. +In the above config setting code the ``TYPE`` ``missed`` matches the ``FEATURES`` key ``missed``. .. _bluetooth-sensor-doc: @@ -1150,8 +1150,8 @@ stddurationactivebout minutes Std duration active bout: The standard .. _DAY_SEGMENTS: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/config.yaml#L13 .. _PHONE_VALID_SENSED_DAYS: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/config.yaml#L60 .. _`Call Config Code`: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/config.yaml#L46 -.. _call_metrics: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/rules/features.snakefile#L13 -.. _call_metrics.R: https://github.com/carissalow/rapids/blob/master/src/features/call_metrics.R +.. _call_features: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/rules/features.snakefile#L13 +.. _call_features.R: https://github.com/carissalow/rapids/blob/master/src/features/call_features.R .. _`Bluetooth Config Code`: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/config.yaml#L76 .. _bluetooth_feature: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/rules/features.snakefile#L63 .. _bluetooth_features.R: https://github.com/carissalow/rapids/blob/765bb462636d5029a05f54d4c558487e3786b90b/src/features/bluetooth_features.R diff --git a/rules/features.snakefile b/rules/features.snakefile index add363ce..07b79f9a 100644 --- a/rules/features.snakefile +++ b/rules/features.snakefile @@ -10,17 +10,17 @@ rule sms_metrics: script: "../src/features/sms_metrics.R" -rule call_metrics: +rule call_features: input: "data/raw/{pid}/calls_with_datetime_unified.csv" params: call_type = "{call_type}", day_segment = "{day_segment}", - metrics = lambda wildcards: config["CALLS"]["METRICS"][wildcards.call_type] + features = lambda wildcards: config["CALLS"]["FEATURES"][wildcards.call_type] output: "data/processed/{pid}/call_{call_type}_{day_segment}.csv" script: - "../src/features/call_metrics.R" + "../src/features/call_features.R" rule battery_deltas: input: diff --git a/src/features/call_metrics.R b/src/features/call_features.R similarity index 59% rename from src/features/call_metrics.R rename to src/features/call_features.R index 1fa04ffa..f4e70b75 100644 --- a/src/features/call_metrics.R +++ b/src/features/call_features.R @@ -16,8 +16,8 @@ Mode <- function(v) { uniqv[which.max(tabulate(match(v, uniqv)))] } -compute_call_feature <- function(calls, metric, day_segment){ - if(metric == "countmostfrequentcontact"){ +compute_call_feature <- function(calls, requested_feature, day_segment){ + if(requested_feature == "countmostfrequentcontact"){ # Get the most frequent contact calls <- calls %>% group_by(trace) %>% mutate(N=n()) %>% @@ -26,41 +26,41 @@ compute_call_feature <- function(calls, metric, day_segment){ return(calls %>% filter_by_day_segment(day_segment) %>% - summarise(!!paste("call", type, day_segment, metric, sep = "_") := n())) + summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := n())) } else { calls <- calls %>% filter_by_day_segment(day_segment) - feature <- switch(metric, - "count" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := n()), - "distinctcontacts" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := n_distinct(trace)), - "meanduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := mean(call_duration)), - "sumduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := sum(call_duration)), - "minduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := min(call_duration)), - "maxduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := max(call_duration)), - "stdduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := sd(call_duration)), - "modeduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := Mode(call_duration)), - "hubermduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := huberM(call_duration)$mu), - "varqnduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := Qn(call_duration)), - "entropyduration" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := entropy.MillerMadow(call_duration)), - "timefirstcall" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := first(local_hour) + (first(local_minute)/60)), - "timelastcall" = calls %>% summarise(!!paste("call", type, day_segment, metric, sep = "_") := last(local_hour) + (last(local_minute)/60))) + feature <- switch(requested_feature, + "count" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := n()), + "distinctcontacts" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := n_distinct(trace)), + "meanduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := mean(call_duration)), + "sumduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := sum(call_duration)), + "minduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := min(call_duration)), + "maxduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := max(call_duration)), + "stdduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := sd(call_duration)), + "modeduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := Mode(call_duration)), + "hubermduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := huberM(call_duration)$mu), + "varqnduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := Qn(call_duration)), + "entropyduration" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := entropy.MillerMadow(call_duration)), + "timefirstcall" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := first(local_hour) + (first(local_minute)/60)), + "timelastcall" = calls %>% summarise(!!paste("call", type, day_segment, requested_feature, sep = "_") := last(local_hour) + (last(local_minute)/60))) return(feature) } } calls <- read.csv(snakemake@input[[1]], stringsAsFactors = FALSE) day_segment <- snakemake@params[["day_segment"]] -metrics <- snakemake@params[["metrics"]] +requested_features <- snakemake@params[["features"]] type <- snakemake@params[["call_type"]] features = data.frame(local_date = character(), stringsAsFactors = FALSE) calls <- calls %>% filter(call_type == ifelse(type == "incoming", "1", ifelse(type == "outgoing", "2", "3"))) -for(metric in metrics){ - feature <- compute_call_feature(calls, metric, day_segment) +for(requested_feature in requested_features){ + feature <- compute_call_feature(calls, requested_feature, day_segment) features <- merge(features, feature, by="local_date", all = TRUE) } -if("countmostfrequentcontact" %in% metrics) +if("countmostfrequentcontact" %in% requested_features) features <- features %>% mutate_at(vars(contains('countmostfrequentcontact')), funs(ifelse(is.na(.), 0, .))) write.csv(features, snakemake@output[[1]], row.names = FALSE)