junos
/
rapids
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update visualization section: add MIN_VALID_BINS_PER_HOUR

pull/95/head
Meng Li 2020-07-26 20:11:25 -04:00
parent 766c94413f
commit af6678c70c
4 changed files with 18 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Snakefile
View File

@ -138,14 +138,14 @@ if config["DORYAB_LOCATION"]["COMPUTE"]:
# visualization for data exploration # visualization for data exploration
if config["HEATMAP_FEATURES_CORRELATIONS"]["PLOT"]: if config["HEATMAP_FEATURES_CORRELATIONS"]["PLOT"]:
files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}h/heatmap_features_correlations.html", min_valid_hours_per_day=config["HEATMAP_FEATURES_CORRELATIONS"]["MIN_VALID_HOURS_PER_DAY"])) files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_features_correlations.html", min_valid_hours_per_day=config["HEATMAP_FEATURES_CORRELATIONS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
if config["HISTOGRAM_VALID_SENSED_HOURS"]["PLOT"]: if config["HISTOGRAM_VALID_SENSED_HOURS"]["PLOT"]:
files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}h/histogram_valid_sensed_hours.html", min_valid_hours_per_day=config["HISTOGRAM_VALID_SENSED_HOURS"]["MIN_VALID_HOURS_PER_DAY"])) files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/histogram_valid_sensed_hours.html", min_valid_hours_per_day=config["HISTOGRAM_VALID_SENSED_HOURS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
if config["HEATMAP_DAYS_BY_SENSORS"]["PLOT"]: if config["HEATMAP_DAYS_BY_SENSORS"]["PLOT"]:
files_to_compute.extend(expand("reports/interim/{min_valid_hours_per_day}h/{pid}/heatmap_days_by_sensors.html", pid=config["PIDS"], min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"])) files_to_compute.extend(expand("reports/interim/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/{pid}/heatmap_days_by_sensors.html", pid=config["PIDS"], min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}h/heatmap_days_by_sensors_all_participants.html", min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"])) files_to_compute.extend(expand("reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_days_by_sensors_all_participants.html", min_valid_hours_per_day=config["HEATMAP_DAYS_BY_SENSORS"]["MIN_VALID_HOURS_PER_DAY"], min_valid_bins_per_hour=config["PHONE_VALID_SENSED_DAYS"]["MIN_VALID_BINS_PER_HOUR"]))
if config["HEATMAP_SENSED_BINS"]["PLOT"]: if config["HEATMAP_SENSED_BINS"]["PLOT"]:
files_to_compute.extend(expand("reports/interim/heatmap_sensed_bins/{pid}/heatmap_sensed_bins.html", pid=config["PIDS"])) files_to_compute.extend(expand("reports/interim/heatmap_sensed_bins/{pid}/heatmap_sensed_bins.html", pid=config["PIDS"]))

5
config.yaml
View File

@ -210,6 +210,7 @@ HEATMAP_FEATURES_CORRELATIONS:
PLOT: False PLOT: False
MIN_ROWS_RATIO: 0.5 MIN_ROWS_RATIO: 0.5
MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen] PHONE_FEATURES: [accelerometer, activity_recognition, applications_foreground, battery, calls_incoming, calls_missed, calls_outgoing, conversation, light, location_doryab, messages_received, messages_sent, screen]
FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep] FITBIT_FEATURES: [fitbit_heartrate, fitbit_step, fitbit_sleep]
CORR_THRESHOLD: 0.1 CORR_THRESHOLD: 0.1
@ -218,10 +219,12 @@ HEATMAP_FEATURES_CORRELATIONS:
HISTOGRAM_VALID_SENSED_HOURS: HISTOGRAM_VALID_SENSED_HOURS:
PLOT: False PLOT: False
MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
HEATMAP_DAYS_BY_SENSORS: HEATMAP_DAYS_BY_SENSORS:
PLOT: False PLOT: False
MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
EXPECTED_NUM_OF_DAYS: -1 EXPECTED_NUM_OF_DAYS: -1
PHONE_SENSORS_TABLES: ["accelerometer", "applications_foreground", "battery", "calls", "light", "locations", "messages", "screen", "plugin_google_activity_recognition", "plugin_studentlife_audio_android"] PHONE_SENSORS_TABLES: ["accelerometer", "applications_foreground", "battery", "calls", "light", "locations", "messages", "screen", "plugin_google_activity_recognition", "plugin_studentlife_audio_android"]
@ -234,8 +237,8 @@ OVERALL_COMPLIANCE_HEATMAP:
ONLY_SHOW_VALID_DAYS: False ONLY_SHOW_VALID_DAYS: False
EXPECTED_NUM_OF_DAYS: -1 EXPECTED_NUM_OF_DAYS: -1
BIN_SIZE: *bin_size BIN_SIZE: *bin_size
MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day MIN_VALID_HOURS_PER_DAY: *min_valid_hours_per_day
MIN_VALID_BINS_PER_HOUR: *min_valid_bins_per_hour
### Example Analysis ################################################################ ### Example Analysis ################################################################
PARAMS_FOR_ANALYSIS: PARAMS_FOR_ANALYSIS:

4
rules/preprocessing.snakefile
View File

@ -25,9 +25,9 @@ PHONE_SENSORS = []
PHONE_SENSORS.extend([config["MESSAGES"]["DB_TABLE"], config["CALLS"]["DB_TABLE"], config["BARNETT_LOCATION"]["DB_TABLE"], config["DORYAB_LOCATION"]["DB_TABLE"], config["BLUETOOTH"]["DB_TABLE"], config["BATTERY"]["DB_TABLE"], config["SCREEN"]["DB_TABLE"], config["LIGHT"]["DB_TABLE"], config["ACCELEROMETER"]["DB_TABLE"], config["APPLICATIONS_FOREGROUND"]["DB_TABLE"], config["CONVERSATION"]["DB_TABLE"]["ANDROID"], config["CONVERSATION"]["DB_TABLE"]["IOS"], config["ACTIVITY_RECOGNITION"]["DB_TABLE"]["ANDROID"], config["ACTIVITY_RECOGNITION"]["DB_TABLE"]["IOS"]]) PHONE_SENSORS.extend([config["MESSAGES"]["DB_TABLE"], config["CALLS"]["DB_TABLE"], config["BARNETT_LOCATION"]["DB_TABLE"], config["DORYAB_LOCATION"]["DB_TABLE"], config["BLUETOOTH"]["DB_TABLE"], config["BATTERY"]["DB_TABLE"], config["SCREEN"]["DB_TABLE"], config["LIGHT"]["DB_TABLE"], config["ACCELEROMETER"]["DB_TABLE"], config["APPLICATIONS_FOREGROUND"]["DB_TABLE"], config["CONVERSATION"]["DB_TABLE"]["ANDROID"], config["CONVERSATION"]["DB_TABLE"]["IOS"], config["ACTIVITY_RECOGNITION"]["DB_TABLE"]["ANDROID"], config["ACTIVITY_RECOGNITION"]["DB_TABLE"]["IOS"]])
if len(config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"]) > 0: if len(config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"]) > 0:
PHONE_SENSORS.extend(config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"]) PHONE_SENSORS.append(config["WIFI"]["DB_TABLE"]["VISIBLE_ACCESS_POINTS"])
if len(config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"]) > 0: if len(config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"]) > 0:
PHONE_SENSORS.extend(config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"]) PHONE_SENSORS.append(config["WIFI"]["DB_TABLE"]["CONNECTED_ACCESS_POINTS"])
rule readable_datetime: rule readable_datetime:

16
rules/reports.snakefile
View File

@ -1,41 +1,41 @@
rule heatmap_features_correlations: rule heatmap_features_correlations:
input: input:
features = expand("data/processed/{pid}/{sensor}_{day_segment}.csv", pid=config["PIDS"], sensor=config["HEATMAP_FEATURES_CORRELATIONS"]["PHONE_FEATURES"]+config["HEATMAP_FEATURES_CORRELATIONS"]["FITBIT_FEATURES"], day_segment=config["DAY_SEGMENTS"]), features = expand("data/processed/{pid}/{sensor}_{day_segment}.csv", pid=config["PIDS"], sensor=config["HEATMAP_FEATURES_CORRELATIONS"]["PHONE_FEATURES"]+config["HEATMAP_FEATURES_CORRELATIONS"]["FITBIT_FEATURES"], day_segment=config["DAY_SEGMENTS"]),
phone_valid_sensed_days = expand("data/interim/{pid}/phone_valid_sensed_days_{{min_valid_hours_per_day}}h.csv", pid=config["PIDS"]) phone_valid_sensed_days = expand("data/interim/{pid}/phone_valid_sensed_days_{{min_valid_hours_per_day}}hours_{{min_valid_bins_per_hour}}bins.csv", pid=config["PIDS"])
params: params:
min_rows_ratio = config["HEATMAP_FEATURES_CORRELATIONS"]["MIN_ROWS_RATIO"], min_rows_ratio = config["HEATMAP_FEATURES_CORRELATIONS"]["MIN_ROWS_RATIO"],
corr_threshold = config["HEATMAP_FEATURES_CORRELATIONS"]["CORR_THRESHOLD"], corr_threshold = config["HEATMAP_FEATURES_CORRELATIONS"]["CORR_THRESHOLD"],
corr_method = config["HEATMAP_FEATURES_CORRELATIONS"]["CORR_METHOD"] corr_method = config["HEATMAP_FEATURES_CORRELATIONS"]["CORR_METHOD"]
output: output:
"reports/data_exploration/{min_valid_hours_per_day}h/heatmap_features_correlations.html" "reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_features_correlations.html"
script: script:
"../src/visualization/heatmap_features_correlations.py" "../src/visualization/heatmap_features_correlations.py"
rule histogram_valid_sensed_hours: rule histogram_valid_sensed_hours:
input: input:
phone_valid_sensed_days = expand("data/interim/{pid}/phone_valid_sensed_days_{{min_valid_hours_per_day}}h.csv", pid=config["PIDS"]) phone_valid_sensed_days = expand("data/interim/{pid}/phone_valid_sensed_days_{{min_valid_hours_per_day}}hours_{{min_valid_bins_per_hour}}bins.csv", pid=config["PIDS"])
output: output:
"reports/data_exploration/{min_valid_hours_per_day}h/histogram_valid_sensed_hours.html" "reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/histogram_valid_sensed_hours.html"
script: script:
"../src/visualization/histogram_valid_sensed_hours.py" "../src/visualization/histogram_valid_sensed_hours.py"
rule heatmap_days_by_sensors: rule heatmap_days_by_sensors:
input: input:
sensors = expand("data/raw/{{pid}}/{sensor}_with_datetime.csv", sensor=config["HEATMAP_DAYS_BY_SENSORS"]["PHONE_SENSORS_TABLES"]), sensors = expand("data/raw/{{pid}}/{sensor}_with_datetime.csv", sensor=config["HEATMAP_DAYS_BY_SENSORS"]["PHONE_SENSORS_TABLES"]),
phone_valid_sensed_days = "data/interim/{pid}/phone_valid_sensed_days_{min_valid_hours_per_day}h.csv" phone_valid_sensed_days = "data/interim/{pid}/phone_valid_sensed_days_{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins.csv"
params: params:
pid = "{pid}", pid = "{pid}",
expected_num_of_days = config["HEATMAP_DAYS_BY_SENSORS"]["EXPECTED_NUM_OF_DAYS"] expected_num_of_days = config["HEATMAP_DAYS_BY_SENSORS"]["EXPECTED_NUM_OF_DAYS"]
output: output:
"reports/interim/{min_valid_hours_per_day}h/{pid}/heatmap_days_by_sensors.html" "reports/interim/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/{pid}/heatmap_days_by_sensors.html"
script: script:
"../src/visualization/heatmap_days_by_sensors.py" "../src/visualization/heatmap_days_by_sensors.py"
rule heatmap_days_by_sensors_all_participants: rule heatmap_days_by_sensors_all_participants:
input: input:
heatmap_rows = expand("reports/interim/{{min_valid_hours_per_day}}h/{pid}/heatmap_days_by_sensors.html", pid=config["PIDS"]) heatmap_rows = expand("reports/interim/{{min_valid_hours_per_day}}hours_{{min_valid_bins_per_hour}}bins/{pid}/heatmap_days_by_sensors.html", pid=config["PIDS"])
output: output:
"reports/data_exploration/{min_valid_hours_per_day}h/heatmap_days_by_sensors_all_participants.html" "reports/data_exploration/{min_valid_hours_per_day}hours_{min_valid_bins_per_hour}bins/heatmap_days_by_sensors_all_participants.html"
script: script:
"../src/visualization/heatmap_days_by_sensors_all_participants.Rmd" "../src/visualization/heatmap_days_by_sensors_all_participants.Rmd"