diff --git a/Snakefile b/Snakefile index 346b1803..eca70344 100644 --- a/Snakefile +++ b/Snakefile @@ -28,7 +28,7 @@ for provider in config["PHONE_DATA_YIELD"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_DATA_YIELD"]["SENSORS"]))) files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_data_yield_features/phone_data_yield_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_data_yield_features/phone_data_yield_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_data_yield.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -37,7 +37,7 @@ for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys(): if config["PHONE_MESSAGES"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_messages_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_messages_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_messages_features/phone_messages_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_MESSAGES"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_messages_features/phone_messages_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_MESSAGES"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_messages.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -46,7 +46,7 @@ for provider in config["PHONE_CALLS"]["PROVIDERS"].keys(): if config["PHONE_CALLS"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_calls_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_calls_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_calls_features/phone_calls_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CALLS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_calls_features/phone_calls_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_CALLS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_calls.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -55,7 +55,7 @@ for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys(): if config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_bluetooth_features/phone_bluetooth_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_bluetooth_features/phone_bluetooth_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_bluetooth.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -67,7 +67,7 @@ for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_features/phone_activity_recognition_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_features/phone_activity_recognition_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_activity_recognition.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -78,7 +78,7 @@ for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_battery_features/phone_battery_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BATTERY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_battery_features/phone_battery_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_BATTERY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_battery.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -94,7 +94,7 @@ for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_screen_features/phone_screen_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_SCREEN"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_screen_features/phone_screen_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_SCREEN"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_screen.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -103,7 +103,7 @@ for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys(): if config["PHONE_LIGHT"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_light_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_light_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_light_features/phone_light_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_light_features/phone_light_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_LIGHT"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_light.csv", pid=config["PIDS"],)) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -112,7 +112,7 @@ for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys(): if config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_accelerometer_features/phone_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_accelerometer_features/phone_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_accelerometer.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -122,7 +122,7 @@ for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime_with_categories.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_applications_foreground_features/phone_applications_foreground_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_applications_foreground_features/phone_applications_foreground_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_foreground.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -131,7 +131,7 @@ for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys(): if config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_visible_features/phone_wifi_visible_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_visible_features/phone_wifi_visible_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_visible.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -140,7 +140,7 @@ for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys(): if config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_connected_features/phone_wifi_connected_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_connected_features/phone_wifi_connected_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_connected.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -149,7 +149,7 @@ for provider in config["PHONE_CONVERSATION"]["PROVIDERS"].keys(): if config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_conversation_features/phone_conversation_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_conversation_features/phone_conversation_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_conversation.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -161,7 +161,7 @@ if isinstance(config["PHONE_APPLICATIONS_CRASHES"]["PROVIDERS"], dict): files_to_compute.extend(expand("data/raw/{pid}/phone_applications_crashes_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_crashes_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_crashes_with_datetime_with_categories.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_applications_crashes_features/phone_applications_crashes_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_CRASHES"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_applications_crashes_features/phone_applications_crashes_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_APPLICATIONS_CRASHES"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_crashes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -172,7 +172,7 @@ if isinstance(config["PHONE_APPLICATIONS_NOTIFICATIONS"]["PROVIDERS"], dict): files_to_compute.extend(expand("data/raw/{pid}/phone_applications_notifications_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_notifications_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_notifications_with_datetime_with_categories.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_applications_notifications_features/phone_applications_notifications_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_NOTIFICATIONS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_applications_notifications_features/phone_applications_notifications_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_APPLICATIONS_NOTIFICATIONS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_notifications.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -182,7 +182,7 @@ if isinstance(config["PHONE_KEYBOARD"]["PROVIDERS"], dict): if config["PHONE_KEYBOARD"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_keyboard_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_keyboard_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_keyboard_features/phone_keyboard_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_KEYBOARD"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_keyboard_features/phone_keyboard_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_KEYBOARD"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_keyboard.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -192,7 +192,7 @@ if isinstance(config["PHONE_LOG"]["PROVIDERS"], dict): if config["PHONE_LOG"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_log_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_log_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_log_features/phone_log_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LOG"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_log_features/phone_log_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_LOG"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_log.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -209,7 +209,7 @@ for provider in config["PHONE_LOCATIONS"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed_with_datetime_with_home.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_locations_features/phone_locations_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_locations_features/phone_locations_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_locations.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -226,7 +226,7 @@ for provider in config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"].keys(): if config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_summary.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -235,7 +235,7 @@ for provider in config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"].keys(): if config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_intraday_features/fitbit_heartrate_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_intraday_features/fitbit_heartrate_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_intraday.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -244,7 +244,7 @@ for provider in config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"].keys(): if config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_summary_features/fitbit_sleep_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_summary_features/fitbit_sleep_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_sleep_summary.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -256,7 +256,7 @@ for provider in config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_features/fitbit_sleep_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_features/fitbit_sleep_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_sleep_intraday.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -265,7 +265,7 @@ for provider in config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"].keys(): if config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_summary_features/fitbit_steps_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_summary_features/fitbit_steps_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_summary.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -274,7 +274,7 @@ for provider in config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"].keys(): if config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_intraday_features/fitbit_steps_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_intraday_features/fitbit_steps_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_intraday.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -284,7 +284,7 @@ for provider in config["EMPATICA_ACCELEROMETER"]["PROVIDERS"].keys(): if config["EMPATICA_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_accelerometer_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_accelerometer_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_accelerometer_features/empatica_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_accelerometer_features/empatica_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_accelerometer.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -293,7 +293,7 @@ for provider in config["EMPATICA_HEARTRATE"]["PROVIDERS"].keys(): if config["EMPATICA_HEARTRATE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_heartrate_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_heartrate_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_heartrate_features/empatica_heartrate_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_HEARTRATE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_heartrate_features/empatica_heartrate_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_HEARTRATE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_heartrate.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -303,7 +303,7 @@ for provider in config["EMPATICA_TEMPERATURE"]["PROVIDERS"].keys(): if config["EMPATICA_TEMPERATURE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_temperature_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_temperature_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_temperature_features/empatica_temperature_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_TEMPERATURE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_temperature_features/empatica_temperature_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_TEMPERATURE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_temperature.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -312,7 +312,7 @@ for provider in config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"].keys(): if config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_electrodermal_activity_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_electrodermal_activity_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_electrodermal_activity.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -321,7 +321,7 @@ for provider in config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"].keys(): if config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_blood_volume_pulse_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_blood_volume_pulse_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_blood_volume_pulse.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -330,7 +330,7 @@ for provider in config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"].keys(): if config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_inter_beat_interval_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_inter_beat_interval_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_inter_beat_interval.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -340,7 +340,7 @@ if isinstance(config["EMPATICA_TAGS"]["PROVIDERS"], dict): if config["EMPATICA_TAGS"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_tags_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_tags_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_tags_features/empatica_tags_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_TAGS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_tags_features/empatica_tags_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_TAGS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_tags.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") diff --git a/config.yaml b/config.yaml index ea1df375..6896e57d 100644 --- a/config.yaml +++ b/config.yaml @@ -67,8 +67,7 @@ PHONE_ACCELEROMETER: RAPIDS: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py PANDA: COMPUTE: False @@ -76,8 +75,7 @@ PHONE_ACCELEROMETER: FEATURES: exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - SRC_FOLDER: "panda" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/panda/main.py # See https://www.rapids.science/latest/features/phone-activity-recognition/ PHONE_ACTIVITY_RECOGNITION: @@ -93,8 +91,7 @@ PHONE_ACTIVITY_RECOGNITION: STATIONARY: ["still", "tilting"] MOBILE: ["on_foot", "walking", "running", "on_bicycle"] VEHICLE: ["in_vehicle"] - SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_activity_recognition/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-crashes/ PHONE_APPLICATIONS_CRASHES: @@ -125,8 +122,7 @@ PHONE_APPLICATIONS_FOREGROUND: EXCLUDED_CATEGORIES: [] EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_applications_foreground/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-notifications/ PHONE_APPLICATIONS_NOTIFICATIONS: @@ -146,8 +142,7 @@ PHONE_BATTERY: RAPIDS: COMPUTE: False FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] - SRC_FOLDER: "rapids" # inside src/features/phone_battery - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_battery/rapids/main.py # See https://www.rapids.science/latest/features/phone-bluetooth/ PHONE_BLUETOOTH: @@ -156,8 +151,8 @@ PHONE_BLUETOOTH: RAPIDS: COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R + DORYAB: COMPUTE: False FEATURES: @@ -173,8 +168,7 @@ PHONE_BLUETOOTH: DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"] SCANS_MOST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] SCANS_LEAST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] - SRC_FOLDER: "doryab" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_bluetooth/doryab/main.py # See https://www.rapids.science/latest/features/phone-calls/ PHONE_CALLS: @@ -187,8 +181,7 @@ PHONE_CALLS: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_calls + SRC_SCRIPT: src/features/phone_calls/rapids/main.R # See https://www.rapids.science/latest/features/phone-conversation/ PHONE_CONVERSATION: @@ -206,8 +199,7 @@ PHONE_CONVERSATION: "unknownexpectedfraction","countconversation"] RECORDING_MINUTES: 1 PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/phone_conversation - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_conversation/rapids/main.py # See https://www.rapids.science/latest/features/phone-data-yield/ PHONE_DATA_YIELD: @@ -217,8 +209,7 @@ PHONE_DATA_YIELD: COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_data_yield + SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/phone-keyboard/ PHONE_KEYBOARD: @@ -232,8 +223,7 @@ PHONE_LIGHT: RAPIDS: COMPUTE: False FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/phone_light - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_light/rapids/main.py # See https://www.rapids.science/latest/features/phone-locations/ PHONE_LOCATIONS: @@ -261,8 +251,7 @@ PHONE_LOCATIONS: CLUSTER_ON: PARTICIPANT_DATASET # PARTICIPANT_DATASET,TIME_SEGMENT CLUSTERING_ALGORITHM: DBSCAN #DBSCAN,OPTICS RADIUS_FOR_HOME: 100 - SRC_FOLDER: "doryab" # inside src/features/phone_locations - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_locations/doryab/main.py BARNETT: COMPUTE: False @@ -270,8 +259,7 @@ PHONE_LOCATIONS: ACCURACY_LIMIT: 100 # meters, drops location coordinates with an accuracy higher than this. This number means there's a 68% probability the true location is within this radius IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features - SRC_FOLDER: "barnett" # inside src/features/phone_locations - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_locations/barnett/main.R # See https://www.rapids.science/latest/features/phone-log/ PHONE_LOG: @@ -290,8 +278,7 @@ PHONE_MESSAGES: FEATURES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_messages + SRC_SCRIPT: src/features/phone_messages/rapids/main.R # See https://www.rapids.science/latest/features/phone-screen/ PHONE_SCREEN: @@ -304,28 +291,25 @@ PHONE_SCREEN: IGNORE_EPISODES_LONGER_THAN: 0 # in minutes, set to 0 to disable FEATURES: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration", "firstuseafter"] # "episodepersensedminutes" needs to be added later EPISODE_TYPES: ["unlock"] - SRC_FOLDER: "rapids" # inside src/features/phone_screen - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_screen/rapids/main.py # See https://www.rapids.science/latest/features/phone-wifi-connected/ PHONE_WIFI_CONNECTED: - CONTAINER: "sensor_wifi" + CONTAINER: sensor_wifi PROVIDERS: RAPIDS: COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_connected/rapids/main.R # See https://www.rapids.science/latest/features/phone-wifi-visible/ PHONE_WIFI_VISIBLE: - CONTAINER: "wifi" + CONTAINER: wifi PROVIDERS: RAPIDS: COMPUTE: False FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R @@ -364,8 +348,7 @@ FITBIT_DATA_YIELD: COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/fitbit_data_yield + SRC_SCRIPT: src/features/fitbit_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/fitbit-heartrate-summary/ @@ -375,8 +358,7 @@ FITBIT_HEARTRATE_SUMMARY: RAPIDS: COMPUTE: False FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/ FITBIT_HEARTRATE_INTRADAY: @@ -385,8 +367,7 @@ FITBIT_HEARTRATE_INTRADAY: RAPIDS: COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_intraday/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-summary/ FITBIT_SLEEP_SUMMARY: @@ -396,8 +377,7 @@ FITBIT_SLEEP_SUMMARY: COMPUTE: False FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"] SLEEP_TYPES: ["main", "nap", "all"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/ FITBIT_SLEEP_INTRADAY: @@ -418,8 +398,7 @@ FITBIT_SLEEP_INTRADAY: SLEEP_TYPES: [main, nap] INCLUDE_SLEEP_LATER_THAN: 0 # a number ranged from 0 (midnight) to 1439 (23:59) REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT" - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py PRICE: COMPUTE: False @@ -432,8 +411,7 @@ FITBIT_SLEEP_INTRADAY: GROUP_EPISODES_WITHIN: # by default: today's 6pm to tomorrow's noon START_TIME: 1080 # number of minutes after the midnight (18:00) 18*60 LENGTH: 1080 # in minutes (18 hours) 18*60 - SRC_FOLDER: "price" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py # See https://www.rapids.science/latest/features/fitbit-steps-summary/ FITBIT_STEPS_SUMMARY: @@ -442,8 +420,7 @@ FITBIT_STEPS_SUMMARY: RAPIDS: COMPUTE: False FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-steps-intraday/ FITBIT_STEPS_INTRADAY: @@ -457,8 +434,7 @@ FITBIT_STEPS_INTRADAY: ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] THRESHOLD_ACTIVE_BOUT: 10 # steps INCLUDE_ZERO_STEP_ROWS: False - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_intraday/rapids/main.py ######################################################################################################################## # EMPATICA # @@ -480,8 +456,7 @@ EMPATICA_ACCELEROMETER: DBDP: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_accelerometer/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-heartrate/ EMPATICA_HEARTRATE: @@ -490,8 +465,7 @@ EMPATICA_HEARTRATE: DBDP: COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_heartrate/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-temperature/ EMPATICA_TEMPERATURE: @@ -500,8 +474,7 @@ EMPATICA_TEMPERATURE: DBDP: COMPUTE: False FEATURES: ["maxtemp", "mintemp", "avgtemp", "mediantemp", "modetemp", "stdtemp", "diffmaxmodetemp", "diffminmodetemp", "entropytemp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_temperature/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-electrodermal-activity/ EMPATICA_ELECTRODERMAL_ACTIVITY: @@ -510,8 +483,7 @@ EMPATICA_ELECTRODERMAL_ACTIVITY: DBDP: COMPUTE: False FEATURES: ["maxeda", "mineda", "avgeda", "medianeda", "modeeda", "stdeda", "diffmaxmodeeda", "diffminmodeeda", "entropyeda"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_electrodermal_activity - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_electrodermal_activity/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-blood-volume-pulse/ EMPATICA_BLOOD_VOLUME_PULSE: @@ -520,8 +492,7 @@ EMPATICA_BLOOD_VOLUME_PULSE: DBDP: COMPUTE: False FEATURES: ["maxbvp", "minbvp", "avgbvp", "medianbvp", "modebvp", "stdbvp", "diffmaxmodebvp", "diffminmodebvp", "entropybvp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_blood_volume_pulse - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_blood_volume_pulse/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-inter-beat-interval/ EMPATICA_INTER_BEAT_INTERVAL: @@ -530,8 +501,7 @@ EMPATICA_INTER_BEAT_INTERVAL: DBDP: COMPUTE: False FEATURES: ["maxibi", "minibi", "avgibi", "medianibi", "modeibi", "stdibi", "diffmaxmodeibi", "diffminmodeibi", "entropyibi"] - SRC_FOLDER: "dbdp" # inside src/features/inter_beat_interval - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_inter_beat_interval/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-tags/ EMPATICA_TAGS: diff --git a/docs/change-log.md b/docs/change-log.md index 3298ead5..06b12ef9 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -8,6 +8,7 @@ - Support to analyze data collected over [multiple time zones](../setup/configuration/#multiple-timezones) - Support for [sleep intraday features](../features/fitbit-sleep-intraday/) from the core team and also from the community (thanks to Stephen Price) - Users can comment on the documentation (powered by utterances). +- `SCR_SCRIPT` and `SRC_LANGUAGE` are replaced by `SRC_SCRIPT`. - Add RAPIDS new logo - Move Citation and Minimal Example page to the Setup section - Add `config.yaml` validation schema and documentation. Now it's more difficult to modify the `config.yaml` file with invalid values. diff --git a/docs/developers/validation-schema-config.md b/docs/developers/validation-schema-config.md index c824d9de..5f01c312 100644 --- a/docs/developers/validation-schema-config.md +++ b/docs/developers/validation-schema-config.md @@ -25,7 +25,7 @@ The schema has three main sections `required`, `definitions`, and `properties`. ### definitions `definitions` lists key/values that are common to different `properties` so we can reuse them. You can define a key/value under `definitions` and use `$ref` to refer to it in any `property`. -For example, every sensor like `[PHONE_ACCELEROMETER]` has one or more providers like `RAPIDS` and `PANDA`, these providers have some common properties like the `COMPUTE` flag or the `SRC_FOLDER` string, therefore we define a common provider "template" that is used by every provider and extended with properties exclusive to each one of them. For example: +For example, every sensor like `[PHONE_ACCELEROMETER]` has one or more providers like `RAPIDS` and `PANDA`, these providers have some common properties like the `COMPUTE` flag or the `SRC_SCRIPT` string. Therefore we define a shared provider "template" that is used by every provider and extended with properties exclusive to each one of them. For example: === "provider definition (template)" The `PROVIDER` definition will be used later on different `properties`. @@ -33,21 +33,19 @@ For example, every sensor like `[PHONE_ACCELEROMETER]` has one or more providers ```yaml PROVIDER: type: object - required: [COMPUTE, SRC_FOLDER, SRC_LANGUAGE, FEATURES] + required: [COMPUTE, SRC_SCRIPT, FEATURES] properties: COMPUTE: type: boolean FEATURES: type: [array, object] - SRC_FOLDER: + SRC_SCRIPT: type: string - SRC_LANGUAGE: - type: string - enum: [python, r] + pattern: "^.*\\.(py|R)$" ``` === "provider reusing and extending the template" - Notice that in this example `RAPIDS` (a provider) is using and extending the `PROVIDER` template. The `FEATURES` key is overriding the `FEATURES` key from the `#/definitions/PROVIDER` template but is keeping the validation for `COMPUTE`, `SRC_FOLDER`, and `SRC_LANGUAGE`. For more details about reusing properties go to this [link](http://json-schema.org/understanding-json-schema/structuring.html#reuse) + Notice that `RAPIDS` (a provider) uses and extends the `PROVIDER` template in this example. The `FEATURES` key is overriding the `FEATURES` key from the `#/definitions/PROVIDER` template but is keeping the validation for `COMPUTE`, and `SRC_SCRIPT`. For more details about reusing properties, go to this [link](http://json-schema.org/understanding-json-schema/structuring.html#reuse) ```yaml hl_lines="9 10" PHONE_ACCELEROMETER: @@ -128,7 +126,7 @@ You can validate different aspects of each key/value in our `config.yaml` file: enum: ["received", "sent"] ``` === "object" - `PARENT` is an object that has two properties. `KID1` is one of those properties that is in turn another object that will reuse the `"#/definitions/PROVIDER"` `definition` **AND** also include (extend) two extra properties `GRAND_KID1` of type `array` and `GRAND_KID2` of type `number`. `KID2` is another property of `PARENT` of type `boolean`. + `PARENT` is an object that has two properties. `KID1` is one of those properties that are, in turn, another object that will reuse the `"#/definitions/PROVIDER"` `definition` **AND** also include (extend) two extra properties `GRAND_KID1` of type `array` and `GRAND_KID2` of type `number`. `KID2` is another property of `PARENT` of type `boolean`. The schema validation looks like this ```yaml @@ -155,8 +153,7 @@ You can validate different aspects of each key/value in our `config.yaml` file: # These four come from the `PROVIDER` definition (template) COMPUTE: False FEATURES: [x, y] # an array - SRC_FOLDER: "any string" - SRC_LANGUAGE: "any string" + SRC_SCRIPT: "a path to a py or R script" # This two come from the extension GRAND_KID1: [a, b] # an array diff --git a/docs/features/add-new-features.md b/docs/features/add-new-features.md index 94321739..67e669ff 100644 --- a/docs/features/add-new-features.md +++ b/docs/features/add-new-features.md @@ -65,7 +65,7 @@ As a tutorial, we will add a new provider for `PHONE_ACCELEROMETER` called `VEGA In this step, you need to add your provider configuration section under the relevant sensor in `config.yaml`. See our example for our tutorial's `VEGA` provider for `PHONE_ACCELEROMETER`: ??? example "Example configuration for a new accelerometer provider `VEGA`" - ```yaml hl_lines="12 13 14 15 16 17" + ```yaml hl_lines="12 13 14 15 16" PHONE_ACCELEROMETER: CONTAINER: accelerometer PROVIDERS: @@ -81,8 +81,7 @@ In this step, you need to add your provider configuration section under the rele COMPUTE: False FEATURES: ["feature1", "feature2", "feature3"] MY_PARAMTER: a_string - SRC_FOLDER: "vega" - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/vega/main.py ``` @@ -91,12 +90,11 @@ In this step, you need to add your provider configuration section under the rele |`[COMPUTE]`| Flag to activate/deactivate your provider |`[FEATURES]`| List of features your provider supports. Your provider code should only return the features on this list |`[MY_PARAMTER]`| An arbitrary parameter that our example provider `VEGA` needs. This can be a boolean, integer, float, string, or an array of any of such types. -|`[SRC_LANGUAGE]`| The programming language of your provider script, it can be `python` or `r`, in our example `python` -|`[SRC_FOLDER]`| The name of your provider in lower case, in our example `vega` (this will be the name of your folder in the next step) +|`[SRC_SCRIPT]`| The relative path from RAPIDS' root folder to an script that computes the features for this provider. It can be implemented in R or Python. ### Create a feature provider script -Create your feature Python or R script called `main.py` or `main.R` in the correct folder, `src/feature/[sensorname]/[providername]/`. RAPIDS automatically loads and executes it based on the config section you added in the last step. For our example, this script is: +Create your feature Python or R script called `main.py` or `main.R` in the correct folder, `src/feature/[sensorname]/[providername]/`. RAPIDS automatically loads and executes it based on the config key `[SRC_SCRIPT]` you added in the last step. For our example, this script is: ```bash src/feature/phone_accelerometer/vega/main.py ``` diff --git a/docs/features/feature-introduction.md b/docs/features/feature-introduction.md index 4d2163d6..25ef2646 100644 --- a/docs/features/feature-introduction.md +++ b/docs/features/feature-introduction.md @@ -23,8 +23,7 @@ PHONE_ACCELEROMETER: # (1) COMPUTE: False # (4) FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "rapids" - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py PANDA: COMPUTE: False @@ -33,8 +32,8 @@ PHONE_ACCELEROMETER: # (1) exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - SRC_FOLDER: "panda" - SRC_LANGUAGE: "python" # (6) + # (6) + SRC_SCRIPT: src/features/phone_accelerometer/panda/main.py ``` --8<--- "docs/snippets/feature_introduction_example.md" diff --git a/docs/migrating-from-old-versions.md b/docs/migrating-from-old-versions.md index baee6104..83b5c4fa 100644 --- a/docs/migrating-from-old-versions.md +++ b/docs/migrating-from-old-versions.md @@ -58,6 +58,10 @@ The most relevant changes to RAPIDS that you need to know about are: ```sql SELECT device_id, device_id as fitbit_id, CONCAT("p", _id) as empatica_id, CONCAT("p", _id) as pid, if(brand = "iPhone", "ios", "android") as platform, CONCAT("p", _id) as label, DATE_FORMAT(FROM_UNIXTIME((timestamp/1000)- 86400), "%Y-%m-%d") as start_date, CURRENT_DATE as end_date from aware_device order by _id; ``` +??? danger "`SCR_SCRIPT` and `SRC_LANGUAGE` are replaced by `SRC_SCRIPT`" + The attributes `SCR_SCRIPT` and `SRC_LANGUAGE` of every sensor `PROVIDER` are replaced by `SRC_SCRIPT`. `SRC_SCRIPT` is a relative path from the RAPIDS root folder to that provider's feature script. We did this to simplify and clarify where the features scripts are stored. + + There are no actions to take unless you created your own feature provider; update it with your feature script path. ## Migrating from RAPIDS beta If you were relying on the [old docs](https://rapidspitt.readthedocs.io/en/latest/) and the most recent version of RAPIDS you are working with is from or before [Oct 13, 2020](https://github.com/carissalow/rapids/commit/640890c7b49492d150accff5c87b1eb25bd97a49) you are using the beta version of RAPIDS. diff --git a/docs/snippets/feature_introduction_example.md b/docs/snippets/feature_introduction_example.md index 6fb3f94f..bc98f332 100644 --- a/docs/snippets/feature_introduction_example.md +++ b/docs/snippets/feature_introduction_example.md @@ -38,4 +38,6 @@ 6. **`PROVIDER` script** - Each `PROVIDER` has a `SRC_FOLDER` and `SRC_LANGUAGE` that point to the script implementing the features of this `PROVIDER`. \ No newline at end of file + Each `PROVIDER` has a `SRC_SCRIPT` that points to the script implementing its behavioral features. + + It has to be a relative path from RAPIDS' root folder and the script's parent folder should be named after the provider, e.g. `panda`. \ No newline at end of file diff --git a/example_profile/Snakefile b/example_profile/Snakefile index cd3f2a5e..f969fdcb 100644 --- a/example_profile/Snakefile +++ b/example_profile/Snakefile @@ -20,14 +20,14 @@ for provider in config["PHONE_DATA_YIELD"]["PROVIDERS"].keys(): if not (set(config["PHONE_DATA_YIELD"]["SENSORS"]) <= set(allowed_phone_sensors)): raise ValueError('\nInvalid sensor(s) for PHONE_DATA_YIELD. config["PHONE_DATA_YIELD"]["SENSORS"] can have ' 'one or more of the following phone sensors: {}.\nInstead you provided "{}".\n' - 'Keep in mind that the sensors\' TABLE attribute must point to a valid database table'\ + 'Keep in mind that the sensors\' CONTAINER attribute must point to a valid database table or file'\ .format(', '.join(allowed_phone_sensors), ', '.join(set(config["PHONE_DATA_YIELD"]["SENSORS"]) - set(allowed_phone_sensors)))) files_to_compute.extend(expand("data/raw/{pid}/{sensor}_raw.csv", pid=config["PIDS"], sensor=map(str.lower, config["PHONE_DATA_YIELD"]["SENSORS"]))) files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_yielded_timestamps_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_data_yield_features/phone_data_yield_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_data_yield_features/phone_data_yield_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_DATA_YIELD"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_data_yield.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -36,7 +36,7 @@ for provider in config["PHONE_MESSAGES"]["PROVIDERS"].keys(): if config["PHONE_MESSAGES"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_messages_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_messages_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_messages_features/phone_messages_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_MESSAGES"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_messages_features/phone_messages_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_MESSAGES"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_messages.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -45,7 +45,7 @@ for provider in config["PHONE_CALLS"]["PROVIDERS"].keys(): if config["PHONE_CALLS"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_calls_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_calls_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_calls_features/phone_calls_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CALLS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_calls_features/phone_calls_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_CALLS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_calls.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -54,7 +54,7 @@ for provider in config["PHONE_BLUETOOTH"]["PROVIDERS"].keys(): if config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_bluetooth_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_bluetooth_features/phone_bluetooth_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_bluetooth_features/phone_bluetooth_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_BLUETOOTH"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_bluetooth.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -66,7 +66,7 @@ for provider in config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_features/phone_activity_recognition_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_activity_recognition_features/phone_activity_recognition_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_ACTIVITY_RECOGNITION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_activity_recognition.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -77,7 +77,7 @@ for provider in config["PHONE_BATTERY"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_battery_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_battery_features/phone_battery_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_BATTERY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_battery_features/phone_battery_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_BATTERY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_battery.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -93,7 +93,7 @@ for provider in config["PHONE_SCREEN"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_screen_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_screen_features/phone_screen_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_SCREEN"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_screen_features/phone_screen_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_SCREEN"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_screen.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -102,7 +102,7 @@ for provider in config["PHONE_LIGHT"]["PROVIDERS"].keys(): if config["PHONE_LIGHT"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_light_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_light_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_light_features/phone_light_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LIGHT"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_light_features/phone_light_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_LIGHT"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_light.csv", pid=config["PIDS"],)) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -111,7 +111,7 @@ for provider in config["PHONE_ACCELEROMETER"]["PROVIDERS"].keys(): if config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_accelerometer_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_accelerometer_features/phone_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_accelerometer_features/phone_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_accelerometer.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -121,7 +121,7 @@ for provider in config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_foreground_with_datetime_with_categories.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_applications_foreground_features/phone_applications_foreground_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_applications_foreground_features/phone_applications_foreground_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_APPLICATIONS_FOREGROUND"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_foreground.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -130,7 +130,7 @@ for provider in config["PHONE_WIFI_VISIBLE"]["PROVIDERS"].keys(): if config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_visible_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_visible_features/phone_wifi_visible_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_visible_features/phone_wifi_visible_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_WIFI_VISIBLE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_visible.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -139,7 +139,7 @@ for provider in config["PHONE_WIFI_CONNECTED"]["PROVIDERS"].keys(): if config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_wifi_connected_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_connected_features/phone_wifi_connected_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_wifi_connected_features/phone_wifi_connected_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_WIFI_CONNECTED"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_wifi_connected.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -148,7 +148,7 @@ for provider in config["PHONE_CONVERSATION"]["PROVIDERS"].keys(): if config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_conversation_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_conversation_features/phone_conversation_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_conversation_features/phone_conversation_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_CONVERSATION"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_conversation.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -160,7 +160,7 @@ if isinstance(config["PHONE_APPLICATIONS_CRASHES"]["PROVIDERS"], dict): files_to_compute.extend(expand("data/raw/{pid}/phone_applications_crashes_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_crashes_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_crashes_with_datetime_with_categories.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_applications_crashes_features/phone_applications_crashes_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_CRASHES"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_applications_crashes_features/phone_applications_crashes_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_APPLICATIONS_CRASHES"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_crashes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -171,7 +171,7 @@ if isinstance(config["PHONE_APPLICATIONS_NOTIFICATIONS"]["PROVIDERS"], dict): files_to_compute.extend(expand("data/raw/{pid}/phone_applications_notifications_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_notifications_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_applications_notifications_with_datetime_with_categories.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_applications_notifications_features/phone_applications_notifications_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_APPLICATIONS_NOTIFICATIONS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_applications_notifications_features/phone_applications_notifications_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_APPLICATIONS_NOTIFICATIONS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_applications_notifications.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -181,7 +181,7 @@ if isinstance(config["PHONE_KEYBOARD"]["PROVIDERS"], dict): if config["PHONE_KEYBOARD"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_keyboard_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_keyboard_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_keyboard_features/phone_keyboard_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_KEYBOARD"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_keyboard_features/phone_keyboard_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_KEYBOARD"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_keyboard.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -191,7 +191,7 @@ if isinstance(config["PHONE_LOG"]["PROVIDERS"], dict): if config["PHONE_LOG"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/phone_log_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/phone_log_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_log_features/phone_log_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LOG"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_log_features/phone_log_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_LOG"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_log.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -208,7 +208,7 @@ for provider in config["PHONE_LOCATIONS"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed_with_datetime.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/phone_locations_processed_with_datetime_with_home.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/phone_locations_features/phone_locations_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/phone_locations_features/phone_locations_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["PHONE_LOCATIONS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/phone_locations.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -225,7 +225,7 @@ for provider in config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"].keys(): if config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_summary_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_summary_features/fitbit_heartrate_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_HEARTRATE_SUMMARY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_summary.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -234,7 +234,7 @@ for provider in config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"].keys(): if config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_heartrate_intraday_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_intraday_features/fitbit_heartrate_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_heartrate_intraday_features/fitbit_heartrate_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_HEARTRATE_INTRADAY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_heartrate_intraday.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -243,7 +243,7 @@ for provider in config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"].keys(): if config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_sleep_summary_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_summary_features/fitbit_sleep_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_summary_features/fitbit_sleep_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_SLEEP_SUMMARY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_sleep_summary.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -255,7 +255,7 @@ for provider in config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"].keys(): files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_episodes.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_episodes_resampled.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_episodes_resampled_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_features/fitbit_sleep_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_sleep_intraday_features/fitbit_sleep_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_SLEEP_INTRADAY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_sleep_intraday.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -264,7 +264,7 @@ for provider in config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"].keys(): if config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_summary_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_summary_features/fitbit_steps_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_summary_features/fitbit_steps_summary_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_STEPS_SUMMARY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_summary.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -273,7 +273,7 @@ for provider in config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"].keys(): if config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/fitbit_steps_intraday_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_intraday_features/fitbit_steps_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/fitbit_steps_intraday_features/fitbit_steps_intraday_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["FITBIT_STEPS_INTRADAY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/fitbit_steps_intraday.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -283,7 +283,7 @@ for provider in config["EMPATICA_ACCELEROMETER"]["PROVIDERS"].keys(): if config["EMPATICA_ACCELEROMETER"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_accelerometer_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_accelerometer_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_accelerometer_features/empatica_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_accelerometer_features/empatica_accelerometer_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_ACCELEROMETER"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_accelerometer.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -292,7 +292,7 @@ for provider in config["EMPATICA_HEARTRATE"]["PROVIDERS"].keys(): if config["EMPATICA_HEARTRATE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_heartrate_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_heartrate_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_heartrate_features/empatica_heartrate_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_HEARTRATE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_heartrate_features/empatica_heartrate_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_HEARTRATE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_heartrate.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -302,7 +302,7 @@ for provider in config["EMPATICA_TEMPERATURE"]["PROVIDERS"].keys(): if config["EMPATICA_TEMPERATURE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_temperature_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_temperature_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_temperature_features/empatica_temperature_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_TEMPERATURE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_temperature_features/empatica_temperature_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_TEMPERATURE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_temperature.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -311,7 +311,7 @@ for provider in config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"].keys(): if config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_electrodermal_activity_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_electrodermal_activity_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_electrodermal_activity_features/empatica_electrodermal_activity_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_ELECTRODERMAL_ACTIVITY"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_electrodermal_activity.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -320,7 +320,7 @@ for provider in config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"].keys(): if config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_blood_volume_pulse_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_blood_volume_pulse_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_blood_volume_pulse_features/empatica_blood_volume_pulse_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_BLOOD_VOLUME_PULSE"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_blood_volume_pulse.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -329,7 +329,7 @@ for provider in config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"].keys(): if config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_inter_beat_interval_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_inter_beat_interval_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_inter_beat_interval_features/empatica_inter_beat_interval_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_INTER_BEAT_INTERVAL"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_inter_beat_interval.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") @@ -339,7 +339,7 @@ if isinstance(config["EMPATICA_TAGS"]["PROVIDERS"], dict): if config["EMPATICA_TAGS"]["PROVIDERS"][provider]["COMPUTE"]: files_to_compute.extend(expand("data/raw/{pid}/empatica_tags_raw.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/raw/{pid}/empatica_tags_with_datetime.csv", pid=config["PIDS"])) - files_to_compute.extend(expand("data/interim/{pid}/empatica_tags_features/empatica_tags_{language}_{provider_key}.csv", pid=config["PIDS"], language=config["EMPATICA_TAGS"]["PROVIDERS"][provider]["SRC_LANGUAGE"].lower(), provider_key=provider.lower())) + files_to_compute.extend(expand("data/interim/{pid}/empatica_tags_features/empatica_tags_{language}_{provider_key}.csv", pid=config["PIDS"], language=get_script_language(config["EMPATICA_TAGS"]["PROVIDERS"][provider]["SRC_SCRIPT"]), provider_key=provider.lower())) files_to_compute.extend(expand("data/processed/features/{pid}/empatica_tags.csv", pid=config["PIDS"])) files_to_compute.extend(expand("data/processed/features/{pid}/all_sensor_features.csv", pid=config["PIDS"])) files_to_compute.append("data/processed/features/all_participants/all_sensor_features.csv") diff --git a/example_profile/example_config.yaml b/example_profile/example_config.yaml index f3bbb44a..8240187b 100644 --- a/example_profile/example_config.yaml +++ b/example_profile/example_config.yaml @@ -44,7 +44,7 @@ TIMEZONE: # PHONE # ######################################################################################################################## -# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration PHONE_DATA_STREAMS: USE: aware_csv @@ -54,6 +54,10 @@ PHONE_DATA_STREAMS: aware_csv: FOLDER: data/external/example_workflow + + aware_influxdb: + DATABASE_GROUP: MY_GROUP + # Sensors ------ # https://www.rapids.science/latest/features/phone-accelerometer/ @@ -63,8 +67,7 @@ PHONE_ACCELEROMETER: RAPIDS: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py PANDA: COMPUTE: False @@ -72,8 +75,7 @@ PHONE_ACCELEROMETER: FEATURES: exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - SRC_FOLDER: "panda" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/panda/main.py # See https://www.rapids.science/latest/features/phone-activity-recognition/ PHONE_ACTIVITY_RECOGNITION: @@ -89,8 +91,7 @@ PHONE_ACTIVITY_RECOGNITION: STATIONARY: ["still", "tilting"] MOBILE: ["on_foot", "walking", "running", "on_bicycle"] VEHICLE: ["in_vehicle"] - SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_activity_recognition/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-crashes/ PHONE_APPLICATIONS_CRASHES: @@ -121,8 +122,7 @@ PHONE_APPLICATIONS_FOREGROUND: EXCLUDED_CATEGORIES: ["system_apps"] EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_applications_foreground/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-notifications/ PHONE_APPLICATIONS_NOTIFICATIONS: @@ -142,8 +142,7 @@ PHONE_BATTERY: RAPIDS: COMPUTE: True FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] - SRC_FOLDER: "rapids" # inside src/features/phone_battery - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_battery/rapids/main.py # See https://www.rapids.science/latest/features/phone-bluetooth/ PHONE_BLUETOOTH: @@ -152,8 +151,8 @@ PHONE_BLUETOOTH: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R + DORYAB: COMPUTE: False FEATURES: @@ -169,8 +168,7 @@ PHONE_BLUETOOTH: DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"] SCANS_MOST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] SCANS_LEAST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] - SRC_FOLDER: "doryab" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_bluetooth/doryab/main.py # See https://www.rapids.science/latest/features/phone-calls/ PHONE_CALLS: @@ -183,8 +181,7 @@ PHONE_CALLS: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_calls + SRC_SCRIPT: src/features/phone_calls/rapids/main.R # See https://www.rapids.science/latest/features/phone-conversation/ PHONE_CONVERSATION: @@ -202,8 +199,7 @@ PHONE_CONVERSATION: "unknownexpectedfraction","countconversation"] RECORDING_MINUTES: 1 PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/phone_conversation - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_conversation/rapids/main.py # See https://www.rapids.science/latest/features/phone-data-yield/ PHONE_DATA_YIELD: @@ -213,8 +209,7 @@ PHONE_DATA_YIELD: COMPUTE: True FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_data_yield + SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/phone-keyboard/ PHONE_KEYBOARD: @@ -228,8 +223,7 @@ PHONE_LIGHT: RAPIDS: COMPUTE: True FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/phone_light - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_light/rapids/main.py # See https://www.rapids.science/latest/features/phone-locations/ PHONE_LOCATIONS: @@ -257,8 +251,7 @@ PHONE_LOCATIONS: CLUSTER_ON: PARTICIPANT_DATASET # PARTICIPANT_DATASET,TIME_SEGMENT CLUSTERING_ALGORITHM: DBSCAN #DBSCAN,OPTICS RADIUS_FOR_HOME: 100 - SRC_FOLDER: "doryab" # inside src/features/phone_locations - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_locations/doryab/main.py BARNETT: COMPUTE: False @@ -266,8 +259,7 @@ PHONE_LOCATIONS: ACCURACY_LIMIT: 51 # meters, drops location coordinates with an accuracy higher than this. This number means there's a 68% probability the true location is within this radius IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features - SRC_FOLDER: "barnett" # inside src/features/phone_locations - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_locations/barnett/main.R # See https://www.rapids.science/latest/features/phone-log/ PHONE_LOG: @@ -286,8 +278,7 @@ PHONE_MESSAGES: FEATURES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_messages + SRC_SCRIPT: src/features/phone_messages/rapids/main.R # See https://www.rapids.science/latest/features/phone-screen/ PHONE_SCREEN: @@ -300,8 +291,7 @@ PHONE_SCREEN: IGNORE_EPISODES_LONGER_THAN: 0 # in minutes, set to 0 to disable FEATURES: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration", "firstuseafter"] # "episodepersensedminutes" needs to be added later EPISODE_TYPES: ["unlock"] - SRC_FOLDER: "rapids" # inside src/features/phone_screen - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_screen/rapids/main.py # See https://www.rapids.science/latest/features/phone-wifi-connected/ PHONE_WIFI_CONNECTED: @@ -310,8 +300,7 @@ PHONE_WIFI_CONNECTED: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_connected/rapids/main.R # See https://www.rapids.science/latest/features/phone-wifi-visible/ PHONE_WIFI_VISIBLE: @@ -320,8 +309,7 @@ PHONE_WIFI_VISIBLE: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R @@ -329,7 +317,7 @@ PHONE_WIFI_VISIBLE: # FITBIT # ######################################################################################################################## -# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: USE: fitbitjson_csv @@ -360,8 +348,7 @@ FITBIT_DATA_YIELD: COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/fitbit_data_yield + SRC_SCRIPT: src/features/fitbit_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/fitbit-heartrate-summary/ @@ -371,8 +358,7 @@ FITBIT_HEARTRATE_SUMMARY: RAPIDS: COMPUTE: True FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/ FITBIT_HEARTRATE_INTRADAY: @@ -381,8 +367,7 @@ FITBIT_HEARTRATE_INTRADAY: RAPIDS: COMPUTE: True FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_intraday/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-summary/ FITBIT_SLEEP_SUMMARY: @@ -393,8 +378,7 @@ FITBIT_SLEEP_SUMMARY: COMPUTE: True FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"] SLEEP_TYPES: ["main", "nap", "all"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/ FITBIT_SLEEP_INTRADAY: @@ -415,8 +399,7 @@ FITBIT_SLEEP_INTRADAY: SLEEP_TYPES: [main, nap] INCLUDE_SLEEP_LATER_THAN: 0 # a number ranged from 0 (midnight) to 1439 (23:59) REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT" - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py PRICE: COMPUTE: False @@ -429,8 +412,7 @@ FITBIT_SLEEP_INTRADAY: GROUP_EPISODES_WITHIN: # by default: today's 6pm to tomorrow's noon START_TIME: 1080 # number of minutes after the midnight (18:00) 18*60 LENGTH: 1080 # in minutes (18 hours) 18*60 - SRC_FOLDER: "price" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py # See https://www.rapids.science/latest/features/fitbit-steps-summary/ FITBIT_STEPS_SUMMARY: @@ -439,8 +421,7 @@ FITBIT_STEPS_SUMMARY: RAPIDS: COMPUTE: True FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-steps-intraday/ FITBIT_STEPS_INTRADAY: @@ -454,8 +435,7 @@ FITBIT_STEPS_INTRADAY: ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] THRESHOLD_ACTIVE_BOUT: 10 # steps INCLUDE_ZERO_STEP_ROWS: False - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_intraday/rapids/main.py ######################################################################################################################## # EMPATICA # @@ -477,8 +457,7 @@ EMPATICA_ACCELEROMETER: DBDP: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_accelerometer/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-heartrate/ EMPATICA_HEARTRATE: @@ -487,8 +466,7 @@ EMPATICA_HEARTRATE: DBDP: COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_heartrate/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-temperature/ EMPATICA_TEMPERATURE: @@ -497,8 +475,7 @@ EMPATICA_TEMPERATURE: DBDP: COMPUTE: False FEATURES: ["maxtemp", "mintemp", "avgtemp", "mediantemp", "modetemp", "stdtemp", "diffmaxmodetemp", "diffminmodetemp", "entropytemp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_temperature/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-electrodermal-activity/ EMPATICA_ELECTRODERMAL_ACTIVITY: @@ -507,8 +484,7 @@ EMPATICA_ELECTRODERMAL_ACTIVITY: DBDP: COMPUTE: False FEATURES: ["maxeda", "mineda", "avgeda", "medianeda", "modeeda", "stdeda", "diffmaxmodeeda", "diffminmodeeda", "entropyeda"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_electrodermal_activity - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_electrodermal_activity/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-blood-volume-pulse/ EMPATICA_BLOOD_VOLUME_PULSE: @@ -517,8 +493,7 @@ EMPATICA_BLOOD_VOLUME_PULSE: DBDP: COMPUTE: False FEATURES: ["maxbvp", "minbvp", "avgbvp", "medianbvp", "modebvp", "stdbvp", "diffmaxmodebvp", "diffminmodebvp", "entropybvp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_blood_volume_pulse - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_blood_volume_pulse/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-inter-beat-interval/ EMPATICA_INTER_BEAT_INTERVAL: @@ -527,8 +502,7 @@ EMPATICA_INTER_BEAT_INTERVAL: DBDP: COMPUTE: False FEATURES: ["maxibi", "minibi", "avgibi", "medianibi", "modeibi", "stdibi", "diffmaxmodeibi", "diffminmodeibi", "entropyibi"] - SRC_FOLDER: "dbdp" # inside src/features/inter_beat_interval - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_inter_beat_interval/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-tags/ EMPATICA_TAGS: diff --git a/rules/common.smk b/rules/common.smk index fac723fb..29ee7b71 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -1,9 +1,21 @@ +def get_script_language(script_path): + from pathlib import Path + script_path = Path(script_path) + if not script_path.exists(): + raise ValueError("The following provider feature script does not exist: " + str(script_path)) + + if script_path.name.endswith(".py"): + return "python" + elif script_path.name.endswith(".R"): + return "r" + + # Features.smk ######################################################################################################### def find_features_files(wildcards): feature_files = [] for provider_key, provider in config[(wildcards.sensor_key).upper()]["PROVIDERS"].items(): if provider["COMPUTE"]: - feature_files.extend(expand("data/interim/{{pid}}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", sensor_key=wildcards.sensor_key.lower(), language=provider["SRC_LANGUAGE"].lower(), provider_key=provider_key.lower())) + feature_files.extend(expand("data/interim/{{pid}}/{sensor_key}_features/{sensor_key}_{language}_{provider_key}.csv", sensor_key=wildcards.sensor_key.lower(), language=get_script_language(provider["SRC_SCRIPT"]), provider_key=provider_key.lower())) return(feature_files) def optional_steps_sleep_input(wildcards): @@ -114,3 +126,4 @@ def pull_wearable_data_input_with_mutation_scripts(wilcards): raise ValueError("Mutate scripts can only be Python or R scripts (.py, .R).\n Instead we got {script} in [{sensor}] of {schema}".format(script=script, sensor=sensor, schema=input.get("stream_format"))) input["mutationscript"+str(idx)] = script return input + diff --git a/src/features/utils/utils.R b/src/features/utils/utils.R index 34c5aeae..629e7eb3 100644 --- a/src/features/utils/utils.R +++ b/src/features/utils/utils.R @@ -53,16 +53,15 @@ fetch_provider_features <- function(provider, provider_key, sensor_key, sensor_d stop(paste0("Provider config[", sensor_key,"][PROVIDERS][", provider_key,"] is missing a FEATURES attribute in config.yaml")) if(provider[["COMPUTE"]] == TRUE){ - code_path <- paste0("src/features/", sensor_key,"/", provider[["SRC_FOLDER"]], "/main.R") - source(code_path) - features_function <- match.fun(paste0(provider[["SRC_FOLDER"]], "_features")) + source(provider[["SRC_SCRIPT"]]) + features_function <- match.fun(paste0(tolower(provider_key), "_features")) time_segments <- time_segments_labels %>% pull(label) for (time_segment in time_segments){ print(paste(rapids_log_tag,"Processing", sensor_key, provider_key, time_segment)) features <- features_function(sensor_data_files, time_segment, provider) if(!"local_segment" %in% colnames(features)) - stop(paste0("The dataframe returned by the ",sensor_key," provider '", provider_key,"' is missing the 'local_segment' column added by the 'filter_data_by_segment()' function. Check the provider script is using such function and is not removing 'local_segment' by accident (", code_path,")\n The 'local_segment' column is used to index a provider's features (each row corresponds to a different time segment instance (e.g. 2020-01-01, 2020-01-02, 2020-01-03, etc.)")) + stop(paste0("The dataframe returned by the ",sensor_key," provider '", provider_key,"' is missing the 'local_segment' column added by the 'filter_data_by_segment()' function. Check the provider script is using such function and is not removing 'local_segment' by accident (", provider[["SRC_SCRIPT"]],")\n The 'local_segment' column is used to index a provider's features (each row corresponds to a different time segment instance (e.g. 2020-01-01, 2020-01-02, 2020-01-03, etc.)")) features <- features %>% rename_at(vars(!matches("local_segment")), ~ paste(sensor_key, provider_key, ., sep = "_")) sensor_features <- merge(sensor_features, features, all = TRUE) } diff --git a/src/features/utils/utils.py b/src/features/utils/utils.py index 0628f0c1..abb8cbd8 100644 --- a/src/features/utils/utils.py +++ b/src/features/utils/utils.py @@ -1,5 +1,19 @@ rapids_log_tag = "RAPIDS:" +import os, sys +import importlib + +def import_path(path): + module_name = os.path.basename(path).replace('-', '_') + spec = importlib.util.spec_from_loader( + module_name, + importlib.machinery.SourceFileLoader(module_name, path) + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[module_name] = module + return module + def filter_data_by_segment(data, time_segment): data.dropna(subset=["assigned_segments"], inplace=True) if(data.shape[0] == 0): # data is empty @@ -85,15 +99,14 @@ def fetch_provider_features(provider, provider_key, sensor_key, sensor_data_file if provider["COMPUTE"] == True: - code_path = sensor_key + "." + provider["SRC_FOLDER"] + ".main" - feature_module = import_module(code_path) - feature_function = getattr(feature_module, provider["SRC_FOLDER"] + "_features") + feature_module = import_path(provider["SRC_SCRIPT"]) + feature_function = getattr(feature_module, provider_key.lower() + "_features") for time_segment in time_segments_labels["label"]: print("{} Processing {} {} {}".format(rapids_log_tag, sensor_key, provider_key, time_segment)) features = feature_function(sensor_data_files, time_segment, provider, filter_data_by_segment=filter_data_by_segment, chunk_episodes=chunk_episodes) if not "local_segment" in features.columns: - raise ValueError("The dataframe returned by the " + sensor_key + " provider '" + provider_key + "' is missing the 'local_segment' column added by the 'filter_data_by_segment()' function. Check the provider script is using such function and is not removing 'local_segment' by accident (" + code_path + ")\n The 'local_segment' column is used to index a provider's features (each row corresponds to a different time segment instance (e.g. 2020-01-01, 2020-01-02, 2020-01-03, etc.)") + raise ValueError("The dataframe returned by the " + sensor_key + " provider '" + provider_key + "' is missing the 'local_segment' column added by the 'filter_data_by_segment()' function. Check the provider script is using such function and is not removing 'local_segment' by accident (" + provider["SRC_SCRIPT"] + ")\n The 'local_segment' column is used to index a provider's features (each row corresponds to a different time segment instance (e.g. 2020-01-01, 2020-01-02, 2020-01-03, etc.)") features.columns = ["{}{}".format("" if col.startswith("local_segment") else (sensor_key + "_"+ provider_key + "_"), col) for col in features.columns] sensor_features = pd.concat([sensor_features, features], axis=0, sort=False) else: diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh index f423e27d..aa96f376 100755 --- a/tests/scripts/run_tests.sh +++ b/tests/scripts/run_tests.sh @@ -8,15 +8,15 @@ run_pipeline() { CONFIG_FILE="./tests/settings/periodic_config.yaml" fi + echo "Copying participant files" + cp -r tests/data/external/participant_files/* data/external/participant_files/ + echo $TYPE echo "Deleting old outputs" snakemake --configfile=$(echo $CONFIG_FILE) --delete-all-output -j1 - echo "Copying participant files" - cp -r tests/data/external/participant_files/* data/external/participant_files/ - echo "Running RAPIDS" - snakemake --configfile=$(echo $CONFIG_FILE) -R pull_phone_data -j12 + snakemake --configfile=$(echo $CONFIG_FILE) -R pull_phone_data -j1 } display_usage() { diff --git a/tests/settings/frequency_config.yaml b/tests/settings/frequency_config.yaml index 9a1575e8..b870a976 100644 --- a/tests/settings/frequency_config.yaml +++ b/tests/settings/frequency_config.yaml @@ -44,7 +44,7 @@ TIMEZONE: # PHONE # ######################################################################################################################## -# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration PHONE_DATA_STREAMS: USE: aware_csv @@ -54,6 +54,10 @@ PHONE_DATA_STREAMS: aware_csv: FOLDER: tests/data/external/aware_csv + + aware_influxdb: + DATABASE_GROUP: MY_GROUP + # Sensors ------ # https://www.rapids.science/latest/features/phone-accelerometer/ @@ -63,8 +67,7 @@ PHONE_ACCELEROMETER: RAPIDS: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py PANDA: COMPUTE: False @@ -72,8 +75,7 @@ PHONE_ACCELEROMETER: FEATURES: exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - SRC_FOLDER: "panda" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/panda/main.py # See https://www.rapids.science/latest/features/phone-activity-recognition/ PHONE_ACTIVITY_RECOGNITION: @@ -89,8 +91,7 @@ PHONE_ACTIVITY_RECOGNITION: STATIONARY: ["still", "tilting"] MOBILE: ["on_foot", "walking", "running", "on_bicycle"] VEHICLE: ["in_vehicle"] - SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_activity_recognition/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-crashes/ PHONE_APPLICATIONS_CRASHES: @@ -121,8 +122,7 @@ PHONE_APPLICATIONS_FOREGROUND: EXCLUDED_CATEGORIES: [] EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_applications_foreground/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-notifications/ PHONE_APPLICATIONS_NOTIFICATIONS: @@ -142,8 +142,7 @@ PHONE_BATTERY: RAPIDS: COMPUTE: False FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] - SRC_FOLDER: "rapids" # inside src/features/phone_battery - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_battery/rapids/main.py # See https://www.rapids.science/latest/features/phone-bluetooth/ PHONE_BLUETOOTH: @@ -152,8 +151,8 @@ PHONE_BLUETOOTH: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R + DORYAB: COMPUTE: False FEATURES: @@ -169,8 +168,7 @@ PHONE_BLUETOOTH: DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"] SCANS_MOST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] SCANS_LEAST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] - SRC_FOLDER: "doryab" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_bluetooth/doryab/main.py # See https://www.rapids.science/latest/features/phone-calls/ PHONE_CALLS: @@ -183,8 +181,7 @@ PHONE_CALLS: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_calls + SRC_SCRIPT: src/features/phone_calls/rapids/main.R # See https://www.rapids.science/latest/features/phone-conversation/ PHONE_CONVERSATION: @@ -202,8 +199,7 @@ PHONE_CONVERSATION: "unknownexpectedfraction","countconversation"] RECORDING_MINUTES: 1 PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/phone_conversation - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_conversation/rapids/main.py # See https://www.rapids.science/latest/features/phone-data-yield/ PHONE_DATA_YIELD: @@ -213,8 +209,7 @@ PHONE_DATA_YIELD: COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_data_yield + SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/phone-keyboard/ PHONE_KEYBOARD: @@ -228,8 +223,7 @@ PHONE_LIGHT: RAPIDS: COMPUTE: True FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/phone_light - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_light/rapids/main.py # See https://www.rapids.science/latest/features/phone-locations/ PHONE_LOCATIONS: @@ -257,8 +251,7 @@ PHONE_LOCATIONS: CLUSTER_ON: PARTICIPANT_DATASET # PARTICIPANT_DATASET,TIME_SEGMENT CLUSTERING_ALGORITHM: DBSCAN #DBSCAN,OPTICS RADIUS_FOR_HOME: 100 - SRC_FOLDER: "doryab" # inside src/features/phone_locations - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_locations/doryab/main.py BARNETT: COMPUTE: False @@ -266,8 +259,7 @@ PHONE_LOCATIONS: ACCURACY_LIMIT: 100 # meters, drops location coordinates with an accuracy higher than this. This number means there's a 68% probability the true location is within this radius IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features - SRC_FOLDER: "barnett" # inside src/features/phone_locations - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_locations/barnett/main.R # See https://www.rapids.science/latest/features/phone-log/ PHONE_LOG: @@ -286,8 +278,7 @@ PHONE_MESSAGES: FEATURES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_messages + SRC_SCRIPT: src/features/phone_messages/rapids/main.R # See https://www.rapids.science/latest/features/phone-screen/ PHONE_SCREEN: @@ -300,8 +291,7 @@ PHONE_SCREEN: IGNORE_EPISODES_LONGER_THAN: 0 # in minutes, set to 0 to disable FEATURES: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration", "firstuseafter"] # "episodepersensedminutes" needs to be added later EPISODE_TYPES: ["unlock"] - SRC_FOLDER: "rapids" # inside src/features/phone_screen - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_screen/rapids/main.py # See https://www.rapids.science/latest/features/phone-wifi-connected/ PHONE_WIFI_CONNECTED: @@ -310,8 +300,7 @@ PHONE_WIFI_CONNECTED: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_connected/rapids/main.R # See https://www.rapids.science/latest/features/phone-wifi-visible/ PHONE_WIFI_VISIBLE: @@ -320,8 +309,7 @@ PHONE_WIFI_VISIBLE: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R @@ -329,7 +317,7 @@ PHONE_WIFI_VISIBLE: # FITBIT # ######################################################################################################################## -# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: USE: fitbitjson_mysql @@ -360,8 +348,7 @@ FITBIT_DATA_YIELD: COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/fitbit_data_yield + SRC_SCRIPT: src/features/fitbit_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/fitbit-heartrate-summary/ @@ -371,8 +358,7 @@ FITBIT_HEARTRATE_SUMMARY: RAPIDS: COMPUTE: False FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/ FITBIT_HEARTRATE_INTRADAY: @@ -381,8 +367,7 @@ FITBIT_HEARTRATE_INTRADAY: RAPIDS: COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_intraday/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-summary/ FITBIT_SLEEP_SUMMARY: @@ -392,8 +377,7 @@ FITBIT_SLEEP_SUMMARY: COMPUTE: False FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"] SLEEP_TYPES: ["main", "nap", "all"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/ FITBIT_SLEEP_INTRADAY: @@ -414,8 +398,7 @@ FITBIT_SLEEP_INTRADAY: SLEEP_TYPES: [main, nap] INCLUDE_SLEEP_LATER_THAN: 0 # a number ranged from 0 (midnight) to 1439 (23:59) REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT" - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py PRICE: COMPUTE: False @@ -428,8 +411,7 @@ FITBIT_SLEEP_INTRADAY: GROUP_EPISODES_WITHIN: # by default: today's 6pm to tomorrow's noon START_TIME: 1080 # number of minutes after the midnight (18:00) 18*60 LENGTH: 1080 # in minutes (18 hours) 18*60 - SRC_FOLDER: "price" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py # See https://www.rapids.science/latest/features/fitbit-steps-summary/ FITBIT_STEPS_SUMMARY: @@ -438,8 +420,7 @@ FITBIT_STEPS_SUMMARY: RAPIDS: COMPUTE: False FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-steps-intraday/ FITBIT_STEPS_INTRADAY: @@ -453,8 +434,7 @@ FITBIT_STEPS_INTRADAY: ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] THRESHOLD_ACTIVE_BOUT: 10 # steps INCLUDE_ZERO_STEP_ROWS: False - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_intraday/rapids/main.py ######################################################################################################################## # EMPATICA # @@ -476,8 +456,7 @@ EMPATICA_ACCELEROMETER: DBDP: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_accelerometer/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-heartrate/ EMPATICA_HEARTRATE: @@ -486,8 +465,7 @@ EMPATICA_HEARTRATE: DBDP: COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_heartrate/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-temperature/ EMPATICA_TEMPERATURE: @@ -496,8 +474,7 @@ EMPATICA_TEMPERATURE: DBDP: COMPUTE: False FEATURES: ["maxtemp", "mintemp", "avgtemp", "mediantemp", "modetemp", "stdtemp", "diffmaxmodetemp", "diffminmodetemp", "entropytemp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_temperature/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-electrodermal-activity/ EMPATICA_ELECTRODERMAL_ACTIVITY: @@ -506,8 +483,7 @@ EMPATICA_ELECTRODERMAL_ACTIVITY: DBDP: COMPUTE: False FEATURES: ["maxeda", "mineda", "avgeda", "medianeda", "modeeda", "stdeda", "diffmaxmodeeda", "diffminmodeeda", "entropyeda"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_electrodermal_activity - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_electrodermal_activity/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-blood-volume-pulse/ EMPATICA_BLOOD_VOLUME_PULSE: @@ -516,8 +492,7 @@ EMPATICA_BLOOD_VOLUME_PULSE: DBDP: COMPUTE: False FEATURES: ["maxbvp", "minbvp", "avgbvp", "medianbvp", "modebvp", "stdbvp", "diffmaxmodebvp", "diffminmodebvp", "entropybvp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_blood_volume_pulse - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_blood_volume_pulse/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-inter-beat-interval/ EMPATICA_INTER_BEAT_INTERVAL: @@ -526,8 +501,7 @@ EMPATICA_INTER_BEAT_INTERVAL: DBDP: COMPUTE: False FEATURES: ["maxibi", "minibi", "avgibi", "medianibi", "modeibi", "stdibi", "diffmaxmodeibi", "diffminmodeibi", "entropyibi"] - SRC_FOLDER: "dbdp" # inside src/features/inter_beat_interval - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_inter_beat_interval/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-tags/ EMPATICA_TAGS: diff --git a/tests/settings/periodic_config.yaml b/tests/settings/periodic_config.yaml index b7292239..cb330a37 100644 --- a/tests/settings/periodic_config.yaml +++ b/tests/settings/periodic_config.yaml @@ -44,7 +44,7 @@ TIMEZONE: # PHONE # ######################################################################################################################## -# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration PHONE_DATA_STREAMS: USE: aware_csv @@ -54,6 +54,10 @@ PHONE_DATA_STREAMS: aware_csv: FOLDER: tests/data/external/aware_csv + + aware_influxdb: + DATABASE_GROUP: MY_GROUP + # Sensors ------ # https://www.rapids.science/latest/features/phone-accelerometer/ @@ -63,8 +67,7 @@ PHONE_ACCELEROMETER: RAPIDS: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "rapids" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/rapids/main.py PANDA: COMPUTE: False @@ -72,8 +75,7 @@ PHONE_ACCELEROMETER: FEATURES: exertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] nonexertional_activity_episode: ["sumduration", "maxduration", "minduration", "avgduration", "medianduration", "stdduration"] - SRC_FOLDER: "panda" # inside src/features/phone_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_accelerometer/panda/main.py # See https://www.rapids.science/latest/features/phone-activity-recognition/ PHONE_ACTIVITY_RECOGNITION: @@ -89,8 +91,7 @@ PHONE_ACTIVITY_RECOGNITION: STATIONARY: ["still", "tilting"] MOBILE: ["on_foot", "walking", "running", "on_bicycle"] VEHICLE: ["in_vehicle"] - SRC_FOLDER: "rapids" # inside src/features/phone_activity_recognition - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_activity_recognition/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-crashes/ PHONE_APPLICATIONS_CRASHES: @@ -121,8 +122,7 @@ PHONE_APPLICATIONS_FOREGROUND: EXCLUDED_CATEGORIES: [] EXCLUDED_APPS: ["com.fitbit.FitbitMobile", "com.aware.plugin.upmc.cancer"] FEATURES: ["count", "timeoffirstuse", "timeoflastuse", "frequencyentropy"] - SRC_FOLDER: "rapids" # inside src/features/phone_applications_foreground - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_applications_foreground/rapids/main.py # See https://www.rapids.science/latest/features/phone-applications-notifications/ PHONE_APPLICATIONS_NOTIFICATIONS: @@ -142,8 +142,7 @@ PHONE_BATTERY: RAPIDS: COMPUTE: True FEATURES: ["countdischarge", "sumdurationdischarge", "countcharge", "sumdurationcharge", "avgconsumptionrate", "maxconsumptionrate"] - SRC_FOLDER: "rapids" # inside src/features/phone_battery - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_battery/rapids/main.py # See https://www.rapids.science/latest/features/phone-bluetooth/ PHONE_BLUETOOTH: @@ -152,8 +151,8 @@ PHONE_BLUETOOTH: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_bluetooth/rapids/main.R + DORYAB: COMPUTE: False FEATURES: @@ -169,8 +168,7 @@ PHONE_BLUETOOTH: DEVICES: ["countscans", "uniquedevices", "meanscans", "stdscans"] SCANS_MOST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] SCANS_LEAST_FREQUENT_DEVICE: ["withinsegments", "acrosssegments", "acrossdataset"] - SRC_FOLDER: "doryab" # inside src/features/phone_bluetooth - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_bluetooth/doryab/main.py # See https://www.rapids.science/latest/features/phone-calls/ PHONE_CALLS: @@ -183,8 +181,7 @@ PHONE_CALLS: missed: [count, distinctcontacts, timefirstcall, timelastcall, countmostfrequentcontact] incoming: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] outgoing: [count, distinctcontacts, meanduration, sumduration, minduration, maxduration, stdduration, modeduration, entropyduration, timefirstcall, timelastcall, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_calls + SRC_SCRIPT: src/features/phone_calls/rapids/main.R # See https://www.rapids.science/latest/features/phone-conversation/ PHONE_CONVERSATION: @@ -202,8 +199,7 @@ PHONE_CONVERSATION: "unknownexpectedfraction","countconversation"] RECORDING_MINUTES: 1 PAUSED_MINUTES : 3 - SRC_FOLDER: "rapids" # inside src/features/phone_conversation - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_conversation/rapids/main.py # See https://www.rapids.science/latest/features/phone-data-yield/ PHONE_DATA_YIELD: @@ -213,8 +209,7 @@ PHONE_DATA_YIELD: COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_data_yield + SRC_SCRIPT: src/features/phone_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/phone-keyboard/ PHONE_KEYBOARD: @@ -228,8 +223,7 @@ PHONE_LIGHT: RAPIDS: COMPUTE: True FEATURES: ["count", "maxlux", "minlux", "avglux", "medianlux", "stdlux"] - SRC_FOLDER: "rapids" # inside src/features/phone_light - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_light/rapids/main.py # See https://www.rapids.science/latest/features/phone-locations/ PHONE_LOCATIONS: @@ -257,8 +251,7 @@ PHONE_LOCATIONS: CLUSTER_ON: PARTICIPANT_DATASET # PARTICIPANT_DATASET,TIME_SEGMENT CLUSTERING_ALGORITHM: DBSCAN #DBSCAN,OPTICS RADIUS_FOR_HOME: 100 - SRC_FOLDER: "doryab" # inside src/features/phone_locations - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_locations/doryab/main.py BARNETT: COMPUTE: False @@ -266,8 +259,7 @@ PHONE_LOCATIONS: ACCURACY_LIMIT: 100 # meters, drops location coordinates with an accuracy higher than this. This number means there's a 68% probability the true location is within this radius IF_MULTIPLE_TIMEZONES: USE_MOST_COMMON MINUTES_DATA_USED: False # Use this for quality control purposes, how many minutes of data (location coordinates gruped by minute) were used to compute features - SRC_FOLDER: "barnett" # inside src/features/phone_locations - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_locations/barnett/main.R # See https://www.rapids.science/latest/features/phone-log/ PHONE_LOG: @@ -286,8 +278,7 @@ PHONE_MESSAGES: FEATURES: received: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] sent: [count, distinctcontacts, timefirstmessage, timelastmessage, countmostfrequentcontact] - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/phone_messages + SRC_SCRIPT: src/features/phone_messages/rapids/main.R # See https://www.rapids.science/latest/features/phone-screen/ PHONE_SCREEN: @@ -300,8 +291,7 @@ PHONE_SCREEN: IGNORE_EPISODES_LONGER_THAN: 0 # in minutes, set to 0 to disable FEATURES: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration", "firstuseafter"] # "episodepersensedminutes" needs to be added later EPISODE_TYPES: ["unlock"] - SRC_FOLDER: "rapids" # inside src/features/phone_screen - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/phone_screen/rapids/main.py # See https://www.rapids.science/latest/features/phone-wifi-connected/ PHONE_WIFI_CONNECTED: @@ -310,8 +300,7 @@ PHONE_WIFI_CONNECTED: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_connected - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_connected/rapids/main.R # See https://www.rapids.science/latest/features/phone-wifi-visible/ PHONE_WIFI_VISIBLE: @@ -320,8 +309,7 @@ PHONE_WIFI_VISIBLE: RAPIDS: COMPUTE: True FEATURES: ["countscans", "uniquedevices", "countscansmostuniquedevice"] - SRC_FOLDER: "rapids" # inside src/features/phone_wifi_visible - SRC_LANGUAGE: "r" + SRC_SCRIPT: src/features/phone_wifi_visible/rapids/main.R @@ -329,7 +317,7 @@ PHONE_WIFI_VISIBLE: # FITBIT # ######################################################################################################################## -# See https://www.rapids.science/latest/setup/configuration/#device-data-source-configuration +# See https://www.rapids.science/latest/setup/configuration/#data-stream-configuration FITBIT_DATA_STREAMS: USE: fitbitjson_mysql @@ -360,8 +348,7 @@ FITBIT_DATA_YIELD: COMPUTE: False FEATURES: [ratiovalidyieldedminutes, ratiovalidyieldedhours] MINUTE_RATIO_THRESHOLD_FOR_VALID_YIELDED_HOURS: 0.5 # 0 to 1, minimum percentage of valid minutes in an hour to be considered valid. - SRC_LANGUAGE: "r" - SRC_FOLDER: "rapids" # inside src/features/fitbit_data_yield + SRC_SCRIPT: src/features/fitbit_data_yield/rapids/main.R # See https://www.rapids.science/latest/features/fitbit-heartrate-summary/ @@ -371,8 +358,7 @@ FITBIT_HEARTRATE_SUMMARY: RAPIDS: COMPUTE: False FEATURES: ["maxrestinghr", "minrestinghr", "avgrestinghr", "medianrestinghr", "moderestinghr", "stdrestinghr", "diffmaxmoderestinghr", "diffminmoderestinghr", "entropyrestinghr"] # calories features' accuracy depend on the accuracy of the participants fitbit profile (e.g. height, weight) use these with care: ["sumcaloriesoutofrange", "maxcaloriesoutofrange", "mincaloriesoutofrange", "avgcaloriesoutofrange", "mediancaloriesoutofrange", "stdcaloriesoutofrange", "entropycaloriesoutofrange", "sumcaloriesfatburn", "maxcaloriesfatburn", "mincaloriesfatburn", "avgcaloriesfatburn", "mediancaloriesfatburn", "stdcaloriesfatburn", "entropycaloriesfatburn", "sumcaloriescardio", "maxcaloriescardio", "mincaloriescardio", "avgcaloriescardio", "mediancaloriescardio", "stdcaloriescardio", "entropycaloriescardio", "sumcaloriespeak", "maxcaloriespeak", "mincaloriespeak", "avgcaloriespeak", "mediancaloriespeak", "stdcaloriespeak", "entropycaloriespeak"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-heartrate-intraday/ FITBIT_HEARTRATE_INTRADAY: @@ -381,8 +367,7 @@ FITBIT_HEARTRATE_INTRADAY: RAPIDS: COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr", "minutesonoutofrangezone", "minutesonfatburnzone", "minutesoncardiozone", "minutesonpeakzone"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_heartrate_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_heartrate_intraday/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-summary/ FITBIT_SLEEP_SUMMARY: @@ -392,8 +377,7 @@ FITBIT_SLEEP_SUMMARY: COMPUTE: False FEATURES: ["countepisode", "avgefficiency", "sumdurationafterwakeup", "sumdurationasleep", "sumdurationawake", "sumdurationtofallasleep", "sumdurationinbed", "avgdurationafterwakeup", "avgdurationasleep", "avgdurationawake", "avgdurationtofallasleep", "avgdurationinbed"] SLEEP_TYPES: ["main", "nap", "all"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-sleep-intraday/ FITBIT_SLEEP_INTRADAY: @@ -414,8 +398,7 @@ FITBIT_SLEEP_INTRADAY: SLEEP_TYPES: [main, nap] INCLUDE_SLEEP_LATER_THAN: 0 # a number ranged from 0 (midnight) to 1439 (23:59) REFERENCE_TIME: MIDNIGHT # chosen from "MIDNIGHT" and "START_OF_THE_SEGMENT" - SRC_FOLDER: "rapids" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/rapids/main.py PRICE: COMPUTE: False @@ -428,8 +411,7 @@ FITBIT_SLEEP_INTRADAY: GROUP_EPISODES_WITHIN: # by default: today's 6pm to tomorrow's noon START_TIME: 1080 # number of minutes after the midnight (18:00) 18*60 LENGTH: 1080 # in minutes (18 hours) 18*60 - SRC_FOLDER: "price" # inside src/features/fitbit_sleep_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_sleep_intraday/price/main.py # See https://www.rapids.science/latest/features/fitbit-steps-summary/ FITBIT_STEPS_SUMMARY: @@ -438,8 +420,7 @@ FITBIT_STEPS_SUMMARY: RAPIDS: COMPUTE: False FEATURES: ["maxsumsteps", "minsumsteps", "avgsumsteps", "mediansumsteps", "stdsumsteps"] - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_summary - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_summary/rapids/main.py # See https://www.rapids.science/latest/features/fitbit-steps-intraday/ FITBIT_STEPS_INTRADAY: @@ -453,8 +434,7 @@ FITBIT_STEPS_INTRADAY: ACTIVE_BOUT: ["countepisode", "sumduration", "maxduration", "minduration", "avgduration", "stdduration"] THRESHOLD_ACTIVE_BOUT: 10 # steps INCLUDE_ZERO_STEP_ROWS: False - SRC_FOLDER: "rapids" # inside src/features/fitbit_steps_intraday - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/fitbit_steps_intraday/rapids/main.py ######################################################################################################################## # EMPATICA # @@ -476,8 +456,7 @@ EMPATICA_ACCELEROMETER: DBDP: COMPUTE: False FEATURES: ["maxmagnitude", "minmagnitude", "avgmagnitude", "medianmagnitude", "stdmagnitude"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_accelerometer - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_accelerometer/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-heartrate/ EMPATICA_HEARTRATE: @@ -486,8 +465,7 @@ EMPATICA_HEARTRATE: DBDP: COMPUTE: False FEATURES: ["maxhr", "minhr", "avghr", "medianhr", "modehr", "stdhr", "diffmaxmodehr", "diffminmodehr", "entropyhr"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_heartrate/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-temperature/ EMPATICA_TEMPERATURE: @@ -496,8 +474,7 @@ EMPATICA_TEMPERATURE: DBDP: COMPUTE: False FEATURES: ["maxtemp", "mintemp", "avgtemp", "mediantemp", "modetemp", "stdtemp", "diffmaxmodetemp", "diffminmodetemp", "entropytemp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_heartrate - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_temperature/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-electrodermal-activity/ EMPATICA_ELECTRODERMAL_ACTIVITY: @@ -506,8 +483,7 @@ EMPATICA_ELECTRODERMAL_ACTIVITY: DBDP: COMPUTE: False FEATURES: ["maxeda", "mineda", "avgeda", "medianeda", "modeeda", "stdeda", "diffmaxmodeeda", "diffminmodeeda", "entropyeda"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_electrodermal_activity - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_electrodermal_activity/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-blood-volume-pulse/ EMPATICA_BLOOD_VOLUME_PULSE: @@ -516,8 +492,7 @@ EMPATICA_BLOOD_VOLUME_PULSE: DBDP: COMPUTE: False FEATURES: ["maxbvp", "minbvp", "avgbvp", "medianbvp", "modebvp", "stdbvp", "diffmaxmodebvp", "diffminmodebvp", "entropybvp"] - SRC_FOLDER: "dbdp" # inside src/features/empatica_blood_volume_pulse - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_blood_volume_pulse/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-inter-beat-interval/ EMPATICA_INTER_BEAT_INTERVAL: @@ -526,8 +501,7 @@ EMPATICA_INTER_BEAT_INTERVAL: DBDP: COMPUTE: False FEATURES: ["maxibi", "minibi", "avgibi", "medianibi", "modeibi", "stdibi", "diffmaxmodeibi", "diffminmodeibi", "entropyibi"] - SRC_FOLDER: "dbdp" # inside src/features/inter_beat_interval - SRC_LANGUAGE: "python" + SRC_SCRIPT: src/features/empatica_inter_beat_interval/dbdp/main.py # See https://www.rapids.science/latest/features/empatica-tags/ EMPATICA_TAGS: diff --git a/tools/config.schema.yaml b/tools/config.schema.yaml index fbd3db66..0d839744 100644 --- a/tools/config.schema.yaml +++ b/tools/config.schema.yaml @@ -40,17 +40,16 @@ required: definitions: PROVIDER: type: object - required: [COMPUTE, SRC_FOLDER, SRC_LANGUAGE, FEATURES] + required: [COMPUTE, SRC_SCRIPT, FEATURES] properties: COMPUTE: type: boolean FEATURES: type: [array, object] - SRC_FOLDER: + SRC_SCRIPT: type: string - SRC_LANGUAGE: - type: string - enum: [python, r] + pattern: "^.*\\.(py|R)$" + DORYAB_BLUETOOTH_FEATURE: type: object