Updated the run_test.sh script

.travis.yml

@@ -69,7 +69,7 @@ jobs:
       - "$RENV_PATHS_ROOT"
       - "$TRAVIS_BUILD_DIR/renv/library"
     script:
-    - bash tests/scripts/run_tests.sh
+    - bash tests/scripts/run_tests.sh all test
   - stage: deploy
     name: Python 3.7 on Xenial Linux Docker
     os: linux

tests/scripts/run_tests.sh

@@ -3,55 +3,168 @@
 
 echo Setting up for testing...
 
-# Uncomment the section below if neccessary to remove old files when testing locally
+clean_old_data() { 
-# echo deleting old data...
+
-# rm -rf data/raw/*
+    echo deleting old data...
-# rm -rf data/processed/*
+    rm -rf data/processed/*
-# rm -rf data/interim/*
+    rm -rf data/interim/*
-# rm -rf data/external/test*
+
+    # echo Backing up preprocessing...
+    # cp rules/preprocessing.smk bak
+} 
+
+run_periodic_pipeline() {
+    
+    echo Running RAPIDS Pipeline periodic segment on testdata...
+    snakemake --profile tests/settings/periodic/ 
+
+    echo Moving produced data from previous pipeline run ...
+    mkdir data/processed/features/periodic
+    mv data/processed/features/test* data/processed/features/periodic/
+    rm -rf data/interim/*
+}
+
+run_frequency_pipeline() {
+
+    echo Running RAPIDS Pipeline frequency segment on testdata...
+    snakemake --profile tests/settings/frequency/ 
+
+    echo Moving produced data from previous pipeline run...
+    mkdir data/processed/features/frequency
+    mv data/processed/features/test* data/processed/features/frequency/
+}
+
+run_periodic_test() {
+
+    echo Running tests on periodic data produced...
+    python -m unittest discover tests/scripts/ -v 
+
+    echo Re-writing the config file being loaded for testing
+    sed -e  's/tests\/settings\/[a-z]*\/testing_config\.yaml/tests\/settings\/periodic\/testing_config\.yaml/' tests/scripts/test_sensor_features.py > test_tmp
+    mv test_tmp tests/scripts/test_sensor_features.py
+}
+
+run_frequency_test() {
+
+    # echo Backing up Testing script...
+    # cp tests/scripts/test_sensor_features.py test_bak
+
+    echo Re-writing the config file being loaded for testing
+    sed -e  's/tests\/settings\/[a-z]*\/testing_config\.yaml/tests\/settings\/frequency\/testing_config\.yaml/' tests/scripts/test_sensor_features.py > test_tmp
+    mv test_tmp tests/scripts/test_sensor_features.py
+
+    echo Running tests on frequency data produced...
+    python -m unittest discover tests/scripts/ -v
+}
+
+display_usage() {
+
+    echo "Usage: run_test.sh [-l] all | periodic | frequency [test]"
+}
 
 echo Copying files... 
 cp -r tests/data/raw/* data/raw
 cp tests/data/external/* data/external
 
-# Uncomment the section below to backup snakemake file when testing locally
-# echo Backing up preprocessing...
-# cp rules/preprocessing.smk bak
-
 echo Disabling downloading of dataset...
 sed -e '27,39 s/^/#/' -e  's/rules.download_dataset.output/"data\/raw\/\{pid\}\/\{sensor\}_raw\.csv"/' rules/preprocessing.smk > tmp
-cp tmp rules/preprocessing.smk
+mv tmp rules/preprocessing.smk
 
-echo Running RAPIDS Pipeline periodic segment on testdata...
+echo $#
-snakemake --profile tests/settings/periodic/ 
 
-echo Moving produced data from previous pipeline run ...
+if [ $# -eq 1 ]
-# rm -rf data/raw/*
+then
-mkdir data/processed/features/periodic
+    if [ $1 == '-l' ] || [ $1 == 'test' ]
-mv data/processed/features/test* data/processed/features/periodic/
+    then
-rm -rf data/interim/*
+        display_usage
-# rm -rf data/external/test*
+    elif [ $1 == 'all' ]
+    then
+        run_periodic_pipeline
+        run_frequency_pipeline
+    elif [ $1 == 'periodic' ]
+    then
+        run_periodic_pipeline
+    elif [ $1 == 'frequency' ]
+    then
+        run_frequency_pipeline
+    else
+        display_usage
+    fi
+elif [ $# -gt 1 ]
+then
+    if [ $1 == '-l' ]
+    then
+        clean_old_data
+        if [ $2 == 'all' ]
+        then
+            run_periodic_pipeline
+            run_frequency_pipeline
+            if [ $# -gt 2 ] && [ $3 == 'test' ]
+            then
+                run_periodic_test
+                run_frequency_test
+            else
+                display_usage
+            fi
+        elif [ $2 == 'periodic' ]
+        then
+            run_periodic_pipeline
+            if [ $# -gt 2 ] && [ $3 == 'test' ]
+            then
+                run_periodic_test
+            else
+                display_usage
+            fi
+        elif [ $2 == 'frequency' ]
+        then
+            run_frequency_pipeline
+            if [ $# -gt 2 ] && [ $3 == 'test' ]
+            then
+                run_frequency_test
+            else
+                display_usage
+            fi
+        else
+            display_usage
+        fi
         
-echo Running RAPIDS Pipeline frequnecy segment on testdata...
+    elif [ $1 == 'all' ]
-snakemake --profile tests/settings/frequency/ 
+    then
-
+        run_periodic_pipeline
-echo Moving produced data from previous pipeline run...
+        run_frequency_pipeline
-mkdir data/processed/features/frequency
+        if [ $2 == 'test' ]
-mv data/processed/features/test* data/processed/features/frequency/
+        then
-
+            run_periodic_test
-echo Running tests on periodic data produced...
+            run_frequency_test
-python -m unittest discover tests/scripts/ -v 
+        else
-
+            display_usage
-echo Backing up Testing script...
+        fi
-cp tests/scripts/test_sensor_features.py test_bak
+    elif [ $1 == 'periodic' ]
-
+    then
-echo Re-writing the config file being loaded for testing
+        run_periodic_pipeline
-sed -e  's/tests\/settings\/periodic\/testing_config\.yaml/tests\/settings\/frequency\/testing_config\.yaml/' tests/scripts/test_sensor_features.py > test_tmp
+        if [ $2 == 'test' ]
-cp test_tmp tests/scripts/test_sensor_features.py
+        then
-
+            run_periodic_test
-echo Running tests on frequency data produced...
+        else
-python -m unittest discover tests/scripts/ -v
+            display_usage
+        fi
+    elif [ $1 == 'frequency' ]
+    then
+        run_frequency_pipeline
+        if [ $2 == 'test' ]
+        then
+            run_frequency_test
+        else
+            display_usage
+        fi
+    else
+        display_usage
+    fi
+else
+    display_usage
+fi
 
+mv bak rules/preprocessing.smk
 # Uncomment to return snakemake back to the original version when testing locally
 # echo Cleaning up...
 # mv bak rules/preprocessing.smk

tests/settings/frequency/config.yaml

@@ -2,4 +2,4 @@ directory: ./
 configfile: ./tests/settings/frequency/testing_config.yaml
 snakefile: ./tests/Snakefile
 cores: 1
-forcerun: [join_features_from_providers]
+forcerun: [compute_day_segments, join_features_from_providers]

tests/settings/periodic/config.yaml

@@ -2,4 +2,4 @@ directory: ./
 configfile: ./tests/settings/periodic/testing_config.yaml
 snakefile: ./tests/Snakefile
 cores: 1
-forcerun: [join_features_from_providers]
+forcerun: [compute_day_segments, join_features_from_providers]