From ac4526df5d8d9706f0b18bdd42cf44501c12a3fd Mon Sep 17 00:00:00 2001 From: JulioV Date: Fri, 18 Dec 2020 15:18:15 -0500 Subject: [PATCH] Update docs, add renv and conda cache,& release CI --- .github/workflows/docs.yaml | 8 +++- .github/workflows/tests.yaml | 63 +++++++++++++++++++++++------- docs/workflow-examples/analysis.md | 2 +- tools/environment-from-history.yml | 17 ++++++++ 4 files changed, 73 insertions(+), 17 deletions(-) create mode 100644 tools/environment-from-history.yml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 39681717..dc71a56c 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -9,14 +9,18 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - if: ${{ github.ref == 'refs/heads/develop' }} #we delay develop because when we release a hotgix (tag + develop push), one of these push will be out of sync + uses: jakejarvis/wait-action@master with: - fetch-depth: 0 + time: '60s' - uses: actions/setup-python@v2 with: python-version: 3.x - run: pip install git+https://${GH_TOKEN}@github.com/carissalow/mkdocs-material-insiders.git - run: pip install mike + - uses: actions/checkout@v2 + with: + fetch-depth: 0 - run: | git config user.name github-actions git config user.email github-actions@github.com diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6f7459d4..37e3eb32 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,7 +2,13 @@ name: tests on: push: - #branches: [ develop ] + branches-ignore: + - "master" + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + pull_request: + branches: + - "develop" env: RENV_PATHS_ROOT: ~/.local/share/renv @@ -13,31 +19,60 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: s-weigand/setup-conda@v1 - - run : conda --version - - run : source activate base + - run: "sed -i 's/name:.*/name: rapidstests/g' environment.yml" + - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - run: echo "RELEASE_VERSION_URL=$(echo $RELEASE_VERSION | sed -e 's/\.//g')" >> $GITHUB_ENV - run : | sudo apt update - sudo apt install libcurl4-openssl-dev - sudo apt install libssl-dev - sudo apt install libxml2-dev + # sudo apt install libcurl4-openssl-dev + # sudo apt install libssl-dev + # sudo apt install libxml2-dev sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' sudo apt install r-base - sudo apt install pandoc - sudo Rscript --vanilla -e 'install.packages("rmarkdown", repos="http://cran.us.r-project.org")' - conda env create -f environment.yml -n test-environment - source activate test-environment - - name: Cache packages + - name: Cache R packages uses: actions/cache@v1 with: path: ${{ env.RENV_PATHS_ROOT }} key: ${{ runner.os }}-renv-${{ hashFiles('**/renv.lock') }} restore-keys: | ${{ runner.os }}-renv- - - name: Restore packages + - name: Restore R packages shell: Rscript {0} run: | if (!requireNamespace("renv", quietly = TRUE)) install.packages("renv") renv::restore() - - run : bash tests/scripts/run_tests.sh all test \ No newline at end of file + - name: Cache conda packages + uses: actions/cache@v1 + env: + # Increase this value to reset cache if environment.yml has not changed + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('**/environment.yml') }} + - name: Restore conda packages + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: rapidstests + environment-file: environment.yml + use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + - name: Run tests + shell: bash -l {0} + run : | + conda activate rapidstests + bash tests/scripts/run_tests.sh all test + - name: Release tag + if: success() && startsWith(github.ref, 'refs/tags') + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + body: | + See [change log](http://www.rapids.science/latest/change-log/#${{ env.RELEASE_VERSION_URL }}) + draft: false + prerelease: false \ No newline at end of file diff --git a/docs/workflow-examples/analysis.md b/docs/workflow-examples/analysis.md index fe226f1a..35f318d6 100644 --- a/docs/workflow-examples/analysis.md +++ b/docs/workflow-examples/analysis.md @@ -37,7 +37,7 @@ In total, our example workflow has nine steps that are in charge of sensor data ## Configure and run the analysis workflow example 1. [Install](../../setup/installation) RAPIDS -2. Configure the [user credentials](../../setup/configuration/#database-credentials) of a local or remote MySQL server with writing permissions in your `.env` file. The example config file is at `example_profile/example_config.yaml`. +2. Configure the [user credentials](../../setup/configuration/#database-credentials) of a local or remote MySQL server with writing permissions in your `.env` file. The config file where you need to modify the `DATABASE_GROUP` is at `example_profile/example_config.yaml`. 3. Unzip the [test database](https://osf.io/skqfv/files/) to `data/external/rapids_example.sql` and run: ```bash ./rapids -j1 restore_sql_file --profile example_profile diff --git a/tools/environment-from-history.yml b/tools/environment-from-history.yml new file mode 100644 index 00000000..1c02d2fe --- /dev/null +++ b/tools/environment-from-history.yml @@ -0,0 +1,17 @@ +name: devtest +channels: + - defaults +dependencies: + - python=3.7 + - pandas=1.1.5 + - scipy=1.5.2 + - plotly=4.14.1 + - astropy=4.2 + - scikit-learn=0.23.2 + - xgboost=0.90 + - lightgbm=3.1.1 + - conda-forge::imbalanced-learn=0.6.2 + - pip=20.3.3 + - pip: + - snakemake==5.30.2 +prefix: /usr/local/Caskroom/miniconda/base/envs/devtest