From 1a321bb2fede27bdc55c313c1a337392c62af7a7 Mon Sep 17 00:00:00 2001 From: Agam Kumar Date: Mon, 17 Aug 2020 03:03:09 -0400 Subject: [PATCH 1/5] Dockerfile + travis.yml changes --- .travis.yml | 7 +++++++ Dockerfile | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 Dockerfile diff --git a/.travis.yml b/.travis.yml index ba118a49..6fdb61c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ services: - mysql + - docker + +sudo: required language: python # this works for Linux but is an error on macOS or Windows @@ -61,6 +64,10 @@ install: script: - bash tests/scripts/run_tests.sh + - docker build -t rapids . + +# after_success: +# need to push built image to appropriate repo in docker hub (to do: encrypt password) branches: only: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..916d326d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# getting base image ubuntu +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && apt install -y \ + libcurl4-openssl-dev \ + libssl-dev \ + libxml2-dev \ + libmysqlclient-dev \ + mysql-server +RUN apt-get update && apt-get install -y gnupg +RUN apt-get update && apt-get install -y software-properties-common +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 +RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' +RUN apt update && apt install -y r-base +RUN apt install -y pandoc +RUN apt install -y git +RUN apt-get update && apt-get install -y vim +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV PATH /opt/conda/bin:$PATH +RUN apt-get update --fix-missing && \ + apt-get install -y wget bzip2 ca-certificates curl git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda clean -tipsy && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc +ENV TINI_VERSION v0.16.1 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini +RUN chmod +x /usr/bin/tini +RUN git clone https://github.com/carissalow/rapids +ENTRYPOINT [ "/usr/bin/tini", "--" ] +CMD [ "/bin/bash" ] +RUN conda update -n base -c defaults conda +WORKDIR /rapids +RUN conda env create -f environment.yml -n rapids +RUN R -e 'renv::restore()' +COPY rapids_example.sql ./data/external +RUN echo "source activate rapids" > ~/.bashrc +ENV PATH /opt/conda/envs/rapids/bin:$PATH \ No newline at end of file From 6cbd73c676918d4f22c3d19549f9197bc59cd517 Mon Sep 17 00:00:00 2001 From: Agam Date: Fri, 28 Aug 2020 15:33:44 -0400 Subject: [PATCH 2/5] update dockerfile - rmarkdown --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 916d326d..904bf988 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,12 @@ RUN apt install -y git RUN apt-get update && apt-get install -y vim ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH + RUN apt-get update --fix-missing && \ apt-get install -y wget bzip2 ca-certificates curl git && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* + RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \ /bin/bash ~/miniconda.sh -b -p /opt/conda && \ rm ~/miniconda.sh && \ @@ -28,6 +30,7 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86 ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ echo "conda activate base" >> ~/.bashrc + ENV TINI_VERSION v0.16.1 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini RUN chmod +x /usr/bin/tini @@ -37,6 +40,7 @@ CMD [ "/bin/bash" ] RUN conda update -n base -c defaults conda WORKDIR /rapids RUN conda env create -f environment.yml -n rapids +RUN Rscript --vanilla -e 'install.packages("rmarkdown", repos="http://cran.us.r-project.org")' RUN R -e 'renv::restore()' COPY rapids_example.sql ./data/external RUN echo "source activate rapids" > ~/.bashrc From faee2bd5401738c77b9deb06734fbaf7cce26717 Mon Sep 17 00:00:00 2001 From: Agam Date: Tue, 1 Sep 2020 00:25:08 -0400 Subject: [PATCH 3/5] updated Dockerfile with git and rapids_example.sql --- Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 904bf988..3bb4b72e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN apt update && apt install -y r-base RUN apt install -y pandoc RUN apt install -y git RUN apt-get update && apt-get install -y vim +RUN apt update && apt install -y unzip ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV PATH /opt/conda/bin:$PATH @@ -42,6 +43,11 @@ WORKDIR /rapids RUN conda env create -f environment.yml -n rapids RUN Rscript --vanilla -e 'install.packages("rmarkdown", repos="http://cran.us.r-project.org")' RUN R -e 'renv::restore()' -COPY rapids_example.sql ./data/external +ADD https://osf.io/587wc/download data/external +RUN mv data/external/download data/external/rapids_example.sql.zip +RUN unzip data/external/rapids_example.sql.zip +RUN cp rapids_example.sql data/external/rapids_example.sql +RUN rm data/external/rapids_example.sql.zip +RUN rm rapids_example.sql RUN echo "source activate rapids" > ~/.bashrc ENV PATH /opt/conda/envs/rapids/bin:$PATH \ No newline at end of file From 6ef0a187900d6bb3fb6f12005145cce1823af9cd Mon Sep 17 00:00:00 2001 From: Agam Date: Tue, 1 Sep 2020 01:27:35 -0400 Subject: [PATCH 4/5] travis docker integration --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6fdb61c6..d03e5ee5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,8 +66,10 @@ script: - bash tests/scripts/run_tests.sh - docker build -t rapids . -# after_success: -# need to push built image to appropriate repo in docker hub (to do: encrypt password) +after_success: + - docker login -u "agamk" -p "Qjp4by3FUUqyXNe5lkBPg6LT0LZIdBobHWS3GTsA6jTk26m0bkuUvmJjsjK+/BtGtIqOcaTy+9e1q2+erpUl3hv2NyG6+yzjCfHyPUjWE5MOLE5ve1u+1fQatQueAtwxcsRCN1JE9zmlzvrMwJ4Xx7hb/0KifLWtkOF1LyiVnLuV4lmMaoBRNgtxrhSEOcSm+gMWw8jTEppThHXVEiN+F7tMKTkQtOsRK+R0wX5eieV8API8a0yiXHORXsVQWUn7BjjGgp69pGCY2QqwJi2xFeOIj4GPKBIv3a0YffYbx3D96uX7lcnQk2vSnM88LBbW2PMXwIBEaBxLWeBAUbn5obquybo8P6sYfPa255dAVOfFnyufKyHvyh2VhNnuc/nEdcd/pGjgPrQjh+G2uofKSzAQNbYSLH4XQDuGoMjXbcw4xcijZjF6o9PslFb9SNWirECL+UPAiddgapW8dfq8kc+4vw8f5zO/MEvYswvq8BEaXTqxUvtHrT7nT+gnDYMrMNEAYZlS/8oaTfdbkcWGbnuzcwMRRvubLQat/rLomY1mZMxGNYR8ZVgswKz7TAEIKmWgFYVuQAXIbBPDoKovVT3EYRlWKSeTvxeQGSj87xGqHnmKeGp+xgeoTnp2IwQTRE4t9RmS27SHwASR/mhdPnhCAaw1GWSH05TVDbQEKHw=" + - docker tag rapids agamk/rapids:travislatest + - docker push agamk/rapids:travislatest branches: only: From db0ce25205c3e50c5081b96248d45ab04ee1c55b Mon Sep 17 00:00:00 2001 From: Agam Date: Tue, 1 Sep 2020 18:14:02 -0400 Subject: [PATCH 5/5] build stages + separate jobs --- .travis.yml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index d03e5ee5..85e970a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,14 @@ services: sudo: required language: python # this works for Linux but is an error on macOS or Windows - +stages: + - test + - dockerize + jobs: include: - - name: "Python 3.7 on Xenial Linux" + - stage: test + name: "Python 3.7 on Xenial Linux" os: linux language: python python: 3.7 @@ -31,7 +35,8 @@ jobs: - $HOME/.local/share/renv - $TRAVIS_BUILD_DIR/renv/library - - name: "Python 3.7 on macOS" + - stage: test + name: "Python 3.7 on macOS" os: osx osx_image: xcode11.3 # Python 3.7 running on macOS 10.14.4 language: generic # 'language: python' is an error on Travis CI macOS @@ -53,6 +58,16 @@ jobs: - $RENV_PATHS_ROOT - $TRAVIS_BUILD_DIR/renv/library + - stage: dockerize + name: "Python 3.7 on Xenial Linux" + os: linux + language: python + script: + - docker build -t rapids . + - docker login -u "agamk" -p "Qjp4by3FUUqyXNe5lkBPg6LT0LZIdBobHWS3GTsA6jTk26m0bkuUvmJjsjK+/BtGtIqOcaTy+9e1q2+erpUl3hv2NyG6+yzjCfHyPUjWE5MOLE5ve1u+1fQatQueAtwxcsRCN1JE9zmlzvrMwJ4Xx7hb/0KifLWtkOF1LyiVnLuV4lmMaoBRNgtxrhSEOcSm+gMWw8jTEppThHXVEiN+F7tMKTkQtOsRK+R0wX5eieV8API8a0yiXHORXsVQWUn7BjjGgp69pGCY2QqwJi2xFeOIj4GPKBIv3a0YffYbx3D96uX7lcnQk2vSnM88LBbW2PMXwIBEaBxLWeBAUbn5obquybo8P6sYfPa255dAVOfFnyufKyHvyh2VhNnuc/nEdcd/pGjgPrQjh+G2uofKSzAQNbYSLH4XQDuGoMjXbcw4xcijZjF6o9PslFb9SNWirECL+UPAiddgapW8dfq8kc+4vw8f5zO/MEvYswvq8BEaXTqxUvtHrT7nT+gnDYMrMNEAYZlS/8oaTfdbkcWGbnuzcwMRRvubLQat/rLomY1mZMxGNYR8ZVgswKz7TAEIKmWgFYVuQAXIbBPDoKovVT3EYRlWKSeTvxeQGSj87xGqHnmKeGp+xgeoTnp2IwQTRE4t9RmS27SHwASR/mhdPnhCAaw1GWSH05TVDbQEKHw=" + - docker tag rapids agamk/rapids:travislatest + - docker push agamk/rapids:travislatest + install: - conda init bash - conda update -q --all --yes conda @@ -64,12 +79,6 @@ install: script: - bash tests/scripts/run_tests.sh - - docker build -t rapids . - -after_success: - - docker login -u "agamk" -p "Qjp4by3FUUqyXNe5lkBPg6LT0LZIdBobHWS3GTsA6jTk26m0bkuUvmJjsjK+/BtGtIqOcaTy+9e1q2+erpUl3hv2NyG6+yzjCfHyPUjWE5MOLE5ve1u+1fQatQueAtwxcsRCN1JE9zmlzvrMwJ4Xx7hb/0KifLWtkOF1LyiVnLuV4lmMaoBRNgtxrhSEOcSm+gMWw8jTEppThHXVEiN+F7tMKTkQtOsRK+R0wX5eieV8API8a0yiXHORXsVQWUn7BjjGgp69pGCY2QqwJi2xFeOIj4GPKBIv3a0YffYbx3D96uX7lcnQk2vSnM88LBbW2PMXwIBEaBxLWeBAUbn5obquybo8P6sYfPa255dAVOfFnyufKyHvyh2VhNnuc/nEdcd/pGjgPrQjh+G2uofKSzAQNbYSLH4XQDuGoMjXbcw4xcijZjF6o9PslFb9SNWirECL+UPAiddgapW8dfq8kc+4vw8f5zO/MEvYswvq8BEaXTqxUvtHrT7nT+gnDYMrMNEAYZlS/8oaTfdbkcWGbnuzcwMRRvubLQat/rLomY1mZMxGNYR8ZVgswKz7TAEIKmWgFYVuQAXIbBPDoKovVT3EYRlWKSeTvxeQGSj87xGqHnmKeGp+xgeoTnp2IwQTRE4t9RmS27SHwASR/mhdPnhCAaw1GWSH05TVDbQEKHw=" - - docker tag rapids agamk/rapids:travislatest - - docker push agamk/rapids:travislatest branches: only: