Add Travis CI

.Rprofile

@@ -0,0 +1,2 @@
+source("renv/activate.R")
+options(renv.settings.snapshot.type = "all")

.gitignore

@@ -85,7 +85,6 @@ target/
 
 # Mypy cache
 .mypy_cache/
-.Rprofile
 packrat/lib*/
 packrat/*
 !packrat/packrat.lock
@@ -108,5 +107,4 @@ reports/
 .RData
 .Rhistory
 sn_profile_*/
-renv/
-
+settings.dcf

.travis.yml

@@ -1,42 +1,59 @@
 services:
   - mysql
 
-os: linux
+language: python            # this works for Linux but is an error on macOS or Windows
 
-dist: xenial
-
-language: python
-python:
-  - '3.7'
-
-cache:
-  directories:
-    - /home/linuxbrew/.linuxbrew
-    - $HOME/.local/share/renv
-    - $TRAVIS_BUILD_DIR/renv/library
-
-before_install:
+jobs:
+  include:
+    - name: "Python 3.7 on Xenial Linux"
+      os: linux
+      language: python
+      python: 3.7
+      before_install:
         - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
         - export PATH=/home/linuxbrew/.linuxbrew/bin:$PATH
         - source ~/.bashrc
         - sudo apt-get install linuxbrew-wrapper
+        - brew tap --shallow linuxbrew/xorg
         - brew install r
-  # - sudo chmod ugo+rw /usr/local/lib/R/site-library
-  - R -e 'install.packages(c( "rmarkdown"), repos = "http://cran.us.r-project.org")'
         - R --version
-
-install:
         - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
         - bash miniconda.sh -b -p $HOME/miniconda
         - source "$HOME/miniconda/etc/profile.d/conda.sh"
         - hash -r
         - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
-  - conda info -a
+      cache:
+        directories:
+          - /home/travis/.linuxbrew
+          - $HOME/.local/share/renv
+          - $TRAVIS_BUILD_DIR/renv/library
+            
+    - name: "Python 3.7 on macOS"
+      os: osx
+      osx_image: xcode11.3  # Python 3.7 running on macOS 10.14.4
+      language: generic       # 'language: python' is an error on Travis CI macOS
+      before_install:
+        - brew install mysql
+        - brew services start mysql
+        - brew install r
+        - R --version
+        - brew cask install miniconda
+        - eval "$(/usr/local/bin/conda shell.bash hook)"
+      env:
+        - RENV_PATHS_ROOT="$HOME/renv/cache"
+      cache:
+        directories:
+          - /usr/local/lib/R
+          - $RENV_PATHS_ROOT
+          - $TRAVIS_BUILD_DIR/renv/library
+
+install:
+  - conda init
+  - conda update -q --all --yes conda
   - conda env create -q -n test-environment python=$TRAVIS_PYTHON_VERSION --file environment.yml
   - conda activate test-environment
   - snakemake renv_install
-  - snakemake renv_init
+  - R -e 'renv::settings$use.cache(FALSE)'
   - snakemake renv_restore 
     
 script:
@@ -47,16 +64,10 @@ script:
 notifications:
   email: false
   slack:
+    # if: branch = travis_test
     secure: cJIpmIjb3zA5AMDBo9axF1v6fYNIgMm6s6UdMNOlHiT511xHGsaLUFej3lACwQLig4Gr94ySI61YdrP+RX1lFcYxusH+kUU/c8LX0PmSKNeKnycM3w/pCM+yTp/6oQG6ZrJD7pNm6zhB0xPL61uSmYhcr+JJ1sh4iLiON+J8/C+IfnAHm1ORkxJ0IxASkiP/LvaiAQDw8lNyYIZNWjSDNZbx68o1VNakyk6Vik3x8omiE3w33rzI2/JAx//QTxOq2J0dtV1AqYYSOWS4iXblV09NLBqgGrhAhrQ6+TbPHSPIyL/4EdhvS+YXO+SBWS7ODD7j/MuL6XiA4SujW72od2rgXNmOjFnlQvIrULO5bzv39BKKDkldvz9+XCyXLcjoLIwA/rmUnwMndNoC7NoD/CkQEevUxswXXB9811BmIFx/7GOHouVxwB2gaMAzkCroZJVwgbrc6ESSOVE5SMcb3wPMbpd8cXOgVZXJcmk5wK206zxXPigCvFfknqOnwDqRgyIWSFoTd/2wHppA7ND3R5U42nQTbEQ7MiONsOo61GlJTTxJELz32sLKl388AuAgOY7+0sqPibxMaHJkF1V4nYVTH0/H5bO/edK4VHMloJ6s0kuyko7LT5EMQf3pBJij5TnYmD2E60t+bSBAxHuH7WA5dvL+igjGEwROnxDc9pc=
     on_success: always
     template:
       - "Repo `%{repository_slug}` *%{result}* build (<%{build_url}|#%{build_number}>) for commit (<%{compare_url}|%{commit}>) on branch `%{branch}`."
       - "Execution time: *%{duration}*"
       - "Message: %{message}"
-
-branches:
-  except:
-  - master
-
-  only:
-  - travis_test

environment.yml

@@ -1,8 +1,8 @@
 name: moshi-env
 channels:
   - anaconda
-  - defaults
   - conda-forge
+  - defaults
 dependencies:
   - appdirs=1.4.3
   - arrow=0.15.2

renv/.gitignore

@@ -0,0 +1,3 @@
+library/
+python/
+staging/

renv/activate.R

@@ -0,0 +1,341 @@
+
+local({
+
+  # the requested version of renv
+  version <- "0.10.0"
+
+  # the project directory
+  project <- getwd()
+
+  # avoid recursion
+  if (!is.na(Sys.getenv("RENV_R_INITIALIZING", unset = NA)))
+    return(invisible(TRUE))
+
+  # signal that we're loading renv during R startup
+  Sys.setenv("RENV_R_INITIALIZING" = "true")
+  on.exit(Sys.unsetenv("RENV_R_INITIALIZING"), add = TRUE)
+
+  # signal that we've consented to use renv
+  options(renv.consent = TRUE)
+
+  # load the 'utils' package eagerly -- this ensures that renv shims, which
+  # mask 'utils' packages, will come first on the search path
+  library(utils, lib.loc = .Library)
+
+  # check to see if renv has already been loaded
+  if ("renv" %in% loadedNamespaces()) {
+
+    # if renv has already been loaded, and it's the requested version of renv,
+    # nothing to do
+    spec <- .getNamespaceInfo(.getNamespace("renv"), "spec")
+    if (identical(spec[["version"]], version))
+      return(invisible(TRUE))
+
+    # otherwise, unload and attempt to load the correct version of renv
+    unloadNamespace("renv")
+
+  }
+
+  # load bootstrap tools   
+  bootstrap <- function(version, library) {
+  
+    # fix up repos
+    repos <- getOption("repos")
+    on.exit(options(repos = repos), add = TRUE)
+    repos[repos == "@CRAN@"] <- "https://cloud.r-project.org"
+    options(repos = repos)
+  
+    # attempt to download renv
+    tarball <- tryCatch(renv_bootstrap_download(version), error = identity)
+    if (inherits(tarball, "error"))
+      stop("failed to download renv ", version)
+  
+    # now attempt to install
+    status <- tryCatch(renv_bootstrap_install(version, tarball, library), error = identity)
+    if (inherits(status, "error"))
+      stop("failed to install renv ", version)
+  
+  }
+  
+  renv_bootstrap_download_impl <- function(url, destfile) {
+  
+    mode <- "wb"
+  
+    # https://bugs.r-project.org/bugzilla/show_bug.cgi?id=17715
+    fixup <-
+      Sys.info()[["sysname"]] == "Windows" &&
+      substring(url, 1L, 5L) == "file:"
+  
+    if (fixup)
+      mode <- "w+b"
+  
+    download.file(
+      url      = url,
+      destfile = destfile,
+      mode     = mode,
+      quiet    = TRUE
+    )
+  
+  }
+  
+  renv_bootstrap_download <- function(version) {
+  
+    methods <- list(
+      renv_bootstrap_download_cran_latest,
+      renv_bootstrap_download_cran_archive,
+      renv_bootstrap_download_github
+    )
+  
+    for (method in methods) {
+      path <- tryCatch(method(version), error = identity)
+      if (is.character(path) && file.exists(path))
+        return(path)
+    }
+  
+    stop("failed to download renv ", version)
+  
+  }
+  
+  renv_bootstrap_download_cran_latest <- function(version) {
+  
+    # check for renv on CRAN matching this version
+    db <- as.data.frame(available.packages(), stringsAsFactors = FALSE)
+    if (!"renv" %in% rownames(db))
+      stop("renv is not available on your declared package repositories")
+  
+    entry <- db["renv", ]
+    if (!identical(entry$Version, version))
+      stop("renv is not available on your declared package repositories")
+  
+    message("* Downloading renv ", version, " from CRAN ... ", appendLF = FALSE)
+  
+    info <- tryCatch(
+      download.packages("renv", destdir = tempdir()),
+      condition = identity
+    )
+  
+    if (inherits(info, "condition")) {
+      message("FAILED")
+      return(FALSE)
+    }
+  
+    message("OK")
+    info[1, 2]
+  
+  }
+  
+  renv_bootstrap_download_cran_archive <- function(version) {
+  
+    name <- sprintf("renv_%s.tar.gz", version)
+    repos <- getOption("repos")
+    urls <- file.path(repos, "src/contrib/Archive/renv", name)
+    destfile <- file.path(tempdir(), name)
+  
+    message("* Downloading renv ", version, " from CRAN archive ... ", appendLF = FALSE)
+  
+    for (url in urls) {
+  
+      status <- tryCatch(
+        renv_bootstrap_download_impl(url, destfile),
+        condition = identity
+      )
+  
+      if (identical(status, 0L)) {
+        message("OK")
+        return(destfile)
+      }
+  
+    }
+  
+    message("FAILED")
+    return(FALSE)
+  
+  }
+  
+  renv_bootstrap_download_github <- function(version) {
+  
+    enabled <- Sys.getenv("RENV_BOOTSTRAP_FROM_GITHUB", unset = "TRUE")
+    if (!identical(enabled, "TRUE"))
+      return(FALSE)
+  
+    # prepare download options
+    pat <- Sys.getenv("GITHUB_PAT")
+    if (nzchar(Sys.which("curl")) && nzchar(pat)) {
+      fmt <- "--location --fail --header \"Authorization: token %s\""
+      extra <- sprintf(fmt, pat)
+      saved <- options("download.file.method", "download.file.extra")
+      options(download.file.method = "curl", download.file.extra = extra)
+      on.exit(do.call(base::options, saved), add = TRUE)
+    } else if (nzchar(Sys.which("wget")) && nzchar(pat)) {
+      fmt <- "--header=\"Authorization: token %s\""
+      extra <- sprintf(fmt, pat)
+      saved <- options("download.file.method", "download.file.extra")
+      options(download.file.method = "wget", download.file.extra = extra)
+      on.exit(do.call(base::options, saved), add = TRUE)
+    }
+  
+    message("* Downloading renv ", version, " from GitHub ... ", appendLF = FALSE)
+  
+    url <- file.path("https://api.github.com/repos/rstudio/renv/tarball", version)
+    name <- sprintf("renv_%s.tar.gz", version)
+    destfile <- file.path(tempdir(), name)
+  
+    status <- tryCatch(
+      renv_bootstrap_download_impl(url, destfile),
+      condition = identity
+    )
+  
+    if (!identical(status, 0L)) {
+      message("FAILED")
+      return(FALSE)
+    }
+  
+    message("Done!")
+    return(destfile)
+  
+  }
+  
+  renv_bootstrap_install <- function(version, tarball, library) {
+  
+    # attempt to install it into project library
+    message("* Installing renv ", version, " ... ", appendLF = FALSE)
+    dir.create(library, showWarnings = FALSE, recursive = TRUE)
+  
+    # invoke using system2 so we can capture and report output
+    bin <- R.home("bin")
+    exe <- if (Sys.info()[["sysname"]] == "Windows") "R.exe" else "R"
+    r <- file.path(bin, exe)
+    args <- c("--vanilla", "CMD", "INSTALL", "-l", shQuote(library), shQuote(tarball))
+    output <- system2(r, args, stdout = TRUE, stderr = TRUE)
+    message("Done!")
+  
+    # check for successful install
+    status <- attr(output, "status")
+    if (is.numeric(status) && !identical(status, 0L)) {
+      header <- "Error installing renv:"
+      lines <- paste(rep.int("=", nchar(header)), collapse = "")
+      text <- c(header, lines, output)
+      writeLines(text, con = stderr())
+    }
+  
+    status
+  
+  }
+  
+  renv_bootstrap_prefix <- function() {
+  
+    # construct version prefix
+    version <- paste(R.version$major, R.version$minor, sep = ".")
+    prefix <- paste("R", numeric_version(version)[1, 1:2], sep = "-")
+  
+    # include SVN revision for development versions of R
+    # (to avoid sharing platform-specific artefacts with released versions of R)
+    devel <-
+      identical(R.version[["status"]],   "Under development (unstable)") ||
+      identical(R.version[["nickname"]], "Unsuffered Consequences")
+  
+    if (devel)
+      prefix <- paste(prefix, R.version[["svn rev"]], sep = "-r")
+  
+    # build list of path components
+    components <- c(prefix, R.version$platform)
+  
+    # include prefix if provided by user
+    prefix <- Sys.getenv("RENV_PATHS_PREFIX")
+    if (nzchar(prefix))
+      components <- c(prefix, components)
+  
+    # build prefix
+    paste(components, collapse = "/")
+  
+  }
+  
+  renv_bootstrap_library_root <- function(project) {
+  
+    path <- Sys.getenv("RENV_PATHS_LIBRARY", unset = NA)
+    if (!is.na(path))
+      return(path)
+  
+    path <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT", unset = NA)
+    if (!is.na(path))
+      return(file.path(path, basename(project)))
+  
+    file.path(project, "renv/library")
+  
+  }
+  
+  renv_bootstrap_validate_version <- function(version) {
+  
+    loadedversion <- utils::packageDescription("renv", fields = "Version")
+    if (version == loadedversion)
+      return(TRUE)
+  
+    # assume four-component versions are from GitHub; three-component
+    # versions are from CRAN
+    components <- strsplit(loadedversion, "[.-]")[[1]]
+    remote <- if (length(components) == 4L)
+      paste("rstudio/renv", loadedversion, sep = "@")
+    else
+      paste("renv", loadedversion, sep = "@")
+  
+    fmt <- paste(
+      "renv %1$s was loaded from project library, but renv %2$s is recorded in lockfile.",
+      "Use `renv::record(\"%3$s\")` to record this version in the lockfile.",
+      "Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library.",
+      sep = "\n"
+    )
+  
+    msg <- sprintf(fmt, loadedversion, version, remote)
+    warning(msg, call. = FALSE)
+  
+    FALSE
+  
+  }
+  
+  renv_bootstrap_load <- function(project, libpath, version) {
+  
+    # try to load renv from the project library
+    if (!requireNamespace("renv", lib.loc = libpath, quietly = TRUE))
+      return(FALSE)
+  
+    # warn if the version of renv loaded does not match
+    renv_bootstrap_validate_version(version)
+  
+    # load the project
+    renv::load(project)
+  
+    TRUE
+  
+  }
+
+  # construct path to library root
+  root <- renv_bootstrap_library_root(project)
+
+  # construct library prefix for platform
+  prefix <- renv_bootstrap_prefix()
+
+  # construct full libpath
+  libpath <- file.path(root, prefix)
+
+  # attempt to load
+  if (renv_bootstrap_load(project, libpath, version))
+    return(TRUE)
+
+  # load failed; attempt to bootstrap
+  bootstrap(version, libpath)
+
+  # try again to load
+  if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) {
+    message("Successfully installed and loaded renv ", version, ".")
+    return(renv::load())
+  }
+
+  # failed to download or load renv; warn the user
+  msg <- c(
+    "Failed to find an renv installation: the project will not be loaded.",
+    "Use `renv::activate()` to re-initialize the project."
+  )
+
+  warning(paste(msg, collapse = "\n"), call. = FALSE)
+
+})

rules/renv.snakefile

@@ -1,12 +1,12 @@
 ## renv_install: installs renv onto machine
 rule renv_install:
     shell:
-        "R -e 'install.packages(\"renv\", repos=\"http://cran.us.r-project.org\")'"
+        "R -e 'if (!requireNamespace(\"renv\", quietly = TRUE)) install.packages(\"renv\", repos=\"http://cran.us.r-project.org\")'"
 
 ## renv_install: initialize a renv environment for this project
 rule renv_init:
     shell:
-        "R -e 'options(renv.consent = TRUE)' && R -e 'renv::init()'"
+        "R -e 'renv::init()'"
 
 ## renv_snap   : Look for new R packages in files & archives them
 rule renv_snap:

tests/data/processed/test01/sms_sent_daily.csv

@@ -1,2 +1,2 @@
 "local_date","sms_sent_daily_countmostfrequentcontact","sms_sent_daily_count","sms_sent_daily_distinctcontacts","sms_sent_daily_timefirstsms","sms_sent_daily_timelastsms"
-"2017-05-14",1,1,1,18.10,18.45
+"2017-05-14",1,1,1,18.45,18.45