From 59e3599a5868343c5786aca641a172e1ed32d459 Mon Sep 17 00:00:00 2001 From: Philip Paquette Date: Tue, 3 Sep 2019 21:31:54 -0400 Subject: Migrated tests from Jenkins to Travis --- .jenkins/Jenkinsfile | 243 -------------------------------------------- .jenkins/get_cache.sh | 44 -------- .jenkins/jenkins-agent.json | 25 ----- .jenkins/merge_pr.sh | 37 ------- .jenkins/set_cache.sh | 25 ----- .travis.yml | 29 ++++++ requirements_dev.txt | 2 +- 7 files changed, 30 insertions(+), 375 deletions(-) delete mode 100644 .jenkins/Jenkinsfile delete mode 100755 .jenkins/get_cache.sh delete mode 100644 .jenkins/jenkins-agent.json delete mode 100755 .jenkins/merge_pr.sh delete mode 100755 .jenkins/set_cache.sh create mode 100644 .travis.yml diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile deleted file mode 100644 index e649b6f..0000000 --- a/.jenkins/Jenkinsfile +++ /dev/null @@ -1,243 +0,0 @@ -// Pipeline -import groovy.json.JsonSlurper -import hudson.Util -import java.util.concurrent.TimeUnit - -// Constants -String PENDING_MSG = "Jenkins started the build." -String SUCCESS_MSG = "The Jenkins build completed successfully." -String FAILURE_MSG = "The Jenkins build failed." - -// Parsing data -def data = new JsonSlurper().parseText(env.payload) - -// Detecting Git URL -String USERNAME = data.repository.owner.login -String REPOSITORY = data.repository.name -String GITHUB_URL = data.repository.ssh_url - -// Determining current pipeline -int exit_status = 0 -String BUILD_TYPE = "" -String COMMIT_HASH = "" -String SHORT_HASH = "" -String PR_BRANCH = "" -String COMMIT_BRANCH = "" -String COMMIT_URL = "" -String BASE_HASH = "0000000" -String SENDER = "" -int PR_NUMBER = 0 -String CHECKOUT_BRANCH = "" -String CHECKOUT_REF = "" -String ROOT_CONTEXT = "jenkins/diplomacy.ai" -String CONTEXT = "" -String MESSAGE = "" - -// ---- Push ---- -if (data.ref != null && data.after.substring(0, 7) != "0000000") { - BUILD_TYPE = "push" - COMMIT_HASH = data.after - SHORT_HASH = COMMIT_HASH.substring(0, 7) - COMMIT_BRANCH = data.ref.replaceAll("refs/heads/", "") - COMMIT_URL = data.compare - SENDER = data.pusher.name - CHECKOUT_BRANCH = COMMIT_HASH - CHECKOUT_REF = "" - CONTEXT = ROOT_CONTEXT + "/push" - MESSAGE = data.head_commit.message - currentBuild.description = "${SHORT_HASH} - ${COMMIT_BRANCH} - ${MESSAGE}" - -// ---- Pull Request ---- -} else if (data.pull_request != null \ - && data.number > 0 \ - && (data.action == "opened" || data.action == "reopened" || data.action == "synchronize")) { - BUILD_TYPE = "pr" - COMMIT_HASH = data.pull_request.head.sha - SHORT_HASH = COMMIT_HASH.substring(0, 7) - PR_BRANCH = data.pull_request.head.label - COMMIT_BRANCH = data.pull_request.base.ref - COMMIT_URL = data.pull_request.html_url - BASE_HASH = data.pull_request.base.sha - SENDER = data.pull_request.user.login - PR_NUMBER = data.pull_request.number - CHECKOUT_BRANCH = COMMIT_HASH - CHECKOUT_REF = "+refs/pull/*:refs/remotes/origin/pr/* +refs/heads/*:refs/remotes/origin/*" - CONTEXT = ROOT_CONTEXT + "/pr" - MESSAGE = data.pull_request.title - currentBuild.description = "${SHORT_HASH} - PR #${PR_NUMBER} - ${MESSAGE}" - -// ---- No need to build ---- -} else { - currentBuild.result = "SUCCESS" - currentBuild.description = "---" - return -} - - -// ====== STAGES ======= -data = null -String STRIPPED_BRANCH = COMMIT_BRANCH.replaceAll("/", "_") - -// ------------------------------- -// 1. Setting Status -try { - githubNotify account: USERNAME, context: ROOT_CONTEXT, credentialsId: 'ppaquette-jenkins-api-as-password', description: PENDING_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'PENDING', targetUrl: '' -} catch(err) { - echo "Error setting status on GitHub commit: ${err}" - exit_status = 1 - currentBuild.result = "FAILURE" -} - -// ------------------------------- -// 2. Running tests -parallel_test = [:] - -// Running Python 3.7 everytime, except for "web", "build", "build_rw" branch -parallel_test['python_3.7'] = { - node('ubuntu-1604-small') { - stage('Python 3.7') { - try { - timeout(time: 20, activity: true, unit: 'MINUTES') { - githubNotify account: USERNAME, context: CONTEXT + "/python3.7", credentialsId: 'ppaquette-jenkins-api-as-password', description: PENDING_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'PENDING', targetUrl: '' - withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-ssh-key', keyFileVariable: 'SSH_KEY_FILE', passphraseVariable: '', usernameVariable: '')]) { - checkout([$class: 'GitSCM', branches: [[name: CHECKOUT_BRANCH]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: CHECKOUT_REF, credentialsId: 'jenkins-ssh-key', url: GITHUB_URL]]]) - sh ".jenkins/merge_pr.sh $PR_NUMBER $COMMIT_HASH $BASE_HASH $SSH_KEY_FILE \ - && wget -nv https://storage.googleapis.com/ppaquette-diplomacy/files/Miniconda3-py37.sh -O miniconda.sh \ - && chmod +x ./miniconda.sh \ - && rm -Rf $HOME/miniconda \ - && ./miniconda.sh -b -p $HOME/miniconda \ - && export PATH=$HOME/miniconda/bin:$PATH \ - && conda create -y -q -n py37 python=3.7 \ - && export PATH=$HOME/miniconda/envs/py37/bin:$PATH \ - && mkdir -p $HOME/.cache/diplomacy \ - && wget -nv https://storage.googleapis.com/ppaquette-diplomacy/cache-jenkins/convoy_paths_cache.pkl -O $HOME/.cache/diplomacy/convoy_paths_cache.pkl \ - && .jenkins/get_cache.sh $PR_NUMBER $STRIPPED_BRANCH \ - && pip install -r requirements.txt \ - && pip install -r requirements_dev.txt \ - && touch run_install_nvm.sh \ - && chmod +x run_install_nvm.sh \ - && ./run_install_nvm.sh \ - && echo '--------------------------------------------------' \ - && pip --version \ - && python --version \ - && which python \ - && git rev-parse HEAD \ - && git log -n 1 \ - && ./run_tests.sh \ - && .jenkins/set_cache.sh $PR_NUMBER $STRIPPED_BRANCH" - } - githubNotify account: USERNAME, context: CONTEXT + "/python3.7", credentialsId: 'ppaquette-jenkins-api-as-password', description: SUCCESS_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'SUCCESS', targetUrl: '' - } - } catch(err) { - echo "Error encountered in parallel (Python 3.7): ${err}" - exit_status = 1 - currentBuild.result = "FAILURE" - githubNotify account: USERNAME, context: CONTEXT + "/python3.7", credentialsId: 'ppaquette-jenkins-api-as-password', description: FAILURE_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'FAILURE', targetUrl: '' - } - } - } -} - -// Only running 3.5, and 3.6 for Pull Requests (and for "jenkins" branch) - -if (BUILD_TYPE == 'pr' || (BUILD_TYPE == 'push' && COMMIT_BRANCH == 'jenkins')) { - parallel_test['python_3.6'] = { - node('ubuntu-1604-small') { - stage('Python 3.6') { - try { - timeout(time: 20, activity: true, unit: 'MINUTES') { - githubNotify account: USERNAME, context: CONTEXT + "/python3.6", credentialsId: 'ppaquette-jenkins-api-as-password', description: PENDING_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'PENDING', targetUrl: '' - withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-ssh-key', keyFileVariable: 'SSH_KEY_FILE', passphraseVariable: '', usernameVariable: '')]) { - checkout([$class: 'GitSCM', branches: [[name: CHECKOUT_BRANCH]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: CHECKOUT_REF, credentialsId: 'jenkins-ssh-key', url: GITHUB_URL]]]) - sh ".jenkins/merge_pr.sh $PR_NUMBER $COMMIT_HASH $BASE_HASH $SSH_KEY_FILE \ - && wget -nv https://storage.googleapis.com/ppaquette-diplomacy/files/Miniconda3-py37.sh -O miniconda.sh \ - && chmod +x ./miniconda.sh \ - && rm -Rf $HOME/miniconda \ - && ./miniconda.sh -b -p $HOME/miniconda \ - && export PATH=$HOME/miniconda/bin:$PATH \ - && conda create -y -q -n py36 python=3.6 \ - && export PATH=$HOME/miniconda/envs/py36/bin:$PATH \ - && mkdir -p $HOME/.cache/diplomacy \ - && wget -nv https://storage.googleapis.com/ppaquette-diplomacy/cache-jenkins/convoy_paths_cache.pkl -O $HOME/.cache/diplomacy/convoy_paths_cache.pkl \ - && .jenkins/get_cache.sh $PR_NUMBER $STRIPPED_BRANCH \ - && pip install -r requirements.txt \ - && pip install -r requirements_dev.txt \ - && touch run_install_nvm.sh \ - && chmod +x run_install_nvm.sh \ - && ./run_install_nvm.sh \ - && echo '--------------------------------------------------' \ - && pip --version \ - && python --version \ - && which python \ - && git rev-parse HEAD \ - && git log -n 1 \ - && ./run_tests.sh \ - && .jenkins/set_cache.sh $PR_NUMBER $STRIPPED_BRANCH" - } - githubNotify account: USERNAME, context: CONTEXT + "/python3.6", credentialsId: 'ppaquette-jenkins-api-as-password', description: SUCCESS_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'SUCCESS', targetUrl: '' - } - } catch(err) { - echo "Error encountered in parallel (Python 3.6): ${err}" - exit_status = 1 - currentBuild.result = "FAILURE" - githubNotify account: USERNAME, context: CONTEXT + "/python3.6", credentialsId: 'ppaquette-jenkins-api-as-password', description: FAILURE_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'FAILURE', targetUrl: '' - } - } - } - } - parallel_test['python_3.5'] = { - node('ubuntu-1604-small') { - stage('Python 3.5') { - try { - timeout(time: 20, activity: true, unit: 'MINUTES') { - githubNotify account: USERNAME, context: CONTEXT + "/python3.5", credentialsId: 'ppaquette-jenkins-api-as-password', description: PENDING_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'PENDING', targetUrl: '' - withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-ssh-key', keyFileVariable: 'SSH_KEY_FILE', passphraseVariable: '', usernameVariable: '')]) { - checkout([$class: 'GitSCM', branches: [[name: CHECKOUT_BRANCH]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CleanBeforeCheckout']], submoduleCfg: [], userRemoteConfigs: [[name: 'origin', refspec: CHECKOUT_REF, credentialsId: 'jenkins-ssh-key', url: GITHUB_URL]]]) - sh ".jenkins/merge_pr.sh $PR_NUMBER $COMMIT_HASH $BASE_HASH $SSH_KEY_FILE \ - && wget -nv https://storage.googleapis.com/ppaquette-diplomacy/files/Miniconda3-py37.sh -O miniconda.sh \ - && chmod +x ./miniconda.sh \ - && rm -Rf $HOME/miniconda \ - && ./miniconda.sh -b -p $HOME/miniconda \ - && export PATH=$HOME/miniconda/bin:$PATH \ - && conda create -y -q -n py35 python=3.5 \ - && export PATH=$HOME/miniconda/envs/py35/bin:$PATH \ - && mkdir -p $HOME/.cache/diplomacy \ - && wget -nv https://storage.googleapis.com/ppaquette-diplomacy/cache-jenkins/convoy_paths_cache.pkl -O $HOME/.cache/diplomacy/convoy_paths_cache.pkl \ - && .jenkins/get_cache.sh $PR_NUMBER $STRIPPED_BRANCH \ - && pip install -r requirements.txt \ - && pip install -r requirements_dev.txt \ - && touch run_install_nvm.sh \ - && chmod +x run_install_nvm.sh \ - && ./run_install_nvm.sh \ - && echo '--------------------------------------------------' \ - && pip --version \ - && python --version \ - && which python \ - && git rev-parse HEAD \ - && git log -n 1 \ - && ./run_tests.sh \ - && .jenkins/set_cache.sh $PR_NUMBER $STRIPPED_BRANCH" - } - githubNotify account: USERNAME, context: CONTEXT + "/python3.5", credentialsId: 'ppaquette-jenkins-api-as-password', description: SUCCESS_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'SUCCESS', targetUrl: '' - } - } catch(err) { - echo "Error encountered in parallel (Python 3.5): ${err}" - exit_status = 1 - currentBuild.result = "FAILURE" - githubNotify account: USERNAME, context: CONTEXT + "/python3.5", credentialsId: 'ppaquette-jenkins-api-as-password', description: FAILURE_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'FAILURE', targetUrl: '' - } - } - } - } -} -parallel parallel_test - -// ------------------------------- -// 3. Updating git status -if (exit_status == 1) { - currentBuild.result = "FAILURE" - githubNotify account: USERNAME, context: ROOT_CONTEXT, credentialsId: 'ppaquette-jenkins-api-as-password', description: FAILURE_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'FAILURE', targetUrl: '' -} else { - currentBuild.result = "SUCCESS" - githubNotify account: USERNAME, context: ROOT_CONTEXT, credentialsId: 'ppaquette-jenkins-api-as-password', description: SUCCESS_MSG, gitApiUrl: '', repo: REPOSITORY, sha: COMMIT_HASH, status: 'SUCCESS', targetUrl: '' -} diff --git a/.jenkins/get_cache.sh b/.jenkins/get_cache.sh deleted file mode 100755 index a589bd2..0000000 --- a/.jenkins/get_cache.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Validating number of arguments -if [ "$#" -ne 2 ]; then - echo "Expected 2 arguments" - echo "Syntax: ./get_cache.sh " - echo "Use PR_NUMBER=0 if not a PR" - exit 1 -fi - -RELEASE=$(lsb_release -c -s) -PYTHON_VERSION=$(python -c "import sys; print('py%d%d' % (sys.version_info.major, sys.version_info.minor))") - -# Trying to download PR cache -if [ "$1" != "0" ]; then - CACHE_FILE="cache-pr_$1-$PYTHON_VERSION-$RELEASE.zip" - CACHE_PATH="gs://ppaquette-diplomacy/cache-jenkins/game-$CACHE_FILE" - gsutil -q stat "$CACHE_PATH" - RET_VALUE=$? - - if [ $RET_VALUE == 0 ]; then - echo "Downloading cache from $CACHE_PATH" - gsutil cp $CACHE_PATH . - unzip -qo $CACHE_FILE -d / - exit 0 - else - echo "No cache found at $CACHE_PATH" - fi -fi - -# Trying to download branch cache -CACHE_FILE="cache-$2-$PYTHON_VERSION-$RELEASE.zip" -CACHE_PATH="gs://ppaquette-diplomacy/cache-jenkins/game-$CACHE_FILE" -gsutil -q stat "$CACHE_PATH" -RET_VALUE=$? - -if [ $RET_VALUE == 0 ]; then - echo "Downloading cache from $CACHE_PATH" - gsutil cp $CACHE_PATH . - unzip -qo $CACHE_FILE -d / - exit 0 -else - echo "No cache found at $CACHE_PATH" -fi diff --git a/.jenkins/jenkins-agent.json b/.jenkins/jenkins-agent.json deleted file mode 100644 index a554e4b..0000000 --- a/.jenkins/jenkins-agent.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "builders": [ - { - "type": "googlecompute", - "project_id": "ppaquette-diplomacy", - "source_image_family": "ubuntu-1604-lts", - "source_image_project_id": "ubuntu-os-cloud", - "zone": "northamerica-northeast1-a", - "disk_size": "10", - "image_name": "jenkins-slave-{{timestamp}}", - "image_family": "jenkins-slave", - "ssh_username": "ubuntu" - } - ], - "provisioners": [ - { - "type": "shell", - "inline": ["sudo apt-get update -y", - "sudo apt-get upgrade -y", - "sudo apt-get install -y default-jdk git wget build-essential zip bzip2", - "sudo apt-get purge -y unattended-upgrades", - "sudo sed -i 's/1/0/g' /etc/apt/apt.conf.d/10periodic"] - } - ] -} diff --git a/.jenkins/merge_pr.sh b/.jenkins/merge_pr.sh deleted file mode 100755 index 4965151..0000000 --- a/.jenkins/merge_pr.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Validating number of arguments -if [ "$#" -ne 4 ]; then - echo "Expected 4 arguments" - echo "Syntax: ./merge_pr.sh " - echo "Use PR_NUMBER=0 if not a PR" - exit 1 -fi - -# Fetching merged head -if [ "$1" != "0" ]; then - PR_NUMBER=$1 - HASH_PR_HEAD=$2 - HASH_BASE_HEAD=$3 - SSH_KEY_PATH=$4 - - # Copying SSH key - mkdir -p $HOME/.ssh - sudo cp $SSH_KEY_PATH $HOME/.ssh/id_rsa - sudo chown $USER:$USER $HOME/.ssh/id_rsa - sudo chmod 400 $HOME/.ssh/id_rsa - - # Setting identity - git config --global user.email "jenkins@diplomacy.ai" - git config --global user.name "Jenkins (diplomacy.ai)" - - # Displaying hashes - echo "PR Head: $HASH_PR_HEAD" - echo "Base Head: $HASH_BASE_HEAD" - echo "PR #${PR_NUMBER} - Merging ${HASH_PR_HEAD::7} into ${HASH_BASE_HEAD::7}" - - # Fetching and merging - git fetch origin +refs/pull/*:refs/remotes/origin/pr/* +refs/heads/*:refs/remotes/origin/* - git checkout -qf $HASH_BASE_HEAD - git merge --no-ff $HASH_PR_HEAD -m "PR #${PR_NUMBER} - Merge ${HASH_PR_HEAD::7} into ${HASH_BASE_HEAD::7}" -fi diff --git a/.jenkins/set_cache.sh b/.jenkins/set_cache.sh deleted file mode 100755 index e51ca77..0000000 --- a/.jenkins/set_cache.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -# Validating number of arguments -if [ "$#" -ne 2 ]; then - echo "Expected 2 arguments" - echo "Syntax: ./set_cache.sh " - echo "Use PR_NUMBER=0 if not a PR" - exit 1 -fi - -RELEASE=$(lsb_release -c -s) -PYTHON_VERSION=$(python -c "import sys; print('py%d%d' % (sys.version_info.major, sys.version_info.minor))") - -# Trying to set cache -if [ "$1" != "0" ]; then - CACHE_FILE="cache-pr_$1-$PYTHON_VERSION-$RELEASE.zip" -else - CACHE_FILE="cache-$2-$PYTHON_VERSION-$RELEASE.zip" -fi - -CACHE_PATH="gs://ppaquette-diplomacy/cache-jenkins/game-$CACHE_FILE" -zip -qr $CACHE_FILE $HOME/.cache/pip/ -echo "Uploading cache to $CACHE_PATH" -gsutil cp ./$CACHE_FILE $CACHE_PATH -exit 0 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2f298dc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +cache: + directories: + - $HOME/.cache/pip + - $HOME/.cache/diplomacy +install: + - mkdir -p $HOME/.cache/diplomacy + - wget -nv https://storage.googleapis.com/ppaquette-diplomacy/cache-jenkins/convoy_paths_cache.pkl -O $HOME/.cache/diplomacy/convoy_paths_cache.pkl + - pip install -r requirements.txt + - pip install -r requirements_dev.txt + - touch run_install_nvm.sh + - chmod +x run_install_nvm.sh + - ./run_install_nvm.sh +script: ./run_tests.sh +jobs: + include: + - os: linux + sudo: required + language: python + python: 3.5 + + - os: linux + sudo: required + language: python + python: 3.6 + + - os: linux + sudo: required + language: python + python: 3.7 diff --git a/requirements_dev.txt b/requirements_dev.txt index c1984ef..30fee7e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,5 @@ -e . -r requirements.txt pylint>=2.3.0 -pytest +pytest>=4.4.0 pytest-xdist -- cgit v1.2.3