updating the nightly build and email templates

author: tgrogers <[email protected]> 2019-06-12 16:18:59 -0400
committer: tgrogers <[email protected]> 2019-06-12 16:18:59 -0400
commit: 933d0aa4f91588d17f8970c578b2c1d2bbc9fc9c (patch)
tree: 69a8efa2b214024c178310c87d14269349962d11
parent: 35b55f5313d079327be9e6c91de5a53a01030e3e (diff)
3 files changed, 112 insertions, 9 deletions
diff --git a/groovy-html.nightly.success.template b/groovy-html.nightly.success.template
new file mode 100644
index 0000000..210b924
--- /dev/null
+++ b/groovy-html.nightly.success.template
@@ -0,0 +1,74 @@
+<STYLE>
+  BODY, TABLE, TD, TH, P {
+    font-family: Courier New, Verdana, Helvetica, sans serif;
+    font-size: 10px;
+    color: black;
+  }
+  .console {
+    font-family: Courier New;
+  }
+  .filesChanged {
+    width: 10%;
+    padding-left: 10px;
+  }
+  .section {
+    width: 100%;
+    border: thin black dotted;
+  }
+  .td-title-main {
+    color: white;
+    font-size: 200%;
+    padding-left: 5px;
+    font-weight: bold;
+  }
+  .td-title {
+    color: white;
+    font-size: 120%;
+    font-weight: bold;
+    padding-left: 5px;
+    text-transform: uppercase;
+  }
+  .td-title-tests {
+    font-weight: bold;
+    font-size: 120%;
+  }
+  .td-header-maven-module {
+    font-weight: bold;
+    font-size: 120%;    
+  }
+  .td-maven-artifact {
+    padding-left: 5px;
+  }
+  .tr-title {
+    background-color: <%= (build.result == null || build.result.toString() == 'SUCCESS') ? '#27AE60' : build.result.toString() == 'FAILURE' ? '#E74C3C' : '#f4e242' %>;
+  }
+  .test {
+    padding-left: 20px;
+  }
+  .test-fixed {
+    color: #27AE60;
+  }
+  .test-failed {
+    color: #E74C3C;
+  }
+</STYLE>
+<BODY>
+
+<%
+    import hudson.model.*
+
+    def JOB_NAME= build.getEnvVars()["JOB_NAME"];
+    def BUILD_NUMBER= build.getEnvVars()["BUILD_NUMBER"];
+    def BUILD_URL= build.getEnvVars()["BUILD_URL"];
+%>
+
+See ${BUILD_URL} for full build detials.
+
+<pre>Statistics changes since last commit:
+<pre>    https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/jenkins/${JOB_NAME}/${BUILD_NUMBER}/
+<pre>        Correlation:
+<pre>            9.1:
+<pre>               https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/jenkins/${JOB_NAME}/${BUILD_NUMBER}/titanv-cycles.cuda-9.1.nightly.html
+<pre>               https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/jenkins/${JOB_NAME}/${BUILD_NUMBER}/titanv-cycles.cuda-9.1.nightly.apps.txt
+<pre>       Correlation also posted to: https://engineering.purdue.edu/tgrogers/group/correlator.html
+</BODY>
diff --git a/groovy-html.success.template b/groovy-html.success.template
index e433a09..974cc61 100644
--- a/groovy-html.success.template
+++ b/groovy-html.success.template
@@ -73,4 +73,5 @@ See ${BUILD_URL} for full build detials.
 <pre>            10.1:
 <pre>               https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/jenkins/${JOB_NAME}/${BUILD_NUMBER}/titanv-cycles.cuda-10.1.html
 <pre>               https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/jenkins/${JOB_NAME}/${BUILD_NUMBER}/titanv-cycles.cuda-10.1.apps.txt
+<pre>       Correlation also posted to: https://engineering.purdue.edu/tgrogers/group/correlator.html
 </BODY>
diff --git a/nightly.jenkinsfile b/nightly.jenkinsfile
index ce2e226..8dc5d28 100644
--- a/nightly.jenkinsfile
+++ b/nightly.jenkinsfile
@@ -37,30 +37,58 @@ pipeline {
         }
         stage('nightly-2B-insn-run'){
             steps {
+                    sh 'rm -rf ./gpgpu-sim_simulations/util/plotting/correl-html && rm -rf gpgpu-sim-results-repo &&\
+                        rm -rf ./gpgpu-sim_simulations/util/plotting/htmls'
+
                     sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\
-                    source `pwd`/setup_environment &&\
-                    ./gpgpu-sim_simulations/util/job_launching/run_simulations.py -B `cat ./gpgpu-sim_simulations/util/job_launching/apps/all-apps.list` -C TITANV-2B -N nightly-$$ && \
-                    PLOTDIR="jenkins/${JOB_NAME}/${BUILD_NUMBER}/getstats" && ssh [email protected] mkdir -p /home/dynamo/a/tgrogers/website/gpgpu-sim-plots/$PLOTDIR && \
-                    ./gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -I -S 1800 -v  -s stats-$$.csv -T 12 -N nightly-$$ && \
-                    ./gpgpu-sim_simulations/util/plotting/plot-get-stats.py -c stats-$$.csv -p [email protected]:~/website/gpgpu-sim-plots/$PLOTDIR -w https://engineering.purdue.edu/tgrogers/gpgpu-sim-plots/$PLOTDIR -n $PLOTDIR'
+                        source `pwd`/setup_environment &&\
+                        ./gpgpu-sim_simulations/util/job_launching/run_simulations.py \
+                            -B `cat ./gpgpu-sim_simulations/util/job_launching/apps/all-apps.list` -C TITANV-2B -N nightly-$$ && \
+                        ./gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -I -S 1800 -v \
+                            -s stats-per-app-9.1.csv -T 12 -N nightly-$$ && \
+                        ./gpgpu-sim_simulations/util/plotting/plot-get-stats.py -c stats-per-app-9.1.csv -P cuda-9.1.nightly'
             }
         }
         stage('nightly-correlate'){
             steps {
+                sh 'git clone [email protected]:purdue-aalp/gpgpu-sim-results-repo.git'
+
+                sh 'if [ ! -d ./gpgpu-sim-results-repo/${JOB_NAME} ]; then \
+                        mkdir -p ./gpgpu-sim-results-repo/${JOB_NAME}/ ; \
+                        cp ./gpgpu-sim-results-repo/purdue-aalp/gpgpu-sim_distribution/dev/* \
+                            ./gpgpu-sim-results-repo/${JOB_NAME}/ ; \
+                    fi && \
+                    ./gpgpu-sim_simulations/run_hw/get_hw_data.sh'
+
                 sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\
+                    ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k \
+                        -B `cat ./gpgpu-sim_simulations/util/job_launching/apps/all-apps.list` \
+                        -C TITANV-2B > stats-per-kernel-9.1.csv &&\
+                    ./gpgpu-sim_simulations/util/plotting/merge-stats.py \
+                        -c ./gpgpu-sim-results-repo/${JOB_NAME}/stats-per-app-9.1.csv,./stats-per-app-9.1.csv -R \
+                        > per-app-merge-9.1.csv &&\
+                    ./gpgpu-sim_simulations/util/plotting/merge-stats.py \
+                        -c ./gpgpu-sim-results-repo/${JOB_NAME}/stats-per-kernel-9.1.csv,./stats-per-kernel-9.1.csv -R \
+                        > per-kernel-merge-9.1.csv &&\
                     source `pwd`/setup_environment &&\
-                    PLOTDIR="jenkins/${JOB_NAME}" &&\
-                    ./gpgpu-sim_simulations/util/job_launching/get_stats.py -R -K -k -B `cat ./gpgpu-sim_simulations/util/job_launching/apps/all-apps.list` -C TITANV-2B > nightly-stats-per-kernel-9.1.csv &&\
-                    ./gpgpu-sim_simulations/util/plotting/correlate_and_publish.sh nightly-stats-per-kernel-9.1.csv $PLOTDIR ${BUILD_NUMBER}'
+                    ./gpgpu-sim_simulations/util/plotting/plot-correlation.py -c per-kernel-merge-9.1.csv \
+                        -p cuda-9.1.nightly | grep "Correl=" | tee correl.9.1.txt'
+
+                sh 'PLOTDIR="/home/dynamo/a/tgrogers/website/gpgpu-sim-plots/jenkins/${JOB_NAME}" &&\
+                    ssh [email protected] mkdir -p $PLOTDIR/${BUILD_NUMBER} && \
+                    scp  ./gpgpu-sim_simulations/util/plotting/correl-html/* [email protected]:$PLOTDIR/${BUILD_NUMBER} &&\
+                    scp  ./gpgpu-sim_simulations/util/plotting/htmls/* [email protected]:$PLOTDIR/${BUILD_NUMBER} &&\
+                    ssh [email protected] "cd $PLOTDIR && rm -rf latest && cp -r ${BUILD_NUMBER} latest"'
             }
         }
     }
     post {
         success {
-            emailext body: "See ${BUILD_URL}",
+            emailext body:'''${SCRIPT, template="groovy-html.nightly.success.template"}''',
                 recipientProviders: [[$class: 'CulpritsRecipientProvider'],
                     [$class: 'RequesterRecipientProvider']],
                 subject: "[AALP Jenkins] Build #${BUILD_NUMBER} - Success!",
+                attachmentsPattern: 'correl.*.txt',
                 to: '[email protected]'
         }
         failure {
author	tgrogers <[email protected]>	2019-06-12 16:18:59 -0400
committer	tgrogers <[email protected]>	2019-06-12 16:18:59 -0400
commit	933d0aa4f91588d17f8970c578b2c1d2bbc9fc9c (patch)
tree	69a8efa2b214024c178310c87d14269349962d11
parent	35b55f5313d079327be9e6c91de5a53a01030e3e (diff)