Continuous Deploy

.gitignore

@@ -30,6 +30,9 @@ desktop.ini
 /code/build
 /code/logs
 /code/graphs
+/code/deploy
+/code/website/_cache
+/code/website/_site
 *.synctex.gz
 *.pyc
 __pycache__

.travis.yml

@@ -22,9 +22,11 @@ branches:
 env:
   global:
     - secure: "zpSgEG8GvxOl8qJl+3OLiW7p5KdvzOOkC9g2ukLftmJ5cQUO2c7Zr7g/L/6ZQak96eDjyinV953laKXnrgy0BhZV1lnwPrnfEnvIV8BRkxIJ2XjMv3oNmwvAX5ayAKzeE7LBqeQyuFf83K+NP8QV5N432Ar9PryoJC5HvK4B/iQ1SuzWYcFSSuowNC1oukQUCKBVyzuhjv88VrZWqj2pLb7QgWlKNfokMsyYgNSytbu4A8xBn3zd51z2wgIjvfQlW9YnmENrloqPd4EcTnnjE+/2g4OSJ5ULWAu3n2QWmO5X5MhI7yGfQsH8fSR1pPSjw6iGYQ8G9woci5kyWCKCYZvYMUWUEZY6+vP+xhknRxkN9X/s9v6S9/Di2kKU0kOxcZC4p+9RF0NT3cZDzValhhRINAiprQYX8ri5G+BOq2fsKyENezvc+OuLvEHXhvskMuuZUJuWYB0ScUYooqIRgJE79Fr3d1EjDM0cQwFrMbOt6LF1P3wgPMo06Zy+4dfgqEx/heeKxX1yNFFbdd5WaSqhWac/7s4Y4v4UKHwbpKwZ1KjlW464VC/Dtwdwc/yS/iHRlcTdBjLl3MNdBA2sdxbDfsuao5osnCLdnt/qQ5B4cRpy3fJkbsmyNfLF57aE3bqpneUWHWrmbhgQ5PSRW6nyKpqgQkyzFD8B0WbfzIM="
+    - secure: "OiCqBb3REAAkRLkynqsC9pN0K3b2oBAOCaFi2wGeeOMWbyql6PTUUaa1effRmb5l9llqVW3MN2g3iaFp3PjghQeoUIYPJnZ6QptR90j6L07UxQI/ZgEHyTYB3HKOxkHPeRk298HEvW36aFNik6Dmt6prnzElYsEeh6sPDiGYHO+cKWtvvUCM89qt9/aFcYE4Rn4Omr3+5uN62L/9Fi0XGKdnA+jORaqt6ARHYmycvqN407xHbvHQNun27ft1VOFLDCmOLFJJa7Nc462yy11DHKtLWcT3bj5oa1cK79o8Frp4KFSbZ8HXKFVz/AdR2fKCE/Iptek0TAJjR01YQOR6PcSrBbHiCMcZ83izyb6svourSJN30/wBxQwGedL7/f/p9NFOdF5AgK90Q+emQYDUDFXp6T4Sl1WFzV41wwdF6scfptrRLMYKNGm0QOxr8/TcyQk8GorlQM4NIOCGZRXx1oXcc2F3jXZ6vnbnGE0bl2m2vG1JYgdXCoTiCZbTJgJn6ZnfVT58JiVXzFCW9D9GPshMHwoTo/3CGfq1EObndyzwgiSgPgNFn58dT699Qu8DVlMgWcbMr+OGa+4hMGCpsD/qHF9qXmcZuJXaXo+T/Bua20a1ShAbP5OUT5ZcofI4N+N1QnxJv0N7pX1ts26fiWJV5OjE27sZP/Kt3X2yRew="
     - LABEL_FILE=/tmp/ci_build_failures
     - MANAGED_LABEL_FILE=/tmp/ci_managed_labels
     - ALL_FUNCTIONS_FILE=$(mktemp)
+    - SHELL_OPTS_FILE=$(mktemp)
 
 # Caching so the next build will be fast too.
 cache:
@@ -76,8 +78,17 @@ script:
      ci_fstep "CI: Code" make code
  - >-
      ci_fstep "CI: Docs" make docs haddockArgs=\"--no-haddock-deps --no-haddock-hyperlink-source\"
+ - >-
+     ci_fstep "CI: Deploy" sh scripts/maybe_test_deploy.sh
  - python3 scripts/pr_label_update.py
 
+deploy:
+  provider: script
+  skip_cleanup: true
+  script: bash scripts/deploy_wrapper.bash
+  on:
+    branch: master
+
 notifications:
   email:
     - szymczdm@mcmaster.ca

code/Makefile

@@ -44,16 +44,18 @@ TEST_E_SUFFIX = _diff
 MOVE_DF_E_SUFFIX = _prog
 TEX_E_SUFFIX = _tex
 CODE_E_SUFFIX = _code
+DCP_E_SUFFIX = _deploy_code_path
 
 GEN_EXAMPLES = $(addsuffix $(GEN_E_SUFFIX), $(EXAMPLES))
 TEST_EXAMPLES =  $(addsuffix $(TEST_E_SUFFIX), $(EXAMPLES))
 MOVE_DF_EXAMPLES =  $(addsuffix $(MOVE_DF_E_SUFFIX), $(EXAMPLES))
 TEX_EXAMPLES = $(addsuffix $(TEX_E_SUFFIX), $(EXAMPLES))
 CODE_EXAMPLES = $(addsuffix $(CODE_E_SUFFIX), $(EXAMPLES))
+DCP_EXAMPLES = $(addsuffix $(DCP_E_SUFFIX), $(EXAMPLES))
 
-EXAMPLE_GEN_TARGET = GEN TEST MOVE_DF TEX CODE
+EXAMPLE_GEN_TARGET = GEN TEST MOVE_DF TEX CODE DCP
 
-GENNED_FOLDERS = $(LOG_FOLDER_NAME) $(BUILD_FOLDER_NAME) $(GRAPH_FOLDER_NAME)
+GENNED_FOLDERS = $(LOG_FOLDER_NAME) $(BUILD_FOLDER_NAME) $(GRAPH_FOLDER_NAME) $(DEPLOY_FOLDER_NAME)
 CLEAN_GF_PREFIX = clean_
 CLEAN_FOLDERS = $(addprefix $(CLEAN_GF_PREFIX), $(GENNED_FOLDERS))
 
@@ -74,7 +76,9 @@ LOG_FOLDER = $(LOG_FOLDER_NAME)/
 SCRIPT_FOLDER = scripts/
 GRAPH_FOLDER_NAME = graphs
 GRAPH_FOLDER = $(GRAPH_FOLDER_NAME)/
-
+DEPLOY_FOLDER_NAME = deploy
+DEPLOY_FOLDER = $(DEPLOY_FOLDER_NAME)/
+DEPLOY_CODE_PATH_KV_SEP = :
 # make command line options
 #  GHC debug options
 PROFALL = --executable-profiling --library-profiling
@@ -186,6 +190,28 @@ $(filter %$(CODE_E_SUFFIX), $(CODE_EXAMPLES)): %$(CODE_E_SUFFIX): %$(MOVE_DF_E_S
 
 code: $(CODE_EXAMPLES)
 
+%$(DCP_E_SUFFIX): EXAMPLE=$(shell echo $* | tr a-z A-Z)
+%$(DCP_E_SUFFIX): EDIR=$($(EXAMPLE)_DIR)
+$(filter %$(DCP_E_SUFFIX), $(DCP_EXAMPLES)): %$(DCP_E_SUFFIX):
+# Print as formatted K/V for deploy to extract
+	@echo "$(EDIR)$(DEPLOY_CODE_PATH_KV_SEP)$(shell git rev-parse --show-prefix)stable/$*/src/"
+
+deploy_code_path: $(DCP_EXAMPLES)
+
+deploy_lite:
+	@mkdir -p $(DEPLOY_FOLDER)
+	@BUILD_FOLDER=$(BUILD_FOLDER) DEPLOY_FOLDER=$(DEPLOY_FOLDER) GRAPH_FOLDER=$(GRAPH_FOLDER) \
+	DEPLOY_CODE_PATH_KV_SEP=$(DEPLOY_CODE_PATH_KV_SEP) MAKE=$(MAKE) "$(SHELL)" $(SCRIPT_FOLDER)deploy_stage.sh
+
+# This rule is for use with developing deployment layout locally. As part of it, it ensures all needed
+# dependencies exist. One of the downsides is we ensure all files are re-generated which means the TeX
+# files appear newer and thus PDF's are regenerated. If you want to "just generate the structure,
+# everything exists," (or you've run `deploy` once already) then `deploy_lite` does just that and is
+# what `deploy.bash` calls. 
+deploy: graphs docs tex
+	$(MAKE) deploy_lite
+
+
 # follow two are for cleaning things out
 $(filter $(CLEAN_GF_PREFIX)%, $(CLEAN_FOLDERS)): $(CLEAN_GF_PREFIX)%:
 	- rm -r "./$*"
@@ -194,4 +220,4 @@ clean: $(CLEAN_FOLDERS)
 	- stack clean
 	- rm $(CACHED_MSV_FILE)
 
-.PHONY: clean code tex doc debug prog test graphs graphmod check_stack all $(ALL_EXPANDED_TARGETS)
+.PHONY: clean code tex doc debug prog test graphs graphmod check_stack graphs deploy_code_path deploy deploy_lite all $(ALL_EXPANDED_TARGETS)

code/scripts/ci_functions.bash

@@ -20,6 +20,7 @@ fi
 
 ci_export_funcs() {
   declare -f > "$ALL_FUNCTIONS_FILE"
+  echo $(shopt | grep -E "on$" | cut -f1 | sed -E "s/^([^ ]*) *$/-O \1/g") > "$SHELL_OPTS_FILE"
 }
 
 ci_fstep() {

code/scripts/deploy.bash

@@ -1,14 +1,16 @@
+SOURCE_BRANCH="master"
 DEPLOY_BRANCH="gh-pages"
-DEPLOY_FOLDER="deploy"
+DEPLOY_FOLDER="deploy/"
 BUILD_NUMBER_FILE=".build-num"
+COMMIT_HASH_FILE=".commit-hash"
 
-if [ "$TRAVIS_EVENT_TYPE" != "push" ]; then
-  echo "Deployment only occurs for push builds."
+if [[ "$TRAVIS_EVENT_TYPE" != "cron" && "$TRAVIS_EVENT_TYPE" != "api" ]]; then
+  echo "Deployment only occurs for cron (or api -- manually run) builds."
   exit 0
 fi
 
-if [ "$TRAVIS_BRANCH" != "master" ]; then
-  echo "Only perform deploys for master."
+if [ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
+  echo "Only perform deploys for $SOURCE_BRANCH."
   echo "Skipping."
   exit 0
 fi
@@ -28,17 +30,15 @@ if [ -z "$BOT_TOKEN" ]; then
   exit 1
 fi
 
-source "$ALL_FUNCTIONS_FILE"
+if [ -z "$BOT_EMAIL" ]; then
+  echo "Assuming dummy email"
+  BOT_EMAIL="drasil-bot@local"
+fi
 
-copy_docs() {
-  rm -r docs
-  DOC_DIR=$(stack path | grep local-doc-root | cut -d":" -f2 | sed -e "s/^ //")/
-  mkdir -p docs
-  cp -r "$DOC_DIR" docs
-}
+source "$ALL_FUNCTIONS_FILE"
 
 try_deploy() {
-  git clone --quiet --branch="$DEPLOY_BRANCH"  --depth=1 "https://github.com/$TRAVIS_REPO_SLUG.git" "$DEPLOY_FOLDER"
+  git clone --quiet --branch="$DEPLOY_BRANCH" --depth=5 "https://github.com/$TRAVIS_REPO_SLUG.git" "$DEPLOY_FOLDER"
   if [ $? = 1 ]; then
     echo "Clone failed. Bailing."
     exit 1
@@ -52,19 +52,34 @@ try_deploy() {
     return 0
   fi
 
+  CLONED_HASH=$(cat "$COMMIT_HASH_FILE")
+
+  if [ $CLONED_HASH = $TRAVIS_COMMIT ]; then
+    echo "Deploy would be from same hash as current. Skipping it."
+    return 0
+  fi
+
   echo $TRAVIS_BUILD_NUMBER > "$BUILD_NUMBER_FILE"
-  copy_docs
+  echo $TRAVIS_COMMIT > "$COMMIT_HASH_FILE"
+  cd "$CUR_DIR"
+  make deploy_lite DEPLOY_FOLDER="$DEPLOY_FOLDER"
+  MAKE_RET=$?
+  if [ $MAKE_RET != 0 ]; then
+    echo "Making the deploy folder failed! Failing deploy."
+    return $MAKE_RET
+  fi
+  cd "$DEPLOY_FOLDER"
 
-  git config user.email "drasil-bot@local"
+  git config user.email "$BOT_EMAIL"
   git config user.name "drasil-bot"
   git add -A .
   # We overwrite history because the artifacts we keep in here are moderately large and woiuld pollute history otherwise.
-  git commit -q --amend --allow-empty --reset-author -m "drasil-bot deploy of master@$TRAVIS_COMMIT"
-  git push --force-with-lease --quiet "https://$BOT_TOKEN@github.com/$TRAVIS_REPO_SLUG.git" "$DEPLOY_BRANCH" >/dev/null 2>&1
+  git commit -q --allow-empty -m "drasil-bot deploy of $SOURCE_BRANCH@$TRAVIS_COMMIT"
+  git push --quiet "https://$BOT_TOKEN@github.com/$TRAVIS_REPO_SLUG.git" "$DEPLOY_BRANCH"
   PUSH_RET=$?
   # Perform some cleanup so we can (optionally retry)
   cd "$CUR_DIR"
-  rm -r "$DEPLOY_FOLDER"
+  rm -rf "$DEPLOY_FOLDER"
   # git push returns >0 if push fails (i.e. we would need to force push)
   return $PUSH_RET
 }

code/scripts/deploy_stage.sh

@@ -0,0 +1,92 @@
+if [ -z "$DEPLOY_FOLDER" ]; then
+	echo "Need DEPLOY_FOLDER to know where to stage deploy."
+	exit 1
+fi
+
+if [ -z "$BUILD_FOLDER" ]; then
+	echo "Missing BUILD_FOLDER."
+	exit 1
+fi
+
+if [ -z "$GRAPH_FOLDER" ]; then
+	echo "Missing GRAPH_FOLDER."
+	exit 1
+fi
+
+if [ -z "$DEPLOY_CODE_PATH_KV_SEP" ]; then
+	echo "Missing DEPLOY_CODE_PATH_KV_SEP."
+	exit 1
+fi
+
+if [ -z "$MAKE" ]; then
+	echo "Missing MAKE. Did you invoke this with \`make deploy(_lite)\`?"
+	exit 1
+fi
+
+DOC_DEST=docs/
+SRS_DEST=srs/
+EXAMPLE_DEST=examples/
+CUR_DIR="$PWD/"
+
+
+copy_docs() {
+  DOC_DIR=$(stack path | grep local-doc-root | cut -d":" -f2 | sed -e "s/^ //")/
+  rm -r "$DOC_DEST" >/dev/null 2>&1  # Printing an error message that a directory doesn't exist isn't the most useful.
+  mkdir -p "$DOC_DEST"
+  cp -r "$DOC_DIR". "$DOC_DEST"
+}
+
+copy_datafiles() {
+	echo "FIXME: Drasil should copy needed images and resources to the appropriate output directory to avoid needing the entirety of datafiles (for HTML)."
+	rm -r datafiles  >/dev/null 2>&1  # Printing an error message that a directory doesn't exist isn't the most useful.
+	mkdir -p datafiles
+	cp -r "$CUR_DIR"datafiles/. datafiles/
+}
+
+copy_graphs() {
+  rm -r "$GRAPH_FOLDER" >/dev/null 2>&1  # Printing an error message that a directory doesn't exist isn't the most useful.
+  cp -r "$CUR_DIR$GRAPH_FOLDER". "$GRAPH_FOLDER"
+}
+
+copy_examples() {
+	for example in "$CUR_DIR$BUILD_FOLDER"*; do
+		example_name=$(basename $example)
+		mkdir -p "$EXAMPLE_DEST$example_name/$SRS_DEST"
+		if [ -d "$example/"SRS ]; then
+			cp "$example/"SRS/*.pdf "$EXAMPLE_DEST$example_name/$SRS_DEST"
+		fi
+		if [ -d "$example/"Website/ ]; then
+			cp -r "$example/"Website/. "$EXAMPLE_DEST$example_name/$SRS_DEST"
+		fi
+		if [ -d "$example/"src ]; then
+			# We don't expose code in deploy. It's more conveneient to link to GitHub's directory
+			# We place a stub file which Hakyll will replace.
+			echo $(cd "$CUR_DIR" && "$MAKE" deploy_code_path | grep "$example_name" | cut -d"$DEPLOY_CODE_PATH_KV_SEP" -f 2-) > "$EXAMPLE_DEST$example_name/src"
+		fi
+	done
+}
+
+build_website() {
+	cd "$CUR_DIR"website
+	make DEPLOY_FOLDER="$CUR_DIR$DEPLOY_FOLDER" DOCS_FOLDER="$DOC_DEST" EXAMPLES_FOLDER="$EXAMPLE_DEST" \
+	SRS_FOLDER_FRAG="$SRS_DEST" GRAPH_FOLDER="$GRAPH_FOLDER"
+	RET=$?
+	if [ $RET != 0 ]; then
+		echo "Build Failed. Bailing."
+		exit 1
+	fi
+	cd "$CUR_DIR$DEPLOY_FOLDER"
+	cp -r "$CUR_DIR"website/_site/. .
+
+	# src stubs were consumed by site generator; safe to delete those.
+	rm "$EXAMPLE_DEST"*/src
+}
+
+
+cd "$DEPLOY_FOLDER"
+copy_docs
+copy_graphs
+copy_datafiles
+copy_examples
+build_website
+cd "$CUR_DIR"

code/scripts/deploy_wrapper.bash

@@ -0,0 +1 @@
+bash $(cat "$SHELL_OPTS_FILE") scripts/deploy.bash

code/scripts/maybe_test_deploy.sh

@@ -0,0 +1,18 @@
+# This script may be a little overzealous in testing deploy (without actually deploying) based on what git diff returns.
+
+if [ -z "$TRAVIS_PULL_REQUEST_BRANCH" ]; then
+  exit 0
+fi
+
+CHANGED_FILES=$(git diff --name-only "$TRAVIS_BRANCH" | grep "website/" | wc -l)
+if [ $CHANGED_FILES -gt 0 ]; then
+  echo "Looks like there were changes to the deploy website."
+  make deploy_lite
+  RET=$?
+  echo "Printing index (in case template splicing went awry)"
+  cat "website/_site/index.html"
+  RET=$(( $RET || $? ))
+  exit $RET
+else
+  echo "No changes to test."
+fi

code/website/Makefile

@@ -0,0 +1,14 @@
+all: run
+
+build:
+	stack build --ghc-options="-Wall" website
+
+run: build
+	stack exec -- site rebuild
+
+clean:
+	-rm -r _site
+	-rm -r _cache
+	-stack clean
+
+.PHONY: all build clean run

code/website/index.html

@@ -0,0 +1,24 @@
+<h2>Drasil - Generate All the Things!</h2>
+<ul>
+$-for(examples)$
+  <li>$name$ Artifacts<br>
+    <ul>
+    $-for(srs)$
+      <li><a href="$url$">$name$ SRS [$type$]</a></li>$sep-$
+    $endfor$
+    $-if(src)$
+      <li><a href="$src-$">Generated Code</a></li>
+    $-endif$
+    </ul>
+  </li>$sep-$
+$endfor$
+</ul>
+$if(docsUrl)-$
+<h4><a href="$docsUrl$">Haskell Documentation</a></h4>
+$-endif$
+Package Dependencies
+<ul>
+  $-for(graphs)$
+    <li><a href="$url$">$name$</a></li>$sep-$
+  $endfor$
+</ul>