Add: [AzurePipeline] introducing a release pipeline

This release pipeline creates all the official release binaries,
and publishes them as artifacts. Currently it can only produce
nightlies and custom builds; stable/testing release binaries are
untested.

This commit also splits up the pipeline in small bits, to both
improve readability, and to share code with the CI pipeline where
possible.

17 files changed, 435 insertions, 0 deletions

diff --git a/azure-pipelines/changelog.sh b/azure-pipelines/changelog.sh
new file mode 100755
index 000000000..71bc56fac
--- /dev/null
+++ b/azure-pipelines/changelog.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+tag=$(git describe --tags 2>/dev/null)
+
+# If we are a tag, show the part of the changelog that matches the tag.
+# In case of a stable, also show all betas and RCs.
+if [ -n "$tag" ]; then
+    grep="."
+    if [ "$(echo $tag | grep '-')" = "" ]; then
+            grep='^[0-9]\.[0-9]\.[0-9][^-]'
+    fi
+    next=$(cat changelog.txt | grep '^[0-9]' | awk 'BEGIN { show="false" } // { if (show=="true") print $0; if ($1=="'$tag'") show="true"} ' | grep "$grep" | head -n1 | sed 's/ .*//')
+    cat changelog.txt | awk 'BEGIN { show="false" } /^[0-9].[0-9].[0-9]/ { if ($1=="'$next'") show="false"; if ($1=="'$tag'") show="true";} // { if (show=="true") print $0 }'
+    exit 0
+fi
+
+# In all other cases, show the git log of the last 7 days
+revdate=$(git log -1 --pretty=format:"%ci")
+last_week=$(date -u -d "$revdate -7days" +"%Y-%m-%d %H:%M")
+git log --after="${last_week}" --pretty=fuller
diff --git a/azure-pipelines/manifest.sh b/azure-pipelines/manifest.sh
new file mode 100755
index 000000000..4d1197277
--- /dev/null
+++ b/azure-pipelines/manifest.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+
+set -ex
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <folder-with-bundles>"
+    exit 1
+fi
+
+FOLDER=$1
+
+if [ ! -e .version ] || [ ! -e .release_date ]; then
+    echo "This script should be executed in the root of an extracted source tarball"
+    exit 1
+fi
+
+# Find the name based on the version
+if [ "${ISSTABLERELEASE}" = "true" ]; then
+    isTesting=$(cat .version | grep "RC\|beta" || true)
+    if [ -z "${isTesting}" ]; then
+        NAME="stable"
+    else
+        NAME="testing"
+    fi
+else
+    NAME=$(cat .version | cut -d- -f2 | cut -d- -f-2)
+fi
+
+# Convert the date to a YAML date
+DATE=$(cat .release_date | tr ' ' T | sed 's/TUTC/:00-00:00/')
+VERSION=$(cat .version)
+BASE="openttd-${VERSION}"
+
+echo "name: ${NAME}" >> manifest.yaml
+echo "date: ${DATE}" >> manifest.yaml
+echo "base: ${BASE}-" >> manifest.yaml
+echo "files:" >> manifest.yaml
+
+error=""
+for i in $(ls ${FOLDER} | grep -v ".txt$\|.md$\|sum$" | sort); do
+    if [ -n "$(echo $i | grep pdb.xz)" ]; then continue; fi
+    if [ -n "$(echo $i | grep dbg.deb)" ]; then continue; fi
+
+    if [ ! -e ${FOLDER}/$i.md5sum ] || [ ! -e ${FOLDER}/$i.sha1sum ] || [ ! -e ${FOLDER}/$i.sha256sum ]; then
+        echo "ERROR: missing checksum file for ${i}" 1>&2
+        error="y"
+        continue
+    fi
+
+    echo "- id: $i" >> manifest.yaml
+    echo "  size: $(stat -c"%s" ${FOLDER}/$i)" >> manifest.yaml
+    echo "  md5sum: $(cat ${FOLDER}/$i.md5sum | cut -d\  -f1)" >> manifest.yaml
+    echo "  sha1sum: $(cat ${FOLDER}/$i.sha1sum | cut -d\  -f1)" >> manifest.yaml
+    echo "  sha256sum: $(cat ${FOLDER}/$i.sha256sum | cut -d\  -f1)" >> manifest.yaml
+done
+
+if [ -n "${error}" ]; then
+    echo "ERROR: exiting due to earlier errors" 1>&2
+    exit 1
+fi
diff --git a/azure-pipelines/templates/ci-git-rebase.yml b/azure-pipelines/templates/ci-git-rebase.yml
new file mode 100644
index 000000000..924302fee
--- /dev/null
+++ b/azure-pipelines/templates/ci-git-rebase.yml
@@ -0,0 +1,9 @@
+steps:
+# Rebase to origin/master for every PR. This means users don't have to
+# rebase every time master changes. As long as the PR applies cleanly, we
+# will validate it.
+- script: |
+    git config user.email 'info@openttd.org'
+    git config user.name 'OpenTTD CI'
+    git rebase origin/master
+  displayName: 'Rebase to master'
diff --git a/azure-pipelines/templates/ci-opengfx.yml b/azure-pipelines/templates/ci-opengfx.yml
new file mode 100644
index 000000000..098f0d2a6
--- /dev/null
+++ b/azure-pipelines/templates/ci-opengfx.yml
@@ -0,0 +1,8 @@
+steps:
+- bash: |
+    set -ex
+    cd bin/baseset
+    curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
+    unzip opengfx-0.5.2-all.zip
+    rm -f opengfx-0.5.2-all.zip
+  displayName: 'Install OpenGFX'
diff --git a/azure-pipelines/templates/linux-build.yml b/azure-pipelines/templates/linux-build.yml
new file mode 100644
index 000000000..0e4fffefb
--- /dev/null
+++ b/azure-pipelines/templates/linux-build.yml
@@ -0,0 +1,18 @@
+parameters:
+  Image: ''
+  Tag: ''
+  ContainerCommand: ''
+
+steps:
+- task: Docker@1
+  ${{ if eq(parameters.Image, 'compile-farm') }}:
+    displayName: 'Build'
+  ${{ if eq(parameters.Image, 'compile-farm-ci') }}:
+    displayName: 'Build and test'
+  inputs:
+    command: 'Run an image'
+    imageName: openttd/${{ parameters.Image }}:${{ parameters.Tag }}
+    volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
+    workingDirectory: '$(Build.SourcesDirectory)'
+    containerCommand: ${{ parameters.ContainerCommand }}
+    runInBackground: false
diff --git a/azure-pipelines/templates/linux-claim-bundles.yml b/azure-pipelines/templates/linux-claim-bundles.yml
new file mode 100644
index 000000000..4434dfcc7
--- /dev/null
+++ b/azure-pipelines/templates/linux-claim-bundles.yml
@@ -0,0 +1,5 @@
+steps:
+# Because we run the compile in a docker (under root), we are not owner
+# of the 'bundles' folder. Fix that by executing a chown on it.
+- bash: sudo chown -R $(id -u):$(id -g) bundles
+  displayName: 'Claim bundles folder back'
diff --git a/azure-pipelines/templates/osx-build.yml b/azure-pipelines/templates/osx-build.yml
new file mode 100644
index 000000000..ae1724125
--- /dev/null
+++ b/azure-pipelines/templates/osx-build.yml
@@ -0,0 +1,5 @@
+steps:
+- script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
+  displayName: 'Configure'
+- script: 'make -j2'
+  displayName: 'Build'
diff --git a/azure-pipelines/templates/osx-dependencies.yml b/azure-pipelines/templates/osx-dependencies.yml
new file mode 100644
index 000000000..c4b723c58
--- /dev/null
+++ b/azure-pipelines/templates/osx-dependencies.yml
@@ -0,0 +1,11 @@
+steps:
+- script: |
+    set -ex
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
+    # Remove the dynamic libraries of these libraries, to ensure we use
+    # the static versions. That is important, as it is unlikely any
+    # end-user has these brew libraries installed.
+    rm /usr/local/Cellar/lzo/*/lib/*.dylib
+    rm /usr/local/Cellar/xz/*/lib/*.dylib
+    rm /usr/local/Cellar/libpng/*/lib/*.dylib
+  displayName: 'Install dependencies'
diff --git a/azure-pipelines/templates/release-bundles.yml b/azure-pipelines/templates/release-bundles.yml
new file mode 100644
index 000000000..9c5a48b96
--- /dev/null
+++ b/azure-pipelines/templates/release-bundles.yml
@@ -0,0 +1,19 @@
+parameters:
+  CalculateChecksums: true
+
+steps:
+- ${{ if eq(parameters.CalculateChecksums, true) }}:
+  - bash: |
+      set -ex
+      cd bundles
+      for i in $(ls); do
+        openssl dgst -r -md5 -hex $i > $i.md5sum
+        openssl dgst -r -sha1 -hex $i > $i.sha1sum
+        openssl dgst -r -sha256 -hex $i > $i.sha256sum
+      done
+    displayName: 'Calculate checksums'
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish bundles'
+  inputs:
+    PathtoPublish: bundles/
+    ArtifactName: bundles
diff --git a/azure-pipelines/templates/release-fetch-source.yml b/azure-pipelines/templates/release-fetch-source.yml
new file mode 100644
index 000000000..8b4398306
--- /dev/null
+++ b/azure-pipelines/templates/release-fetch-source.yml
@@ -0,0 +1,20 @@
+# Fetch the source tarball as prepared by an earlier job. In there is the
+# version predefined. This ensures we are all going to compile the same
+# source with the same version.
+
+steps:
+- checkout: none
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download source'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/openttd-*-source.tar.xz'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- bash: tar --xz -xf ../a/bundles/openttd-*-source.tar.xz --strip-components=1
+  displayName: 'Extracting source'
+- bash: |
+    set -e
+    VERSION=$(cat .version)
+    echo "${VERSION}"
+    echo "##vso[build.updatebuildnumber]${VERSION}"
+  displayName: 'Change BuildNumber to version'
diff --git a/azure-pipelines/templates/release-manifest.yml b/azure-pipelines/templates/release-manifest.yml
new file mode 100644
index 000000000..5c076b30e
--- /dev/null
+++ b/azure-pipelines/templates/release-manifest.yml
@@ -0,0 +1,13 @@
+steps:
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download all bundles'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/*'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- script: |
+    set -ex
+    ./azure-pipelines/manifest.sh ../a/bundles/
+    mkdir -p bundles
+    mv manifest.yaml bundles/
+  displayName: 'Create manifest.yaml'
diff --git a/azure-pipelines/templates/release-prepare-source.yml b/azure-pipelines/templates/release-prepare-source.yml
new file mode 100644
index 000000000..22ec126b1
--- /dev/null
+++ b/azure-pipelines/templates/release-prepare-source.yml
@@ -0,0 +1,24 @@
+# Set the revisions, and remove the VCS files.
+# This ensures everything else picks up on the predefined versions, and not
+# that because of some build process the version all of a sudden changes.
+
+steps:
+- script: |
+    set -ex
+    git checkout -B ${BUILD_SOURCEBRANCHNAME}
+    ./findversion.sh > .ottdrev
+    ./azure-pipelines/changelog.sh > .changelog
+    TZ='UTC' date +"%Y-%m-%d %H:%M UTC" > .release_date
+    cat .ottdrev | cut -f 1 -d$'\t' > .version
+    echo "Release Date: $(cat .release_date)"
+    echo "Revision: $(cat .ottdrev)"
+    echo "Version: $(cat .version)"
+  displayName: 'Create version files'
+- script: |
+    set -e
+    VERSION=$(cat .version)
+    echo "${VERSION}"
+    echo "##vso[build.updatebuildnumber]${VERSION}"
+  displayName: 'Change BuildNumber to version'
+- script: find . -iname .hg -or -iname .git -or -iname .svn | xargs rm -rf
+  displayName: 'Remove VCS information'
diff --git a/azure-pipelines/templates/release.yml b/azure-pipelines/templates/release.yml
new file mode 100644
index 000000000..4daf66ded
--- /dev/null
+++ b/azure-pipelines/templates/release.yml
@@ -0,0 +1,169 @@
+parameters:
+  # If this is false, not all targets are triggered. For example:
+  # The NSIS installer for Windows and the creation of debs only work for
+  # releases. Not for any other type of binary. So they are skilled if this
+  # is set to false.
+  IsStableRelease: false
+
+jobs:
+- job: source
+  displayName: 'Source'
+  pool:
+    vmImage: 'ubuntu-16.04'
+
+  steps:
+  - template: release-prepare-source.yml
+  - script: |
+      set -ex
+
+      # Rename the folder to openttd-NNN-source
+      mkdir openttd-$(Build.BuildNumber)
+      find . -maxdepth 1 -not -name . -not -name openttd-$(Build.BuildNumber) -exec mv {} openttd-$(Build.BuildNumber)/ \;
+      # Copy back release_date, as it is needed for the template 'release-bundles'
+      cp openttd-$(Build.BuildNumber)/.release_date .release_date
+
+      mkdir bundles
+      tar --xz -cf bundles/openttd-$(Build.BuildNumber)-source.tar.xz openttd-$(Build.BuildNumber)
+      zip -9 -r -q bundles/openttd-$(Build.BuildNumber)-source.zip openttd-$(Build.BuildNumber)
+    displayName: 'Create bundle'
+  - template: release-bundles.yml
+
+- job: meta
+  displayName: 'Metadata'
+  pool:
+    vmImage: 'ubuntu-16.04'
+  dependsOn: source
+
+  steps:
+  - template: release-fetch-source.yml
+  - script: |
+      set -ex
+
+      mkdir -p bundles
+      cp .changelog bundles/changelog.txt
+      cp .release_date bundles/released.txt
+      cp README.md bundles/README.md
+    displayName: 'Copy meta files'
+  - template: release-bundles.yml
+    parameters:
+      CalculateChecksums: false
+
+- job: docs
+  displayName: 'Docs'
+  pool:
+    vmImage: 'ubuntu-16.04'
+  dependsOn: source
+
+  steps:
+    - template: release-fetch-source.yml
+    - template: linux-build.yml
+      parameters:
+        Image: compile-farm
+        ContainerCommand: '$(Build.BuildNumber)'
+        Tag: docs
+    - template: linux-claim-bundles.yml
+    - template: release-bundles.yml
+
+- job: windows
+  displayName: 'Windows'
+  pool:
+    vmImage: 'VS2017-Win2016'
+  dependsOn: source
+
+  strategy:
+    matrix:
+      Win32:
+        BuildPlatform: 'Win32'
+        BundlePlatform: 'win32'
+      Win64:
+        BuildPlatform: 'x64'
+        BundlePlatform: 'win64'
+
+  steps:
+  - template: release-fetch-source.yml
+  - template: windows-dependencies.yml
+  - template: windows-dependency-zip.yml
+  - ${{ if eq(parameters.IsStableRelease, true) }}:
+    - template: windows-dependency-nsis.yml
+  - template: windows-build.yml
+    parameters:
+      BuildPlatform: $(BuildPlatform)
+  - bash: |
+      set -ex
+      make -f Makefile.msvc bundle_pdb bundle_zip PLATFORM=$(BundlePlatform) BUNDLE_NAME=openttd-$(Build.BuildNumber)-windows-$(BundlePlatform)
+    displayName: 'Create bundles'
+  - ${{ if eq(parameters.IsStableRelease, true) }}:
+    - bash: |
+        set -ex
+        # NSIS will be part of the Hosted image in the next update. Till then, we set the PATH ourself
+        export PATH="${PATH}:/c/Program Files (x86)/NSIS"
+        make -f Makefile.msvc bundle_exe PLATFORM=$(BundlePlatform) BUNDLE_NAME=openttd-$(Build.BuildNumber)-windows-$(BundlePlatform)
+      displayName: 'Create installer bundle'
+  - template: release-bundles.yml
+
+- ${{ if eq(parameters.IsStableRelease, true) }}:
+  - job: linux_stable
+    displayName: 'Linux'
+    pool:
+      vmImage: 'ubuntu-16.04'
+    dependsOn: source
+
+    strategy:
+      matrix:
+        linux-ubuntu-xenial-i386-gcc: {}
+        linux-ubuntu-xenial-amd64-gcc: {}
+        linux-ubuntu-bionic-i386-gcc: {}
+        linux-ubuntu-bionic-amd64-gcc: {}
+        linux-debian-jessie-i386-gcc: {}
+        linux-debian-jessie-amd64-gcc: {}
+        linux-debian-stretch-i386-gcc: {}
+        linux-debian-stretch-amd64-gcc: {}
+
+    steps:
+    - template: release-fetch-source.yml
+    - template: linux-build.yml
+      parameters:
+        Image: compile-farm
+        ContainerCommand: '$(Build.BuildNumber)'
+        Tag: $(Agent.JobName)
+    - template: linux-claim-bundles.yml
+    - template: release-bundles.yml
+
+- job: macos
+  displayName: 'MacOS'
+  pool:
+    vmImage: 'macOS-10.13'
+  dependsOn: source
+
+  steps:
+  - template: release-fetch-source.yml
+  - template: osx-dependencies.yml
+  - template: osx-build.yml
+  - script: 'make bundle_zip bundle_dmg BUNDLE_NAME=openttd-$(Build.BuildNumber)-macosx'
+    displayName: 'Create bundles'
+  - template: release-bundles.yml
+
+- job: manifest
+  displayName: 'Manifest'
+  pool:
+    vmImage: 'ubuntu-16.04'
+  dependsOn:
+  - source
+  - docs
+  - windows
+  - ${{ if eq(parameters.IsStableRelease, true) }}:
+    - linux_stable
+  - macos
+  # "Skipped" is not a status, and is not succeeded. So it seems to be
+  # considered failed. So we trigger if all the earlier jobs are done (which
+  # might be succeeded, failed, or skipped), and run this job. This is not
+  # optimal, but given the rules, it is the only way to get this to work (as
+  # some jobs might be skipped).
+  condition: succeededOrFailed()
+
+  steps:
+  - template: release-fetch-source.yml
+  - template: release-manifest.yml
+  - template: release-bundles.yml
+    parameters:
+      CalculateChecksums: false
diff --git a/azure-pipelines/templates/windows-build.yml b/azure-pipelines/templates/windows-build.yml
new file mode 100644
index 000000000..5e12f2243
--- /dev/null
+++ b/azure-pipelines/templates/windows-build.yml
@@ -0,0 +1,11 @@
+parameters:
+  BuildPlatform: ''
+
+steps:
+- task: VSBuild@1
+  displayName: 'Build'
+  inputs:
+    solution: 'projects/openttd_vs141.sln'
+    platform: ${{ parameters.BuildPlatform }}
+    configuration: Release
+    maximumCpuCount: true
diff --git a/azure-pipelines/templates/windows-dependencies.yml b/azure-pipelines/templates/windows-dependencies.yml
new file mode 100644
index 000000000..06f56e626
--- /dev/null
+++ b/azure-pipelines/templates/windows-dependencies.yml
@@ -0,0 +1,10 @@
+steps:
+- bash: |
+    set -ex
+    curl -L https://github.com/OpenTTD/OpenTTD-CF/releases/download/latest/windows-dependencies.zip > windows-dependencies.zip
+    unzip windows-dependencies.zip
+    rm -f windows-dependencies.zip
+  displayName: 'Download dependencies'
+  workingDirectory: $(Build.ArtifactStagingDirectory)
+- script: $(Build.ArtifactStagingDirectory)\windows-dependencies\vcpkg.exe integrate install
+  displayName: 'Install dependencies'
diff --git a/azure-pipelines/templates/windows-dependency-nsis.yml b/azure-pipelines/templates/windows-dependency-nsis.yml
new file mode 100644
index 000000000..a1ee8f81e
--- /dev/null
+++ b/azure-pipelines/templates/windows-dependency-nsis.yml
@@ -0,0 +1,28 @@
+parameters:
+  condition: true
+
+steps:
+- bash: |
+    set -ex
+    # NSIS will be part of the Hosted image in the next update. Till then, we install it ourself
+    choco install nsis -y
+
+    mkdir nsis-plugin; cd nsis-plugin
+    curl -L https://devs.openttd.org/~truebrain/nsis-plugins/Nsis7z.zip > Nsis7z.zip
+    unzip Nsis7z.zip
+    cp -R Plugins/* "/c/Program Files (x86)/NSIS/Plugins/"
+    cd ..; rm -rf nsis-plugin
+
+    mkdir nsis-plugin; cd nsis-plugin
+    curl -L https://devs.openttd.org/~truebrain/nsis-plugins/NsisGetVersion.zip > NsisGetVersion.zip
+    unzip NsisGetVersion.zip
+    cp -R Plugins/* "/c/Program Files (x86)/NSIS/Plugins/x86-ansi/"
+    cd ..; rm -rf nsis-plugin
+
+    mkdir nsis-plugin; cd nsis-plugin
+    curl -L https://devs.openttd.org/~truebrain/nsis-plugins/NsisFindProc.zip > NsisFindProc.zip
+    unzip NsisFindProc.zip
+    cp -R *.dll "/c/Program Files (x86)/NSIS/Plugins/x86-ansi/"
+    cd ..; rm -rf nsis-plugin
+  displayName: 'Install NSIS with the 7z, GetVersion, and FindProc plugins'
+  condition: and(succeeded(), ${{ parameters.condition }})
diff --git a/azure-pipelines/templates/windows-dependency-zip.yml b/azure-pipelines/templates/windows-dependency-zip.yml
new file mode 100644
index 000000000..e2ae06dbe
--- /dev/null
+++ b/azure-pipelines/templates/windows-dependency-zip.yml
@@ -0,0 +1,5 @@
+steps:
+- bash: |
+    set -ex
+    choco install zip
+  displayName: 'Install zip'