CI improvements, add deploy to c_library_v1/2 back in (#1568)

* workflows: remove Python 3.5 which is EOL

I'll leave Python 2.7 so that we don't accidentally break it.

* scripts: separate tests into functions

This way we can run parts of it in respective CI instances.

* workflows: separate test into format, python, node

This way we don't have an explosion of versions to test.

* scripts: use consistent shebang

* workflows: use newer action

* workflows: ignore node 15 for now

* workflows: add fake deploy step

* workflows/scripts: update deploy script

.github/workflows/test.yml

@@ -1,39 +0,0 @@
-name: test mavlink
-
-on: [push, pull_request]
-# paths:
-# - "*"
-# - "!README.md" <-- don't rebuild on doc change
-
-jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false  # don't cancel if a job from the matrix fails
-      matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
-        node: ['12', '13', '14']
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install future lxml
-          sudo apt update
-          sudo apt install -y libxml2-dev libxml2-utils
-      - uses: actions/setup-node@v2-beta
-        with:
-          node-version: ${{ matrix.node }}
-      - run: npm install
-      - name : Test mavlink
-        run: |
-          ./scripts/test.sh
-

.github/workflows/test_and_deploy.yml

@@ -0,0 +1,98 @@
+name: Test and deploy
+
+on:
+  push:
+    branches:
+    - 'master'
+  pull_request:
+    branches:
+    - '*'
+
+# paths:
+# - "*"
+# - "!README.md" <-- don't rebuild on doc change
+
+jobs:
+  format:
+    name: Formatting check
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - run: |
+          sudo apt update
+          sudo apt install -y libxml2-dev libxml2-utils
+      - name: Check formatting
+        run: |
+          ./scripts/test.sh format
+
+  python-tests:
+    name: Python ${{ matrix.python-version }} tests
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [2.7, 3.6, 3.7, 3.8]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install future lxml
+      - name : Test Python generator
+        run: |
+          ./scripts/test.sh py
+
+  node-tests:
+    name: Node ${{ matrix.node-version }} test
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: ['12', '14'] # 15 fails for some weird reason
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          pip install future lxml
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: npm install
+      - name : Test mavlink
+        run: |
+          ./scripts/test.sh node
+
+  deploy:
+    name: Generate and push C headers
+    needs: [format, python-tests, node-tests]
+    runs-on: ubuntu-20.04
+    if: github.ref == 'refs/heads/master'
+    env:
+      GH_TOKEN: ${{ secrets.GH_TOKEN }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          pip install future lxml
+      - name: Run deploy script
+        run: |
+          ./scripts/update_generated_repos.sh

scripts/format_xml.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 # A POSIX variable

scripts/test.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 set -e
 
 SRC_DIR=$(pwd)
@@ -7,14 +7,16 @@ SRC_DIR=$(pwd)
 # on the build tree. Otherwise the testing is invalid and may not
 # indicate the code actually works
 
-# check format
-sep="##############################################"
-echo $sep
-echo "FORMAT TEST"
-echo $sep
-cd "$SRC_DIR"
-./scripts/format_xml.sh -c
-echo PASS
+test_format() {
+    # check format
+    sep="##############################################"
+    echo $sep
+    echo "FORMAT TEST"
+    echo $sep
+    cd "$SRC_DIR"
+    ./scripts/format_xml.sh -c
+    echo PASS
+}
 
 generate_mavlink() {
     echo $sep
@@ -30,34 +32,59 @@ generate_mavlink() {
     echo PASS
 }
 
-cd "$SRC_DIR"
-for msg_def in message_definitions/v1.0/*.xml
-do
-    [ -e "$msg_def" ] || continue
-    wire_protocol="1.0"
-    for lang in Python C CS WLua Java
+test_py() {
+    cd "$SRC_DIR"
+    for msg_def in message_definitions/v1.0/*.xml
     do
-        generate_mavlink
+        [ -e "$msg_def" ] || continue
+        wire_protocol="1.0"
+        for lang in Python C CS WLua Java
+        do
+            generate_mavlink
+        done
+        wire_protocol="2.0"
+        for lang in Python C C++11 CS WLua Java
+        do
+            generate_mavlink
+        done
     done
-    wire_protocol="2.0"
-    for lang in Python C C++11 CS WLua Java
-    do
-        generate_mavlink
-    done
-done
-# Avoid `spurious errors` caused by ~/.npm permission issues
-# ref: https://github.com/travis-ci/travis-ci/issues/2244
-# ref: https://github.com/npm/npm/issues/4815
-# Does it already exist? Who owns? What permissions?
-ls -lah ~/.npm || mkdir ~/.npm
-# Make sure we own it
-# $USER references the current user in Travis env
-chown -R "$USER" ~/.npm
-if [ -f /usr/bin/nodejs ]
-then
-	mkdir -p ~/bin
-	ln -sf /usr/bin/nodejs ~/bin/node
-	. ~/.bashrc	
-fi
-cd "$SRC_DIR/pymavlink/generator/javascript" && npm test
+}
+
+test_node() {
+    # Avoid `spurious errors` caused by ~/.npm permission issues
+    # ref: https://github.com/travis-ci/travis-ci/issues/2244
+    # ref: https://github.com/npm/npm/issues/4815
+    # Does it already exist? Who owns? What permissions?
+    ls -lah ~/.npm || mkdir ~/.npm
+    # Make sure we own it
+    # $USER references the current user in Travis env
+    chown -R "$USER" ~/.npm
+    if [ -f /usr/bin/nodejs ]
+    then
+        mkdir -p ~/bin
+        ln -sf /usr/bin/nodejs ~/bin/node
+        . ~/.bashrc
+    fi
+    cd "$SRC_DIR/pymavlink/generator/javascript" && npm test
+}
+
+if [ "$#" -eq 1 ]; then
+    if [ "$1" == "format" ]; then
+        test_format
+    elif [ "$1" == "py" ]; then
+        test_py
+    elif [ "$1" == "node" ]; then
+        test_node
+    else
+        echo "Error: unknown argument '$1'"
+        echo ""
+        echo "Usage:"
+        echo "  $0 [py|node|format]"
+        exit 1
+    fi
+else
+    test_format
+    test_py
+    test_node
+fi
 

scripts/update_c_library.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # c_library repository update script
 # Author: Thomas Gubler <thomasgubler@gmail.com>

scripts/update_generated_repos.sh

@@ -1,27 +1,4 @@
-#!/bin/bash
-
-# Do only build for Python 2.7
-# as we only want to deploy for one
-# unique generator.
-PYTHONVER=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:3])))')
-
-if [[ $PYTHONVER != "2.7"* ]]
-then
-	echo -e "Skipping header generation for Python $PYTHONVER"
-	exit 0
-fi
-
-# Do not build pull requests
-if [[ $TRAVIS_PULL_REQUEST != "false" ]]
-then
-	exit 0
-fi
-
-# Do only build master branch
-if [[ $TRAVIS_BRANCH != "master" ]]
-then
-	exit 0
-fi
+#!/usr/bin/env bash
 
 # Config for auto-building
 git remote rename origin upstream
@@ -44,5 +21,3 @@ cd include/mavlink/v1.0
 git clone https://github.com/mavlink/c_library_v1.git
 cd ../../..
 ./scripts/update_c_library.sh 1
-
-# XXX add build steps for other libraries as well