adding AI & ML test scripts

tests/docker/testing/Dockerfile

+FROM ubuntu:18.04
+
+# Prefere local python binaries over the distributed ones
+ENV PATH /usr/local/bin:$PATH
+
+# Needed for Python 3
+ENV LANG C.UTF-8
+
+RUN apt update \
+    && apt-get --yes install tzdata \
+    && ln -sf /usr/share/zoneinfo/Europe/Berlin /etc/localtime \
+    && apt install --yes python python-dev python-pip wget\
+                         python3 python3-dev python3-pip \
+                         build-essential checkinstall python3-setuptools\
+                         libreadline-gplv2-dev libncursesw5-dev zlib1g-dev\
+                         libssl-dev libsqlite3-dev tk-dev python-smbus\
+                         libgdbm-dev libc6-dev libbz2-dev libffi-dev\
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy build script
+COPY build_python.sh /build_python.sh
+
+# PYTHON VERSIONS
+ENV PYTHON_VERSIONS 3.7.0
+ENV PYTHON_PIP_VERSION 10.0.1
+
+# Build python versions using the script
+ RUN bash -ex /build_python.sh
+
+# Installing tox for one python version is sufficient
+RUN pip3 install tox
+
+# Copy script for testing
+COPY test.sh /test.sh
+
+# Set ENTRYPOINT
+ENTRYPOINT ["sh", "-ex", "/test.sh"]

tests/docker/testing/build_python.sh

+#!/bin/bash
+
+#
+# Helper for building python versions
+#
+# Based on https://github.com/xvzf/tox-docker/blob/master/build_python.sh
+#
+
+
+gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
+
+
+function clean_pyc {
+    find /usr/local -depth \
+    \( \
+        \( -type d -a \( -name test -o -name tests \) \) \
+        -o \
+        \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
+    \) -exec rm -rf '{}' +
+}
+
+
+# Install python
+for PYTHON_VERSION in $PYTHON_VERSIONS; do
+
+    cd /tmp
+
+    # Get source
+    wget -O "python_${PYTHON_VERSION}.tar.xz" \
+       "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-${PYTHON_VERSION}.tar.xz"
+
+
+    # Extract source
+    mkdir -p "/usr/src/python_${PYTHON_VERSION}"
+    tar xJC "/usr/src/python_${PYTHON_VERSION}" --strip-components=1 \
+        -f python_${PYTHON_VERSION}.tar.xz
+
+    rm "python_${PYTHON_VERSION}.tar.xz"
+
+
+    # Configure, build and install
+    cd "/usr/src/python_${PYTHON_VERSION}"
+    ./configure \
+        --build="$gnuArch" \
+        --enable-loadable-sqlite-extensions \
+        --enable-shared \
+        --with-system-expat \
+        --with-system-ffi \
+        --without-ensurepip \
+        --enable-unicode=ucs4 \
+        --prefix=/usr/local \
+        LDFLAGS="-Wl,--rpath=/usr/local/lib"
+
+    make -j "$(nproc)"
+
+    EXTRA_CFLAGS="-DTHREAD_STACK_SIZE=0x100000" make install
+
+    # Cleanup
+    rm -Rf "/usr/src/python_${PYTHON_VERSION}"
+    clean_pyc
+
+
+done
+
+
+# Install pip
+cd /tmp
+wget -O get-pip.py "https://bootstrap.pypa.io/get-pip.py"
+for PYTHON_VERSION in $PYTHON_VERSIONS; do
+
+    # Install pip
+
+    python${PYTHON_VERSION%.*} get-pip.py \
+        --disable-pip-version-check \
+        --no-cache-dir \
+        "pip==$PYTHON_PIP_VERSION"
+
+    # Cleanup
+    clean_pyc
+done

tests/docker/testing/test.sh

+#!/bin/sh
+
+# We do NOT want to test in the actual mountpoint, therefore create a copy!
+cp -Rf /src /srccopy && cd /srccopy
+
+# Run tox for testing
+tox --skip-missing-interpreters

tests/templates/call.j2

+
+async def {{ name.lower_camel_case }}(self):
+    """ Generated by dcsdkgen
+
+    :returns: Tuple[Success, Response]
+    """
+
+    response = await self._stub.{{ name.upper_camel_case }}({{ plugin_name.lower_snake_case }}_pb2.{{ name.upper_camel_case }}Request())
+
+    return self._response_success(response), response

tests/templates/enum.j2

+class {{ name.upper_camel_case }}:
+    """ Generated by dcsdkgen """
+
+    @staticmethod
+    def parse_response(response):
+        """ Parses a gRPC response """
+        return {
+            {%- for value in values %}
+                {{ loop.index - 1 }}: "{{ value.uppercase }}",
+            {%- endfor %}
+            }.get(response.{{ name.lower_snake_case }}, None)

tests/templates/file.j2

+# -*- coding: utf-8 -*-
+from .._base import AsyncBase
+from ..generated import {{ plugin_name.lower_snake_case }}_pb2, {{ plugin_name.lower_snake_case }}_pb2_grpc
+
+
+{%- for enum in enums %}
+
+
+{{ enum }}
+{%- endfor %}
+
+
+{%- for struct in structs %}
+
+
+{{ struct }}
+{%- endfor %}
+
+
+class {{ plugin_name.upper_camel_case }}(AsyncBase):
+    """ Generated by dcsdkgen - DronecodeSDK {{ plugin_name.upper_camel_case }} API """
+
+    # Plugin name
+    name = "{{ plugin_name.upper_camel_case }}"
+
+    def _setup_stub(self, channel):
+        """ Setups the api stub """
+        self._stub = {{ plugin_name.lower_snake_case }}_pb2_grpc.{{ plugin_name.upper_camel_case }}ServiceStub(channel)
+
+    def _response_success(self, response):
+        """ Checks if the request was successfull """
+        return (response.{{ plugin_name.lower_snake_case }}_result.result ==
+                {{ plugin_name.lower_snake_case }}_pb2.{{ plugin_name.upper_camel_case }}Result.SUCCESS)
+
+    {%- for method in methods -%}
+    {{ '\n' + indent(method, 1) }}
+
+    {%- endfor %}

tests/templates/request.j2

+
+async def {{ name.lower_camel_case }}(self):
+    """ Generated by dcsdkgen
+
+    :returns: requested value
+    """
+
+    response = await self._stub.{{ name.lower_camel_case }}({{ plugin_name.lower_snake_case }}_pb2.{{ name.upper_camel_case }}Request())
+
+    return response

tests/templates/stream.j2

+
+async def {{ name.lower_snake_case }}(self{% for param in params %}, {{ param.name }}: {{ param.type }}{% endfor %}):
+    """ Generated by dcsdkgen """
+    {{ name.lower_snake_case }}_stream = self._stub.Subscribe{{ name.upper_camel_case }}(
+                {{ plugin_name.lower_snake_case }}_pb2.Subscribe{{ name.upper_camel_case }}Request())
+
+    try:
+        async for response in {{ name.lower_snake_case }}_stream:
+        {%- if return_type.is_primitive %}
+            yield response.{{ return_name.lower_snake_case }}
+        {%- else %}
+            yield {{ return_type.name }}.parse_response(response)
+        {%- endif %}
+    finally:
+        {{ name.lower_snake_case }}_stream.cancel()

tests/templates/struct.j2

+class {{ name.upper_camel_case }}:
+    """ Generated by dcsdkgen """
+
+    def __init__(
+            self,
+            {%- for field in fields %}
+            {%- if field == fields|last %}
+            {{ field.name.lower_snake_case }}
+            {%- else %}
+            {{ field.name.lower_snake_case }},
+            {%- endif %}
+            {%- endfor %}):
+        """ Initializes the {{ name.upper_camel_case }} object """
+        {%- for field in fields %}
+        self.{{ field.name.lower_snake_case }} = {{ field.name.lower_snake_case }}
+        {%- endfor %}
+
+    def __equals__(self, to_compare):
+        """ Checks if two {{ name.upper_camel_case }} are the same """
+        try:
+            # Try to compare - this likely fails when it is compared to a non
+            # {{ name.upper_camel_case }} object
+            return \
+                {%- for field in fields %}
+                {%- if field == fields|last %}
+                (self.{{ field.name.lower_snake_case }} == to_compare.{{ field.name.lower_snake_case }})
+                {%- else %}
+                (self.{{ field.name.lower_snake_case }} == to_compare.{{ field.name.lower_snake_case }}) and \
+                {%- endif %}
+                {%- endfor %}
+
+        except AttributeError:
+            return False
+
+    def __repr__(self):
+        """ {{ name.upper_camel_case }} in string representation """
+        return "{{ name.upper_camel_case }}" + ", ".join(
+                {%- for field in fields %}
+                {%- if field == fields|last %}
+                self.{{ field.name.lower_snake_case }}
+                {%- else %}
+                self.{{ field.name.lower_snake_case }},
+                {%- endif %}
+                {%- endfor %})
+
+    @staticmethod
+    def parse_response(response):
+        """ Parses a gRPC response """
+        return {{ name.upper_camel_case }}(
+                {%- for field in fields %}
+                {%- if field == fields|last %}
+                response.{{ name.lower_snake_case }}.{{ field.name.lower_snake_case }}
+                {%- else %}
+                response.{{ name.lower_snake_case }}.{{ field.name.lower_snake_case }},
+                {%- endif %}
+                {%- endfor %})

tests/tools/run_protoc.sh

+#!/bin/bash
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+WORK_DIR="${SCRIPT_DIR}/../../"
+PROTO_DIR="${WORK_DIR}/proto"
+GENERATED_DIR="${WORK_DIR}/dronecode_sdk/generated"
+PLUGIN_DIR="${WORK_DIR}/dronecode_sdk/plugins"
+PLUGIN_INIT="${PLUGIN_DIR}/__init__.py"
+export TEMPLATE_PATH="${WORK_DIR}/other/templates/"
+
+function generate {
+    echo -e "# -*- coding: utf-8 -*-\n" > $PLUGIN_INIT
+
+    for PROTO_FILE in `find ${PROTO_DIR} -name "*.proto" -type f`; do
+
+        # Generate bindings for each file individually
+        python3 -m grpc_tools.protoc -I${GENERATED_DIR} \
+                                     --proto_path=$(dirname ${PROTO_FILE}) \
+                                     --python_out=${GENERATED_DIR} \
+                                     --grpc_python_out=${GENERATED_DIR} \
+                                     ${PROTO_FILE}
+
+        # For some reason the import is broken with Python3.5.x and works fine
+        # with Python3.6.x, set an absolute path and everything is fine
+        PROTO_IMPORT_NAME="$(basename -- ${PROTO_FILE%.*})_pb2"
+
+        # We need to create the .original backup files, otherwise we're not compatible with
+        # BSD sed.
+        sed -i'.sedoriginal' -e "s/import ${PROTO_IMPORT_NAME}/from . import ${PROTO_IMPORT_NAME}/" \
+            "${GENERATED_DIR}/${PROTO_IMPORT_NAME}_grpc.py"
+        # Clean up the backup files.
+        find ${GENERATED_DIR} -name '*.sedoriginal' -delete
+
+        echo " -> [+] Generated protobuf and gRPC bindings for ${PROTO_IMPORT_NAME%_*}"
+
+        # Generate plugin
+        python3 -m grpc_tools.protoc -I$(dirname ${PROTO_FILE}) \
+                                     --plugin=protoc-gen-custom=$(which dcsdkgen) \
+                                     --custom_out=${PLUGIN_DIR} \
+                                     --custom_opt=py \
+                                     ${PROTO_FILE}
+
+        WANTED_PLUGIN_NAME="$(echo ${PROTO_FILE} | sed "s#.*/\(.*\).proto#\1#g").py"
+        # protoc generates java like filenames, we don't want that with python
+
+        # @TODO: cleanup this script and figure out a way to make it cross-os friendly perhaps rewrite in python
+        # capilalization as trivial as this needs a workaround for macos bash 3.2
+        # this solution avoids using bash ^ substitution (from bash >4.0) and is using python 3 instead
+        CAPITALIZED_PLUGIN_NAME=$(python3 -c "import sys;print(sys.argv[1].capitalize())" "$WANTED_PLUGIN_NAME")
+        mv ${PLUGIN_DIR}/${CAPITALIZED_PLUGIN_NAME} ${PLUGIN_DIR}/${WANTED_PLUGIN_NAME}
+
+        # Add to imports
+        echo "from .${WANTED_PLUGIN_NAME%.py} import *" >> $PLUGIN_INIT
+        echo " -> [+] Generated plugin for ${PROTO_IMPORT_NAME%_*}"
+
+    done
+}
+
+function install_dcsdkgen {
+    cd ${PROTO_DIR}/pb_plugins
+    if [[ "$VIRTUAL_ENV" != "" ]]
+    then
+        pip3 install .
+    else
+        pip3  install --user .
+    fi
+}
+
+echo "[+] Installing the DronecodeSDK autogenerator"
+install_dcsdkgen
+echo "[+] Done"
+echo "[+] Generating plugins from "
+generate
+echo "[+] Done"

tests/tools/run_tests.sh

+#!/bin/bash
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+WORK_DIR="${SCRIPT_DIR}/../.."
+DOCKER_DIR="${WORK_DIR}/other/docker/testing/"
+DOCKER_IMAGE_NAME="tox-dronecode-sdk-python"
+DOCKER_BUILD_LOG_PATH="/tmp/dronecode-sdk-python-docker-build.log"
+
+function build_docker_image {
+    cd $DOCKER_DIR
+    echo "-> [+] Building the docker image used for testing multiple python versions (this can take some time)"
+    docker build . -t $DOCKER_IMAGE_NAME &> $DOCKER_BUILD_LOG_PATH
+    if [ $? -eq 0 ]; then
+        echo "-> [+] Done!"
+    else
+        echo "-> [-] Failed to build the docker image, see ${DOCKER_BUILD_LOG_PATH}"
+        exit 1
+    fi
+    cd $WORK_DIR
+}
+
+function run_tox {
+    echo "-> [+] Running the test suite!"
+    docker run --rm -v ${WORK_DIR}:/src $DOCKER_IMAGE_NAME
+}
+
+echo "[+] Building docker image"
+build_docker_image
+
+echo "[+] Running tests"
+run_tox