Commit 5f4f6414 authored by Shinny Upatree's avatar Shinny Upatree 🎅🏻

adding AI & ML test scripts

parent 12c5eca8
Pipeline #136 canceled with stages
FROM ubuntu:18.04
# Prefere local python binaries over the distributed ones
ENV PATH /usr/local/bin:$PATH
# Needed for Python 3
ENV LANG C.UTF-8
RUN apt update \
&& apt-get --yes install tzdata \
&& ln -sf /usr/share/zoneinfo/Europe/Berlin /etc/localtime \
&& apt install --yes python python-dev python-pip wget\
python3 python3-dev python3-pip \
build-essential checkinstall python3-setuptools\
libreadline-gplv2-dev libncursesw5-dev zlib1g-dev\
libssl-dev libsqlite3-dev tk-dev python-smbus\
libgdbm-dev libc6-dev libbz2-dev libffi-dev\
&& rm -rf /var/lib/apt/lists/*
# Copy build script
COPY build_python.sh /build_python.sh
# PYTHON VERSIONS
ENV PYTHON_VERSIONS 3.7.0
ENV PYTHON_PIP_VERSION 10.0.1
# Build python versions using the script
RUN bash -ex /build_python.sh
# Installing tox for one python version is sufficient
RUN pip3 install tox
# Copy script for testing
COPY test.sh /test.sh
# Set ENTRYPOINT
ENTRYPOINT ["sh", "-ex", "/test.sh"]
#!/bin/bash
#
# Helper for building python versions
#
# Based on https://github.com/xvzf/tox-docker/blob/master/build_python.sh
#
gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
function clean_pyc {
find /usr/local -depth \
\( \
\( -type d -a \( -name test -o -name tests \) \) \
-o \
\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
\) -exec rm -rf '{}' +
}
# Install python
for PYTHON_VERSION in $PYTHON_VERSIONS; do
cd /tmp
# Get source
wget -O "python_${PYTHON_VERSION}.tar.xz" \
"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-${PYTHON_VERSION}.tar.xz"
# Extract source
mkdir -p "/usr/src/python_${PYTHON_VERSION}"
tar xJC "/usr/src/python_${PYTHON_VERSION}" --strip-components=1 \
-f python_${PYTHON_VERSION}.tar.xz
rm "python_${PYTHON_VERSION}.tar.xz"
# Configure, build and install
cd "/usr/src/python_${PYTHON_VERSION}"
./configure \
--build="$gnuArch" \
--enable-loadable-sqlite-extensions \
--enable-shared \
--with-system-expat \
--with-system-ffi \
--without-ensurepip \
--enable-unicode=ucs4 \
--prefix=/usr/local \
LDFLAGS="-Wl,--rpath=/usr/local/lib"
make -j "$(nproc)"
EXTRA_CFLAGS="-DTHREAD_STACK_SIZE=0x100000" make install
# Cleanup
rm -Rf "/usr/src/python_${PYTHON_VERSION}"
clean_pyc
done
# Install pip
cd /tmp
wget -O get-pip.py "https://bootstrap.pypa.io/get-pip.py"
for PYTHON_VERSION in $PYTHON_VERSIONS; do
# Install pip
python${PYTHON_VERSION%.*} get-pip.py \
--disable-pip-version-check \
--no-cache-dir \
"pip==$PYTHON_PIP_VERSION"
# Cleanup
clean_pyc
done
#!/bin/sh
# We do NOT want to test in the actual mountpoint, therefore create a copy!
cp -Rf /src /srccopy && cd /srccopy
# Run tox for testing
tox --skip-missing-interpreters
async def {{ name.lower_camel_case }}(self):
""" Generated by dcsdkgen
:returns: Tuple[Success, Response]
"""
response = await self._stub.{{ name.upper_camel_case }}({{ plugin_name.lower_snake_case }}_pb2.{{ name.upper_camel_case }}Request())
return self._response_success(response), response
class {{ name.upper_camel_case }}:
""" Generated by dcsdkgen """
@staticmethod
def parse_response(response):
""" Parses a gRPC response """
return {
{%- for value in values %}
{{ loop.index - 1 }}: "{{ value.uppercase }}",
{%- endfor %}
}.get(response.{{ name.lower_snake_case }}, None)
# -*- coding: utf-8 -*-
from .._base import AsyncBase
from ..generated import {{ plugin_name.lower_snake_case }}_pb2, {{ plugin_name.lower_snake_case }}_pb2_grpc
{%- for enum in enums %}
{{ enum }}
{%- endfor %}
{%- for struct in structs %}
{{ struct }}
{%- endfor %}
class {{ plugin_name.upper_camel_case }}(AsyncBase):
""" Generated by dcsdkgen - DronecodeSDK {{ plugin_name.upper_camel_case }} API """
# Plugin name
name = "{{ plugin_name.upper_camel_case }}"
def _setup_stub(self, channel):
""" Setups the api stub """
self._stub = {{ plugin_name.lower_snake_case }}_pb2_grpc.{{ plugin_name.upper_camel_case }}ServiceStub(channel)
def _response_success(self, response):
""" Checks if the request was successfull """
return (response.{{ plugin_name.lower_snake_case }}_result.result ==
{{ plugin_name.lower_snake_case }}_pb2.{{ plugin_name.upper_camel_case }}Result.SUCCESS)
{%- for method in methods -%}
{{ '\n' + indent(method, 1) }}
{%- endfor %}
async def {{ name.lower_camel_case }}(self):
""" Generated by dcsdkgen
:returns: requested value
"""
response = await self._stub.{{ name.lower_camel_case }}({{ plugin_name.lower_snake_case }}_pb2.{{ name.upper_camel_case }}Request())
return response
async def {{ name.lower_snake_case }}(self{% for param in params %}, {{ param.name }}: {{ param.type }}{% endfor %}):
""" Generated by dcsdkgen """
{{ name.lower_snake_case }}_stream = self._stub.Subscribe{{ name.upper_camel_case }}(
{{ plugin_name.lower_snake_case }}_pb2.Subscribe{{ name.upper_camel_case }}Request())
try:
async for response in {{ name.lower_snake_case }}_stream:
{%- if return_type.is_primitive %}
yield response.{{ return_name.lower_snake_case }}
{%- else %}
yield {{ return_type.name }}.parse_response(response)
{%- endif %}
finally:
{{ name.lower_snake_case }}_stream.cancel()
class {{ name.upper_camel_case }}:
""" Generated by dcsdkgen """
def __init__(
self,
{%- for field in fields %}
{%- if field == fields|last %}
{{ field.name.lower_snake_case }}
{%- else %}
{{ field.name.lower_snake_case }},
{%- endif %}
{%- endfor %}):
""" Initializes the {{ name.upper_camel_case }} object """
{%- for field in fields %}
self.{{ field.name.lower_snake_case }} = {{ field.name.lower_snake_case }}
{%- endfor %}
def __equals__(self, to_compare):
""" Checks if two {{ name.upper_camel_case }} are the same """
try:
# Try to compare - this likely fails when it is compared to a non
# {{ name.upper_camel_case }} object
return \
{%- for field in fields %}
{%- if field == fields|last %}
(self.{{ field.name.lower_snake_case }} == to_compare.{{ field.name.lower_snake_case }})
{%- else %}
(self.{{ field.name.lower_snake_case }} == to_compare.{{ field.name.lower_snake_case }}) and \
{%- endif %}
{%- endfor %}
except AttributeError:
return False
def __repr__(self):
""" {{ name.upper_camel_case }} in string representation """
return "{{ name.upper_camel_case }}" + ", ".join(
{%- for field in fields %}
{%- if field == fields|last %}
self.{{ field.name.lower_snake_case }}
{%- else %}
self.{{ field.name.lower_snake_case }},
{%- endif %}
{%- endfor %})
@staticmethod
def parse_response(response):
""" Parses a gRPC response """
return {{ name.upper_camel_case }}(
{%- for field in fields %}
{%- if field == fields|last %}
response.{{ name.lower_snake_case }}.{{ field.name.lower_snake_case }}
{%- else %}
response.{{ name.lower_snake_case }}.{{ field.name.lower_snake_case }},
{%- endif %}
{%- endfor %})
#!/bin/bash
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORK_DIR="${SCRIPT_DIR}/../../"
PROTO_DIR="${WORK_DIR}/proto"
GENERATED_DIR="${WORK_DIR}/dronecode_sdk/generated"
PLUGIN_DIR="${WORK_DIR}/dronecode_sdk/plugins"
PLUGIN_INIT="${PLUGIN_DIR}/__init__.py"
export TEMPLATE_PATH="${WORK_DIR}/other/templates/"
function generate {
echo -e "# -*- coding: utf-8 -*-\n" > $PLUGIN_INIT
for PROTO_FILE in `find ${PROTO_DIR} -name "*.proto" -type f`; do
# Generate bindings for each file individually
python3 -m grpc_tools.protoc -I${GENERATED_DIR} \
--proto_path=$(dirname ${PROTO_FILE}) \
--python_out=${GENERATED_DIR} \
--grpc_python_out=${GENERATED_DIR} \
${PROTO_FILE}
# For some reason the import is broken with Python3.5.x and works fine
# with Python3.6.x, set an absolute path and everything is fine
PROTO_IMPORT_NAME="$(basename -- ${PROTO_FILE%.*})_pb2"
# We need to create the .original backup files, otherwise we're not compatible with
# BSD sed.
sed -i'.sedoriginal' -e "s/import ${PROTO_IMPORT_NAME}/from . import ${PROTO_IMPORT_NAME}/" \
"${GENERATED_DIR}/${PROTO_IMPORT_NAME}_grpc.py"
# Clean up the backup files.
find ${GENERATED_DIR} -name '*.sedoriginal' -delete
echo " -> [+] Generated protobuf and gRPC bindings for ${PROTO_IMPORT_NAME%_*}"
# Generate plugin
python3 -m grpc_tools.protoc -I$(dirname ${PROTO_FILE}) \
--plugin=protoc-gen-custom=$(which dcsdkgen) \
--custom_out=${PLUGIN_DIR} \
--custom_opt=py \
${PROTO_FILE}
WANTED_PLUGIN_NAME="$(echo ${PROTO_FILE} | sed "s#.*/\(.*\).proto#\1#g").py"
# protoc generates java like filenames, we don't want that with python
# @TODO: cleanup this script and figure out a way to make it cross-os friendly perhaps rewrite in python
# capilalization as trivial as this needs a workaround for macos bash 3.2
# this solution avoids using bash ^ substitution (from bash >4.0) and is using python 3 instead
CAPITALIZED_PLUGIN_NAME=$(python3 -c "import sys;print(sys.argv[1].capitalize())" "$WANTED_PLUGIN_NAME")
mv ${PLUGIN_DIR}/${CAPITALIZED_PLUGIN_NAME} ${PLUGIN_DIR}/${WANTED_PLUGIN_NAME}
# Add to imports
echo "from .${WANTED_PLUGIN_NAME%.py} import *" >> $PLUGIN_INIT
echo " -> [+] Generated plugin for ${PROTO_IMPORT_NAME%_*}"
done
}
function install_dcsdkgen {
cd ${PROTO_DIR}/pb_plugins
if [[ "$VIRTUAL_ENV" != "" ]]
then
pip3 install .
else
pip3 install --user .
fi
}
echo "[+] Installing the DronecodeSDK autogenerator"
install_dcsdkgen
echo "[+] Done"
echo "[+] Generating plugins from "
generate
echo "[+] Done"
#!/bin/bash
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
WORK_DIR="${SCRIPT_DIR}/../.."
DOCKER_DIR="${WORK_DIR}/other/docker/testing/"
DOCKER_IMAGE_NAME="tox-dronecode-sdk-python"
DOCKER_BUILD_LOG_PATH="/tmp/dronecode-sdk-python-docker-build.log"
function build_docker_image {
cd $DOCKER_DIR
echo "-> [+] Building the docker image used for testing multiple python versions (this can take some time)"
docker build . -t $DOCKER_IMAGE_NAME &> $DOCKER_BUILD_LOG_PATH
if [ $? -eq 0 ]; then
echo "-> [+] Done!"
else
echo "-> [-] Failed to build the docker image, see ${DOCKER_BUILD_LOG_PATH}"
exit 1
fi
cd $WORK_DIR
}
function run_tox {
echo "-> [+] Running the test suite!"
docker run --rm -v ${WORK_DIR}:/src $DOCKER_IMAGE_NAME
}
echo "[+] Building docker image"
build_docker_image
echo "[+] Running tests"
run_tox
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment