Merge remote-tracking branch 'origin/master' into rblaczkowski/change…

…-code-base-to-OpenVino

.ci/jenkins/ci.groovy

@@ -24,19 +24,18 @@ catch (Exception e) {
         [ sku : "skx", backends : ["cpu", "interpreter"] ],
         [ sku : "clx", backends : ["cpu", "interpreter"] ],
         [ sku : "bdw", backends : ["cpu", "interpreter"] ]
-        // [ sku: "iris", backend : "igpu" ]
     ]
 }
 echo "BACKEND_SKU_CONFIGURATIONS=${BACKEND_SKU_CONFIGURATIONS}"
 
 // --- CI constants ---
 NGRAPH_ONNX_REPO_ADDRESS="[email protected]:NervanaSystems/ngraph-onnx.git"
 NGRAPH_REPO_ADDRESS="[email protected]:NervanaSystems/ngraph.git"
-DLDT_REPO_ADDRESS = "git@gitlab-icv.inn.intel.com:inference-engine/dldt.git"
+OPENVINO_REPO_ADDRESS = "git@github.com:openvinotoolkit/openvino.git"
 
 CI_LABELS = "ngraph_onnx && ci"
 CI_DIR = "ngraph-onnx/.ci/jenkins"
-DOCKER_CONTAINER_NAME = "jenkins_ngraph-onnx_ci"
+DOCKER_CONTAINER_PREFIX = "jenkins_ngraph-onnx_ci"
 
 JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"
 JENKINS_GITLAB_CREDENTIAL_ID = "1caab8d7-1d0c-4b8a-9438-b65336862ead"
@@ -51,6 +50,7 @@ CONFIGURATION_WORKFLOW = { configuration ->
         timeout(activity: true, time: 60) {
             WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
             DOCKER_HOME = "/home/${USER}"
+            DOCKER_CONTAINER_NAME="${DOCKER_CONTAINER_PREFIX}_${EXECUTOR_NUMBER}"
             try {
                 stage("Clone repositories") {
                     dir (WORKDIR) {
@@ -60,11 +60,11 @@ CONFIGURATION_WORKFLOW = { configuration ->
                         gitClone("Clone ngraph", NGRAPH_REPO_ADDRESS, configuration.ngraphBranch)
                     }
                     dir (WORKDIR) {
-                        gitClone("Clone dldt", DLDT_REPO_ADDRESS, configuration.dldtBranch)
+                        gitClone("Clone openvino", OPENVINO_REPO_ADDRESS, configuration.openvinoBranch)
                     }
-                    gitSubmoduleUpdate("dldt")
+                    gitSubmoduleUpdate("openvino")
                 }
-                String imageName = "${DOCKER_REGISTRY}/aibt/aibt/ngraph_cpp/${configuration.os}/ubuntu_18_04"
+                String imageName = "${DOCKER_REGISTRY}/aibt/aibt/ngraph_cpp/${configuration.os}/ubuntu_18_04_test"
                 stage("Prepare Docker image") {
                     pullDockerImage(imageName)
                     appendUserToDockerImage(imageName)
@@ -73,22 +73,7 @@ CONFIGURATION_WORKFLOW = { configuration ->
                     runDockerContainer(imageName)
                 }
                 stage("Prepare environment") {
-                    prepareEnvironment(configuration.backends, configuration.ngraphBranch)
-                }
-                for (backend in configuration.backends) {
-                    try {
-                        stage("Run ${backend} tests") {
-                            runToxTests(backend)
-                        }
-                    }
-                    catch(e) {
-                        // If cause of exception was job abortion - throw exception
-                        if ("$e".contains("143")) {
-                            throw e
-                        } else {
-                            currentBuild.result = "FAILURE"
-                        }
-                    }
+                    prepareEnvironment()
                 }
             }
             catch(e) {
@@ -172,7 +157,7 @@ def runDockerContainer(String imageName) {
         mkdir -p ${HOME}/ONNX_CI
         docker run -id --privileged \
                 --user ${USER} \
-                --name ${DOCKER_CONTAINER_NAME}  \
+                --name ${DOCKER_CONTAINER_NAME} \
                 --volume ${WORKDIR}:/logs \
                 --volume ${HOME}/ONNX_CI/onnx_models/.onnx:${dockerOnnxModels} \
                 --volume ${HOME}/ONNX_CI/cache:${dockerCache} \
@@ -181,20 +166,17 @@ def runDockerContainer(String imageName) {
     """
 }
 
-def prepareEnvironment(List<String> backends, String ngraph_branch) {
-    String backendsString = backends.join(",")
+def prepareEnvironment() {
     sh """
         docker exec ${DOCKER_CONTAINER_NAME} bash -c "${DOCKER_HOME}/${CI_DIR}/prepare_environment.sh \
-                                                                            --build-dir=${DOCKER_HOME} \
-                                                                            --backends=${backendsString} \
-                                                                            --ngraph-branch=${ngraph_branch}"
+                                                                            --build-dir=${DOCKER_HOME}
     """
 }
 
 def runToxTests(String backend) {
     String toxEnvVar = "TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL}"
     String backendEnvVar = "NGRAPH_BACKEND=${backend.toUpperCase()}"
-    String libraryVar = (backend == "ie") ? "LD_LIBRARY_PATH=${DOCKER_HOME}/dldt_dist/deployment_tools/inference_engine/external/tbb/lib:${DOCKER_HOME}/dldt_dist/deployment_tools/inference_engine/lib/intel64:${DOCKER_HOME}/dldt_dist/deployment_tools/inference_engine/external/mkltiny_lnx/lib:${DOCKER_HOME}/dldt_dist/deployment_tools/ngraph/lib" : "LD_LIBRARY_PATH="
+    String libraryVar = (backend == "ie") ? "LD_LIBRARY_PATH=${DOCKER_HOME}/openvino_dist/deployment_tools/inference_engine/external/tbb/lib:${DOCKER_HOME}/openvino_dist/deployment_tools/inference_engine/lib/intel64:${DOCKER_HOME}/openvino_dist/deployment_tools/inference_engine/external/mkltiny_lnx/lib:${DOCKER_HOME}/openvino_dist/deployment_tools/ngraph/lib" : "LD_LIBRARY_PATH="
 
 
     if (backend == "ie") {
@@ -219,7 +201,7 @@ def cleanup() {
     deleteDir()
 }
 
-def getConfigurationsMap(String dockerfilesPath, String ngraphOnnxBranch, String ngraphBranch, String dldtBranch) {
+def getConfigurationsMap(String dockerfilesPath, String ngraphOnnxBranch, String ngraphBranch, String openvinoBranch) {
     def configurationsMap = [:]
     def osImages = sh (script: "find ${dockerfilesPath} -maxdepth 1 -name '*.dockerfile' -printf '%f\n'",
                     returnStdout: true).trim().replaceAll(".dockerfile","").split("\n") as List
@@ -230,7 +212,7 @@ def getConfigurationsMap(String dockerfilesPath, String ngraphOnnxBranch, String
             configuration.os = os
             configuration.ngraphOnnxBranch = ngraphOnnxBranch
             configuration.ngraphBranch = ngraphBranch
-            configuration.dldtBranch = dldtBranch
+            configuration.openvinoBranch = openvinoBranch
             String backendLabels = configuration.backends.join(" && ")
             configuration.label = "${backendLabels} && ${configuration.sku} && ${CI_LABELS}"
             configuration.name = "${configuration.sku}-${configuration.os}"

.ci/jenkins/dockerfiles/ubuntu_18_04.dockerfile

@@ -55,7 +55,8 @@ RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.d
 
 RUN pip3 install --upgrade pip==19.0.3 \
         setuptools==41.0.0 \
-        wheel==0.33.1
+        wheel==0.33.1 \
+        cython
 
 # ONNX dependencies
 RUN apt-get -y --no-install-recommends install \

.ci/jenkins/prepare_environment.sh

@@ -19,77 +19,58 @@
 set -x
 set -e
 
-function build_ngraph() {
+function build_open_vino() {
     set -x
     local directory="$1"
-    local backends="$2"
-    CMAKE_ARGS="-DNGRAPH_TOOLS_ENABLE=FALSE -DNGRAPH_WARNINGS_AS_ERRORS=TRUE -DCMAKE_BUILD_TYPE=Release -DNGRAPH_UNIT_TEST_ENABLE=FALSE -DNGRAPH_USE_PREBUILT_LLVM=TRUE -DNGRAPH_ONNX_IMPORT_ENABLE=TRUE -DCMAKE_INSTALL_PREFIX=${directory}/ngraph_dist"
-    cd "${directory}/ngraph"
+    CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug \
+                -DENABLE_VALIDATION_SET=OFF \
+                -DENABLE_VPU=OFF \
+                -DENABLE_DLIA=OFF \
+                -DENABLE_GNA=OFF \
+                -DENABLE_CPPLINT=OFF \
+                -DENABLE_TESTS=OFF \
+                -DENABLE_BEH_TESTS=OFF \
+                -DENABLE_FUNCTIONAL_TESTS=OFF \
+                -DENABLE_MKL_DNN=ON \
+                -DENABLE_CLDNN=OFF \
+                -DENABLE_PROFILING_ITT=OFF \
+                -DENABLE_SAMPLES=OFF \
+                -DENABLE_SPEECH_DEMO=OFF \
+                -DENABLE_PYTHON=ON \
+                -DPYTHON_EXECUTABLE=`which python3` \
+                -DNGRAPH_ONNX_IMPORT_ENABLE=ON \
+                -DNGRAPH_IE_ENABLE=ON \
+                -DNGRAPH_INTERPRETER_ENABLE=ON \
+                -DNGRAPH_DEBUG_ENABLE=OFF \
+                -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON \
+                -DCMAKE_INSTALL_PREFIX=${directory}/openvino_dist"
 
-    # CMAKE args for nGraph backends
-    if [[ ${backends} == *"igpu"* ]]; then
-        echo "Building nGraph for Intel GPU."
-        CMAKE_ARGS="${CMAKE_ARGS} -DNGRAPH_INTERPRETER_ENABLE=TRUE"
-    fi
-    if [[ ${backends} == *"interpreter"* ]]; then
-        echo "Building nGraph for INTERPRETER backend."
-        CMAKE_ARGS="${CMAKE_ARGS} -DNGRAPH_INTELGPU_ENABLE=TRUE"
-    fi
-
-    cd "${directory}/ngraph"
+    cd "${directory}/openvino"
+
     mkdir -p ./build
     cd ./build
     cmake ${CMAKE_ARGS} ..  || return 1
     make -j $(lscpu --parse=CORE | grep -v '#' | sort | uniq | wc -l) || return 1
     make install || return 1
-    cd "${directory}/ngraph/python"
+
+    cd "${directory}/openvino/ngraph/python"
     if [ ! -d ./pybind11 ]; then
         git clone --recursive https://github.com/pybind/pybind11.git
     fi
-    rm -f "${directory}/ngraph/python/dist/ngraph*.whl"
-    rm -rf "${directory}/ngraph/python/*.so ${directory}/ngraph/python/build"
-    export PYBIND_HEADERS_PATH="${directory}/ngraph/python/pybind11"
-    export NGRAPH_CPP_BUILD_PATH="${directory}/ngraph_dist"
+    virtualenv -p `which python3` venv
+    . venv/bin/activate
+    rm -f "${directory}/openvino/ngraph/python/dist/ngraph*.whl"
+    rm -rf "${directory}/openvino/ngraph/python/*.so ${directory}/openvino/ngraph/python/build"
+    export PYBIND_HEADERS_PATH="${directory}/openvino/ngraph/python/pybind11"
+    export NGRAPH_CPP_BUILD_PATH="${directory}/openvino_dist"
     export NGRAPH_ONNX_IMPORT_ENABLE="TRUE"
-    python3 setup.py bdist_wheel || return 1
-    # Clean build artifacts
-    rm -rf "${directory}/ngraph_dist"
-    return 0
-}
-
-function build_dldt() {
-    set -x
-    local directory="$1"
-    local ngraph_branch="$2"
-    CMAKE_ARGS="-DNGRAPH_LIBRARY_OUTPUT_DIRECTORY=${directory}/dldt_dist \
-                -DNGRAPH_COMPONENT_PREFIX=deployment_tools/ngraph/ -DNGRAPH_USE_PREBUILT_LLVM=TRUE \
-                -DNGRAPH_TOOLS_ENABLE=TRUE -DNGRAPH_WARNINGS_AS_ERRORS=TRUE -DNGRAPH_UNIT_TEST_ENABLE=FALSE \
-                -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=OFF -DENABLE_RPATH=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-                -DENABLE_PERFORMANCE_TESTS=ON -DENABLE_TESTS=ON -DNGRAPH_DEBUG_ENABLE=OFF \
-                -DENABLE_SAMPLES=OFF -DENABLE_FUNCTIONAL_TESTS=ON -DENABLE_MODELS=OFF -DENABLE_PRIVATE_MODELS=OFF \
-                -DENABLE_GNA=OFF -DENABLE_VPU=OFF -DENABLE_SANITIZER=OFF -DENABLE_MYRIAD=OFF -DENABLE_MKL_DNN=ON \
-                -DENABLE_CLDNN=OFF -DENABLE_VALIDATION_SET=OFF -DPYTHON_EXECUTABLE=`which python` \
-                -DNGRAPH_ONNX_IMPORT_ENABLE=ON -DNGRAPH_UNIT_TEST_OPENVINO_ENABLE=TRUE -DNGRAPH_IE_ENABLE=ON \
-                -DCMAKE_INSTALL_PREFIX=${directory}/dldt_dist -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON"
-    cd "${directory}/dldt/ngraph"
-    git checkout "${ngraph_branch}"
-
-    cd "${directory}/dldt"
-
-    mkdir -p ./build
-    cd ./build
-    git lfs install
-    cmake ${CMAKE_ARGS} ..  || return 1
-    make -j $(lscpu --parse=CORE | grep -v '#' | sort | uniq | wc -l) || return 1
-    make install || return 1
+    mv "${directory}/ngraph-onnx/.ci/jenkins/setup.py" .
+    python3 setup.py develop || return 1
     return 0
 }
 
 function main() {
-    # By default copy stored nGraph master and use it to build PR branch
-    BACKENDS="cpu"
-
-    NUM_PARAMETERS="3"
+    NUM_PARAMETERS="1"
     if [ $# -lt "${NUM_PARAMETERS}" ]; then
         echo "ERROR: Expected at least ${NUM_PARAMETERS} parameter got $#"
         exit 1
@@ -99,27 +80,19 @@ function main() {
     for i in "$@"
     do
         case $i in
-            --backends=*)
-                BACKENDS="${i//${PATTERN}/}"
-                ;;
             --build-dir=*)
                 BUILD_DIR="${i//${PATTERN}/}"
                 ;;
-            --ngraph-branch=*)
-                NGRAPH_BRANCH="${i//${PATTERN}/}"
-                ;;
             *)
                 echo "Parameter $i not recognized."
                 exit 1
                 ;;
         esac
     done
 
-    BUILD_NGRAPH_CALL="build_ngraph \"${BUILD_DIR}\" \"${BACKENDS}\""
-    BUILD_DLDT_CALL="build_dldt \"${BUILD_DIR}\" \"${NGRAPH_BRANCH}\""
+    BUILD_OV_CALL="build_open_vino \"${BUILD_DIR}\""
 
-    eval "${BUILD_NGRAPH_CALL}"
-    # eval "${BUILD_DLDT_CALL}"
+    eval "${BUILD_OV_CALL}"
 
 }