Integrate GenAI into OV Docker images (#312)

* Initial commit

* arg_parser: Fixed regex for new GenAI archives

* Add more debug for arg_parser

* Replaced f-string with format because of old py version.

* Reworked regex. Updated debug info

* Fixed regex. Reworked whole script with black linter

* Updated copy.dockerfile with possible tar.gz extension instead of tgz

* Fixed wheels version install for openvino-tokenizers & openvino-genai

* arg_parser: Fixed regex for build_id

* Reset changes for RHEL

* Fixed installing OV wheels for RHEL8 dev image in case of OPENVINO_WHEELS_URL

* Fixed find cmd for installing from archive

* arg_parser.py: Fixed bandit scan

* arg_parser: Bandit scan fix

* Added ignoring errors B603 B607 for subprocess.call

* Actualized test_arg_parser.py

* Updated GH workflows with upload-artifact@v4

* test_arg_parser.py: Added passing OS

* test_arg_parser.py: Added distribution

* codestyle.yml: Make unique name for the artifacts workflow

* test_arg_parser.py: Fixed old URL names

* arg_parser.py: Fixed code style checks

* arg_parser.py: Fixed regex

* Rollback quotes via black.exe and ignore it in flake8

* disabled C812

* Linter fix. Updated messages in tests

* arg_parser.py: Fixed regex

* test_arg_parser.py: Switched from python38 to python310

* test_arg_parser.py: Fixed dockerfile_name

* Fixed tests

.github/workflows/build_test_lin.yml

@@ -41,7 +41,7 @@ jobs:
         run: docker images
       - name: Collecting artifacts
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: |

.github/workflows/build_test_win.yml

@@ -41,7 +41,7 @@ jobs:
         run: docker images
       - name: Collecting artifacts
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: |

.github/workflows/codestyle.yml

@@ -42,8 +42,8 @@ jobs:
 
       - name: Collecting codestyle artifacts
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
-          name: codestyle_checks
+          name: codestyle_checks_${{ matrix.os }}
           path: |
             ./utils_unittests.html

.github/workflows/hadolint_check.yml

@@ -37,7 +37,7 @@ jobs:
           done <  <(find dockerfiles/ \( -name '*.dockerfile' -o -name 'Dockerfile' \) -print0)
           exit "$has_issues"
       - name: Collecting artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: hadolint_results

.github/workflows/images_build_check.yml

@@ -47,7 +47,7 @@ jobs:
         run: docker images
       - name: Collecting artifacts
         if: ${{ always() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: |

.github/workflows/manual_publish.yml

@@ -82,7 +82,7 @@ jobs:
         run: docker logout openvino.azurecr.io
       - name: Collecting artifacts
         if: ${{ matrix.os == github.event.inputs.host_os }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: |

.github/workflows/update_dockerfiles.yml

@@ -66,7 +66,7 @@ jobs:
           echo "Pull Request URL - ${{ steps.create_pr.outputs.pull-request-url }}"
           echo "Pull Request Operation - ${{ steps.create_pr.outputs.pull-request-operation }}"
       - name: Collecting artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: |

setup.cfg

@@ -9,6 +9,7 @@ ignore = DAR101, DAR201, DAR401, D107, D415, I201, T001, S404, S603, G004, B009,
 enable-extensions=G
 per-file-ignores =
     tests/*: D100,D101,D102,D104,S108,CFQ004,PT023
+    utils/*: Q000,C812
     tests/conftest.py: D100,D101,D102,D103,D104,CFQ004
 
 [pydocstyle]

templates/rhel8/dist/dev.dockerfile.j2

@@ -5,10 +5,11 @@ ARG OPENVINO_WHEELS_URL
 # hadolint ignore=SC2102,DL3033
 RUN yum install -y cmake git && yum clean all && \
     if [ -z "$OPENVINO_WHEELS_URL" ]; then \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --extra-index-url https://download.pytorch.org/whl/cpu; \
     else \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu ; \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu ; \
     fi
 
 # download source for PyPi LGPL packages

templates/ubuntu18/install/copy.dockerfile.j2

@@ -5,7 +5,7 @@ ENV INTEL_OPENVINO_DIR /opt/intel/openvino
 # Creating user openvino and adding it to groups"users"
 RUN useradd -ms /bin/bash -G users openvino
 
-RUN tar -xzf "${TEMP_DIR}"/*.tgz && \
+RUN find "${TEMP_DIR}" -name "*.tgz" -o -name "*.tar.gz" -exec tar -xzf {} \; && \
     OV_BUILD="$(find . -maxdepth 1 -type d -name "*openvino*" | grep -oP '(?<=_)\d+.\d+.\d.\d+')" && \
     OV_YEAR="$(echo $OV_BUILD | grep -oP '^[^\d]*(\d+)')" && \
     OV_FOLDER="$(find . -maxdepth 1 -type d -name "*openvino*")" && \
@@ -14,7 +14,7 @@ RUN tar -xzf "${TEMP_DIR}"/*.tgz && \
     rm -rf "${TEMP_DIR:?}"/"$OV_FOLDER" && \
     ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino && \
     ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino_"$OV_YEAR" && \
-    rm -rf ${INTEL_OPENVINO_DIR}/tools/workbench && rm -rf ${TEMP_DIR} && \
+    rm -rf ${TEMP_DIR} && \
     chown -R openvino /opt/intel/openvino_"$OV_BUILD"
 {% if no_samples is defined or 'dev_no_samples' == distribution %}
 RUN rm -rf ${INTEL_OPENVINO_DIR}/samples

templates/ubuntu20/dist/dev.dockerfile.j2

@@ -5,11 +5,15 @@ ARG OPENVINO_WHEELS_URL
 # hadolint ignore=SC2102
 RUN apt-get update && apt-get install -y --no-install-recommends cmake make git && rm -rf /var/lib/apt/lists/* && \
     if [ -z "$OPENVINO_WHEELS_URL" ]; then \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="$OPENVINO_WHEELS_VERSION" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-genai=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --extra-index-url https://download.pytorch.org/whl/cpu; \
     else \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-genai=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
     fi
 
 WORKDIR ${INTEL_OPENVINO_DIR}/licensing

templates/ubuntu20/env/dev_env.dockerfile.j2

@@ -8,4 +8,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig

templates/ubuntu20/env/runtime_env.dockerfile.j2

@@ -8,4 +8,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig

templates/ubuntu20/install/copy.dockerfile.j2

@@ -5,7 +5,7 @@ ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
 # Creating user openvino and adding it to groups"users"
 RUN useradd -ms /bin/bash -G users openvino
 
-RUN tar -xzf "${TEMP_DIR}"/*.tgz && \
+RUN find "${TEMP_DIR}" \( -name "*.tgz" -o -name "*.tar.gz" \) -exec tar -xzf {} \; && \
     OV_BUILD="$(find . -maxdepth 1 -type d -name "*openvino*" | grep -oP '(?<=_)\d+.\d+.\d.\d+')" && \
     OV_YEAR="$(echo "$OV_BUILD" | grep -oP '^[^\d]*(\d+)')" && \
     OV_FOLDER="$(find . -maxdepth 1 -type d -name "*openvino*")" && \
@@ -14,7 +14,7 @@ RUN tar -xzf "${TEMP_DIR}"/*.tgz && \
     rm -rf "${TEMP_DIR:?}"/"$OV_FOLDER" && \
     ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino && \
     ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino_"$OV_YEAR" && \
-    rm -rf "${INTEL_OPENVINO_DIR}/tools/workbench" && rm -rf "${TEMP_DIR}" && \
+    rm -rf "${TEMP_DIR}" && \
     chown -R openvino /opt/intel/openvino_"$OV_BUILD"
 {% if no_samples is defined or 'dev_no_samples' == distribution %}
 RUN rm -rf "${INTEL_OPENVINO_DIR}/samples"

templates/ubuntu22/dist/dev.dockerfile.j2

@@ -5,11 +5,15 @@ ARG OPENVINO_WHEELS_URL
 # hadolint ignore=SC2102
 RUN apt-get update && apt-get install -y --no-install-recommends cmake make git && rm -rf /var/lib/apt/lists/* && \
     if [ -z "$OPENVINO_WHEELS_URL" ]; then \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="$OPENVINO_WHEELS_VERSION" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino-genai=="${OPENVINO_WHEELS_VERSION}" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --extra-index-url https://download.pytorch.org/whl/cpu; \
     else \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
-        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="$OPENVINO_WHEELS_VERSION" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-tokenizers=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino-genai=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" && \
+        ${PYTHON_VER} -m pip install --no-cache-dir --pre openvino_dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=="${OPENVINO_WHEELS_VERSION}" --trusted-host=* --find-links "$OPENVINO_WHEELS_URL" --extra-index-url https://download.pytorch.org/whl/cpu; \
     fi
 
 WORKDIR ${INTEL_OPENVINO_DIR}/licensing

templates/ubuntu22/env/dev_env.dockerfile.j2

@@ -7,4 +7,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig

templates/ubuntu22/env/runtime_env.dockerfile.j2

@@ -7,4 +7,5 @@ ENV TBB_DIR=/opt/intel/openvino/runtime/3rdparty/tbb/cmake
 ENV ngraph_DIR=/opt/intel/openvino/runtime/cmake
 ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake
 ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV OV_TOKENIZER_PREBUILD_EXTENSION_PATH=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so
 ENV PKG_CONFIG_PATH=/opt/intel/openvino/runtime/lib/intel64/pkgconfig

templates/ubuntu22/install/copy.dockerfile.j2

@@ -5,7 +5,7 @@ ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
 # Creating user openvino and adding it to groups"users"
 RUN useradd -ms /bin/bash -G users openvino
 
-RUN tar -xzf "${TEMP_DIR}"/*.tgz && \
+RUN find "${TEMP_DIR}" \( -name "*.tgz" -o -name "*.tar.gz" \) -exec tar -xzf {} \; && \
     OV_BUILD="$(find . -maxdepth 1 -type d -name "*openvino*" | grep -oP '(?<=_)\d+.\d+.\d.\d+')" && \
     OV_YEAR="$(echo "$OV_BUILD" | grep -oP '^[^\d]*(\d+)')" && \
     OV_FOLDER="$(find . -maxdepth 1 -type d -name "*openvino*")" && \
@@ -14,7 +14,7 @@ RUN tar -xzf "${TEMP_DIR}"/*.tgz && \
     rm -rf "${TEMP_DIR:?}"/"$OV_FOLDER" && \
     ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino && \
     ln --symbolic /opt/intel/openvino_"$OV_BUILD"/ /opt/intel/openvino_"$OV_YEAR" && \
-    rm -rf "${INTEL_OPENVINO_DIR}/tools/workbench" && rm -rf "${TEMP_DIR}" && \
+    rm -rf "${TEMP_DIR}" && \
     chown -R openvino /opt/intel/openvino_"$OV_BUILD"
 {% if no_samples is defined or 'dev_no_samples' == distribution %}
 RUN rm -rf "${INTEL_OPENVINO_DIR}/samples"