code/06.E2E/E2E_OpenVino_Chat_Phi3-instruct.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the optimum-cli tool to export a model to the OpenVINO format\n",
    "optimum-cli export openvino \\\n",
    "    # Specify the model to be exported\n",
    "    --model \"microsoft/Phi-3-mini-4k-instruct\" \\\n",
    "    # Define the task for the model, which is text generation with past context\n",
    "    --task text-generation-with-past \\\n",
    "    # Set the weight format to 4-bit integers for quantization\n",
    "    --weight-format int4 \\\n",
    "    # Define the group size for quantization\n",
    "    --group-size 128 \\\n",
    "    # Set the ratio parameter, possibly for optimization\n",
    "    --ratio 0.6 \\\n",
    "    # Use symmetric quantization\n",
    "    --sym \\\n",
    "    # Allow execution of remote code if necessary\n",
    "    --trust-remote-code \\\n",
    "    # Specify the output directory for the exported model\n",
    "    ./model/phi3-instruct/int4\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   1% 1/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   1% 1/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:00 • 0:00:28\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:00 • 0:00:28\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:28\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:28\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:28\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:28\n",
      "Mixed-Precision assignment ---------------------   2% 3/128 • 0:00:01 • 0:00:47\n",
      "Mixed-Precision assignment ---------------------   2% 3/128 • 0:00:01 • 0:00:47\n",
      "Mixed-Precision assignment  --------------------   3% 4/128 • 0:00:01 • 0:00:45\n",
      "Mixed-Precision assignment  --------------------   3% 4/128 • 0:00:01 • 0:00:45\n",
      "Mixed-Precision assignment  --------------------   4% 5/128 • 0:00:01 • 0:00:40\n",
      "Mixed-Precision assignment  --------------------   4% 5/128 • 0:00:01 • 0:00:40\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:35\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:35\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:35\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:35\n",
      "Mixed-Precision assignment - -------------------   5% 7/128 • 0:00:02 • 0:00:40\n",
      "Mixed-Precision assignment - -------------------   5% 7/128 • 0:00:02 • 0:00:40\n",
      "Mixed-Precision assignment - -------------------   5% 7/128 • 0:00:02 • 0:00:40\n",
      "Mixed-Precision assignment - -------------------   6% 8/128 • 0:00:02 • 0:00:39\n",
      "Mixed-Precision assignment - -------------------   6% 8/128 • 0:00:02 • 0:00:39\n",
      "Mixed-Precision assignment - -------------------   7% 9/128 • 0:00:03 • 0:00:37\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:03 • 0:00:34\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:03 • 0:00:34\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:03 • 0:00:34\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:03 • 0:00:34\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:03 • 0:00:34\n",
      "Mixed-Precision assignment - ------------------   9% 11/128 • 0:00:03 • 0:00:37\n",
      "Mixed-Precision assignment - ------------------   9% 11/128 • 0:00:03 • 0:00:37\n",
      "Mixed-Precision assignment - ------------------   9% 12/128 • 0:00:03 • 0:00:36\n",
      "Mixed-Precision assignment - ------------------   9% 12/128 • 0:00:04 • 0:00:36\n",
      "Mixed-Precision assignment - ------------------   9% 12/128 • 0:00:04 • 0:00:36\n",
      "Mixed-Precision assignment -- -----------------  10% 13/128 • 0:00:04 • 0:00:36\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:04 • 0:00:33\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:04 • 0:00:33\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:04 • 0:00:33\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:04 • 0:00:33\n",
      "Mixed-Precision assignment -- -----------------  12% 15/128 • 0:00:04 • 0:00:34\n",
      "Mixed-Precision assignment -- -----------------  12% 15/128 • 0:00:04 • 0:00:34\n",
      "Mixed-Precision assignment -- -----------------  12% 15/128 • 0:00:04 • 0:00:34\n",
      "Mixed-Precision assignment -- -----------------  12% 16/128 • 0:00:05 • 0:00:34\n",
      "Mixed-Precision assignment -- -----------------  12% 16/128 • 0:00:05 • 0:00:34\n",
      "Mixed-Precision assignment -- -----------------  13% 17/128 • 0:00:05 • 0:00:33\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:05 • 0:00:32\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:05 • 0:00:32\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:05 • 0:00:32\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:05 • 0:00:32\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:05 • 0:00:32\n",
      "Mixed-Precision assignment -- -----------------  15% 19/128 • 0:00:05 • 0:00:33\n",
      "Mixed-Precision assignment -- -----------------  15% 19/128 • 0:00:06 • 0:00:33\n",
      "Mixed-Precision assignment -- -----------------  15% 19/128 • 0:00:06 • 0:00:33\n",
      "Mixed-Precision assignment --- ----------------  16% 20/128 • 0:00:06 • 0:00:33\n",
      "Mixed-Precision assignment --- ----------------  16% 20/128 • 0:00:06 • 0:00:33\n",
      "Mixed-Precision assignment --- ----------------  16% 20/128 • 0:00:06 • 0:00:33\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:06 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:06 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:06 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:06 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:07 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  18% 23/128 • 0:00:07 • 0:00:32\n",
      "Mixed-Precision assignment --- ----------------  18% 23/128 • 0:00:07 • 0:00:32\n",
      "Mixed-Precision assignment --- ----------------  18% 23/128 • 0:00:07 • 0:00:32\n",
      "Mixed-Precision assignment --- ----------------  19% 24/128 • 0:00:07 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  19% 24/128 • 0:00:07 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  20% 25/128 • 0:00:07 • 0:00:31\n",
      "Mixed-Precision assignment --- ----------------  20% 25/128 • 0:00:07 • 0:00:31\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:07 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:08 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:08 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:08 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  21% 27/128 • 0:00:08 • 0:00:31\n",
      "Mixed-Precision assignment ---- ---------------  21% 27/128 • 0:00:08 • 0:00:31\n",
      "Mixed-Precision assignment ---- ---------------  21% 27/128 • 0:00:08 • 0:00:31\n",
      "Mixed-Precision assignment ---- ---------------  22% 28/128 • 0:00:08 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  22% 28/128 • 0:00:08 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  22% 28/128 • 0:00:08 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  23% 29/128 • 0:00:09 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:09 • 0:00:29\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:09 • 0:00:29\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:09 • 0:00:29\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:09 • 0:00:29\n",
      "Mixed-Precision assignment ---- ---------------  24% 31/128 • 0:00:09 • 0:00:30\n",
      "Mixed-Precision assignment ---- ---------------  24% 31/128 • 0:00:09 • 0:00:30\n",
      "Mixed-Precision assignment ----- --------------  25% 32/128 • 0:00:09 • 0:00:29\n",
      "Mixed-Precision assignment ----- --------------  25% 32/128 • 0:00:09 • 0:00:29\n",
      "Mixed-Precision assignment ----- --------------  26% 33/128 • 0:00:10 • 0:00:29\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  27% 35/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  27% 35/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  28% 36/128 • 0:00:10 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  28% 36/128 • 0:00:11 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  28% 36/128 • 0:00:11 • 0:00:28\n",
      "Mixed-Precision assignment ----- --------------  29% 37/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ------ -------------  30% 39/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ------ -------------  30% 39/128 • 0:00:11 • 0:00:27\n",
      "Mixed-Precision assignment ------ -------------  31% 40/128 • 0:00:12 • 0:00:26\n",
      "Mixed-Precision assignment ------ -------------  31% 40/128 • 0:00:12 • 0:00:26\n",
      "Mixed-Precision assignment ------ -------------  31% 40/128 • 0:00:12 • 0:00:26\n",
      "Mixed-Precision assignment ------ -------------  32% 41/128 • 0:00:12 • 0:00:26\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:12 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:12 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:12 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:12 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  34% 43/128 • 0:00:12 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  34% 43/128 • 0:00:13 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  34% 43/128 • 0:00:13 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  34% 44/128 • 0:00:13 • 0:00:25\n",
      "Mixed-Precision assignment ------ -------------  34% 44/128 • 0:00:13 • 0:00:25\n",
      "Mixed-Precision assignment ------- ------------  35% 45/128 • 0:00:13 • 0:00:25\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:13 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:13 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:13 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:13 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  37% 47/128 • 0:00:14 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  37% 47/128 • 0:00:14 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  38% 48/128 • 0:00:14 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  38% 48/128 • 0:00:14 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  38% 48/128 • 0:00:14 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  38% 49/128 • 0:00:14 • 0:00:24\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:14 • 0:00:23\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:14 • 0:00:23\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:14 • 0:00:23\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:15 • 0:00:23\n",
      "Mixed-Precision assignment ------- ------------  40% 51/128 • 0:00:15 • 0:00:23\n",
      "Mixed-Precision assignment ------- ------------  40% 51/128 • 0:00:15 • 0:00:23\n",
      "Mixed-Precision assignment ------- ------------  40% 51/128 • 0:00:15 • 0:00:23\n",
      "Mixed-Precision assignment -------- -----------  41% 52/128 • 0:00:15 • 0:00:23\n",
      "Mixed-Precision assignment -------- -----------  41% 52/128 • 0:00:15 • 0:00:23\n",
      "Mixed-Precision assignment -------- -----------  41% 53/128 • 0:00:15 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:15 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:15 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:16 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:16 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:16 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  43% 55/128 • 0:00:16 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  43% 55/128 • 0:00:16 • 0:00:22\n",
      "Mixed-Precision assignment -------- -----------  44% 56/128 • 0:00:16 • 0:00:21\n",
      "Mixed-Precision assignment -------- -----------  44% 56/128 • 0:00:16 • 0:00:21\n",
      "Mixed-Precision assignment -------- -----------  44% 56/128 • 0:00:16 • 0:00:21\n",
      "Mixed-Precision assignment -------- -----------  45% 57/128 • 0:00:16 • 0:00:21\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:16 • 0:00:21\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:17 • 0:00:21\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:17 • 0:00:21\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:17 • 0:00:21\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:17 • 0:00:21\n",
      "Mixed-Precision assignment --------- ----------  46% 59/128 • 0:00:17 • 0:00:21\n",
      "Mixed-Precision assignment --------- ----------  47% 60/128 • 0:00:17 • 0:00:20\n",
      "Mixed-Precision assignment --------- ----------  47% 60/128 • 0:00:17 • 0:00:20\n",
      "Mixed-Precision assignment --------- ----------  47% 60/128 • 0:00:17 • 0:00:20\n",
      "Mixed-Precision assignment --------- ----------  48% 61/128 • 0:00:17 • 0:00:20\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment --------- ----------  49% 63/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment --------- ----------  49% 63/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment --------- ----------  49% 63/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  50% 64/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  50% 64/128 • 0:00:18 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  51% 65/128 • 0:00:19 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  52% 67/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  52% 67/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  53% 68/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  53% 68/128 • 0:00:19 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  53% 68/128 • 0:00:20 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  54% 69/128 • 0:00:20 • 0:00:18\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:20 • 0:00:17\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:20 • 0:00:17\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:20 • 0:00:17\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:20 • 0:00:17\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:20 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  55% 71/128 • 0:00:20 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  55% 71/128 • 0:00:20 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  56% 72/128 • 0:00:21 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  56% 72/128 • 0:00:21 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  56% 72/128 • 0:00:21 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  57% 73/128 • 0:00:21 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:21 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:21 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:21 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:21 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  59% 75/128 • 0:00:21 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  59% 75/128 • 0:00:22 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  59% 75/128 • 0:00:22 • 0:00:16\n",
      "Mixed-Precision assignment ----------- --------  59% 76/128 • 0:00:22 • 0:00:15\n",
      "Mixed-Precision assignment ----------- --------  59% 76/128 • 0:00:22 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  60% 77/128 • 0:00:22 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:22 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:22 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:22 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:22 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:23 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  62% 79/128 • 0:00:23 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  62% 79/128 • 0:00:23 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  62% 80/128 • 0:00:23 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  62% 80/128 • 0:00:23 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  62% 80/128 • 0:00:23 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:23 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:23 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:23 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:23 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:24 • 0:00:14\n",
      "Mixed-Precision assignment ------------ -------  65% 83/128 • 0:00:24 • 0:00:13\n",
      "Mixed-Precision assignment ------------ -------  65% 83/128 • 0:00:24 • 0:00:13\n",
      "Mixed-Precision assignment ------------- ------  66% 84/128 • 0:00:24 • 0:00:13\n",
      "Mixed-Precision assignment ------------- ------  66% 84/128 • 0:00:24 • 0:00:13\n",
      "Mixed-Precision assignment ------------- ------  66% 84/128 • 0:00:24 • 0:00:13\n",
      "Mixed-Precision assignment ------------- ------  66% 85/128 • 0:00:24 • 0:00:13\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:24 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:24 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  68% 87/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  68% 87/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  69% 88/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  69% 88/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment ------------- ------  69% 88/128 • 0:00:25 • 0:00:12\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:25 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  71% 91/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  71% 91/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  72% 92/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  72% 92/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  72% 92/128 • 0:00:26 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  73% 93/128 • 0:00:27 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment -------------- -----  74% 95/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  75% 96/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  75% 96/128 • 0:00:27 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  75% 96/128 • 0:00:28 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  76% 97/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ----  77% 99/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ----  77% 99/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ----  77% 99/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment -------------- ----  78% 100/128 • 0:00:28 • 0:00:09\n",
      "Mixed-Precision assignment -------------- ----  78% 100/128 • 0:00:29 • 0:00:09\n",
      "Mixed-Precision assignment -------------- ----  78% 100/128 • 0:00:29 • 0:00:09\n",
      "Mixed-Precision assignment -------------- ----  79% 101/128 • 0:00:29 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:29 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:29 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:29 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:29 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:29 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 103/128 • 0:00:29 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 103/128 • 0:00:30 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  81% 104/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  81% 104/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  82% 105/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:30 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  84% 107/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  84% 108/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  84% 108/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  84% 108/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  85% 109/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:31 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:32 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  87% 111/128 • 0:00:32 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  87% 111/128 • 0:00:32 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  88% 112/128 • 0:00:32 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  88% 112/128 • 0:00:32 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  88% 112/128 • 0:00:32 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  88% 113/128 • 0:00:32 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:32 • 0:00:04\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:32 • 0:00:04\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  90% 115/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  90% 115/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  90% 115/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  91% 116/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  91% 117/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  91% 117/128 • 0:00:33 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:33 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  93% 119/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  93% 119/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  94% 120/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  94% 120/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  94% 120/128 • 0:00:34 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  95% 121/128 • 0:00:34 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   96% 123/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   96% 123/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   97% 124/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   97% 124/128 • 0:00:35 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   98% 125/128 • 0:00:35 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:36 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:36 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:36 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:36 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:36 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   99% 127/128 • 0:00:36 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   99% 127/128 • 0:00:36 • 0:00:01\n",
      "Mixed-Precision assignment ------------------- 100% 128/128 • 0:00:36 • 0:00:00\n",
      "Mixed-Precision assignment ------------------- 100% 128/128 • 0:00:36 • 0:00:00\n",
      "INFO:nncf:Statistics of the bitwidth distribution:\n",
      "+----------------+-----------------------------+----------------------------------------+\n",
      "|   Num bits (N) | % all parameters (layers)   | % ratio-defining parameters (layers)   |\n",
      "+================+=============================+========================================+\n",
      "|              8 | 44% (56 / 130)              | 41% (54 / 128)                         |\n",
      "+----------------+-----------------------------+----------------------------------------+\n",
      "|              4 | 56% (74 / 130)              | 59% (74 / 128)                         |\n",
      "+----------------+-----------------------------+----------------------------------------+\n",
      "\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   1% 1/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   1% 1/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   1% 1/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   1% 1/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   1% 1/130 • 0:00:01 • -:--:--\n",
      "Applying Weight Compression --------------------   2% 2/130 • 0:00:01 • 0:01:08\n",
      "Applying Weight Compression --------------------   2% 2/130 • 0:00:01 • 0:01:08\n",
      "Applying Weight Compression --------------------   2% 3/130 • 0:00:01 • 0:00:47\n",
      "Applying Weight Compression --------------------   2% 3/130 • 0:00:01 • 0:00:47\n",
      "Applying Weight Compression --------------------   2% 3/130 • 0:00:01 • 0:00:47\n",
      "Applying Weight Compression --------------------   2% 3/130 • 0:00:02 • 0:00:47\n",
      "Applying Weight Compression  -------------------   3% 4/130 • 0:00:03 • 0:01:35\n",
      "Applying Weight Compression  -------------------   3% 4/130 • 0:00:03 • 0:01:35\n",
      "Applying Weight Compression  -------------------   4% 5/130 • 0:00:04 • 0:01:55\n",
      "Applying Weight Compression  -------------------   4% 5/130 • 0:00:04 • 0:01:55\n",
      "Applying Weight Compression  -------------------   4% 5/130 • 0:00:04 • 0:01:55\n",
      "Applying Weight Compression  -------------------   5% 6/130 • 0:00:05 • 0:02:06\n",
      "Applying Weight Compression  -------------------   5% 6/130 • 0:00:05 • 0:02:06\n",
      "Applying Weight Compression  -------------------   5% 6/130 • 0:00:06 • 0:02:06\n",
      "Applying Weight Compression  -------------------   5% 6/130 • 0:00:07 • 0:02:06\n",
      "Applying Weight Compression - ------------------   5% 7/130 • 0:00:07 • 0:02:13\n",
      "Applying Weight Compression - ------------------   5% 7/130 • 0:00:07 • 0:02:13\n",
      "Applying Weight Compression - ------------------   6% 8/130 • 0:00:07 • 0:01:57\n",
      "Applying Weight Compression - ------------------   7% 9/130 • 0:00:07 • 0:01:45\n",
      "Applying Weight Compression - ------------------   7% 9/130 • 0:00:07 • 0:01:45\n",
      "Applying Weight Compression - ------------------   7% 9/130 • 0:00:07 • 0:01:45\n",
      "Applying Weight Compression - ------------------   7% 9/130 • 0:00:07 • 0:01:45\n",
      "Applying Weight Compression - -----------------   8% 10/130 • 0:00:09 • 0:01:52\n",
      "Applying Weight Compression - -----------------   8% 10/130 • 0:00:09 • 0:01:52\n",
      "Applying Weight Compression - -----------------   8% 10/130 • 0:00:09 • 0:01:52\n",
      "Applying Weight Compression - -----------------   8% 11/130 • 0:00:10 • 0:01:57\n",
      "Applying Weight Compression - -----------------   8% 11/130 • 0:00:10 • 0:01:57\n",
      "Applying Weight Compression - -----------------   9% 12/130 • 0:00:10 • 0:01:48\n",
      "Applying Weight Compression - -----------------   9% 12/130 • 0:00:10 • 0:01:48\n",
      "Applying Weight Compression - -----------------  10% 13/130 • 0:00:10 • 0:01:40\n",
      "Applying Weight Compression - -----------------  10% 13/130 • 0:00:10 • 0:01:40\n",
      "Applying Weight Compression - -----------------  10% 13/130 • 0:00:11 • 0:01:40\n",
      "Applying Weight Compression -- ----------------  11% 14/130 • 0:00:11 • 0:01:34\n",
      "Applying Weight Compression -- ----------------  11% 14/130 • 0:00:11 • 0:01:34\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:11 • 0:01:29\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:11 • 0:01:29\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:11 • 0:01:29\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:12 • 0:01:29\n",
      "Applying Weight Compression -- ----------------  12% 16/130 • 0:00:12 • 0:01:33\n",
      "Applying Weight Compression -- ----------------  12% 16/130 • 0:00:13 • 0:01:33\n",
      "Applying Weight Compression -- ----------------  12% 16/130 • 0:00:14 • 0:01:33\n",
      "Applying Weight Compression -- ----------------  13% 17/130 • 0:00:14 • 0:01:36\n",
      "Applying Weight Compression -- ----------------  14% 18/130 • 0:00:14 • 0:01:31\n",
      "Applying Weight Compression -- ----------------  14% 18/130 • 0:00:14 • 0:01:31\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:14 • 0:01:27\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:14 • 0:01:27\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:14 • 0:01:27\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:15 • 0:01:27\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:16 • 0:01:27\n",
      "Applying Weight Compression -- ----------------  15% 20/130 • 0:00:16 • 0:01:30\n",
      "Applying Weight Compression -- ----------------  15% 20/130 • 0:00:16 • 0:01:30\n",
      "Applying Weight Compression --- ---------------  16% 21/130 • 0:00:16 • 0:01:26\n",
      "Applying Weight Compression --- ---------------  16% 21/130 • 0:00:16 • 0:01:26\n",
      "Applying Weight Compression --- ---------------  16% 21/130 • 0:00:16 • 0:01:26\n",
      "Applying Weight Compression --- ---------------  17% 22/130 • 0:00:16 • 0:01:23\n",
      "Applying Weight Compression --- ---------------  17% 22/130 • 0:00:16 • 0:01:23\n",
      "Applying Weight Compression --- ---------------  18% 23/130 • 0:00:16 • 0:01:20\n",
      "Applying Weight Compression --- ---------------  18% 23/130 • 0:00:17 • 0:01:20\n",
      "Applying Weight Compression --- ---------------  18% 23/130 • 0:00:17 • 0:01:20\n",
      "Applying Weight Compression --- ---------------  18% 24/130 • 0:00:17 • 0:01:17\n",
      "Applying Weight Compression --- ---------------  18% 24/130 • 0:00:17 • 0:01:17\n",
      "Applying Weight Compression --- ---------------  19% 25/130 • 0:00:17 • 0:01:14\n",
      "Applying Weight Compression --- ---------------  19% 25/130 • 0:00:17 • 0:01:14\n",
      "Applying Weight Compression --- ---------------  19% 25/130 • 0:00:17 • 0:01:14\n",
      "Applying Weight Compression --- ---------------  20% 26/130 • 0:00:17 • 0:01:12\n",
      "Applying Weight Compression --- ---------------  20% 26/130 • 0:00:17 • 0:01:12\n",
      "Applying Weight Compression --- ---------------  20% 26/130 • 0:00:18 • 0:01:12\n",
      "Applying Weight Compression --- ---------------  21% 27/130 • 0:00:18 • 0:01:10\n",
      "Applying Weight Compression --- ---------------  21% 27/130 • 0:00:18 • 0:01:10\n",
      "Applying Weight Compression ---- --------------  22% 28/130 • 0:00:18 • 0:01:07\n",
      "Applying Weight Compression ---- --------------  22% 28/130 • 0:00:18 • 0:01:07\n",
      "Applying Weight Compression ---- --------------  22% 28/130 • 0:00:18 • 0:01:07\n",
      "Applying Weight Compression ---- --------------  22% 28/130 • 0:00:19 • 0:01:07\n",
      "Applying Weight Compression ---- --------------  22% 29/130 • 0:00:19 • 0:01:10\n",
      "Applying Weight Compression ---- --------------  22% 29/130 • 0:00:20 • 0:01:10\n",
      "Applying Weight Compression ---- --------------  23% 30/130 • 0:00:21 • 0:01:12\n",
      "Applying Weight Compression ---- --------------  23% 30/130 • 0:00:21 • 0:01:12\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:21 • 0:01:09\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:21 • 0:01:09\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:21 • 0:01:09\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:22 • 0:01:09\n",
      "Applying Weight Compression ---- --------------  25% 32/130 • 0:00:23 • 0:01:11\n",
      "Applying Weight Compression ---- --------------  25% 32/130 • 0:00:23 • 0:01:11\n",
      "Applying Weight Compression ---- --------------  25% 32/130 • 0:00:24 • 0:01:11\n",
      "Applying Weight Compression ---- --------------  25% 33/130 • 0:00:24 • 0:01:13\n",
      "Applying Weight Compression ---- --------------  25% 33/130 • 0:00:24 • 0:01:13\n",
      "Applying Weight Compression ---- --------------  25% 33/130 • 0:00:24 • 0:01:13\n",
      "Applying Weight Compression ---- --------------  25% 33/130 • 0:00:25 • 0:01:13\n",
      "Applying Weight Compression ---- --------------  26% 34/130 • 0:00:25 • 0:01:14\n",
      "Applying Weight Compression ---- --------------  26% 34/130 • 0:00:25 • 0:01:14\n",
      "Applying Weight Compression ----- -------------  27% 35/130 • 0:00:26 • 0:01:13\n",
      "Applying Weight Compression ----- -------------  27% 35/130 • 0:00:26 • 0:01:13\n",
      "Applying Weight Compression ----- -------------  28% 36/130 • 0:00:27 • 0:01:12\n",
      "Applying Weight Compression ----- -------------  28% 36/130 • 0:00:27 • 0:01:12\n",
      "Applying Weight Compression ----- -------------  28% 36/130 • 0:00:28 • 0:01:12\n",
      "Applying Weight Compression ----- -------------  28% 37/130 • 0:00:28 • 0:01:12\n",
      "Applying Weight Compression ----- -------------  29% 38/130 • 0:00:28 • 0:01:11\n",
      "Applying Weight Compression ----- -------------  29% 38/130 • 0:00:29 • 0:01:11\n",
      "Applying Weight Compression ----- -------------  30% 39/130 • 0:00:29 • 0:01:10\n",
      "Applying Weight Compression ----- -------------  30% 39/130 • 0:00:29 • 0:01:10\n",
      "Applying Weight Compression ----- -------------  30% 39/130 • 0:00:30 • 0:01:10\n",
      "Applying Weight Compression ----- -------------  30% 39/130 • 0:00:30 • 0:01:10\n",
      "Applying Weight Compression ----- -------------  31% 40/130 • 0:00:30 • 0:01:10\n",
      "Applying Weight Compression ----- -------------  31% 40/130 • 0:00:31 • 0:01:10\n",
      "Applying Weight Compression ----- -------------  32% 41/130 • 0:00:31 • 0:01:11\n",
      "Applying Weight Compression ------ ------------  32% 42/130 • 0:00:32 • 0:01:08\n",
      "Applying Weight Compression ------ ------------  32% 42/130 • 0:00:32 • 0:01:08\n",
      "Applying Weight Compression ------ ------------  32% 42/130 • 0:00:32 • 0:01:08\n",
      "Applying Weight Compression ------ ------------  33% 43/130 • 0:00:33 • 0:01:06\n",
      "Applying Weight Compression ------ ------------  34% 44/130 • 0:00:33 • 0:01:05\n",
      "Applying Weight Compression ------ ------------  34% 44/130 • 0:00:33 • 0:01:05\n",
      "Applying Weight Compression ------ ------------  34% 44/130 • 0:00:34 • 0:01:05\n",
      "Applying Weight Compression ------ ------------  35% 45/130 • 0:00:34 • 0:01:03\n",
      "Applying Weight Compression ------ ------------  35% 46/130 • 0:00:35 • 0:01:03\n",
      "Applying Weight Compression ------ ------------  35% 46/130 • 0:00:35 • 0:01:03\n",
      "Applying Weight Compression ------ ------------  36% 47/130 • 0:00:36 • 0:01:01\n",
      "Applying Weight Compression ------- -----------  37% 48/130 • 0:00:36 • 0:00:59\n",
      "Applying Weight Compression ------- -----------  38% 49/130 • 0:00:36 • 0:00:57\n",
      "Applying Weight Compression ------- -----------  38% 49/130 • 0:00:36 • 0:00:57\n",
      "Applying Weight Compression ------- -----------  38% 50/130 • 0:00:36 • 0:00:55\n",
      "Applying Weight Compression ------- -----------  38% 50/130 • 0:00:36 • 0:00:55\n",
      "Applying Weight Compression ------- -----------  38% 50/130 • 0:00:37 • 0:00:55\n",
      "Applying Weight Compression ------- -----------  39% 51/130 • 0:00:37 • 0:00:57\n",
      "Applying Weight Compression ------- -----------  40% 52/130 • 0:00:37 • 0:00:54\n",
      "Applying Weight Compression ------- -----------  40% 52/130 • 0:00:37 • 0:00:54\n",
      "Applying Weight Compression ------- -----------  42% 54/130 • 0:00:37 • 0:00:50\n",
      "Applying Weight Compression ------- -----------  42% 54/130 • 0:00:38 • 0:00:50\n",
      "Applying Weight Compression ------- -----------  42% 54/130 • 0:00:38 • 0:00:50\n",
      "Applying Weight Compression -------- ----------  42% 55/130 • 0:00:38 • 0:00:50\n",
      "Applying Weight Compression -------- ----------  43% 56/130 • 0:00:39 • 0:00:49\n",
      "Applying Weight Compression -------- ----------  44% 57/130 • 0:00:39 • 0:00:46\n",
      "Applying Weight Compression -------- ----------  44% 57/130 • 0:00:39 • 0:00:46\n",
      "Applying Weight Compression -------- ----------  44% 57/130 • 0:00:39 • 0:00:46\n",
      "Applying Weight Compression -------- ----------  45% 58/130 • 0:00:40 • 0:00:46\n",
      "Applying Weight Compression -------- ----------  45% 58/130 • 0:00:40 • 0:00:46\n",
      "Applying Weight Compression -------- ----------  45% 58/130 • 0:00:40 • 0:00:46\n",
      "Applying Weight Compression -------- ----------  45% 59/130 • 0:00:40 • 0:00:47\n",
      "Applying Weight Compression -------- ----------  46% 60/130 • 0:00:41 • 0:00:46\n",
      "Applying Weight Compression -------- ----------  47% 61/130 • 0:00:41 • 0:00:45\n",
      "Applying Weight Compression -------- ----------  47% 61/130 • 0:00:41 • 0:00:45\n",
      "Applying Weight Compression --------- ---------  48% 62/130 • 0:00:42 • 0:00:44\n",
      "Applying Weight Compression --------- ---------  48% 63/130 • 0:00:42 • 0:00:43\n",
      "Applying Weight Compression --------- ---------  49% 64/130 • 0:00:42 • 0:00:42\n",
      "Applying Weight Compression --------- ---------  49% 64/130 • 0:00:42 • 0:00:42\n",
      "Applying Weight Compression --------- ---------  50% 65/130 • 0:00:42 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  50% 65/130 • 0:00:43 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  50% 65/130 • 0:00:43 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  50% 65/130 • 0:00:43 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  51% 66/130 • 0:00:44 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  52% 67/130 • 0:00:44 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  52% 67/130 • 0:00:44 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  52% 67/130 • 0:00:45 • 0:00:39\n",
      "Applying Weight Compression --------- ---------  52% 68/130 • 0:00:45 • 0:00:38\n",
      "Applying Weight Compression ---------- --------  53% 69/130 • 0:00:46 • 0:00:38\n",
      "Applying Weight Compression ---------- --------  53% 69/130 • 0:00:46 • 0:00:38\n",
      "Applying Weight Compression ---------- --------  53% 69/130 • 0:00:46 • 0:00:38\n",
      "Applying Weight Compression ---------- --------  54% 70/130 • 0:00:46 • 0:00:39\n",
      "Applying Weight Compression ---------- --------  55% 71/130 • 0:00:47 • 0:00:40\n",
      "Applying Weight Compression ---------- --------  55% 71/130 • 0:00:47 • 0:00:40\n",
      "Applying Weight Compression ---------- --------  55% 72/130 • 0:00:48 • 0:00:40\n",
      "Applying Weight Compression ---------- --------  55% 72/130 • 0:00:48 • 0:00:40\n",
      "Applying Weight Compression ---------- --------  55% 72/130 • 0:00:48 • 0:00:40\n",
      "Applying Weight Compression ---------- --------  56% 73/130 • 0:00:49 • 0:00:38\n",
      "Applying Weight Compression ---------- --------  56% 73/130 • 0:00:49 • 0:00:38\n",
      "Applying Weight Compression ---------- --------  57% 74/130 • 0:00:49 • 0:00:38\n",
      "Applying Weight Compression ---------- --------  58% 75/130 • 0:00:50 • 0:00:36\n",
      "Applying Weight Compression ---------- --------  58% 75/130 • 0:00:50 • 0:00:36\n",
      "Applying Weight Compression ----------- -------  58% 76/130 • 0:00:51 • 0:00:35\n",
      "Applying Weight Compression ----------- -------  58% 76/130 • 0:00:51 • 0:00:35\n",
      "Applying Weight Compression ----------- -------  58% 76/130 • 0:00:51 • 0:00:35\n",
      "Applying Weight Compression ----------- -------  59% 77/130 • 0:00:52 • 0:00:34\n",
      "Applying Weight Compression ----------- -------  60% 78/130 • 0:00:52 • 0:00:34\n",
      "Applying Weight Compression ----------- -------  60% 78/130 • 0:00:52 • 0:00:34\n",
      "Applying Weight Compression ----------- -------  60% 78/130 • 0:00:53 • 0:00:34\n",
      "Applying Weight Compression ----------- -------  61% 79/130 • 0:00:53 • 0:00:32\n",
      "Applying Weight Compression ----------- -------  62% 80/130 • 0:00:53 • 0:00:32\n",
      "Applying Weight Compression ----------- -------  62% 80/130 • 0:00:54 • 0:00:32\n",
      "Applying Weight Compression ----------- -------  62% 81/130 • 0:00:54 • 0:00:31\n",
      "Applying Weight Compression ----------- -------  62% 81/130 • 0:00:54 • 0:00:31\n",
      "Applying Weight Compression ----------- -------  63% 82/130 • 0:00:55 • 0:00:30\n",
      "Applying Weight Compression ----------- -------  63% 82/130 • 0:00:55 • 0:00:30\n",
      "Applying Weight Compression ----------- -------  63% 82/130 • 0:00:56 • 0:00:30\n",
      "Applying Weight Compression ------------ ------  64% 83/130 • 0:00:56 • 0:00:30\n",
      "Applying Weight Compression ------------ ------  65% 84/130 • 0:00:57 • 0:00:29\n",
      "Applying Weight Compression ------------ ------  65% 84/130 • 0:00:57 • 0:00:29\n",
      "Applying Weight Compression ------------ ------  65% 84/130 • 0:00:57 • 0:00:29\n",
      "Applying Weight Compression ------------ ------  65% 85/130 • 0:00:58 • 0:00:28\n",
      "Applying Weight Compression ------------ ------  65% 85/130 • 0:00:58 • 0:00:28\n",
      "Applying Weight Compression ------------ ------  66% 86/130 • 0:00:58 • 0:00:28\n",
      "Applying Weight Compression ------------ ------  66% 86/130 • 0:00:59 • 0:00:28\n",
      "Applying Weight Compression ------------ ------  67% 87/130 • 0:00:59 • 0:00:27\n",
      "Applying Weight Compression ------------ ------  67% 87/130 • 0:00:59 • 0:00:27\n",
      "Applying Weight Compression ------------ ------  67% 87/130 • 0:01:00 • 0:00:27\n",
      "Applying Weight Compression ------------ ------  68% 88/130 • 0:01:00 • 0:00:26\n",
      "Applying Weight Compression ------------- -----  68% 89/130 • 0:01:01 • 0:00:26\n",
      "Applying Weight Compression ------------- -----  68% 89/130 • 0:01:01 • 0:00:26\n",
      "Applying Weight Compression ------------- -----  69% 90/130 • 0:01:01 • 0:00:25\n",
      "Applying Weight Compression ------------- -----  69% 90/130 • 0:01:01 • 0:00:25\n",
      "Applying Weight Compression ------------- -----  70% 91/130 • 0:01:02 • 0:00:24\n",
      "Applying Weight Compression ------------- -----  70% 91/130 • 0:01:02 • 0:00:24\n",
      "Applying Weight Compression ------------- -----  70% 91/130 • 0:01:03 • 0:00:24\n",
      "Applying Weight Compression ------------- -----  71% 92/130 • 0:01:03 • 0:00:24\n",
      "Applying Weight Compression ------------- -----  72% 93/130 • 0:01:03 • 0:00:23\n",
      "Applying Weight Compression ------------- -----  72% 93/130 • 0:01:04 • 0:00:23\n",
      "Applying Weight Compression ------------- -----  72% 94/130 • 0:01:04 • 0:00:22\n",
      "Applying Weight Compression ------------- -----  72% 94/130 • 0:01:04 • 0:00:22\n",
      "Applying Weight Compression ------------- -----  73% 95/130 • 0:01:05 • 0:00:22\n",
      "Applying Weight Compression ------------- -----  73% 95/130 • 0:01:05 • 0:00:22\n",
      "Applying Weight Compression ------------- -----  73% 95/130 • 0:01:06 • 0:00:22\n",
      "Applying Weight Compression -------------- ----  74% 96/130 • 0:01:06 • 0:00:21\n",
      "Applying Weight Compression -------------- ----  75% 97/130 • 0:01:06 • 0:00:22\n",
      "Applying Weight Compression -------------- ----  75% 97/130 • 0:01:07 • 0:00:22\n",
      "Applying Weight Compression -------------- ----  75% 98/130 • 0:01:07 • 0:00:22\n",
      "Applying Weight Compression ------------- ----  77% 100/130 • 0:01:07 • 0:00:20\n",
      "Applying Weight Compression ------------- ----  77% 100/130 • 0:01:08 • 0:00:20\n",
      "Applying Weight Compression -------------- ---  79% 103/130 • 0:01:08 • 0:00:17\n",
      "Applying Weight Compression -------------- ---  80% 104/130 • 0:01:08 • 0:00:16\n",
      "Applying Weight Compression -------------- ---  82% 107/130 • 0:01:08 • 0:00:14\n",
      "Applying Weight Compression --------------- --  84% 109/130 • 0:01:08 • 0:00:12\n",
      "Applying Weight Compression --------------- --  85% 111/130 • 0:01:08 • 0:00:11\n",
      "Applying Weight Compression --------------- --  85% 111/130 • 0:01:09 • 0:00:11\n",
      "Applying Weight Compression --------------- --  85% 111/130 • 0:01:09 • 0:00:11\n",
      "Applying Weight Compression --------------- --  87% 113/130 • 0:01:09 • 0:00:10\n",
      "Applying Weight Compression --------------- --  88% 115/130 • 0:01:09 • 0:00:08\n",
      "Applying Weight Compression ---------------- -  90% 117/130 • 0:01:09 • 0:00:07\n",
      "Applying Weight Compression ---------------- -  92% 119/130 • 0:01:09 • 0:00:06\n",
      "Applying Weight Compression ---------------- -  94% 122/130 • 0:01:09 • 0:00:04\n",
      "Applying Weight Compression -----------------   95% 124/130 • 0:01:10 • 0:00:03\n",
      "Applying Weight Compression -----------------   96% 125/130 • 0:01:10 • 0:00:03\n",
      "Applying Weight Compression -----------------   98% 127/130 • 0:01:10 • 0:00:02\n",
      "Applying Weight Compression -----------------   98% 127/130 • 0:01:10 • 0:00:02\n",
      "Applying Weight Compression -----------------   98% 128/130 • 0:01:11 • 0:00:01\n",
      "Applying Weight Compression ------------------ 100% 130/130 • 0:01:11 • 0:00:00\n",
      "Applying Weight Compression ------------------ 100% 130/130 • 0:01:11 • 0:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kinfeylo\\AppData\\Roaming\\Python\\Python311\\site-packages\\threadpoolctl.py:1214: RuntimeWarning: \n",
      "Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at\n",
      "the same time. Both libraries are known to be incompatible and this\n",
      "can cause random crashes or deadlocks on Linux when loaded in the\n",
      "same Python program.\n",
      "Using threadpoolctl may cause crashes or deadlocks. For more\n",
      "information and possible workarounds, please see\n",
      "    https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md\n",
      "\n",
      "  warnings.warn(msg, RuntimeWarning)\n",
      "Framework not specified. Using pt to export the model.\n",
      "`flash-attention` package not found, consider installing for better performance: No module named 'flash_attn'.\n",
      "Current `flash-attention` does not support `window_size`. Either upgrade or use `attn_implementation='eager'`.\n",
      "\n",
      "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\n",
      "Loading checkpoint shards:  50%|█████     | 1/2 [00:20<00:20, 20.16s/it]\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:28<00:00, 13.17s/it]\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:28<00:00, 14.22s/it]\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Using framework PyTorch: 2.4.0+cpu\n",
      "WARNING:root:Cannot apply model.to_bettertransformer because of the exception:\n",
      "The model type phi3 is not yet supported to be used with BetterTransformer. Feel free to open an issue at https://github.com/huggingface/optimum/issues if you would like this model type to be supported. Currently supported models are: dict_keys(['albert', 'bark', 'bart', 'bert', 'bert-generation', 'blenderbot', 'bloom', 'camembert', 'blip-2', 'clip', 'codegen', 'data2vec-text', 'deit', 'distilbert', 'electra', 'ernie', 'fsmt', 'gpt2', 'gptj', 'gpt_neo', 'gpt_neox', 'hubert', 'layoutlm', 'm2m_100', 'marian', 'markuplm', 'mbart', 'opt', 'pegasus', 'rembert', 'prophetnet', 'roberta', 'roc_bert', 'roformer', 'splinter', 'tapas', 't5', 'vilt', 'vit', 'vit_mae', 'vit_msn', 'wav2vec2', 'xlm-roberta', 'yolos']).. Usage model with stateful=True may be non-effective if model does not contain torch.functional.scaled_dot_product_attention\n",
      "Overriding 1 configuration item(s)\n",
      "\t- use_cache -> True\n",
      "C:\\Users\\kinfeylo\\AppData\\Roaming\\Python\\Python311\\site-packages\\transformers\\modeling_attn_mask_utils.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if (input_shape[-1] > 1 or self.sliding_window is not None) and self.is_causal:\n",
      "C:\\Users\\kinfeylo\\.conda\\envs\\openvinodev\\Lib\\site-packages\\optimum\\exporters\\onnx\\model_patcher.py:307: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if past_key_values_length > 0:\n",
      "OpenVINO Tokenizers is not available. To deploy models in production with C++ code, please follow installation instructions: https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#installation\n",
      "\n",
      "Tokenizer won't be converted.\n"
     ]
    }
   ],
   "source": [
    "! optimum-cli export openvino --model \"microsoft/Phi-3-mini-4k-instruct\" --task text-generation-with-past --weight-format int4 --group-size 128 --ratio 0.6  --sym  --trust-remote-code ./model/phi3-instruct/int4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kinfeylo\\AppData\\Roaming\\Python\\Python311\\site-packages\\threadpoolctl.py:1214: RuntimeWarning: \n",
      "Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at\n",
      "the same time. Both libraries are known to be incompatible and this\n",
      "can cause random crashes or deadlocks on Linux when loaded in the\n",
      "same Python program.\n",
      "Using threadpoolctl may cause crashes or deadlocks. For more\n",
      "information and possible workarounds, please see\n",
      "    https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md\n",
      "\n",
      "  warnings.warn(msg, RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "# Import the AutoConfig and AutoTokenizer classes from the transformers library\n",
    "from transformers import AutoConfig, AutoTokenizer\n",
    "\n",
    "# Import the OVModelForCausalLM class from the optimum.intel.openvino module\n",
    "from optimum.intel.openvino import OVModelForCausalLM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the directory path where the model is stored\n",
    "model_dir = './model/phi3-instruct/int4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a configuration dictionary for the OpenVINO model\n",
    "ov_config = {\n",
    "    # Set the performance hint to prioritize low latency\n",
    "    \"PERFORMANCE_HINT\": \"LATENCY\",\n",
    "    # Specify the number of inference streams to use\n",
    "    \"NUM_STREAMS\": \"1\",\n",
    "    # Set the directory for caching intermediate data (empty string means no caching)\n",
    "    \"CACHE_DIR\": \"\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Compiling the model to GPU.0 ...\n"
     ]
    }
   ],
   "source": [
    "ov_model = OVModelForCausalLM.from_pretrained(\n",
    "    model_dir,\n",
    "    device='GPU.0',\n",
    "    ov_config=ov_config,\n",
    "    config=AutoConfig.from_pretrained(model_dir, trust_remote_code=True),\n",
    "    trust_remote_code=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "# Load the tokenizer from the specified model directory, allowing remote code execution if necessary\n",
    "tok = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a dictionary of keyword arguments for the tokenizer\n",
    "tokenizer_kwargs = {\n",
    "    # Specify not to add special tokens to the tokenized output\n",
    "    \"add_special_tokens\": False\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the prompt string with special tokens indicating different roles in the conversation\n",
    "prompt = \"<|system|>You are a helpful AI assistant.<|end|><|user|>can you introduce yourself?<|end|><|assistant|>\"\n",
    "\n",
    "# Tokenize the prompt string, returning the result as PyTorch tensors and applying additional tokenizer settings\n",
    "input_tokens = tok(prompt, return_tensors=\"pt\", **tokenizer_kwargs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate a response using the OpenVINO model, with a maximum of 1024 new tokens\n",
    "answer = ov_model.generate(**input_tokens, max_new_tokens=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"You are a helpful AI assistant. can you introduce yourself? Hello! I'm Phi, Microsoft's language model. I'm here to help answer your questions, provide information, and engage in conversation on a wide range of topics. How can I assist you today?\""
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Decode the generated tokens into a string, skipping special tokens, and get the first result\n",
    "decoded_answer = tok.batch_decode(answer, skip_special_tokens=True)[0]\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openvinodev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}