[Android] Speech recognition example (#236)

Add speech recognition example for Android using Whisper model.

Thanks @MaanavD for help with the UI.

---------

Co-authored-by: MaanavD <[email protected]>
Co-authored-by: maanavd <[email protected]>

Author: edgchen1

ci_build/azure_pipelines/mobile-examples-pipeline.yml

@@ -82,6 +82,27 @@ jobs:
       scheme: 'OrtBasicUsage'
       args: $(XcodeArgs)
 
+# mobile/examples/speech_recognition/android
+- job: SpeechRecognitionAndroid
+  pool:
+    vmImage: "macOS-12"
+
+  steps:
+  - template: templates/use-python-step.yml
+
+  - template: templates/use-jdk-step.yml
+    parameters:
+      jdkVersion: "17"
+
+  - template: templates/run-with-android-emulator-steps.yml
+    parameters:
+      steps:
+      - bash: |
+          set -e
+          ./gradlew connectedDebugAndroidTest --no-daemon
+        workingDirectory: mobile/examples/speech_recognition/android
+        displayName: "Build and run tests"
+
 # mobile/examples/speech_recognition/ios
 - job: SpeechRecognitionIos
   pool:
@@ -185,12 +206,7 @@ jobs:
   steps:
   - template: templates/use-python-step.yml
 
-  - task: JavaToolInstaller@0
-    displayName: Use jdk 11
-    inputs:
-      versionSpec: "11"
-      jdkArchitectureOption: "x64"
-      jdkSourceOption: "PreInstalled"
+  - template: templates/use-jdk-step.yml
 
   - bash: |
       set -e
@@ -208,25 +224,12 @@ jobs:
       cat ${GRADLE_FILE}
     displayName: "Update build.gradle"
 
-  - script: |
-      python3 ./ci_build/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --create-avd --system-image "system-images;android-30;default;x86_64" \
-        --start --emulator-extra-args="-partition-size 4096" \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: "Start Android emulator"
-
-  - bash: ./gradlew testDebugUnitTest connectedDebugAndroidTest
-    workingDirectory: mobile/examples/image_classification/android
-    displayName: "Build and run tests"
-
-  - script: |
-      python3 ./ci_build/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --stop \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: "Stop Android emulator"
-    condition: always()
+  - template: templates/run-with-android-emulator-steps.yml
+    parameters:
+      steps:
+      - bash: ./gradlew connectedDebugAndroidTest --no-daemon
+        workingDirectory: mobile/examples/image_classification/android
+        displayName: "Build and run tests"
 
 # Note: start with testing with the included aar package, 
 # can update with testing with Full/Mobile packages as the other samples later.
@@ -238,36 +241,17 @@ jobs:
 
   steps:
   - template: templates/use-python-step.yml
-
-  - script: |
-      python3 ./ci_build/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --create-avd --system-image "system-images;android-30;default;x86_64" \
-        --start --emulator-extra-args="-partition-size 4096" \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: "Start Android emulator"
 
-  - task: JavaToolInstaller@0
-    displayName: Use jdk 11
-    inputs:
-      versionSpec: "11"
-      jdkArchitectureOption: "x64"
-      jdkSourceOption: "PreInstalled"
+  - template: templates/use-jdk-step.yml
 
-  - bash: |
-      set -e
-      chmod +x ./gradlew 
-      ./gradlew connectedDebugAndroidTest --no-daemon
-    workingDirectory: mobile/examples/super_resolution/android
-    displayName: "Build and run tests"
-
-  - script: |
-      python ./ci_build/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --stop \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: "Stop Android emulator"
-    condition: always()
+  - template: templates/run-with-android-emulator-steps.yml
+    parameters:
+      steps:
+      - bash: |
+          set -e
+          ./gradlew connectedDebugAndroidTest --no-daemon
+        workingDirectory: mobile/examples/super_resolution/android
+        displayName: "Build and run tests"
 
 # Note: start with testing with the pre-release version pods.
 # can update with testing with Full/Mobile pods as the other samples later.
@@ -288,6 +272,7 @@ jobs:
       xcWorkspacePath: 'mobile/examples/super_resolution/ios/ORTSuperResolution/ORTSuperResolution.xcworkspace'
       scheme: 'ORTSuperResolution'
 
+# mobile/examples/question_answering/android
 - job: QuestionAnsweringAndroid
   pool:
     vmImage: "macOS-12"
@@ -299,36 +284,19 @@ jobs:
       bash ./prepare_model.sh
     workingDirectory: 'mobile/examples/question_answering/android'
     displayName: "Generate model"
-    
-  - script: |
-      python3 ./ci_build/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --create-avd --system-image "system-images;android-30;default;x86_64" \
-        --start --emulator-extra-args="-partition-size 4096" \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: "Start Android emulator"
   
-  - task: JavaToolInstaller@0
-    displayName: Use jdk 11
-    inputs:
-      versionSpec: "11"
-      jdkArchitectureOption: "x64"
-      jdkSourceOption: "PreInstalled"
-
-  - bash: |
-      set -e
-      ./gradlew connectedDebugAndroidTest --no-daemon
-    workingDirectory: mobile/examples/question_answering/android
-    displayName: "Build and run tests"
-
-  - script: |
-      python ./ci_build/python/run_android_emulator.py \
-        --android-sdk-root ${ANDROID_SDK_ROOT} \
-        --stop \
-        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
-    displayName: "Stop Android emulator"
-    condition: always()
+  - template: templates/use-jdk-step.yml
 
+  - template: templates/run-with-android-emulator-steps.yml
+    parameters:
+      steps:
+      - bash: |
+          set -e
+          ./gradlew connectedDebugAndroidTest --no-daemon
+        workingDirectory: mobile/examples/question_answering/android
+        displayName: "Build and run tests"
+
+# mobile/examples/question_answering/ios
 - job: QuestionAnsweringIos
   pool:
     vmImage: "macOS-12"
@@ -341,4 +309,4 @@ jobs:
   - template: templates/xcode-build-and-test-step.yml
     parameters:
       xcWorkspacePath: 'mobile/examples/question_answering/ios/ORTQuestionAnswering/ORTQuestionAnswering.xcworkspace'
-      scheme: 'ORTQuestionAnswering'
+      scheme: 'ORTQuestionAnswering'

ci_build/azure_pipelines/templates/run-with-android-emulator-steps.yml

@@ -0,0 +1,35 @@
+parameters:
+- name: steps
+  type: stepList
+
+steps:
+- bash: |
+    set -e -x
+
+    ORT_EXAMPLES_BUILD_ANDROID_EMULATOR_PID_FILE="$(Build.BinariesDirectory)/android_emulator.pid"
+
+    python ./ci_build/python/run_android_emulator.py \
+      --android-sdk-root "${ANDROID_SDK_ROOT}" \
+      --create-avd --system-image "system-images;android-30;default;x86_64" \
+      --start --emulator-extra-args="-partition-size 4096" \
+      --emulator-pid-file "${ORT_EXAMPLES_BUILD_ANDROID_EMULATOR_PID_FILE}"
+
+    set +x  # do not output next line or it may be parsed again with a trailing quote
+    echo "##vso[task.setvariable variable=ORT_EXAMPLES_BUILD_ANDROID_EMULATOR_PID_FILE]${ORT_EXAMPLES_BUILD_ANDROID_EMULATOR_PID_FILE}"
+  displayName: "Create and start Android emulator"
+
+- ${{ parameters.steps }}
+
+- bash: |
+    set -e -x
+
+    if [[ -n "${ORT_EXAMPLES_BUILD_ANDROID_EMULATOR_PID_FILE-}" ]]; then
+      python ./ci_build/python/run_android_emulator.py \
+        --android-sdk-root "${ANDROID_SDK_ROOT}" \
+        --stop \
+        --emulator-pid-file "${ORT_EXAMPLES_BUILD_ANDROID_EMULATOR_PID_FILE}"
+
+      rm "${ORT_EXAMPLES_BUILD_ANDROID_EMULATOR_PID_FILE}"
+    fi
+  displayName: "Stop Android emulator"
+  condition: always()

ci_build/azure_pipelines/templates/use-jdk-step.yml

@@ -0,0 +1,12 @@
+parameters:
+- name: jdkVersion
+  type: string
+  default: "11"
+
+steps:
+- task: JavaToolInstaller@0
+  displayName: Use jdk ${{ parameters.jdkVersion }}
+  inputs:
+    versionSpec: "${{ parameters.jdkVersion }}"
+    jdkArchitectureOption: "x64"
+    jdkSourceOption: "PreInstalled"

mobile/README.md

@@ -40,6 +40,7 @@ The example app uses image classification which is able to continuously classify
 
 The example app uses speech recognition to transcribe speech from audio recorded by the device.
 
+- [Android Speech Recognition](examples/speech_recognition/android)
 - [iOS Speech Recognition](examples/speech_recognition/ios)
 
 ### Object Detection
@@ -65,4 +66,4 @@ The example application accomplishes the task of recovering a high resolution (H
 The example app gives a demo of introducing question answering models with pre/post processing into mobile scenario. Currently supports on platform Android and iOS.
 
 - [Android Question answering](examples/question_answering/android)
-- [iOS Question answering](examples/question_answering/ios)
+- [iOS Question answering](examples/question_answering/ios)

mobile/examples/speech_recognition/android/.gitignore

@@ -0,0 +1,15 @@
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

mobile/examples/speech_recognition/android/app/.gitignore

@@ -0,0 +1 @@
+/build

mobile/examples/speech_recognition/android/app/build.gradle

@@ -0,0 +1,50 @@
+plugins {
+    id 'com.android.application'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    namespace 'ai.onnxruntime.example.speechrecognition'
+    compileSdk 33
+
+    defaultConfig {
+        applicationId "ai.onnxruntime.example.speechrecognition"
+        minSdk 24
+        targetSdk 33
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+}
+
+dependencies {
+    implementation 'androidx.core:core-ktx:1.10.0'
+    implementation 'androidx.appcompat:appcompat:1.6.1'
+    implementation 'com.google.android.material:material:1.8.0'
+    implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
+
+    // TODO use released onnxruntime and onnxruntime-extensions packages
+    //implementation 'com.microsoft.onnxruntime:onnxruntime-android:latest.release'
+    //implementation 'com.microsoft.onnxruntime:onnxruntime-extensions-android:latest.release'
+    implementation(name: "onnxruntime-android-1.15.0", ext: "aar")
+    implementation(name: "onnxruntime-extensions-android-0.8.0", ext: "aar")
+
+    testImplementation 'junit:junit:4.13.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.5'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
+}

mobile/examples/speech_recognition/android/app/libs/onnxruntime-android-1.15.0.aar

@@ -0,0 +1,2 @@
+This directory contains pre-release versions of the onnxruntime and onnxruntime-extensions Android packages (AARs).
+When onnxruntime 1.15.0 and onnxruntime-extensions 0.8.0 are released, the released packages should be used directly.

mobile/examples/speech_recognition/android/app/libs/onnxruntime-extensions-android-0.8.0.aar

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

mobile/examples/speech_recognition/android/app/libs/readme.md

@@ -0,0 +1,49 @@
+package ai.onnxruntime.example.speechrecognition
+
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.ext.junit.runners.AndroidJUnit4
+
+import org.junit.Test
+import org.junit.runner.RunWith
+
+import org.junit.Assert.*
+
+/**
+ * Instrumented test, which will execute on an Android device.
+ *
+ * See [testing documentation](http://d.android.com/tools/testing).
+ */
+@RunWith(AndroidJUnit4::class)
+class SpeechRecognitionInstrumentedTest {
+    @Test
+    fun useAppContext() {
+        // Context of the app under test.
+        val appContext = InstrumentationRegistry.getInstrumentation().targetContext
+        assertEquals("ai.onnxruntime.example.speechrecognition", appContext.packageName)
+    }
+
+    @Test
+    fun runModelWithPrerecordedAudio() {
+        val appContext = InstrumentationRegistry.getInstrumentation().targetContext
+
+        val modelBytes: ByteArray =
+            appContext.resources.openRawResource(R.raw.whisper_cpu_int8_model).use {
+                it.readBytes()
+            }
+
+        SpeechRecognizer(modelBytes).use { speechRecognizer ->
+            val audioTensor =
+                appContext.resources.openRawResource(R.raw.audio_mono_16khz_f32le).use {
+                    AudioTensorSource.fromRawPcmBytes(it.readBytes())
+                }
+
+            val result = audioTensor.use { speechRecognizer.run(audioTensor) }
+            assertTrue(
+                result.text.contains(
+                    "welcome to the speech recognition example application",
+                    ignoreCase = true
+                )
+            )
+        }
+    }
+}