Back to Runanywhere Sdks

RunAnywhere Kotlin SDK - API Documentation

sdk/runanywhere-kotlin/docs/Documentation.md

0.20.1245.7 KB
Original Source

RunAnywhere Kotlin SDK - API Documentation

Complete API reference for the RunAnywhere Kotlin SDK. All public APIs are accessible through the RunAnywhere object via extension functions.


Table of Contents

  1. Quick Start
  2. Core API
  3. Text Generation (LLM)
  4. Speech-to-Text (STT)
  5. Text-to-Speech (TTS)
  6. Voice Activity Detection (VAD)
  7. Voice Agent
  8. Model Management
  9. Event System
  10. Types & Enums
  11. Error Handling

Quick Start

Installation (Maven Central)

kotlin
// build.gradle.kts
dependencies {
    // Core SDK with native libraries
    implementation("io.github.sanchitmonga22:runanywhere-sdk:0.20.11")

    // LlamaCPP backend for LLM text generation
    implementation("io.github.sanchitmonga22:runanywhere-llamacpp:0.20.11")

    // ONNX backend for STT/TTS/VAD
    implementation("io.github.sanchitmonga22:runanywhere-onnx:0.20.11")
}
kotlin
// settings.gradle.kts - add repositories
dependencyResolutionManagement {
    repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS)
    repositories {
        google()
        mavenCentral()
    }
}

Initialize SDK

kotlin
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.SDKEnvironment

// In your Application.onCreate() or Activity
RunAnywhere.initialize(environment = SDKEnvironment.DEVELOPMENT)

Register & Load Models

The starter app uses these specific model IDs and URLs. Loading is done through the canonical proto-backed lifecycle service (loadModel); the removed v1 helpers (loadLLMModel / loadSTTModel / loadTTSVoice) no longer exist.

kotlin
import ai.runanywhere.proto.v1.ModelCategory
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.registerModel
import com.runanywhere.sdk.public.extensions.downloadModel
import com.runanywhere.sdk.public.extensions.loadModel
import com.runanywhere.sdk.public.types.RAInferenceFramework
import com.runanywhere.sdk.public.types.RAModelLoadRequest

// LLM Model - SmolLM2 360M (small, fast, good for demos)
RunAnywhere.registerModel(
    id = "smollm2-360m-instruct-q8_0",
    name = "SmolLM2 360M Instruct Q8_0",
    url = "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct-GGUF/resolve/main/smollm2-360m-instruct-q8_0.gguf",
    framework = RAInferenceFramework.INFERENCE_FRAMEWORK_LLAMA_CPP,
    modality = ModelCategory.MODEL_CATEGORY_LANGUAGE,
    memoryRequirement = 400_000_000, // ~400MB
)

// STT Model - Whisper Tiny English (fast transcription)
RunAnywhere.registerModel(
    id = "sherpa-onnx-whisper-tiny.en",
    name = "Sherpa Whisper Tiny (ONNX)",
    url = "https://github.com/RunanywhereAI/sherpa-onnx/releases/download/runanywhere-models-v1/sherpa-onnx-whisper-tiny.en.tar.gz",
    framework = RAInferenceFramework.INFERENCE_FRAMEWORK_ONNX,
    modality = ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION,
)

// TTS Model - Piper TTS (US English - Medium quality)
RunAnywhere.registerModel(
    id = "vits-piper-en_US-lessac-medium",
    name = "Piper TTS (US English - Medium)",
    url = "https://github.com/RunanywhereAI/sherpa-onnx/releases/download/runanywhere-models-v1/vits-piper-en_US-lessac-medium.tar.gz",
    framework = RAInferenceFramework.INFERENCE_FRAMEWORK_ONNX,
    modality = ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS,
)

// Download model (returns Flow<DownloadProgress>)
RunAnywhere.downloadModel("smollm2-360m-instruct-q8_0")
    .catch { e -> println("Download failed: ${e.message}") }
    .collect { progress ->
        println("Download: ${(progress.progress * 100).toInt()}%")
    }

// Load through the proto-backed lifecycle service.
val loadResult = RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "smollm2-360m-instruct-q8_0",
        category = ModelCategory.MODEL_CATEGORY_LANGUAGE,
    ),
)
require(loadResult.success) { "load failed: ${loadResult.error_message}" }

Text Generation (LLM)

kotlin
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.generate
import com.runanywhere.sdk.public.types.RALLMGenerationOptions

// Canonical generation API: returns RALLMGenerationResult with text +
// metrics. There is no longer a separate `chat()` helper.
val result = RunAnywhere.generate(
    prompt = "What is AI?",
    options = RALLMGenerationOptions(max_tokens = 64),
)
println(result.text)

Speech-to-Text (STT)

kotlin
import ai.runanywhere.proto.v1.ModelCategory
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.loadModel
import com.runanywhere.sdk.public.extensions.transcribe
import com.runanywhere.sdk.public.types.RAModelLoadRequest
import com.runanywhere.sdk.public.types.RASTTOptions

// Load STT model through the canonical lifecycle service.
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "sherpa-onnx-whisper-tiny.en",
        category = ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION,
    ),
)

// Transcribe audio (16kHz, mono, 16-bit PCM ByteArray).
val output = RunAnywhere.transcribe(audioData, RASTTOptions(language = "en"))
println("You said: ${output.text}")

Text-to-Speech (TTS)

kotlin
import ai.runanywhere.proto.v1.ModelCategory
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.loadModel
import com.runanywhere.sdk.public.extensions.synthesize
import com.runanywhere.sdk.public.types.RAModelLoadRequest
import com.runanywhere.sdk.public.types.RATTSOptions

// Load TTS voice through the canonical lifecycle service.
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "vits-piper-en_US-lessac-medium",
        category = ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS,
    ),
)

// Synthesize audio - returns TTSOutput with audioData
val output = RunAnywhere.synthesize("Hello, world!", RATTSOptions())
// output.audio_data carries the WAV audio bytes (a Wire `ByteString`).

Voice Pipeline (STT → LLM → TTS)

Option 1: Streaming Voice Session (Recommended)

The streamVoiceSession() API handles everything automatically:

  • Audio level calculation for visualization
  • Speech detection (when audio level > threshold)
  • Automatic silence detection (triggers processing after 1.5s of silence)
  • Full STT → LLM → TTS orchestration
  • Continuous conversation mode
kotlin
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.streamVoiceSession
import com.runanywhere.sdk.public.extensions.VoiceAgent.VoiceSessionConfig
import com.runanywhere.sdk.public.extensions.VoiceAgent.VoiceSessionEvent
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.*

// Ensure all 3 models are loaded first via the canonical lifecycle service.
import ai.runanywhere.proto.v1.ModelCategory
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "sherpa-onnx-whisper-tiny.en",
        category = ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION,
    ),
)
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "smollm2-360m-instruct-q8_0",
        category = ModelCategory.MODEL_CATEGORY_LANGUAGE,
    ),
)
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "vits-piper-en_US-lessac-medium",
        category = ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS,
    ),
)

// Compose the agent from the models loaded above. The SDK owns mic capture,
// VAD gating, turn segmentation, and TTS playback from here on.
RunAnywhere.initializeVoiceAgentWithLoadedModels()

// Collect pipeline events. VoiceEvent is a Wire oneof envelope: exactly one
// payload field is set per event, so check for null rather than matching on
// sealed subclasses.
sessionJob = scope.launch {
    try {
        RunAnywhere.streamVoiceAgent().collect { event ->
            event.state?.let { state ->
                sessionState = state.to_state
                if (state.to_state != PipelineState.PIPELINE_STATE_LISTENING) {
                    audioLevel = 0f
                }
            }

            event.vad?.let { vad ->
                audioLevel = vad.energy
            }

            event.user_said?.let { said ->
                // User's speech was transcribed
                showTranscript(said.text)
            }

            event.assistant_token?.let { token ->
                // LLM response, streamed token by token
                appendResponse(token.text)
            }

            event.audio?.let { frame ->
                // Synthesized audio frame; the SDK plays it unless you opted out
                enqueueAudio(frame.samples)
            }

            event.error?.let { error ->
                errorMessage = error.message
            }
        }
    } catch (e: CancellationException) {
        // Expected when stopping
    } catch (e: Exception) {
        errorMessage = "Session error: ${e.message}"
        sessionState = PipelineState.PIPELINE_STATE_IDLE
    }
}

// To stop the session:
suspend fun stopSession() {
    sessionJob?.cancel()
    sessionJob = null
    RunAnywhere.cleanupVoiceAgent()
    sessionState = PipelineState.PIPELINE_STATE_IDLE
}

Audio Capture Service (single-modality STT only)

The voice agent above captures its own audio. You only need your own recorder for standalone transcribe / detectVoiceActivity calls:

kotlin
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.awaitClose
import kotlinx.coroutines.flow.*

class AudioCaptureService {
    private var audioRecord: AudioRecord? = null

    @Volatile
    private var isCapturing = false

    companion object {
        const val SAMPLE_RATE = 16000
        const val CHUNK_SIZE_MS = 100 // Emit chunks every 100ms
    }

    fun startCapture(): Flow<ByteArray> = callbackFlow {
        val bufferSize = AudioRecord.getMinBufferSize(
            SAMPLE_RATE,
            AudioFormat.CHANNEL_IN_MONO,
            AudioFormat.ENCODING_PCM_16BIT
        )
        val chunkSize = (SAMPLE_RATE * 2 * CHUNK_SIZE_MS) / 1000

        try {
            audioRecord = AudioRecord(
                MediaRecorder.AudioSource.MIC,
                SAMPLE_RATE,
                AudioFormat.CHANNEL_IN_MONO,
                AudioFormat.ENCODING_PCM_16BIT,
                maxOf(bufferSize, chunkSize * 2)
            )

            if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) {
                close(IllegalStateException("AudioRecord initialization failed"))
                return@callbackFlow
            }

            audioRecord?.startRecording()
            isCapturing = true

            val readJob = launch(Dispatchers.IO) {
                val buffer = ByteArray(chunkSize)
                while (isActive && isCapturing) {
                    val bytesRead = audioRecord?.read(buffer, 0, chunkSize) ?: -1
                    if (bytesRead > 0) {
                        trySend(buffer.copyOf(bytesRead))
                    }
                }
            }

            awaitClose {
                readJob.cancel()
                stopCapture()
            }
        } catch (e: Exception) {
            stopCapture()
            close(e)
        }
    }

    fun stopCapture() {
        isCapturing = false
        try {
            audioRecord?.stop()
            audioRecord?.release()
        } catch (_: Exception) {}
        audioRecord = null
    }
}

Play WAV Audio (Required for Voice Pipeline)

kotlin
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioTrack
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext

suspend fun playWavAudio(wavData: ByteArray) = withContext(Dispatchers.IO) {
    if (wavData.size < 44) return@withContext

    val headerSize = if (wavData.size > 44 &&
        wavData[0] == 'R'.code.toByte() &&
        wavData[1] == 'I'.code.toByte()) 44 else 0

    val pcmData = wavData.copyOfRange(headerSize, wavData.size)
    val sampleRate = 22050 // Piper TTS default sample rate

    val bufferSize = AudioTrack.getMinBufferSize(
        sampleRate, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT
    )

    val audioTrack = AudioTrack.Builder()
        .setAudioAttributes(
            AudioAttributes.Builder()
                .setUsage(AudioAttributes.USAGE_MEDIA)
                .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
                .build()
        )
        .setAudioFormat(
            AudioFormat.Builder()
                .setSampleRate(sampleRate)
                .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
                .setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
                .build()
        )
        .setBufferSizeInBytes(maxOf(bufferSize, pcmData.size))
        .setTransferMode(AudioTrack.MODE_STATIC)
        .build()

    audioTrack.write(pcmData, 0, pcmData.size)
    audioTrack.play()

    val durationMs = (pcmData.size.toLong() * 1000) / (sampleRate * 2)
    delay(durationMs + 100)

    audioTrack.stop()
    audioTrack.release()
}

Option 2: Manual Processing

For more control, use processVoiceTurn() with your own silence detection:

kotlin
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.processVoiceTurn

// Record audio (app responsibility - use AudioRecord)
val audioData: ByteArray = recordAudio() // 16kHz, mono, 16-bit PCM

// Process one turn through the full pipeline - SDK handles orchestration
val result = RunAnywhere.processVoiceTurn(audioData)

if (result.speech_detected) {
    println("You said: ${result.transcription}")
    println("AI response: ${result.assistant_response}")

    // Play synthesized audio (app responsibility)
    result.synthesized_audio?.let { playWavAudio(it.toByteArray()) }
}

VoiceAgentResult is a Wire-generated proto type, so its fields use snake_case and synthesized_audio is a ByteString.

Voice Session Events

EventDescription
StartedSession started and ready
Listening(audioLevel)Listening with real-time audio level (0.0 - 1.0)
SpeechStartedSpeech detected, accumulating audio
ProcessingSilence detected, processing audio
Transcribed(text)STT completed
Responded(text)LLM response generated
SpeakingPlaying TTS audio
TurnCompleted(transcript, response, audio)Full turn complete with audio
StoppedSession ended
Error(message)Error occurred

Complete Voice Pipeline Example

See the Kotlin Starter Example app for a complete working implementation: starter_apps/kotlinstarterexample/app/src/main/java/com/runanywhere/kotlin_starter_example/ui/screens/VoicePipelineScreen.kt


Core API

RunAnywhere Object

The main entry point for all SDK functionality.

kotlin
package com.runanywhere.sdk.public

object RunAnywhere

Properties

PropertyTypeDescription
isInitializedBooleanWhether Phase 1 initialization is complete
areServicesReadyBooleanWhether Phase 2 (services) initialization is complete
isActiveBooleanWhether SDK is initialized and has an environment
versionStringCurrent SDK version string
environmentSDKEnvironment?Current environment (null if not initialized)
eventsEventBusEvent subscription system

Initialization

kotlin
/**
 * Initialize the RunAnywhere SDK (Phase 1).
 * Fast synchronous initialization (~1-5ms).
 *
 * @param apiKey API key (optional for development)
 * @param baseURL Backend API base URL (optional)
 * @param environment SDK environment (default: DEVELOPMENT)
 */
fun initialize(
    apiKey: String? = null,
    baseURL: String? = null,
    environment: SDKEnvironment = SDKEnvironment.DEVELOPMENT
)

/**
 * Initialize SDK for development mode (convenience method).
 */
fun initializeForDevelopment(apiKey: String? = null)

/**
 * Complete services initialization (Phase 2).
 * Called automatically on first API call, or can be awaited explicitly.
 */
suspend fun completeServicesInitialization()

Lifecycle

kotlin
/**
 * Reset SDK state. Clears all initialization state and releases resources.
 */
suspend fun reset()

/**
 * Cleanup SDK resources without full reset.
 */
suspend fun cleanup()

SDKEnvironment

kotlin
enum class SDKEnvironment {
    DEVELOPMENT,  // Debug logging, local testing
    STAGING,      // Info logging, staging backend
    PRODUCTION    // Warning logging only, production backend
}

Text Generation (LLM)

Extension functions for text generation using Large Language Models.

Basic Generation

kotlin
/**
 * Generate text with full metrics. Returns the proto-backed
 * `RALLMGenerationResult` (alias for `ai.runanywhere.proto.v1.LLMGenerationResult`).
 *
 * @param prompt The text prompt
 * @param options Generation options (defaults to null → SDK defaults)
 * @return RALLMGenerationResult with text and metrics
 */
suspend fun RunAnywhere.generate(
    prompt: String,
    options: RALLMGenerationOptions? = null,
): RALLMGenerationResult

The v1 chat(prompt) convenience helper has been removed. Construct the same call as generate(prompt, RALLMGenerationOptions(...)) and read the resulting result.text.

Streaming Generation

kotlin
/**
 * Streaming text generation. One event per generated token plus a
 * terminal event with `is_final == true` carrying `finish_reason` and
 * any `error_message`. The v1 `generateStreamWithMetrics()` variant has
 * been removed; derive metrics from the event sequence (e.g. record TTFT
 * on the first non-empty `event.token_`).
 *
 * @param prompt The text prompt
 * @param options Generation options (defaults to null → SDK defaults)
 * @return Flow<RALLMStreamEvent>
 */
fun RunAnywhere.generateStream(
    prompt: String,
    options: RALLMGenerationOptions? = null,
): Flow<RALLMStreamEvent>

Generation Control

kotlin
/**
 * Cancel any ongoing text generation.
 */
fun RunAnywhere.cancelGeneration()

LLM Types

RALLMGenerationOptions / RALLMGenerationResult / RALLMStreamEvent are typealiases for the Wire-generated proto types under ai.runanywhere.proto.v1.*. They are the canonical structured-options surface — there are no longer hand-rolled value-types (LLMGenerationOptions, LLMStreamingResult, LLMConfiguration).

kotlin
// From sdk/runanywhere-kotlin/src/main/.../types/SwiftAliases.kt
typealias RALLMGenerationOptions = ai.runanywhere.proto.v1.LLMGenerationOptions
typealias RALLMGenerationResult  = ai.runanywhere.proto.v1.LLMGenerationResult
typealias RALLMStreamEvent       = ai.runanywhere.proto.v1.LLMStreamEvent

Common RALLMGenerationOptions fields (proto-snake_case in Kotlin):

kotlin
val max_tokens: Int
val temperature: Float
val top_p: Float
val stop_sequences: List<String>
val system_prompt: String

RALLMGenerationResult carries text, token counts, latency, framework, and tokens-per-second; see the proto definition for the full schema.


Speech-to-Text (STT)

Extension functions for speech recognition.

Transcription

kotlin
/**
 * Transcribe a one-shot audio buffer. Returns the proto-backed `RASTTOutput`
 * (alias for `ai.runanywhere.proto.v1.STTOutput`).
 *
 * @param audio Raw audio data (16kHz PCM by default)
 * @param options Transcription options (defaults to `RASTTOptions()`)
 */
suspend fun RunAnywhere.transcribe(
    audio: ByteArray,
    options: RASTTOptions = RASTTOptions(),
): RASTTOutput

The v1 helpers loadSTTModel(modelId) / unloadSTTModel() / isSTTModelLoaded() / currentSTTModelId / transcribeWithOptions() have been removed. Load STT models through the canonical lifecycle service (loadModel(RAModelLoadRequest(category = MODEL_CATEGORY_SPEECH_RECOGNITION))) and pass options directly to transcribe().

Streaming Transcription

kotlin
/**
 * Stream transcription results from a flow of audio chunks. One
 * `RASTTPartialResult` per incremental update; the stream closes after
 * the final event (`is_final == true`) or on error.
 *
 * @param audio Flow of audio chunk byte arrays
 * @param options Transcription options (null → SDK defaults)
 */
fun RunAnywhere.transcribeStream(
    audio: Flow<ByteArray>,
    options: RASTTOptions? = null,
): Flow<RASTTPartialResult>

STT Types

RASTTOptions / RASTTOutput / RASTTPartialResult are typealiases for the Wire-generated proto types under ai.runanywhere.proto.v1.*. Common fields use the proto's snake_case names:

kotlin
// RASTTOptions (= ai.runanywhere.proto.v1.STTOptions) — selected fields
val language: String           // e.g. "en"
val detect_language: Boolean
val enable_punctuation: Boolean
val enable_timestamps: Boolean
val sample_rate: Int           // default 16000

// RASTTOutput (= ai.runanywhere.proto.v1.STTOutput) — selected fields
val text: String
val confidence: Float
val detected_language: String

Text-to-Speech (TTS)

Extension functions for speech synthesis.

Voice Management

TTS voices are loaded through the canonical proto-backed lifecycle service. The v1 helpers (loadTTSVoice, unloadTTSVoice, isTTSVoiceLoaded, currentTTSVoiceId, availableTTSVoices()) have been removed.

kotlin
import ai.runanywhere.proto.v1.ModelCategory

RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "vits-piper-en_US-lessac-medium",
        category = ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS,
    ),
)

Below is the historical reference for the deleted v1 surface (not present at HEAD):

kotlin
/**
 * Check if a TTS voice is loaded.
 */
suspend fun RunAnywhere.isTTSVoiceLoaded(): Boolean

/**
 * Get the currently loaded TTS voice ID (synchronous).
 */
val RunAnywhere.currentTTSVoiceId: String?

/**
 * Get available TTS voices.
 */
suspend fun RunAnywhere.availableTTSVoices(): List<String>

Synthesis

kotlin
/**
 * Synthesize text to speech audio. Returns the proto-backed
 * `RATTSOutput` (alias for `ai.runanywhere.proto.v1.TTSOutput`).
 *
 * @param text Text to synthesize
 * @param options Synthesis options (defaults to `RATTSOptions()`)
 */
suspend fun RunAnywhere.synthesize(
    text: String,
    options: RATTSOptions = RATTSOptions(),
): RATTSOutput

/**
 * Stream synthesis for long text. One `RATTSOutput` chunk per
 * generated segment. The previous callback-based signature was
 * removed in favor of a Flow.
 */
fun RunAnywhere.synthesizeStream(
    text: String,
    options: RATTSOptions = RATTSOptions(),
): Flow<RATTSOutput>

/**
 * Stop current TTS synthesis.
 */
suspend fun RunAnywhere.stopSynthesis()

Simple Speak API

kotlin
/**
 * Speak text aloud — synthesizes and plays through the platform audio
 * output. Returns the proto `TTSSpeakResult` (latency, duration, etc).
 */
suspend fun RunAnywhere.speak(
    text: String,
    options: RATTSOptions = RATTSOptions(),
): TTSSpeakResult

/**
 * Whether speech is currently playing — sync property read.
 */
val RunAnywhere.isSpeaking: Boolean

/**
 * Stop current speech playback.
 */
suspend fun RunAnywhere.stopSpeaking()

TTS Types

RATTSOptions / RATTSOutput / TTSSpeakResult are typealiases for the Wire-generated proto types under ai.runanywhere.proto.v1.*. Common fields use the proto's snake_case names:

kotlin
typealias RATTSOptions = ai.runanywhere.proto.v1.TTSOptions
typealias RATTSOutput  = ai.runanywhere.proto.v1.TTSOutput

// RATTSOptions selected fields:
val voice: String          // optional voice id override
val language: String       // e.g. "en-US"
val rate: Float            // 0.0 .. 2.0
val pitch: Float           // 0.0 .. 2.0
val volume: Float          // 0.0 .. 1.0
val sample_rate: Int       // default 22050
val use_ssml: Boolean

RATTSOutput exposes the synthesized audio bytes as a Wire ByteString on audio_data; convert with .toByteArray() for JVM audio APIs.

TTSSynthesisMetadata

kotlin
data class TTSSynthesisMetadata(
    val voice: String,
    val language: String,
    val processingTime: Double,        // Processing time in seconds
    val characterCount: Int
) {
    val charactersPerSecond: Double
}

TTSSpeakResult

kotlin
data class TTSSpeakResult(
    val duration: Double,              // Duration in seconds
    val format: AudioFormat,
    val audioSizeBytes: Int,
    val metadata: TTSSynthesisMetadata,
    val timestamp: Long
)

Voice Activity Detection (VAD)

Extension functions for detecting speech in audio.

Detection

kotlin
/**
 * Detect voice activity in audio data.
 *
 * @param audioData Audio data to analyze
 * @return VADResult with detection info
 */
suspend fun RunAnywhere.detectVoiceActivity(audioData: ByteArray): VADResult

/**
 * Stream VAD results from audio samples.
 *
 * @param audio Flow of raw PCM audio chunks
 * @return Flow of VAD results
 */
fun RunAnywhere.streamVAD(audio: Flow<ByteArray>, options: RAVADOptions? = null): Flow<RAVADResult>

Configuration

kotlin
/**
 * Detect speech in one buffer.
 *
 * @param audioData 16 kHz mono PCM16
 * @param options Per-call thresholds; null uses the component defaults
 */
suspend fun RunAnywhere.detectVoiceActivity(
    audioData: ByteArray,
    options: RAVADOptions? = null,
): RAVADResult

/**
 * Detect speech across a stream of buffers.
 */
fun RunAnywhere.streamVAD(
    audio: Flow<ByteArray>,
    options: RAVADOptions? = null,
): Flow<RAVADResult>

/**
 * Reset VAD state.
 */
suspend fun RunAnywhere.resetVAD()

VAD is configured per call through RAVADOptions. There is no separate configureVAD, statistics accessor, or ambient-noise calibration call.

VAD Types

VADConfiguration

kotlin
data class VADConfiguration(
    val threshold: Float = 0.5f,
    val minSpeechDurationMs: Int = 250,
    val minSilenceDurationMs: Int = 300,
    val sampleRate: Int = 16000,
    val frameSizeMs: Int = 30
)

VADResult

kotlin
data class VADResult(
    val hasSpeech: Boolean,            // Speech detected
    val confidence: Float,             // Detection confidence
    val speechStartMs: Long?,          // Speech start time
    val speechEndMs: Long?,            // Speech end time
    val frameIndex: Int,               // Audio frame index
    val timestamp: Long
)

Voice Agent

Extension functions for full voice conversation pipelines.

Setup

kotlin
/**
 * Initialize the voice agent from an explicit compose config.
 *
 * @param config Proto-backed compose config naming the STT, LLM, and TTS models
 */
suspend fun RunAnywhere.initializeVoiceAgent(config: RAVoiceAgentComposeConfig)

/**
 * Initialize the voice agent from the models already loaded in each component.
 *
 * @param ttsVoiceId Optional TTS voice override
 * @param ensureVAD Load the default VAD model when none is present (default: true)
 */
suspend fun RunAnywhere.initializeVoiceAgentWithLoadedModels(
    ttsVoiceId: String? = null,
    ensureVAD: Boolean = true,
)

/**
 * Per-component readiness snapshot (VAD, STT, LLM, TTS).
 */
suspend fun RunAnywhere.getVoiceAgentComponentStates(): RAVoiceAgentComponentStates

/**
 * Load the default VAD model if the pipeline does not have one yet.
 *
 * @return true when a VAD model is available afterwards
 */
suspend fun RunAnywhere.ensureDefaultVAD(modelID: String? = null): Boolean

/**
 * Tear down the voice agent and release its component handles.
 */
suspend fun RunAnywhere.cleanupVoiceAgent()

Voice Processing

kotlin
/**
 * Process one turn through the full pipeline (VAD → STT → LLM → TTS).
 *
 * @param audioData Audio data to process
 * @return VoiceAgentResult with the full response
 */
suspend fun RunAnywhere.processVoiceTurn(audioData: ByteArray): VoiceAgentResult

Voice Session

The SDK owns the audio pipeline. Subscribe to the event stream rather than driving start/stop yourself:

kotlin
/**
 * Stream voice pipeline events for the initialized agent.
 * Emits state, VAD, userSaid, assistantToken, audio, and error payloads.
 */
fun RunAnywhere.streamVoiceAgent(): Flow<VoiceEvent>

Cancel the collecting coroutine to end the session, then call cleanupVoiceAgent() to release handles. There is no separate startVoiceSession / stopVoiceSession pair, and conversation history plus the system prompt are configured through the compose config rather than through setter calls.

VoiceSessionEvent

kotlin
sealed class VoiceSessionEvent {
    /** Session started and ready */
    data object Started : VoiceSessionEvent()

    /** Listening for speech with current audio level (0.0 - 1.0) */
    data class Listening(val audioLevel: Float) : VoiceSessionEvent()

    /** Speech detected, started accumulating audio */
    data object SpeechStarted : VoiceSessionEvent()

    /** Speech ended, processing audio */
    data object Processing : VoiceSessionEvent()

    /** Got transcription from STT */
    data class Transcribed(val text: String) : VoiceSessionEvent()

    /** Got response from LLM */
    data class Responded(val text: String) : VoiceSessionEvent()

    /** Playing TTS audio */
    data object Speaking : VoiceSessionEvent()

    /** Complete turn result with transcript, response, and audio */
    data class TurnCompleted(
        val transcript: String,
        val response: String,
        val audio: ByteArray?
    ) : VoiceSessionEvent()

    /** Session stopped */
    data object Stopped : VoiceSessionEvent()

    /** Error occurred */
    data class Error(val message: String) : VoiceSessionEvent()
}

VoiceAgentResult

kotlin
data class VoiceAgentResult(
    /** Whether speech was detected in the input audio */
    val speechDetected: Boolean = false,
    /** Transcribed text from STT */
    val transcription: String? = null,
    /** Generated response text from LLM */
    val response: String? = null,
    /** Synthesized audio data from TTS (WAV format) */
    val synthesizedAudio: ByteArray? = null
)

Model Management

Extension functions for model registration, download, and lifecycle.

Model Registration

kotlin
/**
 * Register a model from a download URL.
 *
 * @param id Explicit model ID (optional, generated from URL if null)
 * @param name Display name for the model
 * @param url Download URL
 * @param framework Target inference framework
 * @param modality Model category (default: LANGUAGE)
 * @param artifactType How model is packaged (inferred if null)
 * @param memoryRequirement Estimated memory in bytes
 * @param supportsThinking Whether model supports reasoning
 * @return Created ModelInfo
 */
fun RunAnywhere.registerModel(
    id: String? = null,
    name: String,
    url: String,
    framework: InferenceFramework,
    modality: ModelCategory = ModelCategory.LANGUAGE,
    artifactType: ModelArtifactType? = null,
    memoryRequirement: Long? = null,
    supportsThinking: Boolean = false
): ModelInfo

Model Discovery

Discovery is proto-backed: requests and results are generated types, not plain lists of ModelInfo.

kotlin
/**
 * List models. Pass a populated ModelListRequest to filter.
 */
suspend fun RunAnywhere.listModels(request: ModelListRequest = ModelListRequest()): ModelListResult

/**
 * Query models with structured criteria (category, framework, downloaded state).
 */
suspend fun RunAnywhere.queryModels(query: ModelQuery): ModelListResult

/**
 * Downloaded models only.
 */
suspend fun RunAnywhere.downloadedModels(): ModelListResult

/**
 * Fetch one model's metadata.
 */
suspend fun RunAnywhere.getModel(request: ModelGetRequest): ModelGetResult

/**
 * Resolve the model currently loaded for a category.
 */
suspend fun RunAnywhere.modelInfoForCategory(category: ModelCategory): RAModelInfo?

A model's downloaded state is read from RAModelInfo.isDownloaded on a discovery result rather than through a separate predicate call.

Model Downloads

Downloads take a RAModelInfo, not a bare id string. Get one from a discovery call first.

kotlin
/**
 * Download a model, suspending until it completes.
 *
 * @param model Model metadata from a discovery call
 */
suspend fun RunAnywhere.downloadModel(model: RAModelInfo, /* ... */)

/**
 * Download a model and observe progress.
 *
 * @return Flow of DownloadProgress
 */
fun RunAnywhere.downloadModelStream(model: RAModelInfo): Flow<DownloadProgress>

Cancel a download by cancelling the coroutine collecting downloadModelStream; there is no separate cancelDownload call.

Model Lifecycle

kotlin
/**
 * Delete a downloaded model.
 */
suspend fun RunAnywhere.deleteModel(modelId: String)

/**
 * Refresh the model registry from remote.
 */
suspend fun RunAnywhere.refreshModelRegistry()

To clear everything, use the storage surface (deleteStorage, clearCache, cleanTempFiles) rather than a bulk model delete.

LLM Model Loading

LLM (and STT/TTS/VAD) loading goes through the canonical proto-backed lifecycle service. The v1 per-modality helpers were deleted; use loadModel(RAModelLoadRequest(...)) plus the category enum.

kotlin
/**
 * Load a model through the canonical lifecycle service. The C++
 * router resolves the path/framework from the registry by `model_id`.
 *
 * @param request RAModelLoadRequest (alias for proto ModelLoadRequest).
 *                Set `model_id` and `category`; other fields are optional.
 * @return RAModelLoadResult — `success`, `error_message`, etc.
 */
suspend fun RunAnywhere.loadModel(request: RAModelLoadRequest): RAModelLoadResult

/**
 * Unload one or all models. `ModelUnloadRequest.unload_all = true`
 * tears every loaded component down; otherwise pass `category`.
 */
suspend fun RunAnywhere.unloadModel(request: ModelUnloadRequest): ModelUnloadResult

/**
 * Snapshot the currently loaded model for a given category. The
 * returned `CurrentModelResult` has `found: Boolean` and `model_id`.
 */
suspend fun RunAnywhere.currentModel(request: CurrentModelRequest = CurrentModelRequest()): CurrentModelResult

/**
 * Per-component lifecycle snapshot — useful for asserting readiness
 * from tests.
 */
suspend fun RunAnywhere.componentLifecycleSnapshot(component: SDKComponent): ComponentLifecycleSnapshot

There are no per-modality accessors. To ask which STT model is loaded, pass the category to the canonical call: modelInfoForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION).

Model Types

ModelInfo

kotlin
data class ModelInfo(
    val id: String,
    val name: String,
    val category: ModelCategory,
    val format: ModelFormat,
    val downloadURL: String?,
    var localPath: String?,
    val artifactType: ModelArtifactType,
    val downloadSize: Long?,
    val framework: InferenceFramework,
    val contextLength: Int?,
    val supportsThinking: Boolean,
    val thinkingPattern: ThinkingTagPattern?,
    val description: String?,
    val source: ModelSource,
    val createdAt: Long,
    var updatedAt: Long
) {
    val isDownloaded: Boolean
    val isAvailable: Boolean
    val isBuiltIn: Boolean
}

DownloadProgress

kotlin
data class DownloadProgress(
    val modelId: String,
    val progress: Float,               // 0.0 to 1.0
    val bytesDownloaded: Long,
    val totalBytes: Long?,
    val state: DownloadState,
    val error: String?
)

enum class DownloadState {
    PENDING, DOWNLOADING, EXTRACTING, COMPLETED, ERROR, CANCELLED
}

ModelCategory

kotlin
enum class ModelCategory {
    LANGUAGE,              // LLMs (text-to-text)
    SPEECH_RECOGNITION,    // STT (voice-to-text)
    SPEECH_SYNTHESIS,      // TTS (text-to-voice)
    VISION,                // Image understanding
    MULTIMODAL,            // Multiple modalities
    AUDIO                  // Audio processing
}

ModelFormat

kotlin
enum class ModelFormat {
    ONNX,      // ONNX Runtime format
    ORT,       // Optimized ONNX Runtime
    GGUF,      // llama.cpp format
    BIN,       // Generic binary
    UNKNOWN
}

Event System

EventBus

kotlin
object EventBus {
    val allEvents: SharedFlow<SDKEvent>
    val llmEvents: SharedFlow<LLMEvent>
    val sttEvents: SharedFlow<STTEvent>
    val ttsEvents: SharedFlow<TTSEvent>
    val modelEvents: SharedFlow<ModelEvent>
    val errorEvents: SharedFlow<ErrorEvent>
}

Event Types

All event types are proto-generated via Wire. The SDK re-exports them as typealiases:

kotlin
// All events use the proto-generated SDKEvent envelope
typealias SDKEvent = ai.runanywhere.proto.v1.SDKEvent
typealias EventCategory = ai.runanywhere.proto.v1.EventCategory
typealias EventDestination = ai.runanywhere.proto.v1.EventDestination

// Typed event payloads (proto-generated)
typealias GenerationEvent = ai.runanywhere.proto.v1.GenerationEvent
typealias ModelEvent = ai.runanywhere.proto.v1.ModelEvent
typealias DownloadEvent = ai.runanywhere.proto.v1.DownloadEvent
typealias VoiceEvent = ai.runanywhere.proto.v1.VoiceEvent
typealias PerformanceEvent = ai.runanywhere.proto.v1.PerformanceEvent
typealias FailureEvent = ai.runanywhere.proto.v1.FailureEvent
typealias NetworkEvent = ai.runanywhere.proto.v1.NetworkEvent
typealias StorageLifecycleEvent = ai.runanywhere.proto.v1.StorageLifecycleEvent
typealias ComponentInitializationEvent = ai.runanywhere.proto.v1.ComponentInitializationEvent
typealias ComponentLifecycleEvent = ai.runanywhere.proto.v1.ComponentLifecycleEvent
typealias ModelRegistryEvent = ai.runanywhere.proto.v1.ModelRegistryEvent

Types & Enums

InferenceFramework

kotlin
enum class InferenceFramework {
    ONNX,              // ONNX Runtime (STT/TTS/VAD)
    LLAMA_CPP,         // llama.cpp (LLM)
    FOUNDATION_MODELS, // Platform foundation models
    SYSTEM_TTS,        // System text-to-speech
    FLUID_AUDIO,       // FluidAudio engine
    BUILT_IN,          // Simple built-in services
    NONE,              // No model needed
    UNKNOWN
}

SDKComponent

kotlin
enum class SDKComponent {
    LLM,        // Language Model
    STT,        // Speech to Text
    TTS,        // Text to Speech
    VAD,        // Voice Activity Detection
    VOICE,      // Voice Agent
    EMBEDDING   // Embedding model
}

AudioFormat

kotlin
enum class AudioFormat {
    PCM, WAV, MP3, AAC, OGG, OPUS, FLAC
}

Error Handling

SDKError

kotlin
data class SDKError(
    val code: ErrorCode,
    val category: ErrorCategory,
    override val message: String,
    override val cause: Throwable?
) : Exception(message, cause)

Error Factory Methods

kotlin
// General
SDKError.general(message, code?, cause?)
SDKError.unknown(message, cause?)

// Initialization
SDKError.notInitialized(component, cause?)
SDKError.alreadyInitialized(component, cause?)

// Model
SDKError.modelNotFound(modelId, cause?)
SDKError.modelNotLoaded(modelId?, cause?)
SDKError.modelLoadFailed(modelId, reason?, cause?)

// LLM
SDKError.llm(message, code?, cause?)
SDKError.llmGenerationFailed(reason?, cause?)

// STT
SDKError.stt(message, code?, cause?)
SDKError.sttTranscriptionFailed(reason?, cause?)

// TTS
SDKError.tts(message, code?, cause?)
SDKError.ttsSynthesisFailed(reason?, cause?)

// VAD
SDKError.vad(message, code?, cause?)
SDKError.vadDetectionFailed(reason?, cause?)

// Network
SDKError.network(message, code?, cause?)
SDKError.networkUnavailable(cause?)
SDKError.timeout(operation, timeoutMs?, cause?)

// Download
SDKError.downloadFailed(url, reason?, cause?)
SDKError.downloadCancelled(url, cause?)

// Storage
SDKError.insufficientStorage(requiredBytes?, cause?)
SDKError.fileNotFound(path, cause?)

// From C++ error codes
SDKError.fromRawValue(rawValue, message?, cause?)
SDKError.fromErrorCode(errorCode, message?, cause?)

ErrorCategory

kotlin
enum class ErrorCategory {
    GENERAL, CONFIGURATION, INITIALIZATION, FILE_RESOURCE, MEMORY,
    STORAGE, OPERATION, NETWORK, MODEL, PLATFORM, LLM, STT, TTS,
    VAD, VOICE_AGENT, DOWNLOAD, AUTHENTICATION
}

ErrorCode

Common error codes include:

  • SUCCESS, UNKNOWN, INVALID_ARGUMENT
  • NOT_INITIALIZED, ALREADY_INITIALIZED
  • MODEL_NOT_FOUND, MODEL_NOT_LOADED, MODEL_LOAD_FAILED
  • LLM_GENERATION_FAILED, STT_TRANSCRIPTION_FAILED, TTS_SYNTHESIS_FAILED
  • NETWORK_ERROR, NETWORK_UNAVAILABLE, TIMEOUT
  • DOWNLOAD_FAILED, DOWNLOAD_CANCELLED
  • INSUFFICIENT_STORAGE, FILE_NOT_FOUND, OUT_OF_MEMORY

Usage Examples

Complete LLM Chat (Matching Starter App)

kotlin
import ai.runanywhere.proto.v1.ModelCategory
import ai.runanywhere.proto.v1.ModelUnloadRequest
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.SDKEnvironment
import com.runanywhere.sdk.public.extensions.*
import com.runanywhere.sdk.public.types.RAInferenceFramework
import com.runanywhere.sdk.public.types.RALLMGenerationOptions
import com.runanywhere.sdk.public.types.RAModelLoadRequest

// Initialize
RunAnywhere.initialize(environment = SDKEnvironment.DEVELOPMENT)

// Register model (same as starter app)
RunAnywhere.registerModel(
    id = "smollm2-360m-instruct-q8_0",
    name = "SmolLM2 360M Instruct Q8_0",
    url = "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct-GGUF/resolve/main/smollm2-360m-instruct-q8_0.gguf",
    framework = RAInferenceFramework.INFERENCE_FRAMEWORK_LLAMA_CPP,
    modality = ModelCategory.MODEL_CATEGORY_LANGUAGE,
    memoryRequirement = 400_000_000,
)

// Download model
RunAnywhere.downloadModel("smollm2-360m-instruct-q8_0")
    .catch { e -> println("Download failed: ${e.message}") }
    .collect { progress ->
        println("Download: ${(progress.progress * 100).toInt()}%")
    }

// Load through the canonical lifecycle service.
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "smollm2-360m-instruct-q8_0",
        category = ModelCategory.MODEL_CATEGORY_LANGUAGE,
    ),
)

// Generate (the v1 `chat()` shortcut was deleted — use `generate`).
val result = RunAnywhere.generate(
    prompt = "Explain AI in simple terms",
    options = RALLMGenerationOptions(max_tokens = 128),
)
println("Response: ${result.text}")

// Cleanup
RunAnywhere.unloadModel(ModelUnloadRequest(unload_all = true))

Complete STT Example (Matching Starter App)

kotlin
import ai.runanywhere.proto.v1.ModelCategory
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.*
import com.runanywhere.sdk.public.types.RAInferenceFramework
import com.runanywhere.sdk.public.types.RAModelLoadRequest
import com.runanywhere.sdk.public.types.RASTTOptions

// Register STT model
RunAnywhere.registerModel(
    id = "sherpa-onnx-whisper-tiny.en",
    name = "Sherpa Whisper Tiny (ONNX)",
    url = "https://github.com/RunanywhereAI/sherpa-onnx/releases/download/runanywhere-models-v1/sherpa-onnx-whisper-tiny.en.tar.gz",
    framework = RAInferenceFramework.INFERENCE_FRAMEWORK_ONNX,
    modality = ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION,
)

// Download and load
RunAnywhere.downloadModel("sherpa-onnx-whisper-tiny.en").collect { progress ->
    println("Download: ${(progress.progress * 100).toInt()}%")
}
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "sherpa-onnx-whisper-tiny.en",
        category = ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION,
    ),
)

// Transcribe audio (16kHz, mono, 16-bit PCM).
val output = RunAnywhere.transcribe(audioData, RASTTOptions(language = "en"))
println("You said: ${output.text}")

Complete TTS Example (Matching Starter App)

kotlin
import ai.runanywhere.proto.v1.ModelCategory
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.*
import com.runanywhere.sdk.public.types.RAInferenceFramework
import com.runanywhere.sdk.public.types.RAModelLoadRequest
import com.runanywhere.sdk.public.types.RATTSOptions

// Register TTS model
RunAnywhere.registerModel(
    id = "vits-piper-en_US-lessac-medium",
    name = "Piper TTS (US English - Medium)",
    url = "https://github.com/RunanywhereAI/sherpa-onnx/releases/download/runanywhere-models-v1/vits-piper-en_US-lessac-medium.tar.gz",
    framework = RAInferenceFramework.INFERENCE_FRAMEWORK_ONNX,
    modality = ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS,
)

// Download and load
RunAnywhere.downloadModel("vits-piper-en_US-lessac-medium").collect { progress ->
    println("Download: ${(progress.progress * 100).toInt()}%")
}
RunAnywhere.loadModel(
    RAModelLoadRequest(
        model_id = "vits-piper-en_US-lessac-medium",
        category = ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS,
    ),
)

// Synthesize audio
val output = RunAnywhere.synthesize("Hello, world!", RATTSOptions())
// output.audio_data carries the WAV audio bytes (Wire ByteString).
playWavAudio(output.audio_data.toByteArray())

Voice Pipeline Session (Matching Starter App)

kotlin
import ai.runanywhere.proto.v1.CurrentModelRequest
import ai.runanywhere.proto.v1.ModelCategory
import ai.runanywhere.proto.v1.VoiceAgentConfig
import com.runanywhere.sdk.public.RunAnywhere
import com.runanywhere.sdk.public.extensions.*

// Ensure all 3 models are loaded via the canonical lifecycle service.
suspend fun isLoaded(category: ModelCategory): Boolean =
    RunAnywhere.currentModel(CurrentModelRequest(category = category)).found

val allModelsLoaded =
    isLoaded(ModelCategory.MODEL_CATEGORY_LANGUAGE) &&
        isLoaded(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION) &&
        isLoaded(ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS)

if (allModelsLoaded) {
    // Initialize the voice agent and consume its unified event stream.
    RunAnywhere.initializeVoiceAgent(
        VoiceAgentConfig(
            stt_model_id = "sherpa-onnx-whisper-tiny.en",
            llm_model_id = "smollm2-360m-instruct-q8_0",
            tts_voice_id = "vits-piper-en_US-lessac-medium",
        ),
    )

    scope.launch {
        RunAnywhere.streamVoiceAgent().collect { event ->
            event.user_said?.let { showTranscript(it.text) }
            event.agent_said?.let { showResponse(it.text) }
            event.synthesized_audio?.let { audio ->
                playWavAudio(audio.audio_data.toByteArray())
            }
            event.state_change?.let { /* update UI state */ }
        }
    }
}

See Also