How I Shrank an Agentic AI to 142KB and Ran It on a $150 Phone — A Practical Guide to Local-First LLM Agents

# generate_tool_data.py
import json
import random
from faker import Faker

fake = Faker()

TOOLS = [
    {"name": "get_weather", "params": {"location": "string"}},
    {"name": "set_timer", "params": {"seconds": "int"}},
    {"name": "send_sms", "params": {"to": "string", "body": "string"}},
    {"name": "get_contact", "params": {"name": "string"}},
]

SYSTEM_PROMPT = """You are a mobile assistant. Use tools when needed.
Available tools:
{tools}

Respond with JSON only:
{"tool": "name", "args": {...}} or {"answer": "text"}"""

def generate_example():
    tool = random.choice(TOOLS)
    params = {}
    if tool["name"] == "get_weather":
        params["location"] = fake.city()
    elif tool["name"] == "set_timer":
        params["seconds"] = random.randint(10, 3600)
    elif tool["name"] == "send_sms":
        params["to"] = fake.phone_number()
        params["body"] = fake.sentence()
    elif tool["name"] == "get_contact":
        params["name"] = fake.name()
    
    user_query = f"Can you {tool['name'].replace('_', ' ')} for {list(params.values())[0]}?"
    assistant_response = json.dumps({"tool": tool["name"], "args": params})
    
    return {
        "messages": [
            {"role": "system", "content": SYSTEM_PROMPT.format(tools=json.dumps(TOOLS))},
            {"role": "user", "content": user_query},
            {"role": "assistant", "content": assistant_response}
        ]
    }

if __name__ == "__main__":
    dataset = [generate_example() for _ in range(2000)]
    with open("tool_calling.jsonl", "w") as f:
        for ex in dataset:
            f.write(json.dumps(ex) + "\n")

# Fine-tune on 1x A100 (takes ~12 minutes)
python -m torchrun --nproc_per_node=1 fine_tune.py \
    --model HuggingFaceTB/SmolLM-135M \
    --dataset tool_calling.jsonl \
    --lora_r 8 --lora_alpha 16 \
    --output_dir ./smollm-tool-lora

# calibrate.py
from datasets import load_dataset
from transformers import AutoTokenizer
import torch

tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM-135M")
tokenizer.pad_token = tokenizer.eos_token

# Use 512 samples from FineWeb for calibration
calib_data = load_dataset("HuggingFaceFW/fineweb", split="train[:512]")

def tokenize_fn(examples):
    return tokenizer(examples["text"], truncation=True, max_length=512, padding="max_length")

calib_tokenized = calib_data.map(tokenize_fn, batched=True, remove_columns=["text"])
calib_tokenized.set_format(type="torch", columns=["input_ids", "attention_mask"])

torch.save(calib_tokenized, "calibration_data.pt")

# quantize_gptq.py
import torch
from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
from transformers import AutoTokenizer

model_id = "HuggingFaceTB/SmolLM-135M"
tokenizer = AutoTokenizer.from_pretrained(model_id)

quantize_config = BaseQuantizeConfig(
    bits=4,
    group_size=32,
    desc_act=False,  # Critical for mobile: no act-order = faster inference
    sym=True,
    true_sequential=True,
)

model = AutoGPTQForCausalLM.from_pretrained(model_id, quantize_config, device_map="auto")

calib_data = torch.load("calibration_data.pt")
model.quantize(calib_data, use_triton=False)

model.save_quantized("./smollm-int4-gptq")
tokenizer.save_pretrained("./smollm-int4-gptq")

# Requires llama.cpp built from source
cd llama.cpp
python convert-hf-to-gguf.py ../smollm-int4-gptq \
    --outfile ../smollm-135m-int4.gguf \
    --outtype q4_k_m

# Verify size
ls -lh ../smollm-135m-int4.gguf
# Should show ~142KB

# micro_agent.py — 87 lines, zero dependencies
import json
import re
from typing import Callable, Dict, Any, List
from dataclasses import dataclass

@dataclass
class Tool:
    name: str
    description: str
    params: Dict[str, str]
    handler: Callable[[Dict], Any]

class MicroAgent:
    def __init__(self, model, tokenizer, tools: List[Tool], max_steps: int = 3):
        self.model = model
        self.tokenizer = tokenizer
        self.tools = {t.name: t for t in tools}
        self.max_steps = max_steps
    
    def build_prompt(self, history: List[Dict], available_tools: List[Tool]) -> str:
        tool_desc = "\n".join([
            f"- {t.name}({', '.join(f'{k}: {v}' for k,v in t.params.items())}): {t.description}"
            for t in available_tools
        ])
        sys = f"You are a phone assistant. Tools:\n{tool_desc}\n\nRespond with ONLY one JSON object per turn:\n{{"tool": "name", "args": {{...}}}} or {{"answer": "text"}}"
        
        messages = [{"role": "system", "content": sys}] + history
        return self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
    
    def parse_response(self, text: str) -> Dict:
        # Extract first valid JSON object
        match = re.search(r'\{.*\}', text, re.DOTALL)
        if not match:
            return {"answer": text.strip()}
        try:
            return json.loads(match.group())
        except json.JSONDecodeError:
            return {"answer": text.strip()}
    
    def run(self, user_input: str) -> str:
        history = [{"role": "user", "content": user_input}]
        
        for _ in range(self.max_steps):
            prompt = self.build_prompt(history, list(self.tools.values()))
            input_ids = self.tokenizer.encode(prompt, return_tensors="pt").to(self.model.device)
            
            with torch.no_grad():
                output_ids = self.model.generate(
                    input_ids,
                    max_new_tokens=128,
                    temperature=0.1,
                    do_sample=True,
                    pad_token_id=self.tokenizer.eos_token_id,
                )
            
            response = self.tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True)
            parsed = self.parse_response(response)
            
            if "answer" in parsed:
                return parsed["answer"]
            
            if "tool" in parsed and parsed["tool"] in self.tools:
                tool = self.tools[parsed["tool"]]
                try:
                    result = tool.handler(parsed.get("args", {}))
                    history.append({"role": "assistant", "content": json.dumps(parsed)})
                    history.append({"role": "tool", "content": json.dumps({"result": result})})
                except Exception as e:
                    history.append({"role": "tool", "content": json.dumps({"error": str(e)})})
            else:
                return "I couldn't understand that request."
        
        return "Max steps reached. Please try a simpler request."

// Tools.kt — Pure Kotlin, no reflection
sealed interface Tool {
    val name: String
    val description: String
    val parameters: Map<String, String>
    fun execute(args: Map<String, Any>): ToolResult
}

data class ToolResult(
    val success: Boolean,
    val data: Any? = null,
    val error: String? = null
)

object WeatherTool : Tool {
    override val name = "get_weather"
    override val description = "Get current weather for a location"
    override val parameters = mapOf("location" to "string")
    
    override fun execute(args: Map<String, Any>): ToolResult {
        val location = args["location"] as? String ?: return ToolResult(false, error = "Missing location")
        // In production: call cached weather API or use on-device barometer
        return ToolResult(true, data = "Weather in $location: 22°C, partly cloudy")
    }
}

object TimerTool : Tool {
    override val name = "set_timer"
    override val description = "Set a countdown timer"
    override val parameters = mapOf("seconds" to "int")
    
    override fun execute(args: Map<String, Any>): ToolResult {
        val seconds = (args["seconds"] as? Number)?.toInt() ?: return ToolResult(false, error = "Missing seconds")
        // Schedule via AlarmManager or WorkManager
        TimerManager.setTimer(seconds)
        return ToolResult(true, data = "Timer set for $seconds seconds")
    }
}

object ContactsTool : Tool {
    override val name = "get_contact"
    override val description = "Look up a contact by name"
    override val parameters = mapOf("name" to "string")
    
    override fun execute(args: Map<String, Any>): ToolResult {
        val name = args["name"] as? String ?: return ToolResult(false, error = "Missing name")
        val cursor = context.contentResolver.query(
            ContactsContract.Contacts.CONTENT_URI,
            arrayOf(ContactsContract.Contacts.DISPLAY_NAME, ContactsContract.Contacts.HAS_PHONE_NUMBER),
            "${ContactsContract.Contacts.DISPLAY_NAME} LIKE ?",
            arrayOf("%$name%"),
            null
        )
        return if (cursor?.moveToFirst() == true) {
            val contactName = cursor.getString(0)
            val hasPhone = cursor.getInt(1) > 0
            var phone = ""
            if (hasPhone) {
                val phoneCursor = context.contentResolver.query(
                    ContactsContract.CommonDataKinds.Phone.CONTENT_URI,
                    null,
                    "${ContactsContract.CommonDataKinds.Phone.CONTACT_ID} = ?",
                    arrayOf(cursor.getString(cursor.getColumnIndex(ContactsContract.Contacts._ID))),
                    null
                )
                phoneCursor?.use { if (it.moveToFirst()) phone = it.getString(it.getColumnIndex(ContactsContract.CommonDataKinds.Phone.NUMBER)) }
            }
            ToolResult(true, data = "$contactName: $phone")
        } else ToolResult(false, error = "Contact not found")
    }
}

# CMakeLists.txt
cmake_minimum_required(VERSION 3.22)
project(tinyagent LANGUAGES C CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_ANDROID_NDK_API 24)

# Minimal llama.cpp build
option(LLAMA_BUILD_TOOLS OFF)
option(LLAMA_BUILD_EXAMPLES OFF)
option(LLAMA_BUILD_TESTS OFF)
option(LLAMA_CURL OFF)
option(LLAMA_BLAS OFF)
option(LLAMA_BLAS_VENDOR "")
option(LLAMA_ACCELERATE OFF)
option(LLAMA_METAL OFF)
option(LLAMA_OPENMP OFF)
option(LLAMA_CUDA OFF)
option(LLAMA_HIPBLAS OFF)
option(LLAMA_RPC OFF)

# Critical for size: strip symbols, LTO
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -flto -ffunction-sections -fdata-sections")
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -Wl,--gc-sections -Wl,--strip-all")

add_subdirectory(llama.cpp)

# Our JNI wrapper
add_library(tinyagent_jni SHARED src/main/cpp/tinyagent_jni.cpp)
target_link_libraries(tinyagent_jni PRIVATE llama)
target_include_directories(tinyagent_jni PRIVATE llama.cpp/include)

// tinyagent_jni.cpp
#include <jni.h>
#include <string>
#include <vector>
#include "llama.h"

static llama_model* g_model = nullptr;
static llama_context* g_ctx = nullptr;
static std::vector<llama_token> g_tokens;
static std::mutex g_mutex;

extern "C" JNIEXPORT jboolean JNICALL
Java_com_tinyagent_TinyAgent_nativeInit(JNIEnv* env, jobject, jstring modelPath, jint nCtx, jint nThreads) {
    std::lock_guard<std::mutex> lock(g_mutex);
    if (g_ctx) return JNI_TRUE;

    const char* path = env->GetStringUTFChars(modelPath, nullptr);
    
    llama_model_params model_params = llama_model_default_params();
    model_params.n_gpu_layers = 0; // CPU only
    g_model = llama_load_model_from_file(path, model_params);
    env->ReleaseStringUTFChars(modelPath, path);
    
    if (!g_model) return JNI_FALSE;

    llama_context_params ctx_params = llama_context_default_params();
    ctx_params.n_ctx = nCtx;
    ctx_params.n_threads = nThreads;
    ctx_params.n_threads_batch = nThreads;
    g_ctx = llama_new_context_with_model(g_model, ctx_params);
    
    return g_ctx != nullptr ? JNI_TRUE : JNI_FALSE;
}

extern "C" JNIEXPORT jstring JNICALL
Java_com_tinyagent_TinyAgent_nativeGenerate(JNIEnv* env, jobject, jstring prompt, jint maxTokens, jfloat temp) {
    std::lock_guard<std::mutex> lock(g_mutex);
    if (!g_ctx) return env->NewStringUTF("Model not initialized");

    const char* promptC = env->GetStringUTFChars(prompt, nullptr);
    
    // Tokenize
    int n_tokens = -llama_tokenize(g_model, promptC, strlen(promptC), nullptr, 0, true, true);
    g_tokens.resize(n_tokens);
    llama_tokenize(g_model, promptC, strlen(promptC), g_tokens.data(), g_tokens.size(), true, true);
    env->ReleaseStringUTFChars(prompt, promptC);

    // Evaluate prompt
    if (llama_decode(g_ctx, llama_batch_get_one(g_tokens.data(), g_tokens.size()))) {
        return env->NewStringUTF("Decode failed");
    }

    // Generate
    std::string output;
    for (int i = 0; i < maxTokens; ++i) {
        llama_token id = llama_sample_token_greedy(g_ctx, nullptr); // temp=0 for tools
        if (temp > 0.0f) {
            auto candidates = llama_sample_token_mirostat(g_ctx, nullptr, temp, 0.1f, 100);
            id = candidates[0].id;
        }
        
        if (llama_token_is_eog(g_model, id)) break;
        
        char buf[32];
        int n = llama_token_to_piece(g_model, id, buf, sizeof(buf), 0, true);
        if (n > 0) output.append(buf, n);
        
        if (llama_decode(g_ctx, llama_batch_get_one(&id, 1))) break;
    }

    return env->NewStringUTF(output.c_str());
}

extern "C" JNIEXPORT void JNICALL
Java_com_tinyagent_TinyAgent_nativeFree(JNIEnv*, jobject) {
    std::lock_guard<std::mutex> lock(g_mutex);
    if (g_ctx) { llama_free(g_ctx); g_ctx = nullptr; }
    if (g_model) { llama_free_model(g_model); g_model = nullptr; }
}

// TinyAgent.kt
package com.tinyagent

import android.content.Context
import android.util.Log
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext

class TinyAgent(private val context: Context) {
    companion object {
        private const val TAG = "TinyAgent"
        private var loaded = false
        init { System.loadLibrary("tinyagent_jni") }
    }

    private external fun nativeInit(modelPath: String, nCtx: Int, nThreads: Int): Boolean
    private external fun nativeGenerate(prompt: String, maxTokens: Int, temp: Float): String
    private external fun nativeFree()

    suspend fun initialize(modelAssetName: String = "smollm-135m-int4.gguf"): Boolean = withContext(Dispatchers.IO) {
        if (loaded) return@withContext true
        val modelPath = copyAssetToFile(modelAssetName)
        val success = nativeInit(modelPath, 512, 2) // 512 ctx, 2 threads
        loaded = success
        Log.i(TAG, "Model loaded: $success")
        success
    }

    suspend fun generate(prompt: String, maxTokens: Int = 128, temperature: Float = 0.1f): String = withContext(Dispatchers.IO) {
        nativeGenerate(prompt, maxTokens, temperature)
    }

    fun shutdown() {
        nativeFree()
        loaded = false
    }

    private fun copyAssetToFile(assetName: String): String {
        val file = File(context.filesDir, assetName)
        if (file.exists()) return file.absolutePath
        context.assets.open(assetName).use { input ->
            file.outputStream().use { output -> input.copyTo(output) }
        }
        file.absolutePath
    }
}

// app/build.gradle.kts
plugins {
    id("com.android.application")
    id("org.jetbrains.kotlin.android")
    id("cpp")
}

android {
    namespace = "com.tinyagent"
    compileSdk = 34

    defaultConfig {
        minSdk = 24
        targetSdk = 34
        versionCode = 1
        versionName = "1.0"

        externalNativeBuild {
            cmake {
                arguments("-DANDROID_STL=c++_shared", "-DLLAMA_BUILD_TOOLS=OFF")
                abiFilters("arm64-v8a", "armeabi-v7a")
            }
        }
    }

    buildTypes {
        release {
            isMinifyEnabled = true
            isShrinkResources = true
            proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
            ndk {
                debugSymbolLevel = "FULL"
            }
        }
    }

    externalNativeBuild {
        cmake {
            path = "src/main/cpp/CMakeLists.txt"
            version = "3.22.1"
        }
    }

    packagingOptions {
        jniLibs {
            useLegacyPackaging = true
        }
        doNotStrip "*/*/libtinyagent_jni.so"
    }
}

dependencies {
    implementation("androidx.core:core-ktx:1.12.0")
    implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.7.0")
    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3")
}

# proguard-rules.pro
-keep class com.tinyagent.TinyAgent { *; }
-keep class com.tinyagent.ToolsKt { *; }
-keep class com.tinyagent.ToolResult { *; }
-dontwarn com.tinyagent.**
-assumenosideeffects class android.util.Log { *; }

// MainActivity.kt
package com.tinyagent

import android.os.Bundle
import android.widget.*
import androidx.appcompat.app.AppCompatActivity
import androidx.lifecycle.lifecycleScope
import kotlinx.coroutines.launch

class MainActivity : AppCompatActivity() {
    private val agent = TinyAgent(this)
    private lateinit var chatView: TextView
    private lateinit var inputEdit: EditText
    private lateinit var sendBtn: Button

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContentView(R.layout.activity_main)

        chatView = findViewById(R.id.chatView)
        inputEdit = findViewById(R.id.inputEdit)
        sendBtn = findViewById(R.id.sendBtn)

        lifecycleScope.launch {
            val loaded = agent.initialize()
            runOnUiThread { sendBtn.isEnabled = loaded }
        }

        sendBtn.setOnClickListener {
            val query = inputEdit.text.toString().trim()
            if (query.isBlank()) return@setOnClickListener
            
            appendChat("You: $query")
            inputEdit.text.clear()
            sendBtn.isEnabled = false
            
            lifecycleScope.launch {
                val response = agent.generate(buildPrompt(query))
                runOnUiThread {
                    appendChat("Agent: $response")
                    sendBtn.isEnabled = true
                }
            }
        }
    }

    private fun buildPrompt(query: String): String {
        // In production: inject tool definitions dynamically
        return query
    }

    private fun appendChat(msg: String) {
        chatView.append("$msg\n\n")
    }

    override fun onDestroy() {
        agent.shutdown()
        super.onDestroy()
    }
}