Setting Up a Local LLM Development Environment

# Install Ollama (macOS/Linux)
curl -fsSL https://ollama.com/install.sh | sh
 
# Pull models for different use cases
ollama pull llama3.1:8b          # Fast general purpose, ~4.7GB
ollama pull codellama:13b        # Code generation, ~7.4GB
ollama pull deepseek-coder-v2:16b # Best code quality, ~8.9GB
ollama pull nomic-embed-text     # Embeddings for RAG, ~274MB
 
# Verify everything is running
ollama list
curl http://localhost:11434/api/tags

# Set environment variables before starting Ollama
# On Linux, add these to /etc/systemd/system/ollama.service.d/override.conf
# On macOS, use launchctl setenv
 
# Keep models loaded in memory longer (default is 5 minutes)
export OLLAMA_KEEP_ALIVE="30m"
 
# Allow parallel requests (useful for editor + terminal simultaneously)
export OLLAMA_NUM_PARALLEL=2
 
# Restrict to specific GPU if you have multiple
export CUDA_VISIBLE_DEVICES=0
 
# Increase context window for larger code files
# Note: larger context = more VRAM usage
export OLLAMA_MAX_LOADED_MODELS=2

{
  "continue.models": [
    {
      "title": "Local CodeLlama",
      "provider": "ollama",
      "model": "codellama:13b",
      "apiBase": "http://localhost:11434"
    },
    {
      "title": "Local Llama 3.1",
      "provider": "ollama",
      "model": "llama3.1:8b",
      "apiBase": "http://localhost:11434"
    }
  ]
}

import OpenAI from "openai";
 
const isLocal = process.env.AI_PROVIDER === "local";
 
export const ai = new OpenAI({
  baseURL: isLocal
    ? "http://localhost:11434/v1"
    : "https://api.openai.com/v1",
  apiKey: isLocal ? "ollama" : process.env.OPENAI_API_KEY!,
});
 
// Usage is identical regardless of provider
export async function generateSummary(content: string): Promise<string> {
  const response = await ai.chat.completions.create({
    model: isLocal ? "llama3.1:8b" : "gpt-4o-mini",
    messages: [
      {
        role: "system",
        content: "Summarize the following content in 2-3 sentences.",
      },
      { role: "user", content },
    ],
    temperature: 0.3,
    max_tokens: 200,
  });
 
  return response.choices[0].message.content ?? "";
}

import { describe, it, expect } from "vitest";
import { ai } from "@/lib/ai-client";
 
const MODEL = process.env.AI_MODEL ?? "llama3.1:8b";
 
describe("prompt evaluation suite", () => {
  it("should extract structured data from unstructured text", async () => {
    const response = await ai.chat.completions.create({
      model: MODEL,
      messages: [
        {
          role: "system",
          content: `Extract contact info as JSON: { "name": string, "email": string, "company": string | null }`,
        },
        {
          role: "user",
          content: "Hi, I'm Sarah Chen from Acme Corp. Reach me at sarah@acme.co",
        },
      ],
      temperature: 0,
      response_format: { type: "json_object" },
    });
 
    const parsed = JSON.parse(response.choices[0].message.content ?? "{}");
    expect(parsed).toHaveProperty("name");
    expect(parsed).toHaveProperty("email");
    expect(parsed.email).toMatch(/@/);
  }, 30_000);
 
  it("should respect output length constraints", async () => {
    const response = await ai.chat.completions.create({
      model: MODEL,
      messages: [
        {
          role: "user",
          content: "Explain TCP/IP in exactly one sentence.",
        },
      ],
      temperature: 0,
      max_tokens: 100,
    });
 
    const text = response.choices[0].message.content ?? "";
    const sentences = text.split(/[.!?]+/).filter(Boolean);
    expect(sentences.length).toBeLessThanOrEqual(2);
  }, 30_000);
});

import { readdir, readFile } from "fs/promises";
import { join } from "path";
 
const OLLAMA_BASE = "http://localhost:11434";
 
async function getEmbedding(text: string): Promise<number[]> {
  const response = await fetch(`${OLLAMA_BASE}/api/embeddings`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: "nomic-embed-text",
      prompt: text,
    }),
  });
  const data = await response.json();
  return data.embedding;
}
 
async function indexDirectory(dir: string) {
  const files = await readdir(dir, { recursive: true });
  const documents: Array<{ path: string; content: string; embedding: number[] }> = [];
 
  for (const file of files) {
    if (!file.endsWith(".md") && !file.endsWith(".mdx")) continue;
 
    const content = await readFile(join(dir, file), "utf-8");
    const chunks = splitIntoChunks(content, 500);
 
    for (const chunk of chunks) {
      const embedding = await getEmbedding(chunk);
      documents.push({ path: file, content: chunk, embedding });
    }
  }
 
  return documents;
}
 
function splitIntoChunks(text: string, maxTokens: number): string[] {
  const paragraphs = text.split(/\n\n+/);
  const chunks: string[] = [];
  let current = "";
 
  for (const para of paragraphs) {
    if ((current + para).length > maxTokens * 4) {
      if (current) chunks.push(current.trim());
      current = para;
    } else {
      current += "\n\n" + para;
    }
  }
  if (current.trim()) chunks.push(current.trim());
  return chunks;
}