feat: Add semantic search with vector embeddings

Add vector-based semantic search to complement keyword search. Users can toggle between "Keyword" and "Semantic" modes in the search modal (Cmd+K, then Tab to switch). Semantic search: - Uses OpenAI text-embedding-ada-002 (1536 dimensions) - Finds content by meaning, not exact words - Shows similarity scores as percentages - ~300ms latency, ~$0.0001/query - Graceful fallback if OPENAI_API_KEY not set New files: - convex/embeddings.ts - Embedding generation actions - convex/embeddingsQueries.ts - Queries/mutations for embeddings - convex/semanticSearch.ts - Vector search action - convex/semanticSearchQueries.ts - Result hydration queries - content/pages/docs-search.md - Keyword search docs - content/pages/docs-semantic-search.md - Semantic search docs Changes: - convex/schema.ts: Add embedding field and by_embedding vectorIndex - SearchModal.tsx: Add mode toggle (TextAa/Brain icons) - sync-posts.ts: Generate embeddings after content sync - global.css: Search mode toggle styles Documentation updated: - changelog.md, TASK.md, files.md, about.md, home.md Configuration: npx convex env set OPENAI_API_KEY sk-your-key Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> Status: Ready to commit. All semantic search files are staged. The TypeScript warnings are pre-existing (unused variables) and don't affect the build.
2026-01-12 04:09:14 +00:00 · 2026-01-05 18:30:48 -08:00
parent 83411ec1b2
commit 5a8df46681
58 changed files with 7024 additions and 2527 deletions
--- a/convex/_generated/api.d.ts
+++ b/convex/_generated/api.d.ts
@@ -11,16 +11,22 @@
 import type * as aiChatActions from "../aiChatActions.js";
 import type * as aiChats from "../aiChats.js";
 import type * as aiImageGeneration from "../aiImageGeneration.js";
+import type * as cms from "../cms.js";
 import type * as contact from "../contact.js";
 import type * as contactActions from "../contactActions.js";
 import type * as crons from "../crons.js";
+import type * as embeddings from "../embeddings.js";
+import type * as embeddingsQueries from "../embeddingsQueries.js";
 import type * as http from "../http.js";
+import type * as importAction from "../importAction.js";
 import type * as newsletter from "../newsletter.js";
 import type * as newsletterActions from "../newsletterActions.js";
 import type * as pages from "../pages.js";
 import type * as posts from "../posts.js";
 import type * as rss from "../rss.js";
 import type * as search from "../search.js";
+import type * as semanticSearch from "../semanticSearch.js";
+import type * as semanticSearchQueries from "../semanticSearchQueries.js";
 import type * as stats from "../stats.js";

 import type {
@@ -33,16 +39,22 @@ declare const fullApi: ApiFromModules<{
  aiChatActions: typeof aiChatActions;
  aiChats: typeof aiChats;
  aiImageGeneration: typeof aiImageGeneration;
+  cms: typeof cms;
  contact: typeof contact;
  contactActions: typeof contactActions;
  crons: typeof crons;
+  embeddings: typeof embeddings;
+  embeddingsQueries: typeof embeddingsQueries;
  http: typeof http;
+  importAction: typeof importAction;
  newsletter: typeof newsletter;
  newsletterActions: typeof newsletterActions;
  pages: typeof pages;
  posts: typeof posts;
  rss: typeof rss;
  search: typeof search;
+  semanticSearch: typeof semanticSearch;
+  semanticSearchQueries: typeof semanticSearchQueries;
  stats: typeof stats;
 }>;

--- a/convex/cms.ts
+++ b/convex/cms.ts
@@ -0,0 +1,412 @@
+import { mutation, query } from "./_generated/server";
+import { v } from "convex/values";
+
+// Shared validator for post data
+const postDataValidator = v.object({
+  slug: v.string(),
+  title: v.string(),
+  description: v.string(),
+  content: v.string(),
+  date: v.string(),
+  published: v.boolean(),
+  tags: v.array(v.string()),
+  readTime: v.optional(v.string()),
+  image: v.optional(v.string()),
+  showImageAtTop: v.optional(v.boolean()),
+  excerpt: v.optional(v.string()),
+  featured: v.optional(v.boolean()),
+  featuredOrder: v.optional(v.number()),
+  authorName: v.optional(v.string()),
+  authorImage: v.optional(v.string()),
+  layout: v.optional(v.string()),
+  rightSidebar: v.optional(v.boolean()),
+  showFooter: v.optional(v.boolean()),
+  footer: v.optional(v.string()),
+  showSocialFooter: v.optional(v.boolean()),
+  aiChat: v.optional(v.boolean()),
+  blogFeatured: v.optional(v.boolean()),
+  newsletter: v.optional(v.boolean()),
+  contactForm: v.optional(v.boolean()),
+  unlisted: v.optional(v.boolean()),
+  docsSection: v.optional(v.boolean()),
+  docsSectionGroup: v.optional(v.string()),
+  docsSectionOrder: v.optional(v.number()),
+  docsSectionGroupOrder: v.optional(v.number()),
+  docsSectionGroupIcon: v.optional(v.string()),
+  docsLanding: v.optional(v.boolean()),
+});
+
+// Shared validator for page data
+const pageDataValidator = v.object({
+  slug: v.string(),
+  title: v.string(),
+  content: v.string(),
+  published: v.boolean(),
+  order: v.optional(v.number()),
+  showInNav: v.optional(v.boolean()),
+  excerpt: v.optional(v.string()),
+  image: v.optional(v.string()),
+  showImageAtTop: v.optional(v.boolean()),
+  featured: v.optional(v.boolean()),
+  featuredOrder: v.optional(v.number()),
+  authorName: v.optional(v.string()),
+  authorImage: v.optional(v.string()),
+  layout: v.optional(v.string()),
+  rightSidebar: v.optional(v.boolean()),
+  showFooter: v.optional(v.boolean()),
+  footer: v.optional(v.string()),
+  showSocialFooter: v.optional(v.boolean()),
+  aiChat: v.optional(v.boolean()),
+  contactForm: v.optional(v.boolean()),
+  newsletter: v.optional(v.boolean()),
+  textAlign: v.optional(v.string()),
+  docsSection: v.optional(v.boolean()),
+  docsSectionGroup: v.optional(v.string()),
+  docsSectionOrder: v.optional(v.number()),
+  docsSectionGroupOrder: v.optional(v.number()),
+  docsSectionGroupIcon: v.optional(v.string()),
+  docsLanding: v.optional(v.boolean()),
+});
+
+// Create a new post via dashboard
+export const createPost = mutation({
+  args: { post: postDataValidator },
+  returns: v.id("posts"),
+  handler: async (ctx, args) => {
+    // Check if slug already exists
+    const existing = await ctx.db
+      .query("posts")
+      .withIndex("by_slug", (q) => q.eq("slug", args.post.slug))
+      .first();
+
+    if (existing) {
+      throw new Error(`Post with slug "${args.post.slug}" already exists`);
+    }
+
+    const postId = await ctx.db.insert("posts", {
+      ...args.post,
+      source: "dashboard",
+      lastSyncedAt: Date.now(),
+    });
+
+    return postId;
+  },
+});
+
+// Update any post (dashboard or synced)
+export const updatePost = mutation({
+  args: {
+    id: v.id("posts"),
+    post: v.object({
+      slug: v.optional(v.string()),
+      title: v.optional(v.string()),
+      description: v.optional(v.string()),
+      content: v.optional(v.string()),
+      date: v.optional(v.string()),
+      published: v.optional(v.boolean()),
+      tags: v.optional(v.array(v.string())),
+      readTime: v.optional(v.string()),
+      image: v.optional(v.string()),
+      showImageAtTop: v.optional(v.boolean()),
+      excerpt: v.optional(v.string()),
+      featured: v.optional(v.boolean()),
+      featuredOrder: v.optional(v.number()),
+      authorName: v.optional(v.string()),
+      authorImage: v.optional(v.string()),
+      layout: v.optional(v.string()),
+      rightSidebar: v.optional(v.boolean()),
+      showFooter: v.optional(v.boolean()),
+      footer: v.optional(v.string()),
+      showSocialFooter: v.optional(v.boolean()),
+      aiChat: v.optional(v.boolean()),
+      blogFeatured: v.optional(v.boolean()),
+      newsletter: v.optional(v.boolean()),
+      contactForm: v.optional(v.boolean()),
+      unlisted: v.optional(v.boolean()),
+      docsSection: v.optional(v.boolean()),
+      docsSectionGroup: v.optional(v.string()),
+      docsSectionOrder: v.optional(v.number()),
+      docsSectionGroupOrder: v.optional(v.number()),
+      docsSectionGroupIcon: v.optional(v.string()),
+      docsLanding: v.optional(v.boolean()),
+    }),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const existing = await ctx.db.get(args.id);
+    if (!existing) {
+      throw new Error("Post not found");
+    }
+
+    // If slug is being changed, check for conflicts
+    const newSlug = args.post.slug;
+    if (newSlug && newSlug !== existing.slug) {
+      const slugConflict = await ctx.db
+        .query("posts")
+        .withIndex("by_slug", (q) => q.eq("slug", newSlug))
+        .first();
+      if (slugConflict) {
+        throw new Error(`Post with slug "${newSlug}" already exists`);
+      }
+    }
+
+    await ctx.db.patch(args.id, {
+      ...args.post,
+      lastSyncedAt: Date.now(),
+    });
+
+    return null;
+  },
+});
+
+// Delete a post
+export const deletePost = mutation({
+  args: { id: v.id("posts") },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const existing = await ctx.db.get(args.id);
+    if (!existing) {
+      throw new Error("Post not found");
+    }
+
+    await ctx.db.delete(args.id);
+    return null;
+  },
+});
+
+// Create a new page via dashboard
+export const createPage = mutation({
+  args: { page: pageDataValidator },
+  returns: v.id("pages"),
+  handler: async (ctx, args) => {
+    // Check if slug already exists
+    const existing = await ctx.db
+      .query("pages")
+      .withIndex("by_slug", (q) => q.eq("slug", args.page.slug))
+      .first();
+
+    if (existing) {
+      throw new Error(`Page with slug "${args.page.slug}" already exists`);
+    }
+
+    const pageId = await ctx.db.insert("pages", {
+      ...args.page,
+      source: "dashboard",
+      lastSyncedAt: Date.now(),
+    });
+
+    return pageId;
+  },
+});
+
+// Update any page (dashboard or synced)
+export const updatePage = mutation({
+  args: {
+    id: v.id("pages"),
+    page: v.object({
+      slug: v.optional(v.string()),
+      title: v.optional(v.string()),
+      content: v.optional(v.string()),
+      published: v.optional(v.boolean()),
+      order: v.optional(v.number()),
+      showInNav: v.optional(v.boolean()),
+      excerpt: v.optional(v.string()),
+      image: v.optional(v.string()),
+      showImageAtTop: v.optional(v.boolean()),
+      featured: v.optional(v.boolean()),
+      featuredOrder: v.optional(v.number()),
+      authorName: v.optional(v.string()),
+      authorImage: v.optional(v.string()),
+      layout: v.optional(v.string()),
+      rightSidebar: v.optional(v.boolean()),
+      showFooter: v.optional(v.boolean()),
+      footer: v.optional(v.string()),
+      showSocialFooter: v.optional(v.boolean()),
+      aiChat: v.optional(v.boolean()),
+      contactForm: v.optional(v.boolean()),
+      newsletter: v.optional(v.boolean()),
+      textAlign: v.optional(v.string()),
+      docsSection: v.optional(v.boolean()),
+      docsSectionGroup: v.optional(v.string()),
+      docsSectionOrder: v.optional(v.number()),
+      docsSectionGroupOrder: v.optional(v.number()),
+      docsSectionGroupIcon: v.optional(v.string()),
+      docsLanding: v.optional(v.boolean()),
+    }),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const existing = await ctx.db.get(args.id);
+    if (!existing) {
+      throw new Error("Page not found");
+    }
+
+    // If slug is being changed, check for conflicts
+    const newSlug = args.page.slug;
+    if (newSlug && newSlug !== existing.slug) {
+      const slugConflict = await ctx.db
+        .query("pages")
+        .withIndex("by_slug", (q) => q.eq("slug", newSlug))
+        .first();
+      if (slugConflict) {
+        throw new Error(`Page with slug "${newSlug}" already exists`);
+      }
+    }
+
+    await ctx.db.patch(args.id, {
+      ...args.page,
+      lastSyncedAt: Date.now(),
+    });
+
+    return null;
+  },
+});
+
+// Delete a page
+export const deletePage = mutation({
+  args: { id: v.id("pages") },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const existing = await ctx.db.get(args.id);
+    if (!existing) {
+      throw new Error("Page not found");
+    }
+
+    await ctx.db.delete(args.id);
+    return null;
+  },
+});
+
+// Export post as markdown with frontmatter
+export const exportPostAsMarkdown = query({
+  args: { id: v.id("posts") },
+  returns: v.string(),
+  handler: async (ctx, args) => {
+    const post = await ctx.db.get(args.id);
+    if (!post) {
+      throw new Error("Post not found");
+    }
+
+    // Build frontmatter
+    const frontmatter: string[] = ["---"];
+    frontmatter.push(`title: "${post.title.replace(/"/g, '\\"')}"`);
+    frontmatter.push(`description: "${post.description.replace(/"/g, '\\"')}"`);
+    frontmatter.push(`date: "${post.date}"`);
+    frontmatter.push(`slug: "${post.slug}"`);
+    frontmatter.push(`published: ${post.published}`);
+    frontmatter.push(`tags: [${post.tags.map((t) => `"${t}"`).join(", ")}]`);
+
+    // Add optional fields
+    if (post.readTime) frontmatter.push(`readTime: "${post.readTime}"`);
+    if (post.image) frontmatter.push(`image: "${post.image}"`);
+    if (post.showImageAtTop !== undefined)
+      frontmatter.push(`showImageAtTop: ${post.showImageAtTop}`);
+    if (post.excerpt)
+      frontmatter.push(`excerpt: "${post.excerpt.replace(/"/g, '\\"')}"`);
+    if (post.featured !== undefined)
+      frontmatter.push(`featured: ${post.featured}`);
+    if (post.featuredOrder !== undefined)
+      frontmatter.push(`featuredOrder: ${post.featuredOrder}`);
+    if (post.authorName) frontmatter.push(`authorName: "${post.authorName}"`);
+    if (post.authorImage) frontmatter.push(`authorImage: "${post.authorImage}"`);
+    if (post.layout) frontmatter.push(`layout: "${post.layout}"`);
+    if (post.rightSidebar !== undefined)
+      frontmatter.push(`rightSidebar: ${post.rightSidebar}`);
+    if (post.showFooter !== undefined)
+      frontmatter.push(`showFooter: ${post.showFooter}`);
+    if (post.footer)
+      frontmatter.push(`footer: "${post.footer.replace(/"/g, '\\"')}"`);
+    if (post.showSocialFooter !== undefined)
+      frontmatter.push(`showSocialFooter: ${post.showSocialFooter}`);
+    if (post.aiChat !== undefined) frontmatter.push(`aiChat: ${post.aiChat}`);
+    if (post.blogFeatured !== undefined)
+      frontmatter.push(`blogFeatured: ${post.blogFeatured}`);
+    if (post.newsletter !== undefined)
+      frontmatter.push(`newsletter: ${post.newsletter}`);
+    if (post.contactForm !== undefined)
+      frontmatter.push(`contactForm: ${post.contactForm}`);
+    if (post.unlisted !== undefined)
+      frontmatter.push(`unlisted: ${post.unlisted}`);
+    if (post.docsSection !== undefined)
+      frontmatter.push(`docsSection: ${post.docsSection}`);
+    if (post.docsSectionGroup)
+      frontmatter.push(`docsSectionGroup: "${post.docsSectionGroup}"`);
+    if (post.docsSectionOrder !== undefined)
+      frontmatter.push(`docsSectionOrder: ${post.docsSectionOrder}`);
+    if (post.docsSectionGroupOrder !== undefined)
+      frontmatter.push(`docsSectionGroupOrder: ${post.docsSectionGroupOrder}`);
+    if (post.docsSectionGroupIcon)
+      frontmatter.push(`docsSectionGroupIcon: "${post.docsSectionGroupIcon}"`);
+    if (post.docsLanding !== undefined)
+      frontmatter.push(`docsLanding: ${post.docsLanding}`);
+
+    frontmatter.push("---");
+
+    return `${frontmatter.join("\n")}\n\n${post.content}`;
+  },
+});
+
+// Export page as markdown with frontmatter
+export const exportPageAsMarkdown = query({
+  args: { id: v.id("pages") },
+  returns: v.string(),
+  handler: async (ctx, args) => {
+    const page = await ctx.db.get(args.id);
+    if (!page) {
+      throw new Error("Page not found");
+    }
+
+    // Build frontmatter
+    const frontmatter: string[] = ["---"];
+    frontmatter.push(`title: "${page.title.replace(/"/g, '\\"')}"`);
+    frontmatter.push(`slug: "${page.slug}"`);
+    frontmatter.push(`published: ${page.published}`);
+
+    // Add optional fields
+    if (page.order !== undefined) frontmatter.push(`order: ${page.order}`);
+    if (page.showInNav !== undefined)
+      frontmatter.push(`showInNav: ${page.showInNav}`);
+    if (page.excerpt)
+      frontmatter.push(`excerpt: "${page.excerpt.replace(/"/g, '\\"')}"`);
+    if (page.image) frontmatter.push(`image: "${page.image}"`);
+    if (page.showImageAtTop !== undefined)
+      frontmatter.push(`showImageAtTop: ${page.showImageAtTop}`);
+    if (page.featured !== undefined)
+      frontmatter.push(`featured: ${page.featured}`);
+    if (page.featuredOrder !== undefined)
+      frontmatter.push(`featuredOrder: ${page.featuredOrder}`);
+    if (page.authorName) frontmatter.push(`authorName: "${page.authorName}"`);
+    if (page.authorImage) frontmatter.push(`authorImage: "${page.authorImage}"`);
+    if (page.layout) frontmatter.push(`layout: "${page.layout}"`);
+    if (page.rightSidebar !== undefined)
+      frontmatter.push(`rightSidebar: ${page.rightSidebar}`);
+    if (page.showFooter !== undefined)
+      frontmatter.push(`showFooter: ${page.showFooter}`);
+    if (page.footer)
+      frontmatter.push(`footer: "${page.footer.replace(/"/g, '\\"')}"`);
+    if (page.showSocialFooter !== undefined)
+      frontmatter.push(`showSocialFooter: ${page.showSocialFooter}`);
+    if (page.aiChat !== undefined) frontmatter.push(`aiChat: ${page.aiChat}`);
+    if (page.contactForm !== undefined)
+      frontmatter.push(`contactForm: ${page.contactForm}`);
+    if (page.newsletter !== undefined)
+      frontmatter.push(`newsletter: ${page.newsletter}`);
+    if (page.textAlign) frontmatter.push(`textAlign: "${page.textAlign}"`);
+    if (page.docsSection !== undefined)
+      frontmatter.push(`docsSection: ${page.docsSection}`);
+    if (page.docsSectionGroup)
+      frontmatter.push(`docsSectionGroup: "${page.docsSectionGroup}"`);
+    if (page.docsSectionOrder !== undefined)
+      frontmatter.push(`docsSectionOrder: ${page.docsSectionOrder}`);
+    if (page.docsSectionGroupOrder !== undefined)
+      frontmatter.push(`docsSectionGroupOrder: ${page.docsSectionGroupOrder}`);
+    if (page.docsSectionGroupIcon)
+      frontmatter.push(`docsSectionGroupIcon: "${page.docsSectionGroupIcon}"`);
+    if (page.docsLanding !== undefined)
+      frontmatter.push(`docsLanding: ${page.docsLanding}`);
+
+    frontmatter.push("---");
+
+    return `${frontmatter.join("\n")}\n\n${page.content}`;
+  },
+});
--- a/convex/embeddings.ts
+++ b/convex/embeddings.ts
@@ -0,0 +1,161 @@
+"use node";
+
+import { v } from "convex/values";
+import { action, internalAction } from "./_generated/server";
+import { internal } from "./_generated/api";
+import OpenAI from "openai";
+
+// Generate embedding for text using OpenAI text-embedding-ada-002
+export const generateEmbedding = internalAction({
+  args: { text: v.string() },
+  returns: v.array(v.float64()),
+  handler: async (_ctx, { text }) => {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      throw new Error("OPENAI_API_KEY not configured in Convex environment");
+    }
+
+    const openai = new OpenAI({ apiKey });
+    const response = await openai.embeddings.create({
+      model: "text-embedding-ada-002",
+      input: text.slice(0, 8000), // Truncate to stay within token limit
+    });
+
+    return response.data[0].embedding;
+  },
+});
+
+// Internal action to generate embeddings for posts without them
+export const generatePostEmbeddings = internalAction({
+  args: {},
+  returns: v.object({ processed: v.number() }),
+  handler: async (ctx) => {
+    const posts = await ctx.runQuery(
+      internal.embeddingsQueries.getPostsWithoutEmbeddings,
+      { limit: 10 }
+    );
+
+    let processed = 0;
+    for (const post of posts) {
+      try {
+        // Combine title and content for embedding
+        const textToEmbed = `${post.title}\n\n${post.content}`;
+        const embedding = await ctx.runAction(internal.embeddings.generateEmbedding, {
+          text: textToEmbed,
+        });
+        await ctx.runMutation(internal.embeddingsQueries.savePostEmbedding, {
+          id: post._id,
+          embedding,
+        });
+        processed++;
+      } catch (error) {
+        console.error(`Failed to generate embedding for post ${post._id}:`, error);
+      }
+    }
+
+    return { processed };
+  },
+});
+
+// Internal action to generate embeddings for pages without them
+export const generatePageEmbeddings = internalAction({
+  args: {},
+  returns: v.object({ processed: v.number() }),
+  handler: async (ctx) => {
+    const pages = await ctx.runQuery(
+      internal.embeddingsQueries.getPagesWithoutEmbeddings,
+      { limit: 10 }
+    );
+
+    let processed = 0;
+    for (const page of pages) {
+      try {
+        // Combine title and content for embedding
+        const textToEmbed = `${page.title}\n\n${page.content}`;
+        const embedding = await ctx.runAction(internal.embeddings.generateEmbedding, {
+          text: textToEmbed,
+        });
+        await ctx.runMutation(internal.embeddingsQueries.savePageEmbedding, {
+          id: page._id,
+          embedding,
+        });
+        processed++;
+      } catch (error) {
+        console.error(`Failed to generate embedding for page ${page._id}:`, error);
+      }
+    }
+
+    return { processed };
+  },
+});
+
+// Public action to generate missing embeddings for all content
+// Called from sync script or manually
+export const generateMissingEmbeddings = action({
+  args: {},
+  returns: v.object({
+    postsProcessed: v.number(),
+    pagesProcessed: v.number(),
+    skipped: v.boolean(),
+  }),
+  handler: async (ctx): Promise<{
+    postsProcessed: number;
+    pagesProcessed: number;
+    skipped: boolean;
+  }> => {
+    // Check for API key first - gracefully skip if not configured
+    if (!process.env.OPENAI_API_KEY) {
+      console.log("OPENAI_API_KEY not set, skipping embedding generation");
+      return { postsProcessed: 0, pagesProcessed: 0, skipped: true };
+    }
+
+    const postsResult: { processed: number } = await ctx.runAction(
+      internal.embeddings.generatePostEmbeddings,
+      {}
+    );
+    const pagesResult: { processed: number } = await ctx.runAction(
+      internal.embeddings.generatePageEmbeddings,
+      {}
+    );
+
+    return {
+      postsProcessed: postsResult.processed,
+      pagesProcessed: pagesResult.processed,
+      skipped: false,
+    };
+  },
+});
+
+// Public action to regenerate embedding for a specific post
+export const regeneratePostEmbedding = action({
+  args: { slug: v.string() },
+  returns: v.object({ success: v.boolean(), error: v.optional(v.string()) }),
+  handler: async (ctx, args) => {
+    if (!process.env.OPENAI_API_KEY) {
+      return { success: false, error: "OPENAI_API_KEY not configured" };
+    }
+
+    // Find the post by slug
+    const post = await ctx.runQuery(internal.embeddingsQueries.getPostBySlug, {
+      slug: args.slug,
+    });
+
+    if (!post) {
+      return { success: false, error: "Post not found" };
+    }
+
+    try {
+      const textToEmbed = `${post.title}\n\n${post.content}`;
+      const embedding = await ctx.runAction(internal.embeddings.generateEmbedding, {
+        text: textToEmbed,
+      });
+      await ctx.runMutation(internal.embeddingsQueries.savePostEmbedding, {
+        id: post._id,
+        embedding,
+      });
+      return { success: true };
+    } catch (error) {
+      return { success: false, error: String(error) };
+    }
+  },
+});
--- a/convex/embeddingsQueries.ts
+++ b/convex/embeddingsQueries.ts
@@ -0,0 +1,105 @@
+import { v } from "convex/values";
+import { internalMutation, internalQuery } from "./_generated/server";
+
+// Internal query to get posts without embeddings
+export const getPostsWithoutEmbeddings = internalQuery({
+  args: { limit: v.number() },
+  returns: v.array(
+    v.object({
+      _id: v.id("posts"),
+      title: v.string(),
+      content: v.string(),
+    })
+  ),
+  handler: async (ctx, args) => {
+    const posts = await ctx.db
+      .query("posts")
+      .withIndex("by_published", (q) => q.eq("published", true))
+      .collect();
+
+    return posts
+      .filter((post) => !post.embedding)
+      .slice(0, args.limit)
+      .map((post) => ({
+        _id: post._id,
+        title: post.title,
+        content: post.content,
+      }));
+  },
+});
+
+// Internal query to get pages without embeddings
+export const getPagesWithoutEmbeddings = internalQuery({
+  args: { limit: v.number() },
+  returns: v.array(
+    v.object({
+      _id: v.id("pages"),
+      title: v.string(),
+      content: v.string(),
+    })
+  ),
+  handler: async (ctx, args) => {
+    const pages = await ctx.db
+      .query("pages")
+      .withIndex("by_published", (q) => q.eq("published", true))
+      .collect();
+
+    return pages
+      .filter((page) => !page.embedding)
+      .slice(0, args.limit)
+      .map((page) => ({
+        _id: page._id,
+        title: page.title,
+        content: page.content,
+      }));
+  },
+});
+
+// Internal mutation to save embedding for a post
+export const savePostEmbedding = internalMutation({
+  args: {
+    id: v.id("posts"),
+    embedding: v.array(v.float64()),
+  },
+  handler: async (ctx, args) => {
+    await ctx.db.patch(args.id, { embedding: args.embedding });
+  },
+});
+
+// Internal mutation to save embedding for a page
+export const savePageEmbedding = internalMutation({
+  args: {
+    id: v.id("pages"),
+    embedding: v.array(v.float64()),
+  },
+  handler: async (ctx, args) => {
+    await ctx.db.patch(args.id, { embedding: args.embedding });
+  },
+});
+
+// Internal query to get post by slug
+export const getPostBySlug = internalQuery({
+  args: { slug: v.string() },
+  returns: v.union(
+    v.object({
+      _id: v.id("posts"),
+      title: v.string(),
+      content: v.string(),
+    }),
+    v.null()
+  ),
+  handler: async (ctx, args) => {
+    const post = await ctx.db
+      .query("posts")
+      .withIndex("by_slug", (q) => q.eq("slug", args.slug))
+      .first();
+
+    if (!post) return null;
+
+    return {
+      _id: post._id,
+      title: post.title,
+      content: post.content,
+    };
+  },
+});
--- a/convex/importAction.ts
+++ b/convex/importAction.ts
@@ -0,0 +1,145 @@
+"use node";
+
+import { v } from "convex/values";
+import { action } from "./_generated/server";
+import { api } from "./_generated/api";
+import FirecrawlApp from "@mendable/firecrawl-js";
+
+/**
+ * Generate a URL-safe slug from a title
+ */
+function generateSlug(title: string): string {
+  return title
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^-|-$/g, "")
+    .substring(0, 60);
+}
+
+/**
+ * Clean up markdown content
+ */
+function cleanMarkdown(content: string): string {
+  return content.replace(/^\s+|\s+$/g, "").replace(/\n{3,}/g, "\n\n");
+}
+
+/**
+ * Calculate reading time from content
+ */
+function calculateReadTime(content: string): string {
+  const wordsPerMinute = 200;
+  const wordCount = content.split(/\s+/).length;
+  const minutes = Math.ceil(wordCount / wordsPerMinute);
+  return `${minutes} min read`;
+}
+
+/**
+ * Import content from a URL using Firecrawl and save directly to database
+ */
+export const importFromUrl = action({
+  args: {
+    url: v.string(),
+    published: v.optional(v.boolean()),
+  },
+  returns: v.object({
+    success: v.boolean(),
+    slug: v.optional(v.string()),
+    title: v.optional(v.string()),
+    error: v.optional(v.string()),
+  }),
+  handler: async (ctx, args) => {
+    const apiKey = process.env.FIRECRAWL_API_KEY;
+    if (!apiKey) {
+      return {
+        success: false,
+        error:
+          "FIRECRAWL_API_KEY not configured. Add it to your Convex environment variables.",
+      };
+    }
+
+    try {
+      const firecrawl = new FirecrawlApp({ apiKey });
+      const result = await firecrawl.scrapeUrl(args.url, {
+        formats: ["markdown"],
+      });
+
+      if (!result.success || !result.markdown) {
+        return {
+          success: false,
+          error: result.error || "Failed to scrape URL - no content returned",
+        };
+      }
+
+      const title = result.metadata?.title || "Imported Post";
+      const description = result.metadata?.description || "";
+      const content = cleanMarkdown(result.markdown);
+      const baseSlug = generateSlug(title);
+      const slug = baseSlug || `imported-${Date.now()}`;
+      const today = new Date().toISOString().split("T")[0];
+
+      // Add source attribution
+      let hostname: string;
+      try {
+        hostname = new URL(args.url).hostname;
+      } catch {
+        hostname = "external source";
+      }
+      const contentWithAttribution = `${content}\n\n---\n\n*Originally published at [${hostname}](${args.url})*`;
+
+      // Create post directly in database using the CMS mutation
+      try {
+        await ctx.runMutation(api.cms.createPost, {
+          post: {
+            slug,
+            title,
+            description,
+            content: contentWithAttribution,
+            date: today,
+            published: args.published ?? false,
+            tags: ["imported"],
+            readTime: calculateReadTime(content),
+          },
+        });
+      } catch (mutationError) {
+        // Handle slug conflict by adding timestamp
+        if (
+          mutationError instanceof Error &&
+          mutationError.message.includes("already exists")
+        ) {
+          const uniqueSlug = `${slug}-${Date.now()}`;
+          await ctx.runMutation(api.cms.createPost, {
+            post: {
+              slug: uniqueSlug,
+              title,
+              description,
+              content: contentWithAttribution,
+              date: today,
+              published: args.published ?? false,
+              tags: ["imported"],
+              readTime: calculateReadTime(content),
+            },
+          });
+          return {
+            success: true,
+            slug: uniqueSlug,
+            title,
+          };
+        }
+        throw mutationError;
+      }
+
+      return {
+        success: true,
+        slug,
+        title,
+      };
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : "Unknown error occurred",
+      };
+    }
+  },
+});
--- a/convex/pages.ts
+++ b/convex/pages.ts
@@ -20,6 +20,7 @@ export const listAll = query({
      featuredOrder: v.optional(v.number()),
      authorName: v.optional(v.string()),
      authorImage: v.optional(v.string()),
+      source: v.optional(v.union(v.literal("dashboard"), v.literal("sync"))),
    }),
  ),
  handler: async (ctx) => {
@@ -48,6 +49,7 @@ export const listAll = query({
      featuredOrder: page.featuredOrder,
      authorName: page.authorName,
      authorImage: page.authorImage,
+      source: page.source,
    }));
  },
 });
@@ -317,6 +319,7 @@ export const getDocsLandingPage = query({
 });

 // Public mutation for syncing pages from markdown files
+// Respects source field: only syncs pages where source !== "dashboard"
 export const syncPagesPublic = mutation({
  args: {
    pages: v.array(
@@ -356,11 +359,13 @@ export const syncPagesPublic = mutation({
    created: v.number(),
    updated: v.number(),
    deleted: v.number(),
+    skipped: v.number(),
  }),
  handler: async (ctx, args) => {
    let created = 0;
    let updated = 0;
    let deleted = 0;
+    let skipped = 0;

    const now = Date.now();
    const incomingSlugs = new Set(args.pages.map((p) => p.slug));
@@ -369,12 +374,17 @@ export const syncPagesPublic = mutation({
    const existingPages = await ctx.db.query("pages").collect();
    const existingBySlug = new Map(existingPages.map((p) => [p.slug, p]));

-    // Upsert incoming pages
+    // Upsert incoming pages (only if source !== "dashboard")
    for (const page of args.pages) {
      const existing = existingBySlug.get(page.slug);

      if (existing) {
-        // Update existing page
+        // Skip dashboard-created pages - don't overwrite them
+        if (existing.source === "dashboard") {
+          skipped++;
+          continue;
+        }
+        // Update existing sync page
        await ctx.db.patch(existing._id, {
          title: page.title,
          content: page.content,
@@ -403,27 +413,29 @@ export const syncPagesPublic = mutation({
          docsSectionGroupOrder: page.docsSectionGroupOrder,
          docsSectionGroupIcon: page.docsSectionGroupIcon,
          docsLanding: page.docsLanding,
+          source: "sync",
          lastSyncedAt: now,
        });
        updated++;
      } else {
-        // Create new page
+        // Create new page with source: "sync"
        await ctx.db.insert("pages", {
          ...page,
+          source: "sync",
          lastSyncedAt: now,
        });
        created++;
      }
    }

-    // Delete pages that no longer exist in the repo
+    // Delete pages that no longer exist in the repo (but not dashboard pages)
    for (const existing of existingPages) {
-      if (!incomingSlugs.has(existing.slug)) {
+      if (!incomingSlugs.has(existing.slug) && existing.source !== "dashboard") {
        await ctx.db.delete(existing._id);
        deleted++;
      }
    }

-    return { created, updated, deleted };
+    return { created, updated, deleted, skipped };
  },
 });
--- a/convex/posts.ts
+++ b/convex/posts.ts
@@ -22,6 +22,7 @@ export const listAll = query({
      featuredOrder: v.optional(v.number()),
      authorName: v.optional(v.string()),
      authorImage: v.optional(v.string()),
+      source: v.optional(v.union(v.literal("dashboard"), v.literal("sync"))),
    }),
  ),
  handler: async (ctx) => {
@@ -49,6 +50,7 @@ export const listAll = query({
      featuredOrder: post.featuredOrder,
      authorName: post.authorName,
      authorImage: post.authorImage,
+      source: post.source,
    }));
  },
 });
@@ -475,6 +477,7 @@ export const syncPosts = internalMutation({
 });

 // Public mutation wrapper for sync script (no auth required for build-time sync)
+// Respects source field: only syncs posts where source !== "dashboard"
 export const syncPostsPublic = mutation({
  args: {
    posts: v.array(
@@ -517,11 +520,13 @@ export const syncPostsPublic = mutation({
    created: v.number(),
    updated: v.number(),
    deleted: v.number(),
+    skipped: v.number(),
  }),
  handler: async (ctx, args) => {
    let created = 0;
    let updated = 0;
    let deleted = 0;
+    let skipped = 0;

    const now = Date.now();
    const incomingSlugs = new Set(args.posts.map((p) => p.slug));
@@ -530,12 +535,17 @@ export const syncPostsPublic = mutation({
    const existingPosts = await ctx.db.query("posts").collect();
    const existingBySlug = new Map(existingPosts.map((p) => [p.slug, p]));

-    // Upsert incoming posts
+    // Upsert incoming posts (only if source !== "dashboard")
    for (const post of args.posts) {
      const existing = existingBySlug.get(post.slug);

      if (existing) {
-        // Update existing post
+        // Skip dashboard-created posts - don't overwrite them
+        if (existing.source === "dashboard") {
+          skipped++;
+          continue;
+        }
+        // Update existing sync post
        await ctx.db.patch(existing._id, {
          title: post.title,
          description: post.description,
@@ -567,28 +577,30 @@ export const syncPostsPublic = mutation({
          docsSectionGroupOrder: post.docsSectionGroupOrder,
          docsSectionGroupIcon: post.docsSectionGroupIcon,
          docsLanding: post.docsLanding,
+          source: "sync",
          lastSyncedAt: now,
        });
        updated++;
      } else {
-        // Create new post
+        // Create new post with source: "sync"
        await ctx.db.insert("posts", {
          ...post,
+          source: "sync",
          lastSyncedAt: now,
        });
        created++;
      }
    }

-    // Delete posts that no longer exist in the repo
+    // Delete posts that no longer exist in the repo (but not dashboard posts)
    for (const existing of existingPosts) {
-      if (!incomingSlugs.has(existing.slug)) {
+      if (!incomingSlugs.has(existing.slug) && existing.source !== "dashboard") {
        await ctx.db.delete(existing._id);
        deleted++;
      }
    }

-    return { created, updated, deleted };
+    return { created, updated, deleted, skipped };
  },
 });

--- a/convex/schema.ts
+++ b/convex/schema.ts
@@ -36,6 +36,8 @@ export default defineSchema({
    docsSectionGroupIcon: v.optional(v.string()), // Phosphor icon name for sidebar group
    docsLanding: v.optional(v.boolean()), // Use as /docs landing page
    lastSyncedAt: v.number(),
+    source: v.optional(v.union(v.literal("dashboard"), v.literal("sync"))), // Content source: "dashboard" (created in UI) or "sync" (from markdown files)
+    embedding: v.optional(v.array(v.float64())), // Vector embedding for semantic search (1536 dimensions, OpenAI text-embedding-ada-002)
  })
    .index("by_slug", ["slug"])
    .index("by_date", ["date"])
@@ -44,6 +46,7 @@ export default defineSchema({
    .index("by_blogFeatured", ["blogFeatured"])
    .index("by_authorName", ["authorName"])
    .index("by_docsSection", ["docsSection"])
+    .index("by_source", ["source"])
    .searchIndex("search_content", {
      searchField: "content",
      filterFields: ["published"],
@@ -51,6 +54,11 @@ export default defineSchema({
    .searchIndex("search_title", {
      searchField: "title",
      filterFields: ["published"],
+    })
+    .vectorIndex("by_embedding", {
+      vectorField: "embedding",
+      dimensions: 1536,
+      filterFields: ["published"],
    }),

  // Static pages (about, projects, contact, etc.)
@@ -84,11 +92,14 @@ export default defineSchema({
    docsSectionGroupIcon: v.optional(v.string()), // Phosphor icon name for sidebar group
    docsLanding: v.optional(v.boolean()), // Use as /docs landing page
    lastSyncedAt: v.number(),
+    source: v.optional(v.union(v.literal("dashboard"), v.literal("sync"))), // Content source: "dashboard" (created in UI) or "sync" (from markdown files)
+    embedding: v.optional(v.array(v.float64())), // Vector embedding for semantic search (1536 dimensions, OpenAI text-embedding-ada-002)
  })
  .index("by_slug", ["slug"])
  .index("by_published", ["published"])
  .index("by_featured", ["featured"])
  .index("by_docsSection", ["docsSection"])
+  .index("by_source", ["source"])
    .searchIndex("search_content", {
      searchField: "content",
      filterFields: ["published"],
@@ -96,6 +107,11 @@ export default defineSchema({
    .searchIndex("search_title", {
      searchField: "title",
      filterFields: ["published"],
+    })
+    .vectorIndex("by_embedding", {
+      vectorField: "embedding",
+      dimensions: 1536,
+      filterFields: ["published"],
    }),

  // View counts for analytics
--- a/convex/semanticSearch.ts
+++ b/convex/semanticSearch.ts
@@ -0,0 +1,156 @@
+"use node";
+
+import { v } from "convex/values";
+import { action } from "./_generated/server";
+import { internal } from "./_generated/api";
+import OpenAI from "openai";
+
+// Search result type matching existing search.ts format
+const searchResultValidator = v.object({
+  _id: v.string(),
+  type: v.union(v.literal("post"), v.literal("page")),
+  slug: v.string(),
+  title: v.string(),
+  description: v.optional(v.string()),
+  snippet: v.string(),
+  score: v.number(), // Similarity score from vector search
+});
+
+// Main semantic search action
+export const semanticSearch = action({
+  args: { query: v.string() },
+  returns: v.array(searchResultValidator),
+  handler: async (ctx, args) => {
+    // Return empty for empty queries
+    if (!args.query.trim()) {
+      return [];
+    }
+
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      // Gracefully return empty if not configured
+      console.log("OPENAI_API_KEY not set, semantic search unavailable");
+      return [];
+    }
+
+    // Generate embedding for search query
+    const openai = new OpenAI({ apiKey });
+    const embeddingResponse = await openai.embeddings.create({
+      model: "text-embedding-ada-002",
+      input: args.query,
+    });
+    const queryEmbedding = embeddingResponse.data[0].embedding;
+
+    // Search posts using vector index
+    const postResults = await ctx.vectorSearch("posts", "by_embedding", {
+      vector: queryEmbedding,
+      limit: 10,
+      filter: (q) => q.eq("published", true),
+    });
+
+    // Search pages using vector index
+    const pageResults = await ctx.vectorSearch("pages", "by_embedding", {
+      vector: queryEmbedding,
+      limit: 10,
+      filter: (q) => q.eq("published", true),
+    });
+
+    // Fetch full document details
+    const posts: Array<{
+      _id: string;
+      slug: string;
+      title: string;
+      description: string;
+      content: string;
+      unlisted?: boolean;
+    }> = await ctx.runQuery(internal.semanticSearchQueries.fetchPostsByIds, {
+      ids: postResults.map((r) => r._id),
+    });
+    const pages: Array<{
+      _id: string;
+      slug: string;
+      title: string;
+      content: string;
+    }> = await ctx.runQuery(internal.semanticSearchQueries.fetchPagesByIds, {
+      ids: pageResults.map((r) => r._id),
+    });
+
+    // Build results with scores
+    const results: Array<{
+      _id: string;
+      type: "post" | "page";
+      slug: string;
+      title: string;
+      description?: string;
+      snippet: string;
+      score: number;
+    }> = [];
+
+    // Map posts with scores
+    for (const result of postResults) {
+      const post = posts.find((p) => p._id === result._id);
+      if (post) {
+        results.push({
+          _id: String(post._id),
+          type: "post",
+          slug: post.slug,
+          title: post.title,
+          description: post.description,
+          snippet: createSnippet(post.content, 120),
+          score: result._score,
+        });
+      }
+    }
+
+    // Map pages with scores
+    for (const result of pageResults) {
+      const page = pages.find((p) => p._id === result._id);
+      if (page) {
+        results.push({
+          _id: String(page._id),
+          type: "page",
+          slug: page.slug,
+          title: page.title,
+          snippet: createSnippet(page.content, 120),
+          score: result._score,
+        });
+      }
+    }
+
+    // Sort by score descending (higher = more similar)
+    results.sort((a, b) => b.score - a.score);
+
+    // Limit to top 15 results
+    return results.slice(0, 15);
+  },
+});
+
+// Check if semantic search is available (API key configured)
+export const isSemanticSearchAvailable = action({
+  args: {},
+  returns: v.boolean(),
+  handler: async () => {
+    return !!process.env.OPENAI_API_KEY;
+  },
+});
+
+// Helper to create snippet from content (same logic as search.ts)
+function createSnippet(content: string, maxLength: number): string {
+  // Remove markdown syntax for cleaner snippets
+  const cleanContent = content
+    .replace(/#{1,6}\s/g, "") // Headers
+    .replace(/\*\*([^*]+)\*\*/g, "$1") // Bold
+    .replace(/\*([^*]+)\*/g, "$1") // Italic
+    .replace(/`([^`]+)`/g, "$1") // Inline code
+    .replace(/```[\s\S]*?```/g, "") // Code blocks
+    .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // Links
+    .replace(/!\[([^\]]*)\]\([^)]+\)/g, "") // Images
+    .replace(/\n+/g, " ") // Newlines to spaces
+    .replace(/\s+/g, " ") // Multiple spaces to single
+    .trim();
+
+  if (cleanContent.length <= maxLength) {
+    return cleanContent;
+  }
+  return cleanContent.slice(0, maxLength) + "...";
+}
--- a/convex/semanticSearchQueries.ts
+++ b/convex/semanticSearchQueries.ts
@@ -0,0 +1,62 @@
+import { v } from "convex/values";
+import { internalQuery } from "./_generated/server";
+
+// Internal query to fetch post details by IDs
+export const fetchPostsByIds = internalQuery({
+  args: { ids: v.array(v.id("posts")) },
+  returns: v.array(
+    v.object({
+      _id: v.id("posts"),
+      slug: v.string(),
+      title: v.string(),
+      description: v.string(),
+      content: v.string(),
+      unlisted: v.optional(v.boolean()),
+    })
+  ),
+  handler: async (ctx, args) => {
+    const results = [];
+    for (const id of args.ids) {
+      const doc = await ctx.db.get(id);
+      if (doc && doc.published && !doc.unlisted) {
+        results.push({
+          _id: doc._id,
+          slug: doc.slug,
+          title: doc.title,
+          description: doc.description,
+          content: doc.content,
+          unlisted: doc.unlisted,
+        });
+      }
+    }
+    return results;
+  },
+});
+
+// Internal query to fetch page details by IDs
+export const fetchPagesByIds = internalQuery({
+  args: { ids: v.array(v.id("pages")) },
+  returns: v.array(
+    v.object({
+      _id: v.id("pages"),
+      slug: v.string(),
+      title: v.string(),
+      content: v.string(),
+    })
+  ),
+  handler: async (ctx, args) => {
+    const results = [];
+    for (const id of args.ids) {
+      const doc = await ctx.db.get(id);
+      if (doc && doc.published) {
+        results.push({
+          _id: doc._id,
+          slug: doc.slug,
+          title: doc.title,
+          content: doc.content,
+        });
+      }
+    }
+    return results;
+  },
+});