diff --git a/TASK.md b/TASK.md index 08756f4..99310cd 100644 --- a/TASK.md +++ b/TASK.md @@ -4,10 +4,17 @@ ## Current Status -v1.20.2 deployed. Write conflict prevention for heartbeat mutations. +v1.20.3 deployed. SEO/AEO/GEO improvements for AI crawlers and search engines. ## Completed +- [x] Raw markdown files now accessible to AI crawlers (ChatGPT, Perplexity) +- [x] Added /raw/ path bypass in botMeta edge function +- [x] Sitemap now includes static pages (about, docs, contact, etc.) +- [x] Security headers added to netlify.toml +- [x] Link header pointing to llms.txt for AI discovery +- [x] Preconnect hints for Convex backend +- [x] Fixed URL consistency in openapi.yaml and robots.txt - [x] Write conflict prevention: increased dedup windows, added heartbeat jitter - [x] Visitor map styling: removed box-shadow, increased land dot contrast and opacity - [x] Real-time visitor map on stats page showing live visitor locations diff --git a/changelog.md b/changelog.md index 2d1e21a..9965d10 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.20.3] - 2025-12-21 + +### Fixed + +- Raw markdown files now accessible to AI crawlers (ChatGPT, Perplexity) + - Added `/raw/` path bypass in botMeta edge function + - AI services were receiving HTML instead of markdown content + +### Added + +- SEO and AEO improvements + - Sitemap now includes static pages (about, docs, contact, etc.) + - Security headers: X-Frame-Options, X-Content-Type-Options, X-XSS-Protection, Referrer-Policy + - Link header pointing to llms.txt for AI discovery + - Raw markdown files served with proper Content-Type and CORS headers + - Preconnect hints for Convex backend (faster API calls) + +### Changed + +- Fixed URL consistency: openapi.yaml and robots.txt now use www.markdown.fast + ## [1.20.2] - 2025-12-21 ### Fixed diff --git a/content/pages/changelog-page.md b/content/pages/changelog-page.md index 391b68b..30b3cd5 100644 --- a/content/pages/changelog-page.md +++ b/content/pages/changelog-page.md @@ -7,6 +7,21 @@ order: 5 All notable changes to this project. +## v1.20.3 + +Released December 21, 2025 + +**SEO, AEO, and GEO improvements** + +- Raw markdown files now accessible to AI crawlers (ChatGPT, Perplexity) +- Added `/raw/` path bypass in botMeta edge function so AI services receive markdown, not HTML +- Sitemap now includes static pages (about, docs, contact, etc.) +- Security headers: X-Frame-Options, X-Content-Type-Options, X-XSS-Protection, Referrer-Policy +- Link header pointing to llms.txt for AI discovery +- Raw markdown files served with proper Content-Type and CORS headers +- Preconnect hints for Convex backend (faster API calls) +- Fixed URL consistency: openapi.yaml and robots.txt now use www.markdown.fast + ## v1.20.2 Released December 21, 2025 diff --git a/convex/http.ts b/convex/http.ts index 7355d38..5230115 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -23,12 +23,13 @@ http.route({ handler: rssFullFeed, }); -// Sitemap.xml endpoint for search engines +// Sitemap.xml endpoint for search engines (includes posts and pages) http.route({ path: "/sitemap.xml", method: "GET", handler: httpAction(async (ctx) => { const posts = await ctx.runQuery(api.posts.getAllPosts); + const pages = await ctx.runQuery(api.pages.getAllPages); const urls = [ // Homepage @@ -44,6 +45,14 @@ http.route({ ${post.date} monthly 0.8 + `, + ), + // All pages + ...pages.map( + (page) => ` + ${SITE_URL}/${page.slug} + monthly + 0.7 `, ), ]; diff --git a/index.html b/index.html index 08d84d5..db67985 100644 --- a/index.html +++ b/index.html @@ -3,6 +3,10 @@ + + + + diff --git a/netlify.toml b/netlify.toml index 4339626..d6b540a 100644 --- a/netlify.toml +++ b/netlify.toml @@ -49,6 +49,24 @@ path = "/*" function = "botMeta" +# Security and SEO headers +[[headers]] + for = "/*" + [headers.values] + X-Frame-Options = "DENY" + X-Content-Type-Options = "nosniff" + X-XSS-Protection = "1; mode=block" + Referrer-Policy = "strict-origin-when-cross-origin" + Link = "; rel=\"author\"" + +# Raw markdown files with proper content-type +[[headers]] + for = "/raw/*.md" + [headers.values] + Content-Type = "text/markdown; charset=utf-8" + Access-Control-Allow-Origin = "*" + X-Robots-Tag = "noindex" + [context.production.environment] NODE_ENV = "production" diff --git a/netlify/edge-functions/botMeta.ts b/netlify/edge-functions/botMeta.ts index 98ddced..fbf2140 100644 --- a/netlify/edge-functions/botMeta.ts +++ b/netlify/edge-functions/botMeta.ts @@ -42,12 +42,13 @@ export default async function handler( // Only intercept post pages for bots const pathParts = url.pathname.split("/").filter(Boolean); - // Skip if it's the home page, static assets, or API routes + // Skip if it's the home page, static assets, API routes, or raw markdown files if ( pathParts.length === 0 || pathParts[0].includes(".") || pathParts[0] === "api" || - pathParts[0] === "_next" + pathParts[0] === "_next" || + pathParts[0] === "raw" ) { return context.next(); } diff --git a/public/images/122.png b/public/images/122.png index 71af916..74b8e31 100644 Binary files a/public/images/122.png and b/public/images/122.png differ diff --git a/public/images/v17.png b/public/images/v17.png index 6de7431..00217d4 100644 Binary files a/public/images/v17.png and b/public/images/v17.png differ diff --git a/public/openapi.yaml b/public/openapi.yaml index e7ed421..63ccb6d 100644 --- a/public/openapi.yaml +++ b/public/openapi.yaml @@ -9,7 +9,7 @@ info: url: https://github.com/waynesutton/markdown-site servers: - - url: https://markdown.fast + - url: https://www.markdown.fast description: Production server paths: diff --git a/public/raw/changelog.md b/public/raw/changelog.md index a12e5f1..70401a0 100644 --- a/public/raw/changelog.md +++ b/public/raw/changelog.md @@ -7,6 +7,21 @@ Date: 2025-12-22 All notable changes to this project. +## v1.20.3 + +Released December 21, 2025 + +**SEO, AEO, and GEO improvements** + +- Raw markdown files now accessible to AI crawlers (ChatGPT, Perplexity) +- Added `/raw/` path bypass in botMeta edge function so AI services receive markdown, not HTML +- Sitemap now includes static pages (about, docs, contact, etc.) +- Security headers: X-Frame-Options, X-Content-Type-Options, X-XSS-Protection, Referrer-Policy +- Link header pointing to llms.txt for AI discovery +- Raw markdown files served with proper Content-Type and CORS headers +- Preconnect hints for Convex backend (faster API calls) +- Fixed URL consistency: openapi.yaml and robots.txt now use www.markdown.fast + ## v1.20.2 Released December 21, 2025 diff --git a/public/robots.txt b/public/robots.txt index 161a18f..0e6d9b5 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -5,7 +5,7 @@ User-agent: * Allow: / # Sitemaps -Sitemap: https://markdown.fast/sitemap.xml +Sitemap: https://www.markdown.fast/sitemap.xml # AI and LLM crawlers User-agent: GPTBot