Divide sitemaps and create a index sitemap (#99)

* sort answers in user page

* remove post.createdAt'

* sort answers in user page

* resolve conflicts

* divide sitemap

* Create sitemap index

* give some spacing

* remove main url from ind ex sitemap

* move index sitemap to posts/sitemap.xml

* move again

Author: anay-208

apps/web/app/post/sitemap-index.xml/route.ts

@@ -0,0 +1,32 @@
+import { getBaseUrl } from '@/utils/urls'
+import { db } from '@nextjs-forum/db'
+import { URL_PER_SITEMAP } from '@/consts'
+
+export const revalidate = 604800 // 1 Week in seconds
+
+export async function GET() {
+  const { postCount } = await db
+    .selectFrom('posts')
+    .select(db.fn.count('id').as('postCount'))
+    .where('isIndexed', '=', true)
+    .executeTakeFirstOrThrow()
+
+  const sitemapCount = Math.ceil(Number(postCount) / URL_PER_SITEMAP)
+
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+    <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      ${Array.from(
+        { length: sitemapCount },
+        (_, index) => `
+        <sitemap>
+          <loc>${getBaseUrl()}/post/sitemap/${index}.xml</loc>
+        </sitemap>`,
+      ).join('\n')}
+    </sitemapindex>`
+
+  return new Response(xml, {
+    headers: {
+      'Content-Type': 'application/xml',
+    },
+  })
+}

apps/web/app/post/sitemap.ts

@@ -0,0 +1,44 @@
+import { MetadataRoute } from 'next'
+import { getBaseUrl } from '@/utils/urls'
+import { db } from '@nextjs-forum/db'
+import { URL_PER_SITEMAP } from '@/consts'
+
+// Update sitemap only once every 6 hours
+export const revalidate = 21600
+
+export async function generateSitemaps() {
+  const { postCount } = await db
+    .selectFrom('posts')
+    .select(db.fn.count('id').as('postCount'))
+    .where('isIndexed', '=', true)
+    .executeTakeFirstOrThrow()
+
+  const sitemapCount = Math.ceil(Number(postCount) / URL_PER_SITEMAP)
+  const sitemaps = Array.from({ length: sitemapCount }, (_, index) => ({
+    id: index,
+  }))
+
+  return sitemaps
+}
+
+export default async function sitemap({
+  id,
+}: {
+  id: number
+}): Promise<MetadataRoute.Sitemap> {
+  const start = id * URL_PER_SITEMAP
+  const posts = await db
+    .selectFrom('posts')
+    .select(['snowflakeId', 'lastActiveAt'])
+    .where('isIndexed', '=', true)
+    .offset(start)
+    .limit(URL_PER_SITEMAP)
+    .execute()
+
+  return posts.map((p) => ({
+    url: `${getBaseUrl()}/post/${p.snowflakeId}`,
+    changeFrequency: 'weekly',
+    priority: 0.9,
+    lastModified: p.lastActiveAt,
+  })) satisfies MetadataRoute.Sitemap
+}

apps/web/app/sitemap.ts

@@ -1,31 +1,12 @@
-import { MetadataRoute } from 'next'
 import { getBaseUrl } from '@/utils/urls'
-import { db } from '@nextjs-forum/db'
-
-// Update sitemap only once every 6 hours
-export const revalidate = 21600
-
-export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-  const posts = await db
-    .selectFrom('posts')
-    .select(['snowflakeId', 'lastActiveAt'])
-    .where('isIndexed', '=', true)
-    .limit(50_000) // we will probably need to chunk the sitemap in the future
-    .execute()
+import { MetadataRoute } from 'next'
 
+export default function sitemap(): MetadataRoute.Sitemap {
   return [
     {
       url: getBaseUrl(),
       changeFrequency: 'daily',
       priority: 1,
     },
-    ...posts.map((p) => {
-      return {
-        url: `${getBaseUrl()}/post/${p.snowflakeId}`,
-        changeFrequency: 'weekly',
-        priority: 0.9,
-        lastModified: p.lastActiveAt,
-      } satisfies MetadataRoute.Sitemap[0]
-    }),
   ]
 }

apps/web/consts.ts

@@ -0,0 +1 @@
+export const URL_PER_SITEMAP = 10_000