Preparing SEO for NextJS 13 site

The Context

The Solution

1. Add a /robots.txt endpoint to your site which tells search engine crawlers which URLs the crawler can access on your site.

2. Add a sitemap.xml endpoint to your site which describes the exact available pages for a search engine to crawl. This can be done dynamically by:
1. Adding a new api endpoint to fetch some data and return XML.
2. Redirecting requests to the /sitemap.xml endpoint to the /api/sitemap endpoint by configuring your next.config.js

3. Going to Google Search Console and submitting your sitemap to be indexed by Google.

1. robots.txt

import { MetadataRoute } from "next";
import { DOMAIN } from "./constants";

export default function robots(): MetadataRoute.Robots {
  return {
    rules: {
      userAgent: "*",
      allow: "/",
      disallow: "/private/", // any paths you don't want to be indexed
    },
    sitemap: `${DOMAIN}/sitemap.xml`,
  };
}

User-Agent: *
Allow: /
Disallow: /private/

Sitemap: https://jameshw.dev/sitemap.xml

2. sitemap.xml

/* eslint-disable @typescript-eslint/restrict-template-expressions */
import type { NextApiRequest, NextApiResponse } from "next";
import { serverSideCmsClient } from "api/services/cms/cms.client";
import { isArticle, isJournalEntry } from "types/guards";
import { DOMAIN, PATHS } from "app/constants";

const getSitemapRoute = (path: string) => {
  return `
    <url>
        <loc>${DOMAIN}${path}</loc>
        <lastmod>${new Date().toISOString().split("T")[0]}</lastmod>
    </url>`;
};

export default async function handler(_: NextApiRequest, res: NextApiResponse) {
  res.statusCode = 200;
  res.setHeader("Content-Type", "text/xml");

  // Instructing the Vercel edge to cache the file
  res.setHeader("Cache-control", "stale-while-revalidate, s-maxage=3600");

  const articles = await serverSideCmsClient.getDatabaseEntries(
    process.env.BLOG_DB_ID,
    isArticle
  );
  const journals = await serverSideCmsClient.getDatabaseEntries(
    process.env.JOURNAL_DB_ID,
    isJournalEntry
  );

  res.send(`<?xml version="1.0" encoding="UTF-8"?>
    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> 
    ${Object.values(PATHS).map((path) => getSitemapRoute(path))}
    ${articles.map(({ slug, published }) =>
      getSitemapRoute(`${PATHS.BLOG}/${published}/${slug}`)
    )}
    ${journals.map(({ slug, date }) =>
      getSitemapRoute(`${PATHS.JOURNAL}/${date}/${slug}`)
    )}
    </urlset>`);
}

/** @type {import('next').NextConfig} */
const nextConfig = {
  ...
  async rewrites() {
    return [
      {
        source: "/sitemap.xml",
        destination: "/api/sitemap",
      },
    ];
  },
};

module.exports = nextConfig;

3. Google Search Console

The Result