From 25598c20cf6e863322caf93508a8ab2cb29d0b51 Mon Sep 17 00:00:00 2001 From: gurkerl83 Date: Mon, 28 Jun 2021 17:17:04 +0200 Subject: [PATCH 1/4] Generate sitemaps dynamically, on request The pages in the project are dynamic; sitemaps are not; they have to be re-generated once a new page gets added. The page server-sitemap is unique; it uses NextJs getServerSideProps to determine any available dynamic page once a request is made and serves a sitemap. --- pages/server-sitemap.xml/index.tsx | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 pages/server-sitemap.xml/index.tsx diff --git a/pages/server-sitemap.xml/index.tsx b/pages/server-sitemap.xml/index.tsx new file mode 100644 index 00000000..c5578220 --- /dev/null +++ b/pages/server-sitemap.xml/index.tsx @@ -0,0 +1,45 @@ +import { UrlUtil } from '@app/utils'; +import isArray from 'lodash/isArray'; +import { getServerSideSitemap, ISitemapField } from 'next-sitemap'; +import { sep } from 'path'; + +import { getPath } from '../../docs/src/lib/getPath'; + +import type { GetServerSideProps, GetServerSidePropsContext } from 'next'; + +export const getServerSideProps: GetServerSideProps = async ( + ctx: GetServerSidePropsContext +) => { + const { defaultLocale, req } = ctx; + + const { origin } = UrlUtil.getAbsoluteUrl(req, '', { https: true }); + + const docsPaths = await getPath('docs'); + + const pages = docsPaths.map(path => { + const { + locale, + params: { slug } + } = path; + + const stringifiedSlug = `${isArray(slug) ? slug.join(sep) || '.' : slug}`; + + if (locale !== defaultLocale) { + return `${locale}/docs/${stringifiedSlug}`; + } + return `docs/${stringifiedSlug}`; + }); + + const sitemapFields: Array = pages.map(page => { + return { + loc: `${origin}/${page}`, + changefreq: 'daily', + priority: '0.7', + lastmod: new Date().toISOString() + }; + }); + + return getServerSideSitemap(ctx, sitemapFields); +}; + +export default getServerSideProps; From b57a466afe04d19f17d97762787cfc039b61ed4a Mon Sep 17 00:00:00 2001 From: gurkerl83 Date: Mon, 28 Jun 2021 17:17:55 +0200 Subject: [PATCH 2/4] Onboard absolute-url helper util --- packages/utils/src/index.ts | 1 + packages/utils/src/url/absolute.ts | 41 ++++++++++++++++++++++++++++++ packages/utils/src/url/index.ts | 1 + 3 files changed, 43 insertions(+) create mode 100644 packages/utils/src/url/absolute.ts create mode 100644 packages/utils/src/url/index.ts diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 6228ac65..56790548 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -5,3 +5,4 @@ export * as MathUtil from './math'; export * as StringUtil from './string'; export * as RouterUtils from './router'; export * as TypeUtil from './type'; +export * as UrlUtil from './url'; diff --git a/packages/utils/src/url/absolute.ts b/packages/utils/src/url/absolute.ts new file mode 100644 index 00000000..3b01bd65 --- /dev/null +++ b/packages/utils/src/url/absolute.ts @@ -0,0 +1,41 @@ +import { IncomingMessage } from 'http'; + +export const getAbsoluteUrl = ( + req?: IncomingMessage, + localhostAddress = 'localhost:3000', + options = { + https: false + } +) => { + let { https } = options; + let host = + (req?.headers ? req.headers.host : window.location.host) || + localhostAddress; + let protocol = https + ? 'https:' // if NODE_ENV is production + : process.env.NODE_ENV == 'production' + ? 'https:' // if Customised Prameter Passed + : 'http:'; + + if ( + req && + req.headers['x-forwarded-host'] && + typeof req.headers['x-forwarded-host'] === 'string' + ) { + host = req.headers['x-forwarded-host']; + } + + if ( + req && + req.headers['x-forwarded-proto'] && + typeof req.headers['x-forwarded-proto'] === 'string' + ) { + protocol = `${req.headers['x-forwarded-proto']}:`; + } + + return { + protocol, + host, + origin: protocol + '//' + host + }; +}; diff --git a/packages/utils/src/url/index.ts b/packages/utils/src/url/index.ts new file mode 100644 index 00000000..0b236eb5 --- /dev/null +++ b/packages/utils/src/url/index.ts @@ -0,0 +1 @@ +export { getAbsoluteUrl } from './absolute'; From 34db4029f9ac13b8d809b77c5f243873a3fecb4d Mon Sep 17 00:00:00 2001 From: gurkerl83 Date: Mon, 28 Jun 2021 17:18:31 +0200 Subject: [PATCH 3/4] Add global sitemap configuration --- next-sitemap.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 next-sitemap.js diff --git a/next-sitemap.js b/next-sitemap.js new file mode 100644 index 00000000..ef11c667 --- /dev/null +++ b/next-sitemap.js @@ -0,0 +1,21 @@ +module.exports = { + siteUrl: 'https://millipede.me', + + changefreq: 'daily', + priority: 0.7, + sitemapSize: 5000, + + generateRobotsTxt: true, + exclude: ['/server-sitemap.xml'], // <= exclude here + robotsTxtOptions: { + // policies: [ + // { + // userAgent: '*', + // allow: '/' + // } + // ], + additionalSitemaps: [ + 'https://millipede.me/server-sitemap.xml' // <==== Add here + ] + } +}; From 05306287f3abbdd94c2d005ac1ad5c4916e182ec Mon Sep 17 00:00:00 2001 From: gurkerl83 Date: Mon, 28 Jun 2021 17:19:11 +0200 Subject: [PATCH 4/4] Add next-sitemap dependency --- package.json | 4 +++- yarn.lock | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index c6041b85..5bd3b837 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "lint": "yarn eslint './docs/src/**/*' './src/**/*' './pages/**/*' './packages/**/src/**/*'", "format:mdx": "prettier --parser mdx --end-of-line lf --print-width 80 --prose-wrap always --single-quote true --write \"docs/src/pages/**/*.mdx\"", "format:json": "prettier --parser json --print-width 80 --use-tabs --write \"locales/**/*.json\"", - "sitemap": "node sitemap-generator" + "sitemap": "node sitemap-generator", + "postbuild": "next-sitemap" }, "dependencies": { "@app/analytics": "workspace:packages/analytics", @@ -91,6 +92,7 @@ "next": "11.0.1", "next-mdx-remote": "^3.0.2", "next-merge-props": "^0.5.0", + "next-sitemap": "^1.6.124", "next-translate": "1.0.7", "query-string": "^7.0.1", "react": "17.0.2", diff --git a/yarn.lock b/yarn.lock index 5b7515dc..25118573 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1668,6 +1668,13 @@ __metadata: languageName: node linkType: hard +"@corex/deepmerge@npm:^2.6.20": + version: 2.6.20 + resolution: "@corex/deepmerge@npm:2.6.20" + checksum: 40f33a87a460f0a04b82d40fd33a9d73105d510bec781b024cdc90d180389a8c628ce57db650e72c0a5215c1c949adb34426b13a838c34647c6ca1d392e805c0 + languageName: node + linkType: hard + "@date-io/core@npm:^2.10.11": version: 2.10.11 resolution: "@date-io/core@npm:2.10.11" @@ -8881,6 +8888,15 @@ __metadata: languageName: node linkType: hard +"matcher@npm:^4.0.0": + version: 4.0.0 + resolution: "matcher@npm:4.0.0" + dependencies: + escape-string-regexp: ^4.0.0 + checksum: d338aff31d8dfd3626873e43777f46b123579734d53bb8d18d64b08a822ba5e8d39f5fe2e23403258e6143aa0cbe20a15662720d825cd0d3af961d5a44230328 + languageName: node + linkType: hard + "md5.js@npm:^1.3.4": version: 1.3.5 resolution: "md5.js@npm:1.3.5" @@ -9169,6 +9185,7 @@ __metadata: next-mdx-remote: ^3.0.2 next-merge-props: ^0.5.0 next-remote-watch: ^1.0.0 + next-sitemap: ^1.6.124 next-translate: 1.0.7 nextjs-sitemap-generator: ^1.3.1 prettier: ^2.3.1 @@ -9486,6 +9503,21 @@ __metadata: languageName: node linkType: hard +"next-sitemap@npm:^1.6.124": + version: 1.6.124 + resolution: "next-sitemap@npm:1.6.124" + dependencies: + "@corex/deepmerge": ^2.6.20 + matcher: ^4.0.0 + minimist: ^1.2.5 + peerDependencies: + next: "*" + bin: + next-sitemap: bin/next-sitemap + checksum: 088741aa51796abbe9c9d86960e513fca75cd5777be101fd763f0de518af1b4a23385e819a9acb928f8ae83390eb9ce7e17d40f94f5ff01065a6804644f5fd45 + languageName: node + linkType: hard + "next-translate@npm:1.0.7": version: 1.0.7 resolution: "next-translate@npm:1.0.7" @@ -12181,15 +12213,15 @@ resolve@^2.0.0-next.3: linkType: hard "terser@npm:^5.7.0": - version: 5.7.0 - resolution: "terser@npm:5.7.0" + version: 5.7.1 + resolution: "terser@npm:5.7.1" dependencies: commander: ^2.20.0 source-map: ~0.7.2 source-map-support: ~0.5.19 bin: terser: bin/terser - checksum: 3abeb551865079b27e2890dbec866054967d1963fc80a81e5e14e414c43db88ff53a5e88844c145df11bed01b28040aa96afd82113c6d1a6ad28409b6cae4fde + checksum: 9354af5c22ade95004c103893606968cbd5390824c9d89df540f9e578c6e6559a5e9c56a1bf23ed055cdb61c60defdbc2c29e773514f9f2fbc7fd7ed335f1f58 languageName: node linkType: hard