Reland static prefetches & fix prefetch bailout behavior (vercel#56228)

Reland vercel#54403

Also modifies the implementation of vercel#55950 to not change the prefetch behavior when there is flight router state - we should only check the entire loader tree in the static prefetch case, otherwise we're inadvertently rendering the component tree for prefetches that match the current flight router state segment. ([slack x-ref](https://vercel.slack.com/archives/C03S8ED1DKM/p1695862974745849))

This includes a few other misc fixes for static prefetch generation:
- `next start` was not serving them (which also meant tests weren't catching a few small bugs)
- the router cache key casing can differ between build and runtime, resulting in a bad cache lookup which results suspending indefinitely during navigation
- We cannot generate static prefetches for pages that opt into `searchParams`, as the prefetch response won't have the right cache key in the RSC payload
- Layouts that use headers/cookies shouldn't use a static prefetch because it can result in unexpected behavior (ie, being redirected to a login page, if the prefetch contains redirect logic for unauthed users)

Closes NEXT-1665
Closes NEXT-1643

Author: ztanner

packages/next/src/build/index.ts

@@ -129,6 +129,7 @@ import { eventSwcPlugins } from '../telemetry/events/swc-plugins'
 import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
 import {
   ACTION,
+  NEXT_ROUTER_PREFETCH,
   RSC,
   RSC_CONTENT_TYPE_HEADER,
   RSC_VARY_HEADER,
@@ -227,6 +228,7 @@ export type RoutesManifest = {
   rsc: {
     header: typeof RSC
     varyHeader: typeof RSC_VARY_HEADER
+    prefetchHeader: typeof NEXT_ROUTER_PREFETCH
   }
   skipMiddlewareUrlNormalize?: boolean
   caseSensitive?: boolean
@@ -795,6 +797,7 @@ export default async function build(
             rsc: {
               header: RSC,
               varyHeader: RSC_VARY_HEADER,
+              prefetchHeader: NEXT_ROUTER_PREFETCH,
               contentTypeHeader: RSC_CONTENT_TYPE_HEADER,
             },
             skipMiddlewareUrlNormalize: config.skipMiddlewareUrlNormalize,
@@ -1055,6 +1058,7 @@ export default async function build(
       const additionalSsgPaths = new Map<string, Array<string>>()
       const additionalSsgPathsEncoded = new Map<string, Array<string>>()
       const appStaticPaths = new Map<string, Array<string>>()
+      const appPrefetchPaths = new Map<string, string>()
       const appStaticPathsEncoded = new Map<string, Array<string>>()
       const appNormalizedPaths = new Map<string, string>()
       const appDynamicParamPaths = new Set<string>()
@@ -1554,6 +1558,14 @@ export default async function build(
                             appDynamicParamPaths.add(originalAppPath)
                           }
                           appDefaultConfigs.set(originalAppPath, appConfig)
+
+                          if (
+                            !isStatic &&
+                            !isAppRouteRoute(originalAppPath) &&
+                            !isDynamicRoute(originalAppPath)
+                          ) {
+                            appPrefetchPaths.set(originalAppPath, page)
+                          }
                         }
                       } else {
                         if (isEdgeRuntime(pageRuntime)) {
@@ -2001,6 +2013,15 @@ export default async function build(
                 })
               })
 
+              for (const [originalAppPath, page] of appPrefetchPaths) {
+                defaultMap[page] = {
+                  page: originalAppPath,
+                  query: {},
+                  _isAppDir: true,
+                  _isAppPrefetch: true,
+                }
+              }
+
               if (i18n) {
                 for (const page of [
                   ...staticPages,

packages/next/src/client/components/router-reducer/create-router-cache-key.ts

@@ -5,7 +5,7 @@ export function createRouterCacheKey(
   withoutSearchParameters: boolean = false
 ) {
   return Array.isArray(segment)
-    ? `${segment[0]}|${segment[1]}|${segment[2]}`
+    ? `${segment[0]}|${segment[1]}|${segment[2]}`.toLowerCase()
     : withoutSearchParameters && segment.startsWith('__PAGE__')
     ? '__PAGE__'
     : segment

packages/next/src/client/components/static-generation-async-storage.external.ts

@@ -30,6 +30,7 @@ export interface StaticGenerationStore {
   dynamicUsageDescription?: string
   dynamicUsageStack?: string
   dynamicUsageErr?: DynamicServerError
+  staticPrefetchBailout?: boolean
 
   nextFetchId?: number
   pathWasRevalidated?: boolean

packages/next/src/client/components/static-generation-bailout.ts

@@ -38,6 +38,12 @@ export const staticGenerationBailout: StaticGenerationBailout = (
 
   if (staticGenerationStore) {
     staticGenerationStore.revalidate = 0
+
+    if (!opts?.dynamic) {
+      // we can statically prefetch pages that opt into dynamic,
+      // but not things like headers/cookies
+      staticGenerationStore.staticPrefetchBailout = true
+    }
   }
 
   if (staticGenerationStore?.isStaticGeneration) {

packages/next/src/export/routes/app-page.ts

@@ -6,6 +6,11 @@ import type { NextParsedUrlQuery } from '../../server/request-meta'
 
 import fs from 'fs/promises'
 import { MockedRequest, MockedResponse } from '../../server/lib/mock-request'
+import {
+  RSC,
+  NEXT_URL,
+  NEXT_ROUTER_PREFETCH,
+} from '../../client/components/app-router-headers'
 import { isDynamicUsageError } from '../helpers/is-dynamic-usage-error'
 import { NEXT_CACHE_TAGS_HEADER } from '../../lib/constants'
 import { hasNextSupport } from '../../telemetry/ci-info'
@@ -19,6 +24,37 @@ const render: AppPageRender = (...args) => {
   )
 }
 
+export async function generatePrefetchRsc(
+  req: MockedRequest,
+  path: string,
+  res: MockedResponse,
+  pathname: string,
+  htmlFilepath: string,
+  renderOpts: RenderOpts
+) {
+  req.headers[RSC.toLowerCase()] = '1'
+  req.headers[NEXT_URL.toLowerCase()] = path
+  req.headers[NEXT_ROUTER_PREFETCH.toLowerCase()] = '1'
+
+  renderOpts.supportsDynamicHTML = true
+  renderOpts.isPrefetch = true
+  delete renderOpts.isRevalidate
+
+  const prefetchRenderResult = await render(req, res, pathname, {}, renderOpts)
+
+  prefetchRenderResult.pipe(res)
+  await res.hasStreamed
+
+  const prefetchRscData = Buffer.concat(res.buffers)
+
+  if ((renderOpts as any).store.staticPrefetchBailout) return
+
+  await fs.writeFile(
+    htmlFilepath.replace(/\.html$/, '.prefetch.rsc'),
+    prefetchRscData
+  )
+}
+
 export async function exportAppPage(
   req: MockedRequest,
   res: MockedResponse,
@@ -29,14 +65,28 @@ export async function exportAppPage(
   renderOpts: RenderOpts,
   htmlFilepath: string,
   debugOutput: boolean,
-  isDynamicError: boolean
+  isDynamicError: boolean,
+  isAppPrefetch: boolean
 ): Promise<ExportPageResult> {
   // If the page is `/_not-found`, then we should update the page to be `/404`.
   if (page === '/_not-found') {
     pathname = '/404'
   }
 
   try {
+    if (isAppPrefetch) {
+      await generatePrefetchRsc(
+        req,
+        path,
+        res,
+        pathname,
+        htmlFilepath,
+        renderOpts
+      )
+
+      return { fromBuildExportRevalidate: 0 }
+    }
+
     const result = await render(req, res, pathname, query, renderOpts)
     const html = result.toUnchunkedString()
     const { metadata } = result
@@ -50,6 +100,17 @@ export async function exportAppPage(
         )
       }
 
+      if (!(renderOpts as any).store.staticPrefetchBailout) {
+        await generatePrefetchRsc(
+          req,
+          path,
+          res,
+          pathname,
+          htmlFilepath,
+          renderOpts
+        )
+      }
+
       const { staticBailoutInfo = {} } = metadata
 
       if (revalidate === 0 && debugOutput && staticBailoutInfo?.description) {

packages/next/src/export/worker.ts

@@ -108,6 +108,9 @@ async function exportPageImpl(input: ExportPageInput) {
     // Check if this is an `app/` page.
     _isAppDir: isAppDir = false,
 
+    // Check if this is an `app/` prefix request.
+    _isAppPrefetch: isAppPrefetch = false,
+
     // Check if this should error when dynamic usage is detected.
     _isDynamicError: isDynamicError = false,
 
@@ -306,7 +309,8 @@ async function exportPageImpl(input: ExportPageInput) {
         renderOpts,
         htmlFilepath,
         debugOutput,
-        isDynamicError
+        isDynamicError,
+        isAppPrefetch
       )
     }
 

packages/next/src/server/app-render/app-render.tsx

@@ -1106,6 +1106,13 @@ export const renderToHTMLOrFlight: AppPageRender = (
           // Explicit refresh
           flightRouterState[3] === 'refetch'
 
+        const shouldSkipComponentTree =
+          isPrefetch &&
+          !Boolean(components.loading) &&
+          (flightRouterState ||
+            // If there is no flightRouterState, we need to check the entire loader tree, as otherwise we'll be only checking the root
+            !hasLoadingComponentInTree(loaderTree))
+
         if (!parentRendered && renderComponentsOnThisLevel) {
           const overriddenSegment =
             flightRouterState &&
@@ -1122,9 +1129,7 @@ export const renderToHTMLOrFlight: AppPageRender = (
                 getDynamicParamFromSegment,
                 query
               ),
-              isPrefetch &&
-              !Boolean(components.loading) &&
-              !hasLoadingComponentInTree(loaderTree)
+              shouldSkipComponentTree
                 ? null
                 : // Create component tree using the slice of the loaderTree
                   // @ts-expect-error TODO-APP: fix async component type
@@ -1147,9 +1152,7 @@ export const renderToHTMLOrFlight: AppPageRender = (
 
                     return <Component />
                   }),
-              isPrefetch &&
-              !Boolean(components.loading) &&
-              !hasLoadingComponentInTree(loaderTree)
+              shouldSkipComponentTree
                 ? null
                 : (() => {
                     const { layoutOrPagePath } =

packages/next/src/server/app-render/types.ts

@@ -131,6 +131,7 @@ export type RenderOptsPartial = {
   ) => Promise<NextConfigComplete>
   serverActionsBodySizeLimit?: SizeLimit
   params?: ParsedUrlQuery
+  isPrefetch?: boolean
 }
 
 export type RenderOpts = LoadComponentsReturnType & RenderOptsPartial

packages/next/src/server/async-storage/static-generation-async-storage-wrapper.ts

@@ -12,6 +12,7 @@ export type StaticGenerationContext = {
     isRevalidate?: boolean
     isOnDemandRevalidate?: boolean
     isBot?: boolean
+    isPrefetch?: boolean
     nextExport?: boolean
     fetchCache?: StaticGenerationStore['fetchCache']
     isDraftMode?: boolean

packages/next/src/server/base-server.ts

@@ -86,6 +86,8 @@ import {
   FLIGHT_PARAMETERS,
   NEXT_RSC_UNION_QUERY,
   ACTION,
+  NEXT_ROUTER_PREFETCH,
+  RSC_CONTENT_TYPE_HEADER,
 } from '../client/components/app-router-headers'
 import {
   MatchOptions,
@@ -2124,6 +2126,27 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       } else if (
         components.routeModule?.definition.kind === RouteKind.APP_PAGE
       ) {
+        const isAppPrefetch = req.headers[NEXT_ROUTER_PREFETCH.toLowerCase()]
+
+        if (isAppPrefetch && process.env.NODE_ENV === 'production') {
+          try {
+            const prefetchRsc = await this.getPrefetchRsc(resolvedUrlPathname)
+
+            if (prefetchRsc) {
+              res.setHeader(
+                'cache-control',
+                'private, no-cache, no-store, max-age=0, must-revalidate'
+              )
+              res.setHeader('content-type', RSC_CONTENT_TYPE_HEADER)
+              res.body(prefetchRsc).send()
+              return null
+            }
+          } catch (_) {
+            // we fallback to invoking the function if prefetch
+            // data is not available
+          }
+        }
+
         const module = components.routeModule as AppPageRouteModule
 
         // Due to the way we pass data by mutating `renderOpts`, we can't extend the