From f73dddd72aeea27eebcbc6661684481fff0bf455 Mon Sep 17 00:00:00 2001 From: Olimi <861921648@qq.com> Date: Thu, 21 Dec 2023 21:11:37 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=83=A8=E5=88=86?= =?UTF-8?q?=E6=96=87=E7=AB=A0=E6=97=A0=E6=B3=95=E5=8A=A0=E8=BD=BD=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E5=8E=9F=E5=9B=A0=E6=98=AFNotion=20?= =?UTF-8?q?API=E5=8A=A0=E8=BD=BD=E6=97=B6block=E4=B8=8A=E9=99=901000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复思路是在加载所有文章时，会首先一次性加载数据库中所有block，此时因为上限1000会发生溢出。之后会读取所有page的IDs，然后逐个获取page对应的block。此时如果发现ID查找不到（说明发生了溢出），则再调用API去逐一获取这些溢出了的page。可能这不是最优解。 修改的代码包括： 1. getDataBaseInfoByNotionAPI中添加上述判断逻辑。 2. 修改getPageProperties函数签名，直接传入block对应的value，而不是block数组。 --- .env.local | 4 ++-- lib/notion/getAllPosts.js | 2 +- lib/notion/getNotionData.js | 42 +++++++++++++++++++++++++++------ lib/notion/getPageProperties.js | 13 +++++----- lib/notion/getPostBlocks.js | 19 +++++++++++++++ package-lock.json | 2 +- 6 files changed, 64 insertions(+), 18 deletions(-) diff --git a/.env.local b/.env.local index 8a0dbf80623..4f36af0ff04 100644 --- a/.env.local +++ b/.env.local @@ -4,12 +4,12 @@ NEXT_PUBLIC_VERSION=4.1.2 # 可在此添加环境变量，去掉最左边的（# ）注释即可 # Notion页面ID,必须 -# NOTION_PAGE_ID= +# NOTION_PAGE_ID=097e5f674880459d8e1b4407758dc4fb # 非必须 # NEXT_PUBLIC_PSEUDO_STATIC= # NEXT_PUBLIC_REVALIDATE_SECOND= -# NEXT_PUBLIC_THEME= +# NEXT_PUBLIC_THEME=matery # NEXT_PUBLIC_THEME_SWITCH= # NEXT_PUBLIC_LANG= # NEXT_PUBLIC_APPEARANCE= diff --git a/lib/notion/getAllPosts.js b/lib/notion/getAllPosts.js index ad4acde6c7d..5d63107ad5d 100644 --- a/lib/notion/getAllPosts.js +++ b/lib/notion/getAllPosts.js @@ -28,7 +28,7 @@ export async function getAllPosts({ notionPageData, from, pageType }) { if (!value) { continue } - const properties = (await getPageProperties(id, block, schema, null, tagOptions)) || null + const properties = (await getPageProperties(id, block[id].value, schema, null, tagOptions)) || null data.push(properties) } diff --git a/lib/notion/getNotionData.js b/lib/notion/getNotionData.js index 5b77e716581..7211c31e2ac 100755 --- a/lib/notion/getNotionData.js +++ b/lib/notion/getNotionData.js @@ -1,6 +1,6 @@ import BLOG from '@/blog.config' import { getDataFromCache, setDataToCache } from '@/lib/cache/cache_manager' -import { getPostBlocks } from '@/lib/notion/getPostBlocks' +import { getPostBlocks, getSingleBlock } from '@/lib/notion/getPostBlocks' import { idToUuid } from 'notion-utils' import { deepClone } from '../utils' import { getAllCategories } from './getAllCategories' @@ -185,7 +185,17 @@ export function getNavPages({ allPages }) { return post && post?.slug && (!post?.slug?.startsWith('http')) && post?.type === 'Post' && post?.status === 'Published' }) - return allNavPages.map(item => ({ id: item.id, title: item.title || '', pageCoverThumbnail: item.pageCoverThumbnail || '', category: item.category || null, tags: item.tags || null, summary: item.summary || null, slug: item.slug, pageIcon: item.pageIcon || '', lastEditedDate: item.lastEditedDate })) + return allNavPages.map(item => ({ + id: item.id, + title: item.title || '', + pageCoverThumbnail: item.pageCoverThumbnail || '', + category: item.category || null, + tags: item.tags || null, + summary: item.summary || null, + slug: item.slug, + pageIcon: item.pageIcon || '', + lastEditedDate: item.lastEditedDate + })) } /** @@ -205,7 +215,15 @@ const EmptyData = (pageId) => { const empty = { notice: null, siteInfo: getSiteInfo({}), - allPages: [{ id: 1, title: `无法获取Notion数据，请检查Notion_ID： \n 当前 ${pageId}`, summary: '访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next', status: 'Published', type: 'Post', slug: '13a171332816461db29d50e9f575b00d', date: { start_date: '2023-04-24', lastEditedDay: '2023-04-24', tagItems: [] } }], + allPages: [{ + id: 1, + title: `无法获取Notion数据，请检查Notion_ID： \n 当前 ${pageId}`, + summary: '访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next', + status: 'Published', + type: 'Post', + slug: '13a171332816461db29d50e9f575b00d', + date: { start_date: '2023-04-24', lastEditedDay: '2023-04-24', tagItems: [] } + }], allNavPages: [], collection: [], collectionQuery: {}, @@ -263,9 +281,17 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) { const id = pageIds[i] const value = block[id]?.value if (!value) { + // 如果找不到文章对应的block，说明发生了溢出，使用pageID再去请求 + const pageBlock = await getSingleBlock(id, from) + if (pageBlock.block[id].value) { + const properties = (await getPageProperties(id, pageBlock.block[id].value, schema, null, getTagOptions(schema))) || null + if (properties) { + collectionData.push(properties) + } + } continue } - const properties = (await getPageProperties(id, block, schema, null, getTagOptions(schema))) || null + const properties = (await getPageProperties(id, block[id].value, schema, null, getTagOptions(schema))) || null if (properties) { collectionData.push(properties) } @@ -280,8 +306,8 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) { postCount++ } return post && post?.slug && - (!post?.slug?.startsWith('http')) && - (post?.status === 'Invisible' || post?.status === 'Published') + (!post?.slug?.startsWith('http')) && + (post?.status === 'Invisible' || post?.status === 'Published') }) // 站点配置优先读取配置表格，否则读取blog.config.js 文件 @@ -294,7 +320,9 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) { }) } - const notice = await getNotice(collectionData.filter(post => { return post && post?.type && post?.type === 'Notice' && post.status === 'Published' })?.[0]) + const notice = await getNotice(collectionData.filter(post => { + return post && post?.type && post?.type === 'Notice' && post.status === 'Published' + })?.[0]) const categoryOptions = getAllCategories({ allPages, categoryOptions: getCategoryOptions(schema) }) const tagOptions = getAllTags({ allPages, tagOptions: getTagOptions(schema) }) // 旧的菜单 diff --git a/lib/notion/getPageProperties.js b/lib/notion/getPageProperties.js index 3ea1ce2856b..6004197701a 100644 --- a/lib/notion/getPageProperties.js +++ b/lib/notion/getPageProperties.js @@ -9,16 +9,15 @@ import { mapImgUrl } from './mapImage' /** * 获取页面元素成员属性 * @param {*} id - * @param {*} block + * @param {*} value * @param {*} schema * @param {*} authToken * @param {*} tagOptions * @returns */ -export default async function getPageProperties(id, block, schema, authToken, tagOptions) { - const rawProperties = Object.entries(block?.[id]?.value?.properties || []) +export default async function getPageProperties(id, value, schema, authToken, tagOptions) { + const rawProperties = Object.entries(value?.properties || []) const excludeProperties = ['date', 'select', 'multi_select', 'person'] - const value = block[id]?.value const properties = {} for (let i = 0; i < rawProperties.length; i++) { const [key, val] = rawProperties[i] @@ -91,9 +90,9 @@ export default async function getPageProperties(id, block, schema, authToken, ta properties.lastEditedDate = new Date(value?.last_edited_time) properties.lastEditedDay = formatDate(new Date(value?.last_edited_time), BLOG.LANG) properties.fullWidth = value.format?.page_full_width ?? false - properties.pageIcon = mapImgUrl(block[id].value?.format?.page_icon, block[id].value) ?? '' - properties.pageCover = mapImgUrl(block[id].value?.format?.page_cover, block[id].value) ?? '' - properties.pageCoverThumbnail = mapImgUrl(block[id].value?.format?.page_cover, block[id].value, 'block', 'pageCoverThumbnail') ?? '' + properties.pageIcon = mapImgUrl(value.value?.format?.page_icon, value.value) ?? '' + properties.pageCover = mapImgUrl(value.value?.format?.page_cover, value.value) ?? '' + properties.pageCoverThumbnail = mapImgUrl(value.value?.format?.page_cover, value.value, 'block', 'pageCoverThumbnail') ?? '' properties.content = value.content ?? [] properties.tagItems = properties?.tags?.map(tag => { return { name: tag, color: tagOptions?.find(t => t.value === tag)?.color || 'gray' } diff --git a/lib/notion/getPostBlocks.js b/lib/notion/getPostBlocks.js index 809dfe7ae6e..e9bbedfd05b 100644 --- a/lib/notion/getPostBlocks.js +++ b/lib/notion/getPostBlocks.js @@ -30,6 +30,25 @@ export async function getPostBlocks(id, from, slice) { return pageBlock } +export async function getSingleBlock(id, from) { + const cacheKey = 'single_block_' + id + let pageBlock = await getDataFromCache(cacheKey) + if (pageBlock) { + console.log('[缓存]:', `from:${from}`, cacheKey) + return pageBlock + } + + const start = new Date().getTime() + pageBlock = await getPageWithRetry(id, from) + const end = new Date().getTime() + console.log('[API耗时]', `${end - start}ms`) + + if (pageBlock) { + await setDataToCache(cacheKey, pageBlock) + } + return pageBlock +} + /** * 调用接口，失败会重试 * @param {*} id diff --git a/package-lock.json b/package-lock.json index 1656ec4b3cf..3fe6eeea105 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "notion-next", - "version": "4.0.15", + "version": "4.1.2", "lockfileVersion": 1, "requires": true, "dependencies": { From 09c85d53a6c5e7ee525233b9944cd3ea53da3f10 Mon Sep 17 00:00:00 2001 From: Lyon Liu Date: Thu, 25 Jan 2024 19:10:31 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dnav=E4=B8=BB=E9=A2=98?= =?UTF-8?q?=E4=B8=8Bcategory=E5=92=8Ctag=E4=B8=8B=E6=96=87=E7=AB=A0?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E4=B8=8D=E5=AF=B9=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- themes/nav/components/BlogPostCard.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/themes/nav/components/BlogPostCard.js b/themes/nav/components/BlogPostCard.js index e9567d76784..736220625ee 100755 --- a/themes/nav/components/BlogPostCard.js +++ b/themes/nav/components/BlogPostCard.js @@ -1,13 +1,14 @@ import Link from 'next/link' import NotionIcon from './NotionIcon' import { useRouter } from 'next/router' +import { siteConfig } from '@/lib/config' const BlogPostCard = ({ post, className }) => { const router = useRouter() const currentSelected = router.asPath.split('?')[0] === '/' + post.slug const pageIcon = post.pageIcon.indexOf('amazonaws.com') !== -1 ? post.pageIcon + '&width=88' : post.pageIcon return ( - +
From 3b81794558cb409015746884c67e781d447361cf Mon Sep 17 00:00:00 2001 From: "tangly1024.com" Date: Tue, 30 Jan 2024 18:33:56 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=83=A8=E5=88=86?= =?UTF-8?q?=E6=96=87=E7=AB=A0=E6=97=A0=E6=B3=95=E5=8A=A0=E8=BD=BD=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E5=8E=9F=E5=9B=A0=E6=98=AFNotion=20?= =?UTF-8?q?API=E5=8A=A0=E8=BD=BD=E6=97=B6block=E4=B8=8A=E9=99=901000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复思路是在加载所有文章时，会首先一次性加载数据库中所有block，此时因为上限1000会发生溢出。之后会读取所有page的IDs，然后逐个获取page对应的block。此时如果发现ID查找不到（说明发生了溢出），则再调用API去逐一获取这些溢出了的page。可能这不是最优解。 修改的代码包括： 1. getDataBaseInfoByNotionAPI中添加上述判断逻辑。 2. 修改getPageProperties函数签名，直接传入block对应的value，而不是block数组。 --- lib/notion/getAllPosts.js | 2 +- lib/notion/getNotionData.js | 42 +++++++++++++++++++++++++++------ lib/notion/getPageProperties.js | 13 +++++----- lib/notion/getPostBlocks.js | 19 +++++++++++++++ 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/lib/notion/getAllPosts.js b/lib/notion/getAllPosts.js index ad4acde6c7d..5d63107ad5d 100644 --- a/lib/notion/getAllPosts.js +++ b/lib/notion/getAllPosts.js @@ -28,7 +28,7 @@ export async function getAllPosts({ notionPageData, from, pageType }) { if (!value) { continue } - const properties = (await getPageProperties(id, block, schema, null, tagOptions)) || null + const properties = (await getPageProperties(id, block[id].value, schema, null, tagOptions)) || null data.push(properties) } diff --git a/lib/notion/getNotionData.js b/lib/notion/getNotionData.js index 5b77e716581..7211c31e2ac 100755 --- a/lib/notion/getNotionData.js +++ b/lib/notion/getNotionData.js @@ -1,6 +1,6 @@ import BLOG from '@/blog.config' import { getDataFromCache, setDataToCache } from '@/lib/cache/cache_manager' -import { getPostBlocks } from '@/lib/notion/getPostBlocks' +import { getPostBlocks, getSingleBlock } from '@/lib/notion/getPostBlocks' import { idToUuid } from 'notion-utils' import { deepClone } from '../utils' import { getAllCategories } from './getAllCategories' @@ -185,7 +185,17 @@ export function getNavPages({ allPages }) { return post && post?.slug && (!post?.slug?.startsWith('http')) && post?.type === 'Post' && post?.status === 'Published' }) - return allNavPages.map(item => ({ id: item.id, title: item.title || '', pageCoverThumbnail: item.pageCoverThumbnail || '', category: item.category || null, tags: item.tags || null, summary: item.summary || null, slug: item.slug, pageIcon: item.pageIcon || '', lastEditedDate: item.lastEditedDate })) + return allNavPages.map(item => ({ + id: item.id, + title: item.title || '', + pageCoverThumbnail: item.pageCoverThumbnail || '', + category: item.category || null, + tags: item.tags || null, + summary: item.summary || null, + slug: item.slug, + pageIcon: item.pageIcon || '', + lastEditedDate: item.lastEditedDate + })) } /** @@ -205,7 +215,15 @@ const EmptyData = (pageId) => { const empty = { notice: null, siteInfo: getSiteInfo({}), - allPages: [{ id: 1, title: `无法获取Notion数据，请检查Notion_ID： \n 当前 ${pageId}`, summary: '访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next', status: 'Published', type: 'Post', slug: '13a171332816461db29d50e9f575b00d', date: { start_date: '2023-04-24', lastEditedDay: '2023-04-24', tagItems: [] } }], + allPages: [{ + id: 1, + title: `无法获取Notion数据，请检查Notion_ID： \n 当前 ${pageId}`, + summary: '访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next', + status: 'Published', + type: 'Post', + slug: '13a171332816461db29d50e9f575b00d', + date: { start_date: '2023-04-24', lastEditedDay: '2023-04-24', tagItems: [] } + }], allNavPages: [], collection: [], collectionQuery: {}, @@ -263,9 +281,17 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) { const id = pageIds[i] const value = block[id]?.value if (!value) { + // 如果找不到文章对应的block，说明发生了溢出，使用pageID再去请求 + const pageBlock = await getSingleBlock(id, from) + if (pageBlock.block[id].value) { + const properties = (await getPageProperties(id, pageBlock.block[id].value, schema, null, getTagOptions(schema))) || null + if (properties) { + collectionData.push(properties) + } + } continue } - const properties = (await getPageProperties(id, block, schema, null, getTagOptions(schema))) || null + const properties = (await getPageProperties(id, block[id].value, schema, null, getTagOptions(schema))) || null if (properties) { collectionData.push(properties) } @@ -280,8 +306,8 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) { postCount++ } return post && post?.slug && - (!post?.slug?.startsWith('http')) && - (post?.status === 'Invisible' || post?.status === 'Published') + (!post?.slug?.startsWith('http')) && + (post?.status === 'Invisible' || post?.status === 'Published') }) // 站点配置优先读取配置表格，否则读取blog.config.js 文件 @@ -294,7 +320,9 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) { }) } - const notice = await getNotice(collectionData.filter(post => { return post && post?.type && post?.type === 'Notice' && post.status === 'Published' })?.[0]) + const notice = await getNotice(collectionData.filter(post => { + return post && post?.type && post?.type === 'Notice' && post.status === 'Published' + })?.[0]) const categoryOptions = getAllCategories({ allPages, categoryOptions: getCategoryOptions(schema) }) const tagOptions = getAllTags({ allPages, tagOptions: getTagOptions(schema) }) // 旧的菜单 diff --git a/lib/notion/getPageProperties.js b/lib/notion/getPageProperties.js index 3ea1ce2856b..6004197701a 100644 --- a/lib/notion/getPageProperties.js +++ b/lib/notion/getPageProperties.js @@ -9,16 +9,15 @@ import { mapImgUrl } from './mapImage' /** * 获取页面元素成员属性 * @param {*} id - * @param {*} block + * @param {*} value * @param {*} schema * @param {*} authToken * @param {*} tagOptions * @returns */ -export default async function getPageProperties(id, block, schema, authToken, tagOptions) { - const rawProperties = Object.entries(block?.[id]?.value?.properties || []) +export default async function getPageProperties(id, value, schema, authToken, tagOptions) { + const rawProperties = Object.entries(value?.properties || []) const excludeProperties = ['date', 'select', 'multi_select', 'person'] - const value = block[id]?.value const properties = {} for (let i = 0; i < rawProperties.length; i++) { const [key, val] = rawProperties[i] @@ -91,9 +90,9 @@ export default async function getPageProperties(id, block, schema, authToken, ta properties.lastEditedDate = new Date(value?.last_edited_time) properties.lastEditedDay = formatDate(new Date(value?.last_edited_time), BLOG.LANG) properties.fullWidth = value.format?.page_full_width ?? false - properties.pageIcon = mapImgUrl(block[id].value?.format?.page_icon, block[id].value) ?? '' - properties.pageCover = mapImgUrl(block[id].value?.format?.page_cover, block[id].value) ?? '' - properties.pageCoverThumbnail = mapImgUrl(block[id].value?.format?.page_cover, block[id].value, 'block', 'pageCoverThumbnail') ?? '' + properties.pageIcon = mapImgUrl(value.value?.format?.page_icon, value.value) ?? '' + properties.pageCover = mapImgUrl(value.value?.format?.page_cover, value.value) ?? '' + properties.pageCoverThumbnail = mapImgUrl(value.value?.format?.page_cover, value.value, 'block', 'pageCoverThumbnail') ?? '' properties.content = value.content ?? [] properties.tagItems = properties?.tags?.map(tag => { return { name: tag, color: tagOptions?.find(t => t.value === tag)?.color || 'gray' } diff --git a/lib/notion/getPostBlocks.js b/lib/notion/getPostBlocks.js index 809dfe7ae6e..e9bbedfd05b 100644 --- a/lib/notion/getPostBlocks.js +++ b/lib/notion/getPostBlocks.js @@ -30,6 +30,25 @@ export async function getPostBlocks(id, from, slice) { return pageBlock } +export async function getSingleBlock(id, from) { + const cacheKey = 'single_block_' + id + let pageBlock = await getDataFromCache(cacheKey) + if (pageBlock) { + console.log('[缓存]:', `from:${from}`, cacheKey) + return pageBlock + } + + const start = new Date().getTime() + pageBlock = await getPageWithRetry(id, from) + const end = new Date().getTime() + console.log('[API耗时]', `${end - start}ms`) + + if (pageBlock) { + await setDataToCache(cacheKey, pageBlock) + } + return pageBlock +} + /** * 调用接口，失败会重试 * @param {*} id From 92828aa5badf80f5de6799ee578828983341d26e Mon Sep 17 00:00:00 2001 From: "tangly1024.com" Date: Tue, 30 Jan 2024 18:36:51 +0800 Subject: [PATCH 4/4] log --- pages/_document.js | 1 - 1 file changed, 1 deletion(-) diff --git a/pages/_document.js b/pages/_document.js index 96c999c344e..976a45fb0c3 100644 --- a/pages/_document.js +++ b/pages/_document.js @@ -20,7 +20,6 @@ class MyDocument extends Document { } {BLOG.FONT_URL?.map((fontUrl, index) => { - console.log(fontUrl) if (fontUrl.endsWith('.css')) { return } else {