From 8916bd99d2d6a65268016d4400000b9f3758701a Mon Sep 17 00:00:00 2001 From: PrinOrange Date: Mon, 12 Aug 2024 10:57:33 +0800 Subject: [PATCH] [fix] fix tokenizer problem --- lib/search.ts | 15 +++-- pages/search.tsx | 131 +++++++++++++++++++++++++---------------- types/search-result.ts | 1 + 3 files changed, 93 insertions(+), 54 deletions(-) diff --git a/lib/search.ts b/lib/search.ts index 025114f..58c974f 100644 --- a/lib/search.ts +++ b/lib/search.ts @@ -7,10 +7,17 @@ import { getPostFileContent, sortedPosts } from "./post-process"; // Due to the flaws of the word tokenizer, // it is necessary to match CJKL symbols only // during the word segmentation process to prevent repeated recognition. -const CJKLRecognizeRegex = /[\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7A3a-zA-Z]+/g; +const NonCJKLRecognizeRegex = + /[^\u4e00-\u9fa5\u3040-\u30ff\uac00-\ud7af\u1100-\u11ff\u3130-\u318f\u31c0-\u31ef\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\u0041-\u005a\u0061-\u007a\u00c0-\u00ff\u0100-\u017f\u0180-\u024f\s ]/g; function tokenizer(str: string) { - const result = cutForSearch(str, true).filter((item) => CJKLRecognizeRegex.test(item)); + const result = cutForSearch(str.replace(NonCJKLRecognizeRegex, " "), true); + for (let i = 0; i < result.length; i++) { + if (result[i].trim() === "") { + result.splice(i, 1); + i--; + } + } return result; } @@ -18,7 +25,7 @@ function makeSearchIndex() { const startTime = Date.now(); let miniSearch = new minisearch({ fields: ["id", "title", "tags", "subtitle", "summary", "content"], - storeFields: ["id", "title", "tags"], + storeFields: ["id", "title", "tags", "summary"], tokenize: tokenizer, }); for (let index = 0; index < sortedPosts.allPostList.length; index++) { @@ -43,4 +50,4 @@ function makeSearchIndex() { return miniSearch; } -export const SearchIndex = makeSearchIndex(); +export const SearchIndex = Object.freeze(makeSearchIndex()); diff --git a/pages/search.tsx b/pages/search.tsx index 41b4185..d9a2d1e 100644 --- a/pages/search.tsx +++ b/pages/search.tsx @@ -10,52 +10,76 @@ import { Config } from "@/data/config"; import { isEmptyString } from "@/lib/utils"; import { TSearchResultItem } from "@/types/search-result"; import axios from "axios"; +import { isArray } from "lodash"; import { nanoid } from "nanoid"; +import { GetServerSideProps } from "next"; import Link from "next/link"; -import { ChangeEvent, KeyboardEvent, useState } from "react"; -import { useQuery } from "react-query"; +import { useRouter } from "next/router"; +import { ChangeEvent, KeyboardEvent, useEffect, useState } from "react"; -export default function SearchPage() { - const [searchText, setSearchText] = useState(""); +type SearchPageProps = { query: string | null }; + +export default function SearchPage(props: SearchPageProps) { + const [searchText, setSearchText] = useState(props.query ?? ""); const [searchResult, setSearchResult] = useState([]); + const [isLoading, setIsLoading] = useState(false); const { toast } = useToast(); + const router = useRouter(); - const fetchSearchAPI = async (param: string) => { - const response = (await axios.get(`/api/search/${param}`)).data; - return response; + useEffect(() => { + if (!isEmptyString(searchText)) { + handleMakeSearch(); + } + }, []); + + const fetchSearchAPI = (param: string): Promise => { + return axios.get(`/api/search/${param}`).then((response) => response.data); }; - const querySearch = useQuery("searchData", () => fetchSearchAPI(searchText), { - enabled: false, - onSuccess: (data) => { - setSearchResult(data); - if (data.length === 0) { - toast({ title: "Empty Result", description: "No results were found for this keyword. Try another keyword." }); - } - }, - onError: () => { - toast({ title: "Network Error", description: "Please try it later." }); - }, - }); - const handleInputSearchText = (event: ChangeEvent) => { setSearchText(event.target.value); }; const handleEnterKeySearch = (event: KeyboardEvent) => { - (event.key === "Go" || event.key === "Enter") && handleMakeSearch(); + if (event.key === "Go" || event.key === "Enter") { + handleMakeSearch(); + } }; const handleMakeSearch = () => { - if (isEmptyString(searchText)) { + const searchQuery = searchText; + + if (isEmptyString(searchQuery)) { toast({ title: "Enter a Keyword", description: "Please enter one keyword at least." }); return; } - if (searchText.length < 4) { + if (searchQuery && searchQuery.length < 4) { toast({ title: "Keywords too short", description: "Keyword length must be at least 5." }); return; } - querySearch.refetch(); + + router.push({ + pathname: router.pathname, + query: { ...router.query, q: searchQuery }, + }); + setIsLoading(true); + + fetchSearchAPI(searchQuery) + .then((data) => { + setSearchResult(data); + if (data.length === 0) { + toast({ + title: "Empty Result", + description: "No results were found for this keyword. Try another keyword.", + }); + } + }) + .catch(() => { + toast({ title: "Network Error", description: "Please try it later." }); + }) + .finally(() => { + setIsLoading(false); + }); }; return ( @@ -64,9 +88,7 @@ export default function SearchPage() { -

- {"SEARCH POSTS"} -

+

{"SEARCH POSTS"}

-
-
- {querySearch.isSuccess && - searchResult.map((item, index) => ( - -
{item.title}
-
- {item.tags?.map((tagitem) => ( -
- {tagitem} -
- ))} -
- - ))} +
+ {searchResult.map((item, index) => ( + +
+
{item.title}
+ {item.summary &&
{item.summary}
} +
+
+ {item.tags?.map((tagitem) => ( +
+ {tagitem} +
+ ))} +
+ + ))}
+
+

{"For search efficiency, only the first 20 results are displayed."}

+