db: search different forms of a query

Now we also search [fuzzed fuzzes fuzzing fuzzer fuzzers fuzz] for fuzzing

.gitignore

@@ -25,4 +25,6 @@ rawdata/
 
 secret/
 data_crawler/failed.json
-test.py
+test.py
+
+*.gob

Dockerfile

@@ -1,13 +1,16 @@
 FROM golang:1.22 AS build
 WORKDIR /src
 COPY . .
-RUN make index
+RUN make lemmas
 RUN make server
+RUN make index
 
 FROM alpine:latest
 WORKDIR /
 COPY --from=build /src/server /server
 COPY --from=build /src/db.cspapers.org /db.cspapers.org
 COPY ./default.server.config /default.server.config
+COPY --from=build /src/lemma.gob /lemma.gob
+COPY --from=build /src/lemmaInv.gob /lemmaInv.gob
 CMD ["/server", "-config", "/default.server.config"]
 EXPOSE 8000

Makefile

@@ -1,3 +1,6 @@
+lemmas:
+	go run ./lemmas
+
 index:
 	go run ./api.cspapers.org/index -config default.index.config
 

README.md

@@ -144,6 +144,8 @@ PR is welcome
 
 Referred to csrankings.org for organizing conferences.
 
+Thanks to https://github.com/michmech/lemmatization-lists for lemma data.
+
 ## Disclaimer
 
 cspapers.org is served as implemented as is in this repository. I do not modify source code or data in and after distributions.  

api.cspapers.org/db/bleve/bleve.go

@@ -11,6 +11,7 @@ import (
 	"github.com/swkim101/cspapers.org/api.cspapers.org/db/dbimpl"
 	"github.com/swkim101/cspapers.org/api.cspapers.org/log"
 	"github.com/swkim101/cspapers.org/api.cspapers.org/types"
+	"github.com/swkim101/cspapers.org/api.cspapers.org/wordforms"
 )
 
 func init() {
@@ -134,6 +135,15 @@ func search(req *types.SearchRequest) *types.SearchResponse {
 				fq.SetFuzziness(2)
 				keywordQuery = append(keywordQuery, fq)
 			}
+			/* try different word forms */
+			if 3 < len(word) {
+				for _, wf := range wordforms.GetWordForms(word) {
+					qsTitle := fmt.Sprintf("title:%v^2", wf)
+					qsAbs := fmt.Sprintf("abstract:%v", wf)
+					keywordQuery = append(keywordQuery, bleve.NewQueryStringQuery(qsTitle))
+					keywordQuery = append(keywordQuery, bleve.NewQueryStringQuery(qsAbs))
+				}
+			}
 		}
 	}
 

api.cspapers.org/server/server.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/swkim101/cspapers.org/api.cspapers.org/log"
 	"github.com/swkim101/cspapers.org/api.cspapers.org/types"
+	"github.com/swkim101/cspapers.org/api.cspapers.org/wordforms"
 )
 
 var (
@@ -28,6 +29,7 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+	wordforms.LoadLemmas(cfg.Lemma, cfg.LemmaInv)
 	runServer(cfg)
 }
 

api.cspapers.org/server/serverConfig.go

@@ -9,9 +9,11 @@ import (
 
 type serverConfig struct {
 	config.Config
-	Host   string `json:"host"`
-	Port   int    `json:"port"`
-	Prefix string `json:"prefix"`
+	Host     string `json:"host"`
+	Port     int    `json:"port"`
+	Prefix   string `json:"prefix"`
+	Lemma    string `json:"lemma"`
+	LemmaInv string `json:"lemmaInv"`
 
 	dbimpl *dbimpl.Type `json:"-"`
 }

api.cspapers.org/wordforms/wordforms.go

@@ -0,0 +1,51 @@
+package wordforms
+
+import (
+	"encoding/gob"
+	"os"
+)
+
+var lemma map[string]string
+var lemmaInv map[string][]string
+
+func LoadLemmas(lemmaPath string, lemmaInvPath string) {
+	lemma = make(map[string]string)
+	lemmaInv = make(map[string][]string)
+
+	lemmaFile, err := os.Open(lemmaPath)
+	if err != nil {
+		panic(err)
+	}
+	dataDecoder := gob.NewDecoder(lemmaFile)
+	err = dataDecoder.Decode(&lemma)
+	if err != nil {
+		panic(err)
+	}
+	lemmaFile.Close()
+
+	lemmaInvFile, err := os.Open(lemmaInvPath)
+	if err != nil {
+		panic(err)
+	}
+	dataDecoder = gob.NewDecoder(lemmaInvFile)
+	err = dataDecoder.Decode(&lemmaInv)
+	if err != nil {
+		panic(err)
+	}
+	lemmaInvFile.Close()
+}
+
+func GetWordForms(word string) []string {
+	lem, ok := lemma[word]
+	wf := []string{}
+
+	if ok {
+		wf = append(wf, lemmaInv[lem]...)
+		wf = append(wf, lem)
+	} else {
+		wf = append(wf, lemmaInv[word]...)
+		wf = append(wf, word)
+	}
+
+	return wf
+}

api.cspapers.org/wordforms/wordforms_test.go

@@ -0,0 +1,32 @@
+package wordforms_test
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/swkim101/cspapers.org/api.cspapers.org/wordforms"
+)
+
+func TestMain(m *testing.M) {
+	wordforms.LoadLemmas("../../lemma.gob", "../../lemmaInv.gob")
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestWords(t *testing.T) {
+	{
+		got := wordforms.GetWordForms("tests")
+		expected := []string{"tested", "testing", "tests", "test"}
+		if !reflect.DeepEqual(got, expected) {
+			t.Fatalf("expected %v, got %v", expected, got)
+		}
+	}
+	{
+		got := wordforms.GetWordForms("fuzzing")
+		expected := []string{"fuzzed", "fuzzes", "fuzzing", "fuzzer", "fuzzers", "fuzz"}
+		if !reflect.DeepEqual(got, expected) {
+			t.Fatalf("expected %v, got %v", expected, got)
+		}
+	}
+}

default.server.config

@@ -3,5 +3,7 @@
   "type":               "bleve",
   "host":               "localhost",
   "port":               8000,
-  "prefix":             "/"
+  "prefix":             "/",
+  "lemma":             "lemma.gob",
+  "lemmaInv":             "lemmaInv.gob"
 }

lemmas/lemmas.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"bufio"
+	"encoding/gob"
+	"log"
+	"os"
+	"strings"
+)
+
+var lemma map[string]string
+var lemmaInv map[string][]string
+
+func main() {
+	lemma = make(map[string]string)
+	lemmaInv = make(map[string][]string)
+
+	readFile("lemmas/lemmatization-en.txt")
+	readFile("lemmas/lemmatization-cspapers.txt")
+
+	lemmaFile, err := os.Create("lemma.gob")
+	if err != nil {
+		panic(err)
+	}
+	defer lemmaFile.Close()
+	enc := gob.NewEncoder(lemmaFile)
+	enc.Encode(lemma)
+
+	lemmaInvFile, err := os.Create("lemmaInv.gob")
+	if err != nil {
+		panic(err)
+	}
+	defer lemmaInvFile.Close()
+	enc = gob.NewEncoder(lemmaInvFile)
+	enc.Encode(lemmaInv)
+}
+
+func readFile(filename string) {
+	file, err := os.Open(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 {
+			continue
+		}
+		words := strings.Fields(line)
+		left := words[0]
+		right := words[1]
+		lemma[right] = left
+
+		if a, ok := lemmaInv[left]; ok {
+			lemmaInv[left] = append(a, right)
+		} else {
+			lemmaInv[left] = []string{right}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+}

lemmas/lemmatization-cspapers.txt

@@ -0,0 +1,2 @@
+fuzz fuzzer
+fuzz fuzzers