Skip to content

Commit

Permalink
Implement full text search
Browse files Browse the repository at this point in the history
Solves: #40
  • Loading branch information
AlexGustafsson committed Dec 29, 2024
1 parent d4f2114 commit bffe2a4
Show file tree
Hide file tree
Showing 8 changed files with 221 additions and 6 deletions.
6 changes: 6 additions & 0 deletions api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ paths:
schema:
type: integer
required: false
- in: query
name: query
description: Full text search query.
schema:
type: string
required: false
responses:
'200':
description: A page of images.
Expand Down
1 change: 1 addition & 0 deletions internal/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ func NewServer(api *store.Store, hub *events.Hub[store.Event], processQueue chan
Page: int(page),
Limit: int(limit),
Sort: store.Sort(sort),
Query: query.Get("query"),
}
response, err := api.ListImages(r.Context(), listOptions)
s.handleJSONResponse(w, r, response, err)
Expand Down
21 changes: 21 additions & 0 deletions internal/store/createTablesIfNotExist.sql
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,27 @@ CREATE TABLE IF NOT EXISTS images (
FOREIGN KEY(reference) REFERENCES raw_images(reference) ON DELETE CASCADE
);

CREATE VIRTUAL TABLE IF NOT EXISTS images_fts USING FTS5(
reference,
description
);

DROP TRIGGER IF EXISTS images_fts_insert;
CREATE TRIGGER images_fts_insert AFTER INSERT ON images BEGIN
INSERT INTO images_fts(reference, description) VALUES (new.reference, new.description);
END;

DROP TRIGGER IF EXISTS images_fts_delete;
CREATE TRIGGER images_fts_delete AFTER DELETE ON images BEGIN
INSERT INTO images_fts(images_fts, reference, description) VALUES('delete', old.reference, old.description);
END;

DROP TRIGGER IF EXISTS images_fts_update;
CREATE TRIGGER images_fts_update AFTER UPDATE ON images BEGIN
INSERT INTO images_fts(images_fts, reference, description) VALUES('delete', old.reference, old.description);
INSERT INTO images_fts(reference, description) VALUES (new.reference, new.description);
END;

CREATE TABLE IF NOT EXISTS images_tags (
reference TEXT NOT NULL,
tag TEXT NOT NULL,
Expand Down
30 changes: 29 additions & 1 deletion internal/store/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -752,6 +753,8 @@ type ListImageOptions struct {
Limit int
// Sort defaults to SortBump.
Sort Sort
// Query is an Sqlite full text search query.
Query string
}

func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*models.ImagePage, error) {
Expand Down Expand Up @@ -886,7 +889,15 @@ func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*mod

whereClause := ""
if len(options.Tags) > 0 {
whereClause = fmt.Sprintf("WHERE images_tags.tag IN (%s)", "?"+strings.Repeat(", ?", len(options.Tags)-1))
whereClause += fmt.Sprintf("WHERE images_tags.tag IN (%s)", "?"+strings.Repeat(", ?", len(options.Tags)-1))
}
if options.Query != "" {
if len(options.Tags) > 0 {
whereClause += " AND "
} else {
whereClause += "WHERE "
}
whereClause += "images.reference IN (SELECT reference from images_fts WHERE images_fts MATCH ?)"
}

groupByClause := "GROUP BY images.reference"
Expand All @@ -906,7 +917,12 @@ func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*mod
for _, tag := range options.Tags {
args = append(args, tag)
}
if options.Query != "" {
args = append(args, ftsEscape(options.Query))
}
args = append(args, len(options.Tags))
} else if options.Query != "" {
args = append(args, ftsEscape(options.Query))
}
res, err = statement.QueryContext(ctx, args...)
statement.Close()
Expand Down Expand Up @@ -941,7 +957,12 @@ func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*mod
for _, tag := range options.Tags {
args = append(args, tag)
}
if options.Query != "" {
args = append(args, strconv.Quote(options.Query))
}
args = append(args, len(options.Tags))
} else if options.Query != "" {
args = append(args, ftsEscape(options.Query))
}
args = append(args, limit)
args = append(args, offset)
Expand Down Expand Up @@ -1042,3 +1063,10 @@ func (s *Store) DeleteNonPresent(ctx context.Context, references []string) (int6
func (s *Store) Close() error {
return s.db.Close()
}

// ftsEscape escapes a string for use with sqlite's full text search.
// It is not a security feature, it just ensures that all searches are full text
// and not using fts' query syntax.
func ftsEscape(s string) string {
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
}
72 changes: 72 additions & 0 deletions internal/store/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,79 @@ func TestListImages(t *testing.T) {
actualPage, err = store.ListImages(context.TODO(), &ListImageOptions{Page: 1, Limit: 1})
require.NoError(t, err)
assert.Equal(t, expectedPage, actualPage)
}

func TestListImagesQuery(t *testing.T) {
store, err := New("file://"+t.TempDir()+"/sqlite.db", false)
require.NoError(t, err)

images := []models.Image{
{
Reference: "mongo:3",
LatestReference: "mongo:4",
Description: "Mongo is a database",
Tags: []string{"docker"},
Links: []models.ImageLink{
{
Type: "docker",
URL: "https://docker.com/_/mongo",
},
},
Vulnerabilities: []models.ImageVulnerability{},
LastModified: time.Date(2024, 10, 05, 18, 39, 0, 0, time.Local),
Image: "https://example.com/logo.png",
},
}

expectedPage := &models.ImagePage{
Images: []models.Image{
{
Reference: "mongo:3",
LatestReference: "mongo:4",
Description: "Mongo is a database",
Tags: []string{"docker"},
Links: []models.ImageLink{
{
Type: "docker",
URL: "https://docker.com/_/mongo",
},
},
Vulnerabilities: []models.ImageVulnerability{},
LastModified: time.Date(2024, 10, 05, 18, 39, 0, 0, time.Local),
Image: "https://example.com/logo.png",
},
},
Summary: models.ImagePageSummary{
Images: 1,
Outdated: 1,
Vulnerable: 0,
Processing: 0,
},
Pagination: models.PaginationMetadata{
Total: 1,
Page: 0,
Size: 30,
Next: "",
Previous: "",
},
}

for _, image := range images {
_, err := store.InsertRawImage(context.TODO(), &models.RawImage{
Reference: image.Reference,
})
require.NoError(t, err)

err = store.InsertImage(context.TODO(), &image)
require.NoError(t, err)
}

page, err := store.ListImages(context.TODO(), &ListImageOptions{
Query: "database",
})
require.NoError(t, err)

assert.Equal(t, expectedPage, page)
}

func TestStoreDeleteNonPresent(t *testing.T) {
Expand Down
5 changes: 5 additions & 0 deletions web/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ interface UseImagesProps {
order?: 'asc' | 'desc'
page?: number
limit?: number
query?: string
}

export function useImages(options?: UseImagesProps): Result<ImagePage> {
Expand All @@ -131,6 +132,9 @@ export function useImages(options?: UseImagesProps): Result<ImagePage> {
if (options?.limit !== undefined) {
query.set('limit', options.limit.toString())
}
if (options?.query !== undefined) {
query.set('query', options.query)
}

fetch(`${import.meta.env.VITE_API_ENDPOINT}/images?${query.toString()}`)
.then((res) => {
Expand All @@ -148,6 +152,7 @@ export function useImages(options?: UseImagesProps): Result<ImagePage> {
options?.order,
options?.page,
options?.limit,
options?.query,
])

return result
Expand Down
60 changes: 59 additions & 1 deletion web/hooks.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
import { type Dispatch, type SetStateAction, useCallback, useMemo } from 'react'
import {
type Dispatch,
type SetStateAction,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react'
import { useSearchParams } from 'react-router-dom'

export function useFilter(): [string[], Dispatch<SetStateAction<string[]>>] {
Expand Down Expand Up @@ -92,3 +100,53 @@ export function useSort(): [

return [property, setProperty, order, setOrder]
}

export function useDebouncedEffect(
effect: React.EffectCallback,
deps?: React.DependencyList
) {
const timeoutRef = useRef<ReturnType<typeof setTimeout>>(null)

// biome-ignore lint/correctness/useExhaustiveDependencies: effect should not be changed
useEffect(() => {
if (timeoutRef.current !== null) {
clearTimeout(timeoutRef.current)
timeoutRef.current = null
}

timeoutRef.current = setTimeout(() => {
effect()
timeoutRef.current = null
}, 200)
}, deps)
}

export function useQuery(): [
string | undefined,
Dispatch<SetStateAction<string | undefined>>,
] {
const [searchParams, setSearchParams] = useSearchParams()

const query = useMemo(() => {
return searchParams.get('query') || undefined
}, [searchParams])

const setQuery = useCallback(
(s?: string | ((current: string | undefined) => string | undefined)) => {
setSearchParams((current) => {
if (typeof s === 'function') {
s = s(current.get('query') || undefined)
}
if (!s) {
current.delete('query')
} else {
current.set('query', s)
}
return current
})
},
[setSearchParams]
)

return [query, setQuery]
}
32 changes: 28 additions & 4 deletions web/pages/Dashboard.tsx
Original file line number Diff line number Diff line change
@@ -1,28 +1,41 @@
import { type JSX, useEffect } from 'react'
import { type JSX, useEffect, useState } from 'react'
import { NavLink, useNavigate, useSearchParams } from 'react-router-dom'

import { useImages, usePagination, useTags } from '../api'
import { ImageCard } from '../components/ImageCard'
import { Select } from '../components/Select'
import { TagSelect } from '../components/TagSelect'
import { useFilter, useSort } from '../hooks'
import { useDebouncedEffect, useFilter, useQuery, useSort } from '../hooks'
import { name, version } from '../oci'

export function Dashboard(): JSX.Element {
const [filter, setFilter] = useFilter()

const [sort, setSort, sortOrder, setSortOrder] = useSort()

const [query, setQuery] = useQuery()

const [queryInput, setQueryInput] = useState('')

const [searchParams, _] = useSearchParams()

const navigate = useNavigate()

useDebouncedEffect(() => {
setQuery(queryInput)
}, [queryInput])

useEffect(() => {
setQueryInput(query || '')
}, [query])

const images = useImages({
tags: filter,
sort: sort,
order: sortOrder,
page: searchParams.get('page') ? Number(searchParams.get('page')) : 0,
limit: 30,
query: query,
})

const pages = usePagination(
Expand Down Expand Up @@ -98,8 +111,17 @@ export function Dashboard(): JSX.Element {

<hr className="my-6 w-3/4" />

<div className="flex justify-between items-center w-full max-w-[800px]">
<div className="flex gap-x-2">
{/* Filters */}
<div className="flex justify-between items-center w-full mt-2 max-w-[800px]">
<div className="flex items-center gap-x-2 w-full">
<input
type="text"
placeholder="Search"
value={queryInput}
onChange={(e) => setQueryInput(e.target.value)}
className="bg-white dark:bg-[#1e1e1e] pl-3 pr-8 py-2 text-sm rounded flex-grow border border-[#e5e5e5] dark:border-[#333333]"
/>

<Select
value={sort}
onChange={(e) => setSort(e.target.value)}
Expand Down Expand Up @@ -127,6 +149,8 @@ export function Dashboard(): JSX.Element {
<TagSelect tags={tags.value} filter={filter} onChange={setFilter} />
</div>
</div>

{/* Images */}
<div className="flex flex-col mt-2 gap-y-4 w-full max-w-[800px]">
{images.value.images.map((x) => (
<NavLink
Expand Down

0 comments on commit bffe2a4

Please sign in to comment.