Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement full text search #41

Merged
merged 1 commit into from
Dec 29, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Implement full text search
Solves: #40
  • Loading branch information
AlexGustafsson committed Dec 29, 2024
commit bffe2a49ea00f51eb3da2edd52e021febe7eeb3c
6 changes: 6 additions & 0 deletions api.yaml
Original file line number Diff line number Diff line change
@@ -56,6 +56,12 @@ paths:
schema:
type: integer
required: false
- in: query
name: query
description: Full text search query.
schema:
type: string
required: false
responses:
'200':
description: A page of images.
1 change: 1 addition & 0 deletions internal/api/server.go
Original file line number Diff line number Diff line change
@@ -91,6 +91,7 @@ func NewServer(api *store.Store, hub *events.Hub[store.Event], processQueue chan
Page: int(page),
Limit: int(limit),
Sort: store.Sort(sort),
Query: query.Get("query"),
}
response, err := api.ListImages(r.Context(), listOptions)
s.handleJSONResponse(w, r, response, err)
21 changes: 21 additions & 0 deletions internal/store/createTablesIfNotExist.sql
Original file line number Diff line number Diff line change
@@ -15,6 +15,27 @@ CREATE TABLE IF NOT EXISTS images (
FOREIGN KEY(reference) REFERENCES raw_images(reference) ON DELETE CASCADE
);

CREATE VIRTUAL TABLE IF NOT EXISTS images_fts USING FTS5(
reference,
description
);

DROP TRIGGER IF EXISTS images_fts_insert;
CREATE TRIGGER images_fts_insert AFTER INSERT ON images BEGIN
INSERT INTO images_fts(reference, description) VALUES (new.reference, new.description);
END;

DROP TRIGGER IF EXISTS images_fts_delete;
CREATE TRIGGER images_fts_delete AFTER DELETE ON images BEGIN
INSERT INTO images_fts(images_fts, reference, description) VALUES('delete', old.reference, old.description);
END;

DROP TRIGGER IF EXISTS images_fts_update;
CREATE TRIGGER images_fts_update AFTER UPDATE ON images BEGIN
INSERT INTO images_fts(images_fts, reference, description) VALUES('delete', old.reference, old.description);
INSERT INTO images_fts(reference, description) VALUES (new.reference, new.description);
END;

CREATE TABLE IF NOT EXISTS images_tags (
reference TEXT NOT NULL,
tag TEXT NOT NULL,
30 changes: 29 additions & 1 deletion internal/store/store.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"

@@ -752,6 +753,8 @@ type ListImageOptions struct {
Limit int
// Sort defaults to SortBump.
Sort Sort
// Query is an Sqlite full text search query.
Query string
}

func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*models.ImagePage, error) {
@@ -886,7 +889,15 @@ func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*mod

whereClause := ""
if len(options.Tags) > 0 {
whereClause = fmt.Sprintf("WHERE images_tags.tag IN (%s)", "?"+strings.Repeat(", ?", len(options.Tags)-1))
whereClause += fmt.Sprintf("WHERE images_tags.tag IN (%s)", "?"+strings.Repeat(", ?", len(options.Tags)-1))
}
if options.Query != "" {
if len(options.Tags) > 0 {
whereClause += " AND "
} else {
whereClause += "WHERE "
}
whereClause += "images.reference IN (SELECT reference from images_fts WHERE images_fts MATCH ?)"
}

groupByClause := "GROUP BY images.reference"
@@ -906,7 +917,12 @@ func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*mod
for _, tag := range options.Tags {
args = append(args, tag)
}
if options.Query != "" {
args = append(args, ftsEscape(options.Query))
}
args = append(args, len(options.Tags))
} else if options.Query != "" {
args = append(args, ftsEscape(options.Query))
}
res, err = statement.QueryContext(ctx, args...)
statement.Close()
@@ -941,7 +957,12 @@ func (s *Store) ListImages(ctx context.Context, options *ListImageOptions) (*mod
for _, tag := range options.Tags {
args = append(args, tag)
}
if options.Query != "" {
args = append(args, strconv.Quote(options.Query))
}
args = append(args, len(options.Tags))
} else if options.Query != "" {
args = append(args, ftsEscape(options.Query))
}
args = append(args, limit)
args = append(args, offset)
@@ -1042,3 +1063,10 @@ func (s *Store) DeleteNonPresent(ctx context.Context, references []string) (int6
func (s *Store) Close() error {
return s.db.Close()
}

// ftsEscape escapes a string for use with sqlite's full text search.
// It is not a security feature, it just ensures that all searches are full text
// and not using fts' query syntax.
func ftsEscape(s string) string {
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
}
72 changes: 72 additions & 0 deletions internal/store/store_test.go
Original file line number Diff line number Diff line change
@@ -288,7 +288,79 @@ func TestListImages(t *testing.T) {
actualPage, err = store.ListImages(context.TODO(), &ListImageOptions{Page: 1, Limit: 1})
require.NoError(t, err)
assert.Equal(t, expectedPage, actualPage)
}

func TestListImagesQuery(t *testing.T) {
store, err := New("file://"+t.TempDir()+"/sqlite.db", false)
require.NoError(t, err)

images := []models.Image{
{
Reference: "mongo:3",
LatestReference: "mongo:4",
Description: "Mongo is a database",
Tags: []string{"docker"},
Links: []models.ImageLink{
{
Type: "docker",
URL: "https://docker.com/_/mongo",
},
},
Vulnerabilities: []models.ImageVulnerability{},
LastModified: time.Date(2024, 10, 05, 18, 39, 0, 0, time.Local),
Image: "https://example.com/logo.png",
},
}

expectedPage := &models.ImagePage{
Images: []models.Image{
{
Reference: "mongo:3",
LatestReference: "mongo:4",
Description: "Mongo is a database",
Tags: []string{"docker"},
Links: []models.ImageLink{
{
Type: "docker",
URL: "https://docker.com/_/mongo",
},
},
Vulnerabilities: []models.ImageVulnerability{},
LastModified: time.Date(2024, 10, 05, 18, 39, 0, 0, time.Local),
Image: "https://example.com/logo.png",
},
},
Summary: models.ImagePageSummary{
Images: 1,
Outdated: 1,
Vulnerable: 0,
Processing: 0,
},
Pagination: models.PaginationMetadata{
Total: 1,
Page: 0,
Size: 30,
Next: "",
Previous: "",
},
}

for _, image := range images {
_, err := store.InsertRawImage(context.TODO(), &models.RawImage{
Reference: image.Reference,
})
require.NoError(t, err)

err = store.InsertImage(context.TODO(), &image)
require.NoError(t, err)
}

page, err := store.ListImages(context.TODO(), &ListImageOptions{
Query: "database",
})
require.NoError(t, err)

assert.Equal(t, expectedPage, page)
}

func TestStoreDeleteNonPresent(t *testing.T) {
5 changes: 5 additions & 0 deletions web/api.ts
Original file line number Diff line number Diff line change
@@ -107,6 +107,7 @@ interface UseImagesProps {
order?: 'asc' | 'desc'
page?: number
limit?: number
query?: string
}

export function useImages(options?: UseImagesProps): Result<ImagePage> {
@@ -131,6 +132,9 @@ export function useImages(options?: UseImagesProps): Result<ImagePage> {
if (options?.limit !== undefined) {
query.set('limit', options.limit.toString())
}
if (options?.query !== undefined) {
query.set('query', options.query)
}

fetch(`${import.meta.env.VITE_API_ENDPOINT}/images?${query.toString()}`)
.then((res) => {
@@ -148,6 +152,7 @@ export function useImages(options?: UseImagesProps): Result<ImagePage> {
options?.order,
options?.page,
options?.limit,
options?.query,
])

return result
60 changes: 59 additions & 1 deletion web/hooks.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
import { type Dispatch, type SetStateAction, useCallback, useMemo } from 'react'
import {
type Dispatch,
type SetStateAction,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react'
import { useSearchParams } from 'react-router-dom'

export function useFilter(): [string[], Dispatch<SetStateAction<string[]>>] {
@@ -92,3 +100,53 @@ export function useSort(): [

return [property, setProperty, order, setOrder]
}

export function useDebouncedEffect(
effect: React.EffectCallback,
deps?: React.DependencyList
) {
const timeoutRef = useRef<ReturnType<typeof setTimeout>>(null)

// biome-ignore lint/correctness/useExhaustiveDependencies: effect should not be changed
useEffect(() => {
if (timeoutRef.current !== null) {
clearTimeout(timeoutRef.current)
timeoutRef.current = null
}

timeoutRef.current = setTimeout(() => {
effect()
timeoutRef.current = null
}, 200)
}, deps)
}

export function useQuery(): [
string | undefined,
Dispatch<SetStateAction<string | undefined>>,
] {
const [searchParams, setSearchParams] = useSearchParams()

const query = useMemo(() => {
return searchParams.get('query') || undefined
}, [searchParams])

const setQuery = useCallback(
(s?: string | ((current: string | undefined) => string | undefined)) => {
setSearchParams((current) => {
if (typeof s === 'function') {
s = s(current.get('query') || undefined)
}
if (!s) {
current.delete('query')
} else {
current.set('query', s)
}
return current
})
},
[setSearchParams]
)

return [query, setQuery]
}
32 changes: 28 additions & 4 deletions web/pages/Dashboard.tsx
Original file line number Diff line number Diff line change
@@ -1,28 +1,41 @@
import { type JSX, useEffect } from 'react'
import { type JSX, useEffect, useState } from 'react'
import { NavLink, useNavigate, useSearchParams } from 'react-router-dom'

import { useImages, usePagination, useTags } from '../api'
import { ImageCard } from '../components/ImageCard'
import { Select } from '../components/Select'
import { TagSelect } from '../components/TagSelect'
import { useFilter, useSort } from '../hooks'
import { useDebouncedEffect, useFilter, useQuery, useSort } from '../hooks'
import { name, version } from '../oci'

export function Dashboard(): JSX.Element {
const [filter, setFilter] = useFilter()

const [sort, setSort, sortOrder, setSortOrder] = useSort()

const [query, setQuery] = useQuery()

const [queryInput, setQueryInput] = useState('')

const [searchParams, _] = useSearchParams()

const navigate = useNavigate()

useDebouncedEffect(() => {
setQuery(queryInput)
}, [queryInput])

useEffect(() => {
setQueryInput(query || '')
}, [query])

const images = useImages({
tags: filter,
sort: sort,
order: sortOrder,
page: searchParams.get('page') ? Number(searchParams.get('page')) : 0,
limit: 30,
query: query,
})

const pages = usePagination(
@@ -98,8 +111,17 @@ export function Dashboard(): JSX.Element {

<hr className="my-6 w-3/4" />

<div className="flex justify-between items-center w-full max-w-[800px]">
<div className="flex gap-x-2">
{/* Filters */}
<div className="flex justify-between items-center w-full mt-2 max-w-[800px]">
<div className="flex items-center gap-x-2 w-full">
<input
type="text"
placeholder="Search"
value={queryInput}
onChange={(e) => setQueryInput(e.target.value)}
className="bg-white dark:bg-[#1e1e1e] pl-3 pr-8 py-2 text-sm rounded flex-grow border border-[#e5e5e5] dark:border-[#333333]"
/>

<Select
value={sort}
onChange={(e) => setSort(e.target.value)}
@@ -127,6 +149,8 @@ export function Dashboard(): JSX.Element {
<TagSelect tags={tags.value} filter={filter} onChange={setFilter} />
</div>
</div>

{/* Images */}
<div className="flex flex-col mt-2 gap-y-4 w-full max-w-[800px]">
{images.value.images.map((x) => (
<NavLink
Loading