Skip to content

Commit

Permalink
Allow software titles for software named the same but with different
Browse files Browse the repository at this point in the history
bundle identifiers

Updated software_titles unique index `idx_sw_titles` to include bundle
identifier.
Changed the column type of `software_titles.source` &
`software_titles.bundle_identifier` to use ascii. Using `utf8mb4` caused
the `idx_sw_titles` index to be too large when including `name`,
`source`,`browser`,`bundle_identifier`.
  • Loading branch information
ksykulev committed Jan 28, 2025
1 parent 9b70a2c commit c56628c
Show file tree
Hide file tree
Showing 6 changed files with 251 additions and 5 deletions.
2 changes: 2 additions & 0 deletions changes/25235-software-titles-uniqueness
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
* Updated software_titles.source & software_titles.bundle_identifier to use ascii
* Updated software_titles unique index idx_sw_titles to include bundle_identifier. Code modified to handle software named the same but have different bundle identifiers
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package tables

import (
"database/sql"
"fmt"
)

func init() {
MigrationClient.AddMigration(Up_20250124194347, Down_20250124194347)
}

func Up_20250124194347(tx *sql.Tx) error {
if _, err := tx.Exec(`
ALTER TABLE software_titles
ADD COLUMN coalesce_bundle_name VARCHAR(255) GENERATED ALWAYS AS (COALESCE(bundle_identifier, name)) STORED;
`); err != nil {
return fmt.Errorf("failed to add generated column: %w", err)
}

if _, err := tx.Exec(`
ALTER TABLE software_titles
DROP INDEX idx_sw_titles,
ADD UNIQUE INDEX idx_sw_titles (source, coalesce_bundle_name, browser);
`); err != nil {
return fmt.Errorf("failed to add vpp_apps_teams_id to policies: %w", err)
}

return nil
}

func Down_20250124194347(tx *sql.Tx) error {
if _, err := tx.Exec(`
ALTER TABLE software_titles
DROP COLUMN coalesce_bundle_name
`); err != nil {
return fmt.Errorf("failed to remove generated column: %w", err)
}

if _, err := tx.Exec(`
ALTER TABLE software_titles
DROP INDEX idx_sw_titles,
ADD UNIQUE KEY idx_sw_titles (name, source, browser);
`); err != nil {
return fmt.Errorf("failed to add vpp_apps_teams_id to policies: %w", err)
}

return nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package tables

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestUp_20250124194347(t *testing.T) {
db := applyUpToPrev(t)

var softwareTitles []struct {
ColumnName string `db:"COLUMN_NAME"`
}

sel := `SELECT COLUMN_NAME
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'software_titles'
AND index_name = 'idx_sw_titles'
ORDER BY seq_in_index;`

err := db.Select(&softwareTitles, sel)
if err != nil {
t.Fatalf("Failed to get index information: %v", err)
}
expected := []struct {
ColumnName string `db:"COLUMN_NAME"`
}{
{ColumnName: "name"},
{ColumnName: "source"},
{ColumnName: "browser"},
}
require.Equal(t, expected, softwareTitles)

applyNext(t, db)

err = db.Select(&softwareTitles, sel)
if err != nil {
t.Fatalf("Failed to get index information: %v", err)
}
expected = []struct {
ColumnName string `db:"COLUMN_NAME"`
}{
{ColumnName: "source"},
{ColumnName: "coalesce_bundle_name"},
{ColumnName: "browser"},
}
require.Equal(t, expected, softwareTitles)
}
7 changes: 4 additions & 3 deletions server/datastore/mysql/schema.sql

Large diffs are not rendered by default.

11 changes: 9 additions & 2 deletions server/datastore/mysql/software.go
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,14 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
argsWithoutBundleIdentifier = append(argsWithoutBundleIdentifier, sw.Name, sw.Source, sw.Browser)
}
// Map software title identifier to software checksums so that we can map checksums to actual titles later.
uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(sw.Name, sw.Source, sw.Browser)] = checksum
uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(
func() string {
if sw.BundleIdentifier != "" {
return sw.BundleIdentifier
}
return sw.Name
}(),
sw.Source, sw.Browser)] = checksum
}

// Get titles for software without bundle_identifier.
Expand Down Expand Up @@ -593,7 +600,7 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
}
// Map software titles to software checksums.
for _, title := range existingSoftwareTitlesForNewSoftwareWithBundleIdentifier {
checksum, ok := uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(title.Name, title.Source, title.Browser)]
checksum, ok := uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(*title.BundleIdentifier, title.Source, title.Browser)]
if ok {
incomingChecksumToTitle[checksum] = title
}
Expand Down
138 changes: 138 additions & 0 deletions server/datastore/mysql/software_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ func TestSoftware(t *testing.T) {
{"SaveHost", testSoftwareSaveHost},
{"CPE", testSoftwareCPE},
{"HostDuplicates", testSoftwareHostDuplicates},
{"DuplicateNameDifferentBundleIdentifier", testSoftwareDuplicateNameDifferentBundleIdentifier},
{"DifferentNameSameBundleIdentifier", testSoftwareDifferentNameSameBundleIdentifier},
{"LoadVulnerabilities", testSoftwareLoadVulnerabilities},
{"ListSoftwareCPEs", testListSoftwareCPEs},
{"NothingChanged", testSoftwareNothingChanged},
Expand Down Expand Up @@ -221,6 +223,142 @@ func testSoftwareCPE(t *testing.T, ds *Datastore) {
require.NoError(t, iterator.Close())
}

func testSoftwareDifferentNameSameBundleIdentifier(t *testing.T, ds *Datastore) {
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())

incoming := make(map[string]fleet.Software)
sw, err := fleet.SoftwareFromOsqueryRow("GoLand.app", "2024.3", "apps", "", "", "", "", "com.jetbrains.goland", "", "", "")
require.NoError(t, err)
soft2Key := sw.ToUniqueStr()
incoming[soft2Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err := ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err := ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

var software []fleet.Software
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 1)
var softwareTitle []fleet.SoftwareTitle
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 1)

incoming = make(map[string]fleet.Software)
sw, err = fleet.SoftwareFromOsqueryRow("GoLand 2.app", "2024.3", "apps", "", "", "", "", "com.jetbrains.goland", "", "", "")
require.NoError(t, err)
soft3Key := sw.ToUniqueStr()
incoming[soft3Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err = ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err = ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 2)
for _, s := range software {
require.NotEmpty(t, s.TitleID)
}

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 1)
}

func testSoftwareDuplicateNameDifferentBundleIdentifier(t *testing.T, ds *Datastore) {
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())

incoming := make(map[string]fleet.Software)
sw, err := fleet.SoftwareFromOsqueryRow("a", "0.0.1", "chrome_extension", "", "", "", "", "bundle_id1", "", "", "")
require.NoError(t, err)
soft2Key := sw.ToUniqueStr()
incoming[soft2Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err := ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err := ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

var software []fleet.Software
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 1)
var softwareTitle []fleet.SoftwareTitle
err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 1)

incoming = make(map[string]fleet.Software)
sw, err = fleet.SoftwareFromOsqueryRow("a", "0.0.1", "chrome_extension", "", "", "", "", "bundle_id2", "", "", "")
require.NoError(t, err)
soft3Key := sw.ToUniqueStr()
incoming[soft3Key] = *sw

currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, err = ds.getExistingSoftware(
context.Background(), make(map[string]fleet.Software), incoming,
)
require.NoError(t, err)
tx, err = ds.writer(context.Background()).Beginx()
require.NoError(t, err)
_, err = ds.insertNewInstalledHostSoftwareDB(
context.Background(), tx, host1.ID, currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle,
)
require.NoError(t, err)
require.NoError(t, tx.Commit())

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&software, `SELECT id, name, bundle_identifier, title_id FROM software`,
)
require.NoError(t, err)
require.Len(t, software, 2)
for _, s := range software {
require.NotEmpty(t, s.TitleID)
}

err = sqlx.SelectContext(context.Background(), ds.reader(context.Background()),
&softwareTitle, `SELECT id, name FROM software_titles`,
)
require.NoError(t, err)
require.Len(t, softwareTitle, 2)
}

func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) {
host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())

Expand Down

0 comments on commit c56628c

Please sign in to comment.