fix(table-core): ensure isSubRowSelected returns false if no subRows are selectable

packages/table-core/src/features/RowSelection.ts

@@ -635,6 +635,7 @@ export function isSubRowSelected<TData extends RowData>(
 
   let allChildrenSelected = true
   let someSelected = false
+  let someSelectable = false
 
   row.subRows.forEach(subRow => {
     // Bail out early if we know both of these
@@ -643,6 +644,7 @@ export function isSubRowSelected<TData extends RowData>(
     }
 
     if (subRow.getCanSelect()) {
+      someSelectable = true
       if (isRowSelected(subRow, selection)) {
         someSelected = true
       } else {
@@ -655,14 +657,18 @@ export function isSubRowSelected<TData extends RowData>(
       const subRowChildrenSelected = isSubRowSelected(subRow, selection, table)
       if (subRowChildrenSelected === 'all') {
         someSelected = true
+        someSelectable = true
       } else if (subRowChildrenSelected === 'some') {
         someSelected = true
         allChildrenSelected = false
+        someSelectable = true
       } else {
         allChildrenSelected = false
       }
     }
   })
 
+  if (!someSelectable) return false
+
   return allChildrenSelected ? 'all' : someSelected ? 'some' : false
 }

packages/table-core/tests/RowSelection.test.ts

@@ -201,6 +201,62 @@ describe('RowSelection', () => {
       expect(result).toEqual(false)
     })
 
+    it('should return false if no sub-rows are selectable', () => {
+      const data = makeData(3, 2)
+      const columns = generateColumns(data)
+
+      const table = createTable<Person>({
+        enableRowSelection: false,
+        onStateChange() {},
+        renderFallbackValue: '',
+        data,
+        getSubRows: row => row.subRows,
+        state: {
+          rowSelection: {},
+        },
+        columns,
+        getCoreRowModel: getCoreRowModel(),
+      })
+
+      const firstRow = table.getCoreRowModel().rows[0]
+
+      const result = RowSelection.isSubRowSelected(
+        firstRow,
+        table.getState().rowSelection,
+        table
+      )
+
+      expect(result).toEqual(false)
+    })
+
+    it('should return some if no children are selectable, but a grand-child is and is selected', () => {
+      const data = makeData(3, 2, 2)
+      const columns = generateColumns(data)
+
+      const table = createTable<Person>({
+        enableRowSelection: row => row.id === '0.0.1',
+        onStateChange() {},
+        renderFallbackValue: '',
+        data,
+        getSubRows: row => row.subRows,
+        state: {
+          rowSelection: { '0.0.1': true },
+        },
+        columns,
+        getCoreRowModel: getCoreRowModel(),
+      })
+
+      const firstRow = table.getCoreRowModel().rows[0]
+
+      const result = RowSelection.isSubRowSelected(
+        firstRow,
+        table.getState().rowSelection,
+        table
+      )
+
+      expect(result).toEqual('some')
+    })
+
     it('should return some if some sub-rows are selected', () => {
       const data = makeData(3, 2)
       const columns = generateColumns(data)