diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py
index f1b0217a5f..2ed59933f1 100644
--- a/holoviews/operation/datashader.py
+++ b/holoviews/operation/datashader.py
@@ -318,13 +318,17 @@ def get_agg_data(cls, obj, category=None):
                 df = pd.concat(paths)
         else:
             df = paths[0] if paths else pd.DataFrame([], columns=[x.name, y.name])
-        if category and df[category].dtype.name != 'category':
-            df[category] = df[category].astype('category')
 
         is_custom = lazy_isinstance(df, "dask.dataframe:DataFrame") or cuDFInterface.applies(df)
-        if any((not is_custom and len(df[d.name]) and isinstance(df[d.name].values[0], cftime_types)) or
-               df[d.name].dtype.kind in ["M", "u"] for d in (x, y)):
+        category_check = category and df[category].dtype.name != 'category'
+        if (
+            category_check or
+            any((not is_custom and len(df[d.name]) and isinstance(df[d.name].values[0], cftime_types)) or
+            df[d.name].dtype.kind in ["M", "u"] for d in (x, y))
+        ):
             df = df.copy()
+        if category_check:
+            df[category] = df[category].astype('category')
 
         for d in (x, y):
             vals = df[d.name]
diff --git a/holoviews/tests/operation/test_datashader.py b/holoviews/tests/operation/test_datashader.py
index 73656196b2..dd36ecc396 100644
--- a/holoviews/tests/operation/test_datashader.py
+++ b/holoviews/tests/operation/test_datashader.py
@@ -1646,3 +1646,12 @@ def test_imagestack_dynspread():
     points = Points(df, ['x','y'], ['language'])
     op = dynspread(rasterize(points, aggregator=ds.by('language', ds.count())))
     render(op)  # should not error out
+
+def test_datashade_count_cat_no_change_inplace():
+    # Test for https://github.com/holoviz/holoviews/issues/6324
+    df = pd.DataFrame({"x": range(3), "y": range(3), "c": list(map(str, range(3)))})
+    assert df["c"].dtype == "object"
+    op = datashade(Points(df), aggregator=ds.count_cat("c"))
+    render(op)
+    # Should not convert to category dtype
+    assert df["c"].dtype == "object"