Add ability to make concepts public. (#624)

Also refactors concepts a little bit so that there's a separated
metadata section for concepts.

<img width="1251" alt="image"
src="https://github.com/lilacai/lilac/assets/1100749/35ef721a-23ba-4e4a-a843-7c339b0b3197">

https://huggingface.co/spaces/lilacai/nikhil_staging

Public nikhil concept:

https://lilacai-nikhil-staging.hf.space/concepts#100712716653593140239/aliens
Private nikhil concept (this should error):

https://lilacai-nikhil-staging.hf.space/concepts#100712716653593140239/aliens_private

<img width="1232" alt="image"
src="https://github.com/lilacai/lilac/assets/1100749/39ec7661-d7c9-40e2-8efb-1b7c10bbd5c6">

lilac/concepts/concept.py

@@ -71,6 +71,14 @@ class Example(ExampleIn):
   id: str
 
 
+class ConceptMetadata(BaseModel):
+  """Metadata associated with a concept."""
+  # True if the concept is publicly visible.
+  is_public: bool = False
+  tags: list[str] = []
+  description: Optional[str] = None
+
+
 class Concept(BaseModel):
   """A concept is a collection of examples."""
   # The namespace of the concept.
@@ -82,8 +90,7 @@ class Concept(BaseModel):
   data: dict[str, Example]
   version: int = 0
 
-  tags: list[str] = []
-  description: Optional[str] = None
+  metadata: Optional[ConceptMetadata] = None
 
   def drafts(self) -> list[DraftId]:
     """Gets all the drafts for the concept."""

lilac/concepts/db_concept.py

@@ -19,10 +19,18 @@
 
 from ..auth import ConceptAuthorizationException, UserInfo
 from ..env import data_path, env
-from ..schema import SignalInputType
 from ..signal import get_signal_cls
 from ..utils import delete_file, file_exists, get_lilac_cache_dir, open_file
-from .concept import DRAFT_MAIN, Concept, ConceptModel, ConceptType, DraftId, Example, ExampleIn
+from .concept import (
+  DRAFT_MAIN,
+  Concept,
+  ConceptMetadata,
+  ConceptModel,
+  ConceptType,
+  DraftId,
+  Example,
+  ExampleIn,
+)
 
 CONCEPTS_DIR = 'concept'
 CONCEPT_JSON_FILENAME = 'concept.json'
@@ -51,10 +59,10 @@ class ConceptInfo(BaseModel):
   """Information about a concept."""
   namespace: str
   name: str
-  description: Optional[str] = None
   type: ConceptType
+  metadata: ConceptMetadata
+
   drafts: list[DraftId]
-  tags: list[str] = []
 
   acls: ConceptACL
 
@@ -99,15 +107,15 @@ def create(self,
              namespace: str,
              name: str,
              type: Union[ConceptType, str],
-             description: Optional[str] = None,
+             metadata: Optional[ConceptMetadata] = None,
              user: Optional[UserInfo] = None) -> Concept:
     """Create a concept.
 
     Args:
       namespace: The namespace of the concept.
       name: The name of the concept.
       type: The type of the concept.
-      description: The description of the concept.
+      metadata: The metadata for the concept, including tags, description, and visibility.
       user: The user creating the concept, if authentication is enabled.
     """
     pass
@@ -121,6 +129,22 @@ def edit(self,
     """Edit a concept. If the concept doesn't exist, throw an error."""
     pass
 
+  @abc.abstractmethod
+  def update_metadata(self,
+                      namespace: str,
+                      name: str,
+                      metadata: ConceptMetadata,
+                      user: Optional[UserInfo] = None) -> None:
+    """Update the metadata of a concept.
+
+    Args:
+      namespace: The namespace of the concept.
+      name: The name of the concept.
+      metadata: The metadata to update.
+      user: The user updating the metadata, if authentication is enabled.
+    """
+    pass
+
   @abc.abstractmethod
   def remove(self, namespace: str, name: str, user: Optional[UserInfo] = None) -> None:
     """Remove a concept."""
@@ -349,9 +373,15 @@ def namespace_acls(self, namespace: str, user: Optional[UserInfo] = None) -> Con
   @override
   def concept_acls(self, namespace: str, name: str, user: Optional[UserInfo] = None) -> ConceptACL:
     namespace_acls = self.namespace_acls(namespace, user=user)
+    read = namespace_acls.read
+    if env('LILAC_AUTH_ENABLED'):
+      concept = self._read_concept(namespace, name)
+      if concept and concept.metadata and concept.metadata.is_public:
+        read = True
+
     # Concept ACL inherit from the namespace ACL. We currently don't have concept-specific
     #  ACL.
-    return ConceptACL(read=namespace_acls.read, write=namespace_acls.write)
+    return ConceptACL(read=read, write=namespace_acls.write)
 
   @override
   def list(self, user: Optional[UserInfo] = None) -> list[ConceptInfo]:
@@ -396,7 +426,9 @@ def get(self, namespace: str, name: str, user: Optional[UserInfo] = None) -> Opt
     if not acls.read:
       raise ConceptAuthorizationException(
         f'Concept "{namespace}/{name}" does not exist or user does not have access.')
+    return self._read_concept(namespace, name)
 
+  def _read_concept(self, namespace: str, name: str) -> Optional[Concept]:
     concept_json_path = _concept_json_path(self._get_base_dir(), namespace, name)
     if not file_exists(concept_json_path):
       return None
@@ -412,7 +444,7 @@ def create(self,
              namespace: str,
              name: str,
              type: Union[ConceptType, str] = ConceptType.TEXT,
-             description: Optional[str] = None,
+             metadata: Optional[ConceptMetadata] = None,
              user: Optional[UserInfo] = None) -> Concept:
     """Create a concept."""
     # If the user does not have access to the write to the concept namespace, throw.
@@ -427,8 +459,7 @@ def create(self,
 
     if isinstance(type, str):
       type = ConceptType(type)
-    concept = Concept(
-      namespace=namespace, concept_name=name, type=type, data={}, description=description)
+    concept = Concept(namespace=namespace, concept_name=name, type=type, data={}, metadata=metadata)
     self._save(concept)
     return concept
 
@@ -439,6 +470,20 @@ def _validate_examples(self, examples: List[Union[ExampleIn, Example]],
       if inferred_type != type:
         raise ValueError(f'Example type "{inferred_type}" does not match concept type "{type}".')
 
+  @override
+  def update_metadata(self,
+                      namespace: str,
+                      name: str,
+                      metadata: ConceptMetadata,
+                      user: Optional[UserInfo] = None) -> None:
+    concept = self.get(namespace, name, user=user)
+    if not concept:
+      raise ValueError('Concept with namespace "{namespace}" and name "{name}" does not exist.')
+
+    concept.metadata = metadata
+
+    self._save(concept)
+
   @override
   def edit(self,
            namespace: str,
@@ -552,13 +597,13 @@ def merge_draft(self,
 
 
 def _info_from_concept(concept: Concept, acls: ConceptACL) -> ConceptInfo:
+  metadata = concept.metadata or ConceptMetadata()
   return ConceptInfo(
     namespace=concept.namespace,
     name=concept.concept_name,
-    description=concept.description,
-    type=SignalInputType.TEXT,
+    metadata=metadata,
+    type=concept.type,
     drafts=concept.drafts(),
-    tags=concept.tags,
     acls=acls)
 
 

lilac/concepts/db_concept_test.py

@@ -17,6 +17,7 @@
 from .concept import (
   DRAFT_MAIN,
   Concept,
+  ConceptMetadata,
   ConceptModel,
   ConceptType,
   DraftId,
@@ -98,7 +99,8 @@ def test_create_concept(self, db_cls: Type[ConceptDB]) -> None:
         name='test_concept',
         type=ConceptType.TEXT,
         drafts=[DRAFT_MAIN],
-        acls=ConceptACL(read=True, write=True))
+        acls=ConceptACL(read=True, write=True),
+        metadata=ConceptMetadata())
     ]
 
     # Make sure list with drafts relects the drafts.
@@ -117,7 +119,42 @@ def test_create_concept(self, db_cls: Type[ConceptDB]) -> None:
         name='test_concept',
         type=ConceptType.TEXT,
         drafts=[DRAFT_MAIN, 'test_draft'],
-        acls=ConceptACL(read=True, write=True))
+        acls=ConceptACL(read=True, write=True),
+        metadata=ConceptMetadata())
+    ]
+
+  def test_update_metadata(self, db_cls: Type[ConceptDB]) -> None:
+    db = db_cls()
+    db.create(namespace='test', name='test_concept', type=ConceptType.TEXT)
+
+    # Remove lilac concepts.
+    concepts = list(filter(lambda c: c.namespace != 'lilac', db.list()))
+
+    assert concepts == [
+      ConceptInfo(
+        namespace='test',
+        name='test_concept',
+        type=ConceptType.TEXT,
+        drafts=[DRAFT_MAIN],
+        acls=ConceptACL(read=True, write=True),
+        metadata=ConceptMetadata())
+    ]
+
+    db.update_metadata(
+      'test', 'test_concept',
+      ConceptMetadata(is_public=True, tags=['test_tag'], description='test description'))
+
+    # Remove lilac concepts.
+    concepts = list(filter(lambda c: c.namespace != 'lilac', db.list()))
+
+    assert concepts == [
+      ConceptInfo(
+        namespace='test',
+        name='test_concept',
+        type=ConceptType.TEXT,
+        drafts=[DRAFT_MAIN],
+        acls=ConceptACL(read=True, write=True),
+        metadata=ConceptMetadata(is_public=True, tags=['test_tag'], description='test description'))
     ]
 
   def test_add_example(self, db_cls: Type[ConceptDB]) -> None:

lilac/concepts/legal-termination/concept.json

@@ -2,7 +2,12 @@
   "namespace": "lilac",
   "concept_name": "legal-termination",
   "type": "text",
-  "tags": ["legal"],
+  "metadata": {
+    "is_public": true,
+    "tags": ["legal"],
+    "description": "Termination or survival clause in a legal document",
+    "version": 33
+  },
   "data": {
     "731b1338cf1949958c3526c555f88058": {
       "label": true,
@@ -179,7 +184,5 @@
       "text": "terminate in accordance with Section 2 of the Investors\u2019 Rights Agreement.",
       "id": "907f92e0d5704418944a559a4bfb96c7"
     }
-  },
-  "version": 33,
-  "description": "Termination or survival clause in a legal document"
+  }
 }

lilac/concepts/negative-sentiment/concept.json

@@ -2,6 +2,10 @@
   "namespace": "lilac",
   "concept_name": "negative-sentiment",
   "type": "text",
+  "metadata": {
+    "is_public": true,
+    "description": "Negative sentiment."
+  },
   "data": {
     "0": {
       "label": true,
@@ -628,7 +632,5 @@
       "text": "Mr Bournelis suggested all 30 lineal metres of blockwork should be removed and replaced, which would require removing and reinstalling the fence. The total cost of his suggested method of rectification was said to be $14,650 for each unit, giving a total cost of rectification of $29,300.",
       "id": "5471008376cf44518f2ff1f67f057c08"
     }
-  },
-  "version": 27,
-  "description": "Negative sentiment"
-}
+  }
+}

lilac/concepts/non-english/concept.json

@@ -2,6 +2,10 @@
   "namespace": "lilac",
   "concept_name": "non-english",
   "type": "text",
+  "metadata": {
+    "is_public": true,
+    "description": "Text that contains non-English."
+  },
   "data": {
     "c727f30a2d2d40f69b81aa981515fb62": {
       "label": true,
@@ -1019,6 +1023,5 @@
       "id": "95cc03b8508b44a18a4aee4b27743f1f"
     }
   },
-  "version": 189,
-  "description": "Text that contains non-English."
-}
+  "version": 189
+}

lilac/concepts/positive-sentiment/concept.json

@@ -2,6 +2,10 @@
   "namespace": "lilac",
   "concept_name": "positive-sentiment",
   "type": "text",
+  "metadata": {
+    "is_public": true,
+    "description": "Positive sentiment."
+  },
   "data": {
     "0": {
       "label": false,
@@ -559,6 +563,5 @@
       "id": "abf2d24c7d8845769b7368be28f2c25d"
     }
   },
-  "version": 11,
-  "description": "Positive sentiment"
+  "version": 11
 }

lilac/concepts/profanity/concept.json

@@ -2,6 +2,10 @@
   "namespace": "lilac",
   "concept_name": "profanity",
   "type": "text",
+  "metadata": {
+    "is_public": true,
+    "description": "Profane or obscene language."
+  },
   "data": {
     "0": {
       "label": true,
@@ -5294,6 +5298,5 @@
       "id": "0e4484fa362a4559a4050e8ef0718684"
     }
   },
-  "version": 34,
-  "description": "Profane or obscene language"
-}
+  "version": 34
+}

lilac/concepts/question/concept.json

@@ -2,6 +2,10 @@
   "namespace": "lilac",
   "concept_name": "question",
   "type": "text",
+  "metadata": {
+    "is_public": true,
+    "description": "Text in the form of a question."
+  },
   "data": {
     "09a1ce39f573494a8d4a47a4fce0f523": {
       "label": true,
@@ -1894,6 +1898,5 @@
       "id": "8e7cff1f58774a45a24c895782d4689b"
     }
   },
-  "version": 373,
-  "description": "A sentence in the form of a question"
-}
+  "version": 373
+}

lilac/concepts/source-code/concept.json

@@ -2,6 +2,10 @@
   "namespace": "lilac",
   "concept_name": "source-code",
   "type": "text",
+  "metadata": {
+    "is_public": true,
+    "description": "Source code from any programming language."
+  },
   "data": {
     "c7d0400c6e5442a59859ea7b0a7d6bab": {
       "label": true,
@@ -384,6 +388,5 @@
       "id": "8d6eb8f1aac745359540c9f454aa9a58"
     }
   },
-  "version": 62,
-  "description": "Source code for a programming language."
-}
+  "version": 62
+}