refactor(kag): refactor builder components

.github/workflows/code-format-check.yml → .github/workflows/code-check.yml

@@ -1,4 +1,4 @@
-name: Code Format Check
+name: CI
 
 on:
   push:
@@ -22,7 +22,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pre-commit
+          pip install pre-commit pytest pytest-cov
+          pip install -r requirements.txt
+          pip install -e .
 
       - name: Run pre-commit
-        run: pre-commit run --all-files
+        run: pre-commit run --all-files
+
+      # - name: Run unit tests
+      #   run: pushd tests/unit && pytest && popd
+

kag/__init__.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # Apache License
 # Version 2.0, January 2004
 # http://www.apache.org/licenses/
@@ -204,6 +205,15 @@
 __package_name__ = "openspg-kag"
 __version__ = "0.5.2-beta1"
 
-from kag.common.env import init_env
+# Register Built-in Components
+from kag.common.conf import init_env
 
 init_env()
+
+import kag.interface
+import kag.builder.component
+import kag.builder.prompt
+import kag.solver.prompt
+import kag.common.vectorize_model
+import kag.common.llm
+import kag.solver

kag/builder/__init__.py

@@ -1,10 +0,0 @@
-# Copyright 2023 OpenSPG Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-# in compliance with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-# or implied.

kag/builder/component/__init__.py

@@ -10,13 +10,63 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied.
 
+from kag.builder.component.external_graph.external_graph import (
+    DefaultExternalGraphLoader,
+)
+from kag.builder.component.extractor.kag_extractor import KAGExtractor
+from kag.builder.component.extractor.spg_extractor import SPGExtractor
+from kag.builder.component.aligner.kag_aligner import KAGAligner
+from kag.builder.component.aligner.spg_aligner import SPGAligner
+from kag.builder.component.postprocessor.kag_postprocessor import KAGPostProcessor
+
 from kag.builder.component.mapping.spg_type_mapping import SPGTypeMapping
 from kag.builder.component.mapping.relation_mapping import RelationMapping
+from kag.builder.component.mapping.spo_mapping import SPOMapping
+from kag.builder.component.reader.csv_reader import CSVReader
+from kag.builder.component.reader.pdf_reader import PDFReader
+from kag.builder.component.reader.json_reader import JSONReader
+from kag.builder.component.reader.markdown_reader import MarkDownReader
+from kag.builder.component.reader.docx_reader import DocxReader
+from kag.builder.component.reader.txt_reader import TXTReader
+from kag.builder.component.reader.dataset_reader import (
+    HotpotqaCorpusReader,
+    TwowikiCorpusReader,
+    MusiqueCorpusReader,
+)
+from kag.builder.component.reader.yuque_reader import YuqueReader
+from kag.builder.component.splitter.length_splitter import LengthSplitter
+from kag.builder.component.splitter.pattern_splitter import PatternSplitter
+from kag.builder.component.splitter.outline_splitter import OutlineSplitter
+from kag.builder.component.splitter.semantic_splitter import SemanticSplitter
+from kag.builder.component.vectorizer.batch_vectorizer import BatchVectorizer
 from kag.builder.component.writer.kg_writer import KGWriter
 
 
 __all__ = [
+    "DefaultExternalGraphLoader",
+    "KAGExtractor",
+    "SPGExtractor",
+    "KAGAligner",
+    "SPGAligner",
+    "KAGPostProcessor",
+    "KGWriter",
     "SPGTypeMapping",
     "RelationMapping",
+    "SPOMapping",
+    "TXTReader",
+    "PDFReader",
+    "MarkDownReader",
+    "JSONReader",
+    "HotpotqaCorpusReader",
+    "MusiqueCorpusReader",
+    "TwowikiCorpusReader",
+    "YuqueReader",
+    "CSVReader",
+    "DocxReader",
+    "LengthSplitter",
+    "PatternSplitter",
+    "OutlineSplitter",
+    "SemanticSplitter",
+    "BatchVectorizer",
     "KGWriter",
 ]

kag/builder/component/aligner/__init__.py

@@ -1,11 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2023 OpenSPG Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-# in compliance with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-# or implied.

kag/builder/component/aligner/kag_post_processor.py → kag/builder/component/aligner/kag_aligner.py

@@ -13,11 +13,12 @@
 from typing import List, Sequence, Dict, Type
 
 from kag.builder.model.sub_graph import SubGraph
-from kag.interface.builder import AlignerABC
+from kag.interface import AlignerABC
 from knext.common.base.runnable import Input, Output
 
 
-class KAGPostProcessorAligner(AlignerABC):
+@AlignerABC.register("kag")
+class KAGAligner(AlignerABC):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 

kag/builder/component/aligner/spg_post_processor.py → kag/builder/component/aligner/spg_aligner.py

@@ -12,19 +12,21 @@
 
 from typing import List, Type, Dict
 
-from kag.interface.builder import AlignerABC
+from kag.interface import AlignerABC
 from knext.schema.client import BASIC_TYPES
+from kag.common.conf import KAG_PROJECT_CONF
 from kag.builder.model.spg_record import SPGRecord
 from kag.builder.model.sub_graph import SubGraph
 from knext.common.base.runnable import Input, Output
 from knext.schema.client import SchemaClient
 from knext.schema.model.base import ConstraintTypeEnum, BaseSpgType
 
 
-class SPGPostProcessorAligner(AlignerABC):
+@AlignerABC.register("spg")
+class SPGAligner(AlignerABC):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.spg_types = SchemaClient(project_id=self.project_id).load()
+        self.spg_types = SchemaClient(project_id=KAG_PROJECT_CONF.project_id).load()
 
     @property
     def input_types(self) -> Type[Input]: