Resolving pathing and build dependencies between lm-buddy and Lumigat…

…or (#220) * path replace * adding build files * fix submission * lint
mozilla-ai · Sep 20, 2024 · d0e24bf · d0e24bf
1 parent d9a2f97
commit d0e24bf
Show file tree

Hide file tree

Showing 47 changed files with 2,418 additions and 2,525 deletions.
diff --git a/3rdparty/python/darwin.lock b/3rdparty/python/darwin.lock
diff --git a/3rdparty/python/linux_cpu.lock b/3rdparty/python/linux_cpu.lock
diff --git a/3rdparty/python/linux_cuda.lock b/3rdparty/python/linux_cuda.lock
diff --git a/3rdparty/python/pyproject.toml b/3rdparty/python/pyproject.toml
@@ -3,23 +3,6 @@ name = "lumigator-3rdparty-python-dependencies"
 version = "0.0.1"
 requires-python = "==3.11.9"
 
-## local lm-buddy can be made with the following url spec.
-## You have to build the wheel manually and clean out the other wheels that
-## are built there - e.g.,
-#
-## pushd $WORKSPACE/lm-buddy && \
-#    # pip wheel . --wheel-dir=$WORKSPACE/wheelhouse  && \
-#    # cd $WORKSPACE/wheelhouse &&  \
-#    # mv lm_buddy-<VERSION>-py3-none-any.whl lm_buddy-<VERSION>-py3-none-any.bak && \
-#    # rm *.whl && mv lm_buddy-<VERSION>-py3-none-any.bak  lm_buddy-<VERSION>-py3-none-any.whl && \
-#    # popd
-## after that you can uncomment the following and put it in the `dependencies` map:
-## "lm-buddy[dev] @ $WHEELS_DIR/",
-
-## warning - if you're developing with a copy of this in your system under $WHEELS_DIR,
-## in pants.toml, pants will use the local version even without the local URI
-## and things will break. make sure to clean it up before building; this is more a note for followups
-
 # this has to be an array of pep 508 strings, not the map style
 dependencies = [
     "accelerate>=0.25.0",

diff --git a/lumigator/python/mzai/lm_buddy/BUILD.pants b/lumigator/python/mzai/lm_buddy/BUILD.pants
@@ -0,0 +1,11 @@
+python_sources(
+    name="lm_buddy_sources",
+    sources=[
+        "**/*.py",
+        "!tests",
+    ],
+)
+
+python_requirements(
+    name="reqs",
+)
diff --git a/lumigator/python/mzai/lm_buddy/__init__.py b/lumigator/python/mzai/lm_buddy/__init__.py
@@ -1,3 +1,3 @@
-from lumigator.python.mzai.lm_buddy.buddy import LMBuddy
+from mzai.lm_buddy.buddy import LMBuddy
 
 __all__ = ["LMBuddy"]
diff --git a/lumigator/python/mzai/lm_buddy/__main__.py b/lumigator/python/mzai/lm_buddy/__main__.py
@@ -3,7 +3,7 @@
 Makes the LM Buddy CLI executable via `python -m lm_buddy`.
 """
 
-from lumigator.python.mzai.lm_buddy.cli import cli
+from mzai.lm_buddy.cli import cli
 
 if __name__ == "__main__":
     cli()
diff --git a/lumigator/python/mzai/lm_buddy/buddy.py b/lumigator/python/mzai/lm_buddy/buddy.py
@@ -1,6 +1,6 @@
 import wandb
 
-from lumigator.python.mzai.lm_buddy.configs.jobs import (
+from mzai.lm_buddy.configs.jobs import (
     EvaluationJobConfig,
     FinetuningJobConfig,
     HuggingFaceEvalJobConfig,
@@ -9,18 +9,18 @@
     PrometheusJobConfig,
     RagasJobConfig,
 )
-from lumigator.python.mzai.lm_buddy.jobs.common import (
+from mzai.lm_buddy.jobs.common import (
     EvaluationResult,
     FinetuningResult,
     JobType,
 )
-from lumigator.python.mzai.lm_buddy.jobs.evaluation.hf_evaluate import run_hf_evaluation
-from lumigator.python.mzai.lm_buddy.jobs.evaluation.lm_harness import run_lm_harness
-from lumigator.python.mzai.lm_buddy.jobs.evaluation.prometheus import run_prometheus
-from lumigator.python.mzai.lm_buddy.jobs.evaluation.ragas import run_ragas
-from lumigator.python.mzai.lm_buddy.jobs.finetuning import run_finetuning
-from lumigator.python.mzai.lm_buddy.paths import strip_path_prefix
-from lumigator.python.mzai.lm_buddy.tracking.run_utils import WandbResumeMode
+from mzai.lm_buddy.jobs.evaluation.hf_evaluate import run_hf_evaluation
+from mzai.lm_buddy.jobs.evaluation.lm_harness import run_lm_harness
+from mzai.lm_buddy.jobs.evaluation.prometheus import run_prometheus
+from mzai.lm_buddy.jobs.evaluation.ragas import run_ragas
+from mzai.lm_buddy.jobs.finetuning import run_finetuning
+from mzai.lm_buddy.paths import strip_path_prefix
+from mzai.lm_buddy.tracking.run_utils import WandbResumeMode
 
 
 class LMBuddy:

diff --git a/lumigator/python/mzai/lm_buddy/cli/__init__.py b/lumigator/python/mzai/lm_buddy/cli/__init__.py
@@ -1,6 +1,6 @@
 import click
 
-from lumigator.python.mzai.lm_buddy.cli import evaluate, finetune
+from mzai.lm_buddy.cli import evaluate, finetune
 
 
 @click.group(name="LM Buddy CLI", help="Entrypoints for the LM Buddy.")

diff --git a/lumigator/python/mzai/lm_buddy/cli/evaluate.py b/lumigator/python/mzai/lm_buddy/cli/evaluate.py
@@ -1,8 +1,8 @@
 import click
 
-from lumigator.python.mzai.lm_buddy import LMBuddy
-from lumigator.python.mzai.lm_buddy.cli.utils import parse_config_option
-from lumigator.python.mzai.lm_buddy.configs.jobs import (
+from mzai.lm_buddy import LMBuddy
+from mzai.lm_buddy.cli.utils import parse_config_option
+from mzai.lm_buddy.configs.jobs import (
     HuggingFaceEvalJobConfig,
     LMHarnessJobConfig,
     PrometheusJobConfig,

diff --git a/lumigator/python/mzai/lm_buddy/cli/finetune.py b/lumigator/python/mzai/lm_buddy/cli/finetune.py
@@ -1,8 +1,8 @@
 import click
 
-from lumigator.python.mzai.lm_buddy import LMBuddy
-from lumigator.python.mzai.lm_buddy.cli.utils import parse_config_option
-from lumigator.python.mzai.lm_buddy.configs.jobs import FinetuningJobConfig
+from mzai.lm_buddy import LMBuddy
+from mzai.lm_buddy.cli.utils import parse_config_option
+from mzai.lm_buddy.configs.jobs import FinetuningJobConfig
 
 
 @click.command(name="finetune", help="Run an LM Buddy finetuning job.")

diff --git a/lumigator/python/mzai/lm_buddy/cli/utils.py b/lumigator/python/mzai/lm_buddy/cli/utils.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import TypeVar
 
-from lumigator.python.mzai.lm_buddy.configs.jobs.common import JobConfig
+from mzai.lm_buddy.configs.jobs.common import JobConfig
 
 ConfigType = TypeVar("ConfigType", bound=JobConfig)
 

diff --git a/lumigator/python/mzai/lm_buddy/configs/huggingface.py b/lumigator/python/mzai/lm_buddy/configs/huggingface.py
@@ -1,11 +1,11 @@
 import dataclasses
 from typing import Any
 
-from lumigator.python.mzai.lm_buddy.configs.common import (
+from mzai.lm_buddy.configs.common import (
     LMBuddyConfig,
     SerializableTorchDtype,
 )
-from lumigator.python.mzai.lm_buddy.paths import AssetPath, PathPrefix
+from mzai.lm_buddy.paths import AssetPath, PathPrefix
 from peft import PeftConfig, PeftType, TaskType
 from pydantic import field_validator, model_validator
 from transformers import BitsAndBytesConfig

diff --git a/lumigator/python/mzai/lm_buddy/configs/jobs/__init__.py b/lumigator/python/mzai/lm_buddy/configs/jobs/__init__.py
@@ -1,9 +1,9 @@
-from lumigator.python.mzai.lm_buddy.configs.jobs.common import JobConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.finetuning import FinetuningJobConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.hf_evaluate import HuggingFaceEvalJobConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.lm_harness import LMHarnessJobConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.prometheus import PrometheusJobConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.ragas import RagasJobConfig
+from mzai.lm_buddy.configs.jobs.common import JobConfig
+from mzai.lm_buddy.configs.jobs.finetuning import FinetuningJobConfig
+from mzai.lm_buddy.configs.jobs.hf_evaluate import HuggingFaceEvalJobConfig
+from mzai.lm_buddy.configs.jobs.lm_harness import LMHarnessJobConfig
+from mzai.lm_buddy.configs.jobs.prometheus import PrometheusJobConfig
+from mzai.lm_buddy.configs.jobs.ragas import RagasJobConfig
 
 EvaluationJobConfig = (
     LMHarnessJobConfig | PrometheusJobConfig | RagasJobConfig | HuggingFaceEvalJobConfig

diff --git a/lumigator/python/mzai/lm_buddy/configs/jobs/common.py b/lumigator/python/mzai/lm_buddy/configs/jobs/common.py
@@ -6,9 +6,9 @@
 from pydantic import Field
 from pydantic_yaml import parse_yaml_file_as, to_yaml_file
 
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
-from lumigator.python.mzai.lm_buddy.configs.wandb import WandbRunConfig
-from lumigator.python.mzai.lm_buddy.paths import AssetPath, PathPrefix
+from mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.configs.wandb import WandbRunConfig
+from mzai.lm_buddy.paths import AssetPath, PathPrefix
 
 
 class JobConfig(LMBuddyConfig):

diff --git a/lumigator/python/mzai/lm_buddy/configs/jobs/finetuning.py b/lumigator/python/mzai/lm_buddy/configs/jobs/finetuning.py
@@ -1,16 +1,16 @@
 from pydantic import Field, field_validator, model_validator
 
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
-from lumigator.python.mzai.lm_buddy.configs.huggingface import (
+from mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.configs.huggingface import (
     AdapterConfig,
     AutoModelConfig,
     AutoTokenizerConfig,
     DatasetConfig,
     QuantizationConfig,
     TrainerConfig,
 )
-from lumigator.python.mzai.lm_buddy.configs.jobs.common import JobConfig
-from lumigator.python.mzai.lm_buddy.paths import AssetPath
+from mzai.lm_buddy.configs.jobs.common import JobConfig
+from mzai.lm_buddy.paths import AssetPath
 
 
 class FinetuningRayConfig(LMBuddyConfig):

diff --git a/lumigator/python/mzai/lm_buddy/configs/jobs/hf_evaluate.py b/lumigator/python/mzai/lm_buddy/configs/jobs/hf_evaluate.py
@@ -1,15 +1,15 @@
 from pydantic import Field, conlist, field_validator, model_validator
 
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
-from lumigator.python.mzai.lm_buddy.configs.huggingface import (
+from mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.configs.huggingface import (
     AutoModelConfig,
     AutoTokenizerConfig,
     DatasetConfig,
     QuantizationConfig,
 )
-from lumigator.python.mzai.lm_buddy.configs.jobs.common import JobConfig
-from lumigator.python.mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
-from lumigator.python.mzai.lm_buddy.paths import AssetPath
+from mzai.lm_buddy.configs.jobs.common import JobConfig
+from mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
+from mzai.lm_buddy.paths import AssetPath
 
 
 class HuggingFaceEvaluationConfig(LMBuddyConfig):

diff --git a/lumigator/python/mzai/lm_buddy/configs/jobs/lm_harness.py b/lumigator/python/mzai/lm_buddy/configs/jobs/lm_harness.py
@@ -2,14 +2,14 @@
 
 from pydantic import conlist, model_validator
 
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
-from lumigator.python.mzai.lm_buddy.configs.huggingface import (
+from mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.configs.huggingface import (
     AutoModelConfig,
     QuantizationConfig,
 )
-from lumigator.python.mzai.lm_buddy.configs.jobs.common import JobConfig
-from lumigator.python.mzai.lm_buddy.configs.vllm import InferenceServerConfig
-from lumigator.python.mzai.lm_buddy.paths import AssetPath
+from mzai.lm_buddy.configs.jobs.common import JobConfig
+from mzai.lm_buddy.configs.vllm import InferenceServerConfig
+from mzai.lm_buddy.paths import AssetPath
 
 
 class LocalChatCompletionsConfig(LMBuddyConfig):

diff --git a/lumigator/python/mzai/lm_buddy/configs/jobs/prometheus.py b/lumigator/python/mzai/lm_buddy/configs/jobs/prometheus.py
@@ -1,10 +1,10 @@
 from pydantic import Field
 
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
-from lumigator.python.mzai.lm_buddy.configs.huggingface import DatasetConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.common import JobConfig
-from lumigator.python.mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
-from lumigator.python.mzai.lm_buddy.paths import AssetPath
+from mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.configs.huggingface import DatasetConfig
+from mzai.lm_buddy.configs.jobs.common import JobConfig
+from mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
+from mzai.lm_buddy.paths import AssetPath
 
 
 class PrometheusEvaluationConfig(LMBuddyConfig):

diff --git a/lumigator/python/mzai/lm_buddy/configs/jobs/ragas.py b/lumigator/python/mzai/lm_buddy/configs/jobs/ragas.py
@@ -1,13 +1,13 @@
 from typing import Literal, get_args
 
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
-from lumigator.python.mzai.lm_buddy.configs.huggingface import (
+from mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.configs.huggingface import (
     AutoModelConfig,
     DatasetConfig,
 )
-from lumigator.python.mzai.lm_buddy.configs.jobs.common import JobConfig
-from lumigator.python.mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
-from lumigator.python.mzai.lm_buddy.paths import AssetPath
+from mzai.lm_buddy.configs.jobs.common import JobConfig
+from mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
+from mzai.lm_buddy.paths import AssetPath
 from pydantic import Field, field_validator
 
 RagasEvaluationMetric = Literal[

diff --git a/lumigator/python/mzai/lm_buddy/configs/vllm.py b/lumigator/python/mzai/lm_buddy/configs/vllm.py
@@ -1,5 +1,5 @@
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
-from lumigator.python.mzai.lm_buddy.paths import AssetPath
+from mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.paths import AssetPath
 
 
 class InferenceServerConfig(LMBuddyConfig):

diff --git a/lumigator/python/mzai/lm_buddy/configs/wandb.py b/lumigator/python/mzai/lm_buddy/configs/wandb.py
@@ -1,7 +1,7 @@
 import os
 import warnings
 
-from lumigator.python.mzai.lm_buddy.configs.common import LMBuddyConfig
+from mzai.lm_buddy.configs.common import LMBuddyConfig
 from pydantic import model_validator
 from wandb.apis.public import Run
 from wandb.util import random_string

diff --git a/lumigator/python/mzai/lm_buddy/jobs/asset_loader.py b/lumigator/python/mzai/lm_buddy/jobs/asset_loader.py
@@ -4,14 +4,14 @@
 from accelerate import Accelerator
 from datasets import Dataset, DatasetDict, load_dataset, load_from_disk
 from loguru import logger
-from lumigator.python.mzai.lm_buddy.configs.huggingface import (
+from mzai.lm_buddy.configs.huggingface import (
     AutoModelConfig,
     AutoTokenizerConfig,
     DatasetConfig,
     QuantizationConfig,
 )
-from lumigator.python.mzai.lm_buddy.paths import AssetPath, PathPrefix, strip_path_prefix
-from lumigator.python.mzai.lm_buddy.tracking.artifact_utils import (
+from mzai.lm_buddy.paths import AssetPath, PathPrefix, strip_path_prefix
+from mzai.lm_buddy.tracking.artifact_utils import (
     get_artifact_directory,
     get_artifact_from_api,
 )

diff --git a/lumigator/python/mzai/lm_buddy/jobs/evaluation/hf_evaluate.py b/lumigator/python/mzai/lm_buddy/jobs/evaluation/hf_evaluate.py
@@ -6,23 +6,23 @@
 
 import s3fs
 from loguru import logger
-from lumigator.python.mzai.lm_buddy.configs.jobs.hf_evaluate import HuggingFaceEvalJobConfig
-from lumigator.python.mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
-from lumigator.python.mzai.lm_buddy.constants import LM_BUDDY_RESULTS_PATH
-from lumigator.python.mzai.lm_buddy.jobs.asset_loader import (
+from mzai.lm_buddy.configs.jobs.hf_evaluate import HuggingFaceEvalJobConfig
+from mzai.lm_buddy.configs.vllm import VLLMCompletionsConfig
+from mzai.lm_buddy.constants import LM_BUDDY_RESULTS_PATH
+from mzai.lm_buddy.jobs.asset_loader import (
     HuggingFaceDatasetLoader,
     HuggingFaceModelLoader,
 )
-from lumigator.python.mzai.lm_buddy.jobs.common import EvaluationResult
-from lumigator.python.mzai.lm_buddy.jobs.evaluation.metrics import EvaluationMetrics
-from lumigator.python.mzai.lm_buddy.jobs.model_clients import (
+from mzai.lm_buddy.jobs.common import EvaluationResult
+from mzai.lm_buddy.jobs.evaluation.metrics import EvaluationMetrics
+from mzai.lm_buddy.jobs.model_clients import (
     BaseModelClient,
     HuggingFaceModelClient,
     MistralModelClient,
     OpenAIModelClient,
     SummarizationPipelineModelClient,
 )
-from lumigator.python.mzai.lm_buddy.jobs.utils import timer
+from mzai.lm_buddy.jobs.utils import timer
 from tqdm import tqdm
 
 

diff --git a/lumigator/python/mzai/lm_buddy/jobs/evaluation/lm_harness.py b/lumigator/python/mzai/lm_buddy/jobs/evaluation/lm_harness.py
@@ -7,14 +7,14 @@
 from lm_eval.models.openai_completions import OpenaiCompletionsLM
 from loguru import logger
 
-from lumigator.python.mzai.lm_buddy.configs.huggingface import AutoModelConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.lm_harness import (
+from mzai.lm_buddy.configs.huggingface import AutoModelConfig
+from mzai.lm_buddy.configs.jobs.lm_harness import (
     LMHarnessJobConfig,
     LocalChatCompletionsConfig,
 )
-from lumigator.python.mzai.lm_buddy.jobs.asset_loader import HuggingFaceModelLoader
-from lumigator.python.mzai.lm_buddy.jobs.common import EvaluationResult
-from lumigator.python.mzai.lm_buddy.tracking.artifact_utils import (
+from mzai.lm_buddy.jobs.asset_loader import HuggingFaceModelLoader
+from mzai.lm_buddy.jobs.common import EvaluationResult
+from mzai.lm_buddy.tracking.artifact_utils import (
     ArtifactType,
     build_table_artifact,
     default_artifact_name,

diff --git a/lumigator/python/mzai/lm_buddy/jobs/evaluation/prometheus.py b/lumigator/python/mzai/lm_buddy/jobs/evaluation/prometheus.py
@@ -9,18 +9,18 @@
 
 from datasets import Dataset
 from loguru import logger
-from lumigator.python.mzai.lm_buddy.configs.huggingface import AutoTokenizerConfig
-from lumigator.python.mzai.lm_buddy.configs.jobs.prometheus import PrometheusJobConfig
-from lumigator.python.mzai.lm_buddy.constants import LM_BUDDY_RESULTS_PATH
-from lumigator.python.mzai.lm_buddy.jobs.asset_loader import (
+from mzai.lm_buddy.configs.huggingface import AutoTokenizerConfig
+from mzai.lm_buddy.configs.jobs.prometheus import PrometheusJobConfig
+from mzai.lm_buddy.constants import LM_BUDDY_RESULTS_PATH
+from mzai.lm_buddy.jobs.asset_loader import (
     HuggingFaceDatasetLoader,
     HuggingFaceModelLoader,
     HuggingFaceTokenizerLoader,
 )
-from lumigator.python.mzai.lm_buddy.jobs.common import EvaluationResult
-from lumigator.python.mzai.lm_buddy.jobs.evaluation.conversation import get_conv_template
-from lumigator.python.mzai.lm_buddy.preprocessing import format_dataset_with_prompt
-from lumigator.python.mzai.lm_buddy.tracking.artifact_utils import (
+from mzai.lm_buddy.jobs.common import EvaluationResult
+from mzai.lm_buddy.jobs.evaluation.conversation import get_conv_template
+from mzai.lm_buddy.preprocessing import format_dataset_with_prompt
+from mzai.lm_buddy.tracking.artifact_utils import (
     ArtifactType,
     build_directory_artifact,
     default_artifact_name,