DatasetRecords: The provided batch size 256 was normalized. Using value 20.\n", + "\n" + ], + "text/plain": [ + "DatasetRecords: The provided batch size \u001b[1;36m256\u001b[0m was normalized. Using value \u001b[1;36m20\u001b[0m.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Sending records...: 100%|██████████| 1/1 [00:00<00:00, 3.50batch/s]\n" + ] + } + ], + "source": [ + "import os\n", + "from uuid import uuid4\n", + "\n", + "import argilla as rg\n", + "\n", + "client = rg.Argilla()\n", + "\n", + "labels = [\"positive\", \"negative\", \"neutral\"]\n", + "\n", + "settings = rg.Settings(\n", + " guidelines=\"Label the sentiment of the text\",\n", + " fields=[\n", + " rg.TextField(\n", + " name=\"text\",\n", + " title=\"Text\",\n", + " description=\"A review of a pc component\",\n", + " )\n", + " ],\n", + " questions=[\n", + " rg.LabelQuestion(\n", + " name=\"sentiment\",\n", + " title=\"sentiment\",\n", + " description=\"Provide a single label for the sentiment of the text\",\n", + " labels=labels,\n", + " )\n", + " ],\n", + " # mapping={\"sentiment\": \"label\"},\n", + ")\n", + "\n", + "dataset_name = f\"pc-component-{uuid4()}\"\n", + "\n", + "dataset = rg.Dataset.from_hub(\n", + " repo_id=\"argilla/pc-components-reviews\",\n", + " name=dataset_name,\n", + " split=\"train[:20]\",\n", + " client=client,\n", + " with_records=True,\n", + " settings=settings,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
DatasetRecords: The provided batch size 256 was normalized. Using value 20.\n", + "\n" + ], + "text/plain": [ + "DatasetRecords: The provided batch size \u001b[1;36m256\u001b[0m was normalized. Using value \u001b[1;36m20\u001b[0m.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Sending records...: 100%|██████████| 1/1 [00:00<00:00, 3.09batch/s]\n" + ] + }, + { + "data": { + "text/plain": [ + "DatasetRecords(Dataset(id=UUID('a27327ea-2ee4-4b44-afcb-ed19724a3866') inserted_at=datetime.datetime(2024, 10, 14, 9, 23, 11, 205431) updated_at=datetime.datetime(2024, 10, 14, 9, 23, 11, 818327) name='pc-component-1b5a4aff-9bc9-4b27-8cec-bc21b57b3e34' status='ready' guidelines='Label the sentiment of the text' allow_extra_metadata=False distribution=OverlapTaskDistributionModel(strategy='overlap', min_submitted=1) workspace_id=UUID('108b045c-a82e-4c75-a61b-0cddfb22c4c8') last_activity_at=datetime.datetime(2024, 10, 14, 9, 23, 11, 818327)))" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "## ADD WITHOUT SUGGESTIONS\n", + "\n", + "# from datasets import load_dataset\n", + "\n", + "# updated_records = []\n", + "\n", + "# for sample in load_dataset(\"argilla/pc-components-reviews\", split=\"train[20:40]\"):\n", + "# _record = {\n", + "# \"text\": sample[\"text\"],\n", + "# }\n", + "# updated_records.append(_record)\n", + "\n", + "# dataset.records.log(updated_records)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Label the datasets\n", + "\n", + "We will now label the datasets. We will use the `ArgillaLabeller` class to label the datasets. This class will use will use a `LlamaCppLLM` LLM to label the datasets. These labels will then be converted into `rg.Suggestion` objects and added to the records. For the sake of the example, we will only label 5 records per time using a while loop that continuesly fetches pending records from Argilla for both datasets and labels them with the LLM. After the labelling, we will update the dataset with the new records." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import Counter\n", + "from distilabel.llms.llamacpp import LlamaCppLLM\n", + "from distilabel.steps.tasks import ArgillaLabeller\n", + "\n", + "dataset = client.datasets(name=dataset_name, workspace=\"argilla\")\n", + "\n", + "example_records = []\n", + "\n", + "counter = Counter()\n", + "max_samples = 16\n", + "\n", + "\n", + "for record in dataset.records(with_responses=True, with_suggestions=True):\n", + " value = record.suggestions[\"sentiment\"].value\n", + " counter[value] += 1\n", + " if counter[value] > 12:\n", + " continue\n", + " record.responses.add(\n", + " rg.Response(question_name=\"sentiment\", value=value, user_id=client.me)\n", + " )\n", + " example_records.append(record)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "example_records = [record.to_dict() for record in example_records]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "20" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(example_records)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "# randomly shuffle the records\n", + "\n", + "import random\n", + "\n", + "random.shuffle(example_records)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Distilabel can label your records\n", + "\n", + "We can use the `process` method of the `ArgillaLabeller` class to label the records. This method will label the records with the LLM and update the records with the new labels. We will use this method to label the records of the datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "\n", + "train_dataset = load_dataset(\"argilla/pc-components-reviews\", split=\"train[:20]\")\n", + "\n", + "train_dataset = train_dataset.remove_columns(\n", + " [col for col in train_dataset.column_names if col not in [\"text\", \"sentiment\"]]\n", + ")\n", + "\n", + "test_dataset = load_dataset(\"argilla/pc-components-reviews\", split=\"train[20:100]\")\n", + "\n", + "test_dataset = test_dataset.remove_columns(\n", + " [col for col in test_dataset.column_names if col not in [\"text\", \"sentiment\"]]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
| \n", + " | sentiment | \n", + "text | \n", + "
|---|---|---|
| 0 | \n", + "positive | \n", + "I finally got my hands on the Ryzen 9 7950X an... | \n", + "
| 1 | \n", + "positive | \n", + "I recently bought the Intel Core i5-10400F and... | \n", + "
| 2 | \n", + "positive | \n", + "I just upgraded to the Ryzen 9 5950X and it's ... | \n", + "
| 3 | \n", + "positive | \n", + "I just swapped out my old GPU for the RTX 4070... | \n", + "
| 4 | \n", + "negative | \n", + "I just upgraded to the new Intel Core i9-13900... | \n", + "
| 5 | \n", + "neutral | \n", + "Just got the new Intel Core i9-13900K, and I m... | \n", + "
| 6 | \n", + "negative | \n", + "I just got the new Ryzen 5 5600X and it’s not ... | \n", + "
| 7 | \n", + "positive | \n", + "Just built my new rig with the Ryzen 5 5600X a... | \n", + "
| 8 | \n", + "neutral | \n", + "I recently upgraded my PC with the AMD Ryzen 5... | \n", + "
| 9 | \n", + "positive | \n", + "I just built my first PC with an AMD Ryzen 5 3... | \n", + "
| 10 | \n", + "negative | \n", + "I just installed the new Ryzen 7 5800X and the... | \n", + "
| 11 | \n", + "neutral | \n", + "Hey, I just got the Ryzen 5 3600 and it’s actu... | \n", + "
| 12 | \n", + "negative | \n", + "I just got this new ASUS ROG Strix Z690-E Gami... | \n", + "
| 13 | \n", + "neutral | \n", + "I recently swapped out my old GPU for the NVID... | \n", + "
| 14 | \n", + "neutral | \n", + "The Ryzen 9 5950X really shines in multi-threa... | \n", + "
| 15 | \n", + "positive | \n", + "I just built my new rig with the Ryzen 9 5900X... | \n", + "
| 16 | \n", + "negative | \n", + "I just built my rig with the Ryzen 5 3600 and ... | \n", + "
| 17 | \n", + "positive | \n", + "I just built my new rig with the Ryzen 7 5800X... | \n", + "
| 18 | \n", + "neutral | \n", + "I recently bought an Intel Core i5 for my new ... | \n", + "
| 19 | \n", + "positive | \n", + "I just assembled a system with the Ryzen 9 595... | \n", + "