Merge branch 'dev' into feature/twitter-integration

autogpt_platform/backend/backend/blocks/nvidia/_auth.py

@@ -0,0 +1,32 @@
+from typing import Literal
+
+from pydantic import SecretStr
+
+from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput
+from backend.integrations.providers import ProviderName
+
+NvidiaCredentials = APIKeyCredentials
+NvidiaCredentialsInput = CredentialsMetaInput[
+    Literal[ProviderName.NVIDIA],
+    Literal["api_key"],
+]
+
+TEST_CREDENTIALS = APIKeyCredentials(
+    id="01234567-89ab-cdef-0123-456789abcdef",
+    provider="nvidia",
+    api_key=SecretStr("mock-nvidia-api-key"),
+    title="Mock Nvidia API key",
+    expires_at=None,
+)
+
+TEST_CREDENTIALS_INPUT = {
+    "provider": TEST_CREDENTIALS.provider,
+    "id": TEST_CREDENTIALS.id,
+    "type": TEST_CREDENTIALS.type,
+    "title": TEST_CREDENTIALS.title,
+}
+
+
+def NvidiaCredentialsField() -> NvidiaCredentialsInput:
+    """Creates an Nvidia credentials input on a block."""
+    return CredentialsField(description="The Nvidia integration requires an API Key.")

autogpt_platform/backend/backend/blocks/nvidia/deepfake.py

@@ -0,0 +1,90 @@
+from backend.blocks.nvidia._auth import (
+    NvidiaCredentials,
+    NvidiaCredentialsField,
+    NvidiaCredentialsInput,
+)
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+from backend.util.request import requests
+
+
+class NvidiaDeepfakeDetectBlock(Block):
+    class Input(BlockSchema):
+        credentials: NvidiaCredentialsInput = NvidiaCredentialsField()
+        image_base64: str = SchemaField(
+            description="Image to analyze for deepfakes", image_upload=True
+        )
+        return_image: bool = SchemaField(
+            description="Whether to return the processed image with markings",
+            default=False,
+        )
+
+    class Output(BlockSchema):
+        status: str = SchemaField(
+            description="Detection status (SUCCESS, ERROR, CONTENT_FILTERED)",
+            default="",
+        )
+        image: str = SchemaField(
+            description="Processed image with detection markings (if return_image=True)",
+            default="",
+            image_output=True,
+        )
+        is_deepfake: float = SchemaField(
+            description="Probability that the image is a deepfake (0-1)",
+            default=0.0,
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="8c7d0d67-e79c-44f6-92a1-c2600c8aac7f",
+            description="Detects potential deepfakes in images using Nvidia's AI API",
+            categories={BlockCategory.SAFETY},
+            input_schema=NvidiaDeepfakeDetectBlock.Input,
+            output_schema=NvidiaDeepfakeDetectBlock.Output,
+        )
+
+    def run(
+        self, input_data: Input, *, credentials: NvidiaCredentials, **kwargs
+    ) -> BlockOutput:
+        url = "https://ai.api.nvidia.com/v1/cv/hive/deepfake-image-detection"
+
+        headers = {
+            "accept": "application/json",
+            "content-type": "application/json",
+            "Authorization": f"Bearer {credentials.api_key.get_secret_value()}",
+        }
+
+        image_data = f"data:image/jpeg;base64,{input_data.image_base64}"
+
+        payload = {
+            "input": [image_data],
+            "return_image": input_data.return_image,
+        }
+
+        try:
+            response = requests.post(url, headers=headers, json=payload)
+            response.raise_for_status()
+            data = response.json()
+
+            result = data.get("data", [{}])[0]
+
+            # Get deepfake probability from first bounding box if any
+            deepfake_prob = 0.0
+            if result.get("bounding_boxes"):
+                deepfake_prob = result["bounding_boxes"][0].get("is_deepfake", 0.0)
+
+            yield "status", result.get("status", "ERROR")
+            yield "is_deepfake", deepfake_prob
+
+            if input_data.return_image:
+                image_data = result.get("image", "")
+                output_data = f"data:image/jpeg;base64,{image_data}"
+                yield "image", output_data
+            else:
+                yield "image", ""
+
+        except Exception as e:
+            yield "error", str(e)
+            yield "status", "ERROR"
+            yield "is_deepfake", 0.0
+            yield "image", ""

autogpt_platform/backend/backend/data/block.py

@@ -61,6 +61,9 @@ class BlockCategory(Enum):
     HARDWARE = "Block that interacts with hardware."
     AGENT = "Block that interacts with other agents."
     CRM = "Block that interacts with CRM services."
+    SAFETY = (
+        "Block that provides AI safety mechanisms such as detecting harmful content"
+    )
 
     def dict(self) -> dict[str, str]:
         return {"category": self.name, "description": self.value}

autogpt_platform/backend/backend/data/model.py

@@ -139,6 +139,8 @@ def SchemaField(
     exclude: bool = False,
     hidden: Optional[bool] = None,
     depends_on: list[str] | None = None,
+    image_upload: Optional[bool] = None,
+    image_output: Optional[bool] = None,
     **kwargs,
 ) -> T:
     if default is PydanticUndefined and default_factory is None:
@@ -154,6 +156,8 @@ def SchemaField(
             "advanced": advanced,
             "hidden": hidden,
             "depends_on": depends_on,
+            "image_upload": image_upload,
+            "image_output": image_output,
         }.items()
         if v is not None
     }

autogpt_platform/backend/backend/integrations/providers.py

@@ -19,6 +19,7 @@ class ProviderName(str, Enum):
     JINA = "jina"
     MEDIUM = "medium"
     NOTION = "notion"
+    NVIDIA = "nvidia"
     OLLAMA = "ollama"
     OPENAI = "openai"
     OPENWEATHERMAP = "openweathermap"

autogpt_platform/backend/backend/integrations/webhooks/utils.py

@@ -7,6 +7,6 @@
 # TODO: add test to assert this matches the actual API route
 def webhook_ingress_url(provider_name: ProviderName, webhook_id: str) -> str:
     return (
-        f"{app_config.platform_base_url}/api/integrations/{provider_name}"
+        f"{app_config.platform_base_url}/api/integrations/{provider_name.value}"
         f"/webhooks/{webhook_id}/ingress"
     )

autogpt_platform/frontend/package.json

@@ -24,8 +24,8 @@
   ],
   "dependencies": {
     "@faker-js/faker": "^9.3.0",
-    "@hookform/resolvers": "^3.9.1",
-    "@next/third-parties": "^15.0.4",
+    "@hookform/resolvers": "^3.10.0",
+    "@next/third-parties": "^15.1.3",
     "@radix-ui/react-alert-dialog": "^1.1.4",
     "@radix-ui/react-avatar": "^1.1.1",
     "@radix-ui/react-checkbox": "^1.1.2",
@@ -59,25 +59,25 @@
     "dotenv": "^16.4.7",
     "elliptic": "6.6.1",
     "embla-carousel-react": "^8.3.0",
-    "framer-motion": "^11.15.0",
+    "framer-motion": "^11.16.0",
     "geist": "^1.3.1",
     "launchdarkly-react-client-sdk": "^3.6.0",
-    "lucide-react": "^0.468.0",
+    "lucide-react": "^0.469.0",
     "moment": "^2.30.1",
     "next": "^14.2.13",
     "next-themes": "^0.4.4",
     "react": "^18",
-    "react-day-picker": "^9.4.4",
+    "react-day-picker": "^9.5.0",
     "react-dom": "^18",
-    "react-hook-form": "^7.54.0",
+    "react-hook-form": "^7.54.2",
     "react-icons": "^5.4.0",
-    "react-markdown": "^9.0.1",
-    "react-modal": "^3.16.1",
+    "react-markdown": "^9.0.3",
+    "react-modal": "^3.16.3",
     "react-shepherd": "^6.1.6",
     "recharts": "^2.14.1",
-    "tailwind-merge": "^2.5.5",
+    "tailwind-merge": "^2.6.0",
     "tailwindcss-animate": "^1.0.7",
-    "uuid": "^11.0.3",
+    "uuid": "^11.0.4",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -94,16 +94,16 @@
     "@storybook/test": "^8.3.5",
     "@storybook/test-runner": "^0.21.0",
     "@types/negotiator": "^0.6.3",
-    "@types/node": "^22.9.0",
+    "@types/node": "^22.10.5",
     "@types/react": "^18",
     "@types/react-dom": "^18",
     "@types/react-modal": "^3.16.3",
     "axe-playwright": "^2.0.3",
-    "chromatic": "^11.12.5",
-    "concurrently": "^9.1.1",
+    "chromatic": "^11.22.0",
+    "concurrently": "^9.1.2",
     "eslint": "^8",
     "eslint-config-next": "15.1.3",
-    "eslint-plugin-storybook": "^0.11.0",
+    "eslint-plugin-storybook": "^0.11.2",
     "msw": "^2.7.0",
     "msw-storybook-addon": "^2.0.3",
     "postcss": "^8",

autogpt_platform/frontend/src/components/integrations/credentials-input.tsx

@@ -60,6 +60,7 @@ export const providerIcons: Record<
   google: FaGoogle,
   groq: fallbackIcon,
   notion: NotionLogoIcon,
+  nvidia: fallbackIcon,
   discord: FaDiscord,
   d_id: fallbackIcon,
   google_maps: FaGoogle,

autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx

@@ -28,6 +28,7 @@ const providerDisplayNames: Record<CredentialsProviderName, string> = {
   jina: "Jina",
   medium: "Medium",
   notion: "Notion",
+  nvidia: "Nvidia",
   ollama: "Ollama",
   openai: "OpenAI",
   openweathermap: "OpenWeatherMap",

autogpt_platform/frontend/src/components/node-input-components.tsx

@@ -103,6 +103,92 @@ const NodeObjectInputTree: FC<NodeObjectInputTreeProps> = ({
 
 export default NodeObjectInputTree;
 
+const NodeImageInput: FC<{
+  selfKey: string;
+  schema: BlockIOStringSubSchema;
+  value?: string;
+  error?: string;
+  handleInputChange: NodeObjectInputTreeProps["handleInputChange"];
+  className?: string;
+  displayName: string;
+}> = ({
+  selfKey,
+  schema,
+  value = "",
+  error,
+  handleInputChange,
+  className,
+  displayName,
+}) => {
+  const handleFileChange = useCallback(
+    async (event: React.ChangeEvent<HTMLInputElement>) => {
+      const file = event.target.files?.[0];
+      if (!file) return;
+
+      // Validate file type
+      if (!file.type.startsWith("image/")) {
+        console.error("Please upload an image file");
+        return;
+      }
+
+      // Convert to base64
+      const reader = new FileReader();
+      reader.onload = (e) => {
+        const base64String = (e.target?.result as string).split(",")[1];
+        handleInputChange(selfKey, base64String);
+      };
+      reader.readAsDataURL(file);
+    },
+    [selfKey, handleInputChange],
+  );
+
+  return (
+    <div className={cn("flex flex-col gap-2", className)}>
+      <div className="nodrag flex flex-col gap-2">
+        <div className="flex items-center gap-2">
+          <Button
+            variant="outline"
+            onClick={() =>
+              document.getElementById(`${selfKey}-upload`)?.click()
+            }
+            className="w-full"
+          >
+            {value ? "Change Image" : `Upload ${displayName}`}
+          </Button>
+          {value && (
+            <Button
+              variant="ghost"
+              className="text-red-500 hover:text-red-700"
+              onClick={() => handleInputChange(selfKey, "")}
+            >
+              <Cross2Icon className="h-4 w-4" />
+            </Button>
+          )}
+        </div>
+
+        <input
+          id={`${selfKey}-upload`}
+          type="file"
+          accept="image/*"
+          onChange={handleFileChange}
+          className="hidden"
+        />
+
+        {value && (
+          <div className="relative mt-2 rounded-md border border-gray-300 p-2 dark:border-gray-600">
+            <img
+              src={`data:image/jpeg;base64,${value}`}
+              alt="Preview"
+              className="max-h-32 w-full rounded-md object-contain"
+            />
+          </div>
+        )}
+      </div>
+      {error && <span className="error-message">{error}</span>}
+    </div>
+  );
+};
+
 const NodeDateTimeInput: FC<{
   selfKey: string;
   schema: BlockIOStringSubSchema;
@@ -499,6 +585,19 @@ export const NodeGenericInputField: FC<{
 
   switch (propSchema.type) {
     case "string":
+      if ("image_upload" in propSchema && propSchema.image_upload === true) {
+        return (
+          <NodeImageInput
+            selfKey={propKey}
+            schema={propSchema}
+            value={currentValue}
+            error={errors[propKey]}
+            className={className}
+            displayName={displayName}
+            handleInputChange={handleInputChange}
+          />
+        );
+      }
       if ("format" in propSchema && propSchema.format === "date-time") {
         return (
           <NodeDateTimeInput