another round of cleanup ...

- ballista.plugin_dir - update to `actions/checkout@v4` relates to apache#1067
milenkovicm · Oct 15, 2024 · 0c67a87 · 0c67a87
1 parent 1c4a38c
commit 0c67a87
Show file tree

Hide file tree

Showing 8 changed files with 24 additions and 32 deletions.
diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml
@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: apache/arrow
       - name: Set up Python
@@ -54,7 +54,7 @@ jobs:
     if: startsWith(github.event.comment.body, '@github-actions rebase')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: r-lib/actions/pr-fetch@master
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: apache/arrow
           submodules: true
@@ -43,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
@@ -55,7 +55,7 @@ jobs:
     name: Use prettier to check formatting of documents
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: "14"

diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml
@@ -29,7 +29,7 @@ jobs:
     name: Process
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Assign GitHub labels
         if: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -24,8 +24,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Installs Rust and Cargo
         run: curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
       - name: Run script
-        run: ./dev/build-ballista-docker.sh
+        run: ./dev/build-ballista-docker.sh
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -38,7 +38,7 @@ jobs:
         # "1" means line tables only, which is useful for panic tracebacks.
         RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Cache Cargo
         uses: actions/cache@v4
         with:
@@ -88,7 +88,7 @@ jobs:
         # "1" means line tables only, which is useful for panic tracebacks.
         RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Install protobuf compiler
@@ -140,7 +140,7 @@ jobs:
         # "1" means line tables only, which is useful for panic tracebacks.
         RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Install protobuf compiler
@@ -190,7 +190,7 @@ jobs:
         os: [windows-latest, macos-latest]
         rust: [stable]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Install protobuf macos compiler
@@ -250,7 +250,7 @@ jobs:
         # "1" means line tables only, which is useful for panic tracebacks.
         RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Install protobuf compiler
@@ -284,7 +284,7 @@ jobs:
     container:
       image: amd64/rust
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup toolchain
         run: |
           rustup toolchain install stable
@@ -304,7 +304,7 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Restore rust artifacts
         uses: actions/download-artifact@v4
         with:
@@ -341,7 +341,7 @@ jobs:
         # "1" means line tables only, which is useful for panic tracebacks.
         RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Cache Cargo
@@ -384,7 +384,7 @@ jobs:
         # "1" means line tables only, which is useful for panic tracebacks.
         RUSTFLAGS: "-C debuginfo=1"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Cache Cargo
@@ -433,7 +433,7 @@ jobs:
 #       arch: [amd64]
 #       rust: [stable]
 #   steps:
-#     - uses: actions/checkout@v3
+#     - uses: actions/checkout@v4
 #       with:
 #         submodules: true
 #     - name: Cache Cargo

diff --git a/ballista/client/src/prelude.rs b/ballista/client/src/prelude.rs
@@ -21,9 +21,9 @@ pub use ballista_core::{
     config::{
         BallistaConfig, BALLISTA_COLLECT_STATISTICS, BALLISTA_DEFAULT_BATCH_SIZE,
         BALLISTA_DEFAULT_SHUFFLE_PARTITIONS, BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE,
-        BALLISTA_JOB_NAME, BALLISTA_PARQUET_PRUNING, BALLISTA_PLUGIN_DIR,
-        BALLISTA_REPARTITION_AGGREGATIONS, BALLISTA_REPARTITION_JOINS,
-        BALLISTA_REPARTITION_WINDOWS, BALLISTA_WITH_INFORMATION_SCHEMA,
+        BALLISTA_JOB_NAME, BALLISTA_PARQUET_PRUNING, BALLISTA_REPARTITION_AGGREGATIONS,
+        BALLISTA_REPARTITION_JOINS, BALLISTA_REPARTITION_WINDOWS,
+        BALLISTA_WITH_INFORMATION_SCHEMA,
     },
     error::{BallistaError, Result},
 };

diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs
@@ -39,8 +39,7 @@ pub const BALLISTA_PARQUET_PRUNING: &str = "ballista.parquet.pruning";
 pub const BALLISTA_COLLECT_STATISTICS: &str = "ballista.collect_statistics";
 
 pub const BALLISTA_WITH_INFORMATION_SCHEMA: &str = "ballista.with_information_schema";
-/// give a plugin files dir, and then the dynamic library files in this dir will be load when scheduler state init.
-pub const BALLISTA_PLUGIN_DIR: &str = "ballista.plugin_dir";
+
 /// max message size for gRPC clients
 pub const BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE: &str =
     "ballista.grpc_client_max_message_size";
@@ -205,9 +204,7 @@ impl BallistaConfig {
                 "Configuration for collecting statistics during scan".to_string(),
                 DataType::Boolean, Some("false".to_string())
             ),
-            ConfigEntry::new(BALLISTA_PLUGIN_DIR.to_string(),
-                             "Sets the plugin dir".to_string(),
-                             DataType::Utf8, Some("".to_string())),
+
             ConfigEntry::new(BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE.to_string(),
                              "Configuration for max message size in gRPC clients".to_string(),
                              DataType::UInt64,
@@ -227,10 +224,6 @@ impl BallistaConfig {
         self.get_usize_setting(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS)
     }
 
-    pub fn default_plugin_dir(&self) -> String {
-        self.get_string_setting(BALLISTA_PLUGIN_DIR)
-    }
-
     pub fn default_batch_size(&self) -> usize {
         self.get_usize_setting(BALLISTA_DEFAULT_BATCH_SIZE)
     }
@@ -290,6 +283,7 @@ impl BallistaConfig {
             v.parse::<bool>().unwrap()
         }
     }
+    #[allow(dead_code)]
     fn get_string_setting(&self, key: &str) -> String {
         if let Some(v) = self.settings.get(key) {
             // infallible because we validate all configs in the constructor
@@ -403,7 +397,6 @@ mod tests {
     fn custom_config_invalid() -> Result<()> {
         let config = BallistaConfig::builder()
             .set(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS, "true")
-            .set(BALLISTA_PLUGIN_DIR, "test_dir")
             .build();
         assert!(config.is_err());
         assert_eq!("General(\"Failed to parse user-supplied value 'ballista.shuffle.partitions' for configuration setting 'true': ParseIntError { kind: InvalidDigit }\")", format!("{:?}", config.unwrap_err()));

diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md
@@ -46,7 +46,6 @@ let ctx = BallistaContext::remote("localhost", 50050, & config).await?;
 | ballista.repartition.windows      | Boolean | true    | When set to true, Ballista will repartition data using the partition keys to execute window functions in parallel using the provided `ballista.shuffle.partitions` level. |
 | ballista.parquet.pruning          | Boolean | true    | Determines whether Parquet pruning should be enabled or not.                                                                                                              |
 | ballista.with_information_schema  | Boolean | true    | Determines whether the `information_schema` should be created in the context. This is necessary for supporting DDL commands such as `SHOW TABLES`.                        |
-| ballista.plugin_dir               | Boolean | true    | Specified a path for plugin files. Dynamic library files in this directory will be loaded when scheduler state initializes.                                               |
 
 ### DataFusion Configuration Settings