Update tokenizer to split on whitespace

civictechdc · Feb 7, 2025 · cd7328e · cd7328e
1 parent e5da070
commit cd7328e
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/analyzers/hashtags/main.py b/analyzers/hashtags/main.py
@@ -43,7 +43,6 @@ def gini(x):
 
 
 def main(context: PrimaryAnalyzerContext):
-
     input_reader = context.input()
     df_input = input_reader.preprocess(pl.read_parquet(input_reader.parquet_path))
 

diff --git a/importing/csv.py b/importing/csv.py
@@ -106,7 +106,9 @@ def _separator_option(previous_value: Optional[str]) -> Optional[str]:
             default=(
                 previous_value
                 if previous_value in [",", ";", "\t"]
-                else "other" if previous_value is not None else None
+                else "other"
+                if previous_value is not None
+                else None
             ),
         )
         if input is None:
@@ -133,7 +135,9 @@ def _quote_char_option(previous_value: Optional[str]) -> Optional[str]:
             default=(
                 previous_value
                 if previous_value in ['"', "'"]
-                else "other" if previous_value is not None else None
+                else "other"
+                if previous_value is not None
+                else None
             ),
         )
         if input is None:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -43,7 +43,6 @@ def gini(x):


		def main(context: PrimaryAnalyzerContext):

		input_reader = context.input()
		df_input = input_reader.preprocess(pl.read_parquet(input_reader.parquet_path))

Expand Down