dont rely on locale in case it is not available (#629)

5.0b2

HISTORY.rst

@@ -5,17 +5,19 @@ Change Log
 This document records all notable changes to `django-sql-explorer <https://github.com/chrisclark/django-sql-explorer>`_.
 This project adheres to `Semantic Versioning <https://semver.org/>`_.
 
-`5.0.0b1`_ (2024-06-07)
+`5.0.b2`_ (2024-06-17)
 ===========================
 
-* Manage DB connections via the UI (and/or Django Admin)
+* Manage DB connections via the UI (and/or Django Admin). Set EXPLORER_DB_CONNECTIONS_ENABLED
+  to True in settings to enable user-facing connection management.
 * Upload CSV or SQLite DBs directly, to create additional connections.
   This functionality has additional dependencies which can be installed with
-  the 'uploads' extra (e.g. pip install django-sql-explorer[uploads])
+  the 'uploads' extra (e.g. pip install django-sql-explorer[uploads]). Then set EXPLORER_USER_UPLOADS_ENABLED
+  to True, and make sure S3_BUCKET is also set up.
 * The above functionality is managed by a new license, restricting the
   ability of 3rd parties resell SQL Explorer (commercial usage is absolutely
   still permitted).
-* Query List home page is sortable.
+* Query List home page is sortable
 * Select all / deselect all with AI assistant
 * Assistant tests run reliably in CI/CD
 * Introduced some branding and styling improvements

explorer/__init__.py

@@ -3,7 +3,7 @@
     "minor": 0,
     "patch": 0,
     "releaselevel": "beta",
-    "serial": 1
+    "serial": 2
 }
 
 

explorer/app_settings.py

@@ -157,10 +157,22 @@
                                     "max_tokens": 128000})
 
 EXPLORER_DB_CONNECTIONS_ENABLED = getattr(settings, "EXPLORER_DB_CONNECTIONS_ENABLED", False)
+EXPLORER_USER_UPLOADS_ENABLED = getattr(settings, "EXPLORER_USER_UPLOADS_ENABLED", False)
 EXPLORER_PRUNE_LOCAL_UPLOAD_COPY_DAYS_INACTIVITY = getattr(settings,
                                                            "EXPLORER_PRUNE_LOCAL_UPLOAD_COPY_DAYS_INACTIVITY", 7)
 # 500mb default max
 EXPLORER_MAX_UPLOAD_SIZE = getattr(settings, "EXPLORER_MAX_UPLOAD_SIZE", 500 * 1024 * 1024)
 
-def has_assistant(): return EXPLORER_AI_API_KEY is not None
-def db_connections_enabled(): return EXPLORER_DB_CONNECTIONS_ENABLED
+
+def has_assistant():
+    return EXPLORER_AI_API_KEY is not None
+
+
+def db_connections_enabled():
+    return EXPLORER_DB_CONNECTIONS_ENABLED
+
+
+def user_uploads_enabled():
+    return (EXPLORER_USER_UPLOADS_ENABLED and
+            EXPLORER_DB_CONNECTIONS_ENABLED and
+            S3_BUCKET is not None)

explorer/ee/db_connections/utils.py

@@ -2,7 +2,6 @@
 from django.db.utils import load_backend
 import os
 
-import locale
 from dateutil import parser
 
 import pandas as pd
@@ -112,12 +111,25 @@ def pandas_to_sqlite(df, local_path="local_database.db"):
 SHORTEST_PLAUSIBLE_DATE_STRING = 5
 
 
+def atof_custom(value):
+    # Remove any thousands separators and convert the decimal point
+    if "," in value and "." in value:
+        if value.index(",") < value.index("."):
+            # 0,000.00 format
+            value = value.replace(",", "")
+        else:
+            # 0.000,00 format
+            value = value.replace(".", "").replace(",", ".")
+    elif "," in value:
+        # No decimal point, only thousands separator
+        value = value.replace(",", "")
+    return float(value)
+
 def csv_to_typed_df(csv_bytes, delimiter=",", has_headers=True):  # noqa
     try:
 
         csv_file = io.BytesIO(csv_bytes)
         df = pd.read_csv(csv_file, sep=delimiter, header=0 if has_headers else None)
-        locale.setlocale(locale.LC_NUMERIC, "en_US.UTF-8")
 
         for column in df.columns:
             values = df[column].dropna().unique()
@@ -130,7 +142,7 @@ def csv_to_typed_df(csv_bytes, delimiter=",", has_headers=True):  # noqa
 
             for value in values:
                 try:
-                    float_val = locale.atof(str(value))
+                    float_val = atof_custom(str(value))
                     if float_val == int(float_val):
                         continue  # This is effectively an integer
                     else:
@@ -163,12 +175,12 @@ def csv_to_typed_df(csv_bytes, delimiter=",", has_headers=True):  # noqa
             if is_date:
                 df[column] = pd.to_datetime(df[column], errors="coerce", utc=True)
             elif is_integer:
-                df[column] = df[column].apply(lambda x: int(locale.atof(str(x))) if pd.notna(x) else x)
+                df[column] = df[column].apply(lambda x: int(atof_custom(str(x))) if pd.notna(x) else x)
                 # If there are NaN / blank values, the column will be converted to float
                 # Convert it back to integer
                 df[column] = df[column].astype("Int64")
             elif is_float:
-                df[column] = df[column].apply(lambda x: locale.atof(str(x)) if pd.notna(x) else x)
+                df[column] = df[column].apply(lambda x: atof_custom(str(x)) if pd.notna(x) else x)
             else:
                 inferred_type = pd.api.types.infer_dtype(values)
                 if inferred_type == "integer":

explorer/src/scss/explorer.scss

@@ -108,27 +108,6 @@ div.sort {
     display: none;
 }
 
-nav.navbar {
-    .nav-link {
-        color: white;
-    }
-
-    nav.nav-link:hover, .nav-link:focus {
-        color: var(--bs-primary-bg-subtle);
-    }
-
-    .nav-pills .nav-link.active, .nav-pills .show > .nav-link {
-        color: var(--bs-nav-pills-link-active-color);
-        border: 1px solid var(--bs-secondary-bg);
-        background: none;
-    }
-
-    .nav-link:hover, .navbar-brand:hover {
-        background: none;
-        color: var(--bs-primary-bg-subtle) !important;
-    }
-}
-
 .dropdown-toggle::after {
     margin-left: 0 !important;
 }
@@ -151,3 +130,11 @@ nav.navbar {
 .logo-image {
     height: 2rem;
 }
+
+.stats-wrapper {
+    td {
+        padding-top: 0;
+        padding-bottom: 0;
+        border: none;
+    }
+}

explorer/templates/connections/database_connection_form.html

@@ -9,7 +9,7 @@ <h2>{% if object %}Edit{% else %}Create New{% endif %} Connection</h2>
         <a href="{% url 'explorer_connections' %}" class="btn btn-secondary">Cancel</a>
         <button type="button" class="btn btn-info" id="test-connection-btn">Test Connection</button>
     </form>
-    {% if not object and s3_enabled %}
+    {% if not object and user_uploads_enabled %}
         <div class="pt-5">
             <h4>...or upload a SQLite DB or CSV File</h4>
             <p>CSV files will get parsed and typed automatically. SQLite databases must <i>not</i> be password protected.</p>

explorer/templates/explorer/assistant.html

@@ -26,7 +26,7 @@
                         data-bs-toggle="tooltip" data-bs-placement="top"
                         data-bs-title="SQL Assistant builds a prompt with your query, your request, relevant schema (tables referenced in your query) and sample data from those tables. You can optionally include other tables (schema + data sample) that you'd like SQL Assistant to know about.">(?)</a>
                     <label for="search_assistant_tables" class="d-none" id="search_assistant_tables_label">Search tables</label>
-                    
+
                     <div id="table-list" class="list"></div>
                 </div>
             </div>

explorer/templates/explorer/base.html

@@ -7,7 +7,7 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>{% trans "SQL Explorer" %}{% if query %} - {{ query.title }}{% elif title %} - {{ title }}{% endif %}</title>
-    <link rel="icon" type="image/svg+xml" href="{% vite_asset 'images/logo.svg' %}">
+    <link rel="icon" type="image/svg+xml" href="{% vite_asset 'images/logo.png' %}">
 
     {% block style %}
         {% vite_asset 'scss/styles.scss' %}
@@ -41,11 +41,11 @@ <h2>This is easy to fix, I promise!</h2>
 {% endif %}
 
 {% block sql_explorer_content_takeover %}
-    <nav class="navbar navbar-expand-lg navbar-dark bg-dark mb-3">
+    <nav class="navbar navbar-expand-lg navbar-dark mb-3">
         <div class="container bd-gutter flex-wrap flex-lg-nowrap">
-            <a class="navbar-brand{% if view_name == 'explorer_index' %} btn text-white{% endif %} d-flex align-items-center"
+            <a class="navbar-brand d-flex align-items-center"
                 href="{% url 'explorer_index' %}">
-                <img src="{% vite_asset 'images/logo.svg' %}" alt="Logo" class="me-2 logo-image" >
+                <img src="{% vite_asset 'images/logo-main.svg' %}" alt="Logo" class="me-2 logo-image" >
                 {% trans "SQL Explorer" %}
             </a>
             <ul class="nav nav-pills">

explorer/templates/explorer/preview_pane.html

@@ -59,7 +59,7 @@
                                         <th class="preview-header counter" style="display: none;"></th>
                                         {% for h in headers %}
                                             <th class="preview-header"><span><i class="sort bi-chevron-expand pe-1" data-sort="{{ forloop.counter0 }}"
-                                                         data-dir="asc"></i>{{ h }}</span></th>
+                                                         data-dir="asc"></i>{{ h }}{% if h.summary %}<i class="stats-expand bi-calculator text-info ps-1"></i>{% endif %}</span></th>
                                         {% endfor %}
                                     </tr>
                                     {% if has_stats %}
@@ -68,11 +68,10 @@
                                             {% for h in headers %}
                                                 <th>
                                                     {% if h.summary %}
-                                                        <i class="stats-expand bi-calculator"></i>
                                                         <table class="stats-wrapper table table-sm fw-normal small" style="display: none;">
                                                             {% for label, value in h.summary.stats.items %}
                                                                 <tr>
-                                                                    <td>{{ label }}</td>
+                                                                    <td class="text-info">{{ label }}</td>
                                                                     <td>{{ value }}</td>
                                                                 </tr>
                                                             {% endfor %}

explorer/templates/explorer/query_list.html

@@ -10,7 +10,7 @@ <h3>
                     Recently Run by You
                 {% endblocktrans %}
             </h3>
-            <table class="table table-striped">
+            <table class="table table-striped table-borderless">
                 <thead>
                     <tr>
                         <th>{% trans "Query" %}</th>
@@ -46,7 +46,7 @@ <h3>{% trans "All Queries" %}</h3>
                 <input class="search" placeholder="{% trans 'Search' %}" style="">
             </div>
         </div>
-        <table class="table table-striped">
+        <table class="table table-striped table-borderless">
             <thead>
                 <tr>
                     <th><i class="sort bi-chevron-expand pe-1" data-sort="sort-name"

explorer/tests/csvs/floats.csv

@@ -6,3 +6,4 @@ Floats,Values
 ,4
 "69.42719764194946",4
 "2,000.0128",5
+"2.000,0128",5

explorer/views/mixins.py

@@ -19,7 +19,7 @@ def gen_ctx(self):
             ),
             "assistant_enabled": app_settings.has_assistant(),
             "db_connections_enabled": app_settings.db_connections_enabled(),
-            "s3_enabled": app_settings.S3_BUCKET is not None,
+            "user_uploads_enabled": app_settings.user_uploads_enabled(),
             "csrf_cookie_name": settings.CSRF_COOKIE_NAME,
             "csrf_token_in_dom": settings.CSRF_COOKIE_HTTPONLY or settings.CSRF_USE_SESSIONS,
             "view_name": self.request.resolver_match.view_name,

test_project/settings.py

@@ -4,7 +4,7 @@
 SECRET_KEY = "shhh"
 DEBUG = True
 STATIC_URL = "/static/"
-VITE_DEV_MODE = False
+VITE_DEV_MODE = True
 
 ALLOWED_HOSTS = ["0.0.0.0", "localhost", "127.0.0.1"]
 
@@ -113,3 +113,4 @@
 EXPLORER_AI_API_KEY = os.environ.get("AI_API_KEY")
 EXPLORER_ASSISTANT_BASE_URL = os.environ.get("AI_BASE_URL")
 EXPLORER_DB_CONNECTIONS_ENABLED = True
+EXPLORER_USER_UPLOADS_ENABLED = True