Add 'import from NDJSON' feature

digitalmethodsinitiative · Jan 11, 2023 · c5ecf45 · c5ecf45
1 parent 386530c
commit c5ecf45
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 10 deletions.
diff --git a/.zenodo.json b/.zenodo.json
@@ -3,7 +3,7 @@
   "license": "MPL-2.0",
   "title": "Zeeschuimer",
   "upload_type": "software",
-  "version": "v1.3",
+  "version": "v1.4",
   "keywords": [
     "scraping", "data capture", "4cat", "instagram", "tiktok"
   ],

diff --git a/js/zs-background.js b/js/zs-background.js
@@ -139,6 +139,7 @@ window.zeeschuimer = {
                             "source_platform": module,
                             "source_platform_url": source_platform_url,
                             "source_url": source_url,
+                            "user_agent": navigator.userAgent,
                             "data": item
                         });
                     }
@@ -155,7 +156,7 @@ window.zeeschuimer = {
      */
     has_tab: async function () {
         const tabs = await browser.tabs.query({});
-        const full_url = browser.runtime.getURL('popup/status.html');
+        const full_url = browser.runtime.getURL('popup/interface.html');
         const zeeschuimer_tab = tabs.filter((tab) => {
             return (tab.url === full_url);
         });
@@ -195,7 +196,7 @@ browser.webNavigation.onCommitted.addListener(
 browser.browserAction.onClicked.addListener(async () => {
     let tab = await zeeschuimer.has_tab();
     if (!tab) {
-        browser.tabs.create({url: 'popup/status.html'});
+        browser.tabs.create({url: 'popup/interface.html'});
     } else if (!tab.active) {
         browser.tabs.update(tab.id, {active: true});
     }

diff --git a/manifest.json b/manifest.json
@@ -3,7 +3,7 @@
   "description": "Collect data while browsing social media platforms and upload it for analysis later",
   "manifest_version": 2,
   "name": "Zeeschuimer",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "homepage_url": "https://github.com/digitalmethodsinitiative/zeeschuimer",
 
   "browser_specific_settings": {

diff --git a/popup/status.html → popup/interface.html b/popup/status.html → popup/interface.html
@@ -1,10 +1,13 @@
 <!DOCTYPE html>
 <head>
     <title>Zeeschuimer</title>
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <meta http-equiv="Content-Security-Policy" content="script-src *;">
     <link rel="stylesheet" href="tooltips.css">
+    <!--<script src="../js/webtorrent.min.js"></script>
+    <script src="../js/ponyfill-2.0.2.js"></script>
+    <script src="../js/streamsaver-2.0.3.js"></script>-->
     <style>
-        /** Colours from https://coolors.co/1446a0-db3069-f5d547-ebebd3-3c3c3b **/
         :root {
             --neutral: #d9c398;
             --neutral-contrast: #3C3C3B;
@@ -13,6 +16,14 @@
             --accent: #e0ba5f;
         }
 
+        *[aria-hidden=true] {
+            display: none;
+        }
+
+        *[aria-hidden=false] {
+            display: auto;
+        }
+
         @font-face {
             font-family: 'Open Sans';
             src: url("../fonts/OpenSans-Regular.ttf")
@@ -199,6 +210,10 @@
             cursor: pointer;
         }
 
+        input[type=file] {
+            width: 15em;
+        }
+
         .reset-all, #clear-history {
             display: block;
             margin: 0 auto;
@@ -210,8 +225,9 @@
 
         nav ul {
             list-style: none;
-            padding: 0;
+            padding: 1em;
             text-align: center;
+            margin: 0;
         }
 
         nav li {
@@ -222,7 +238,7 @@
             content: ' \2022';
         }
 
-        .fourcat-url-container, .zeeschuimer-master-switch {
+        .fourcat-url-container, .zeeschuimer-master-switch, .import-container {
             text-align: center;
             margin-bottom: 0.5em;
         }
@@ -326,7 +342,7 @@
 <article>
     <header>
         <h1>Zeeschuimer</h1>
-        <span class="version">v1.2.2</span>
+        <span class="version">v1.4.0</span>
     </header>
     <section id="status">
         <h2><span>Captured data objects</span></h2>
@@ -371,6 +387,16 @@ <h2><span>Uploaded datasets</span></h2>
             <tbody></tbody>
         </table>
     </section>
+    <section id="advanced-mode" aria-hidden="true">
+        <h2><span>Import from NDJSON</span></h2>
+        <p class="import-container">
+            <label>
+                NDJSON file:
+                <input name="ndjson-file" id="ndjson-file" type="file">
+                <button id="import-button">Import</button>
+            </label>
+        </p>
+    </section>
 </article>
 <footer>
     <nav>
@@ -380,9 +406,10 @@ <h2><span>Uploaded datasets</span></h2>
             <li><a href="https://4cat.nl">4CAT</a></li>
             <li><a href="https://cat4smr.humanities.uva.nl">CAT4SMR</a></li>
             <li><a href="https://doi.org/10.5281/zenodo.6826877">Cite/DOI</a></li>
+            <li><a href="#" id="toggle-advanced-mode">Show advanced options</a></li>
         </ul>
     </nav>
 </footer>
 <script type="text/javascript" src="tooltips.js"></script>
-<script type="text/javascript" src="popup.js"></script>
+<script type="text/javascript" src="interface.js"></script>
 </body>
diff --git a/popup/popup.js → popup/interface.js b/popup/popup.js → popup/interface.js
@@ -337,6 +337,59 @@ async function button_handler(event) {
         xhr.abort();
         xhr.aborted = true;
         status.innerHTML = '';
+
+    } else if(event.target.matches('#import-button')) {
+        if(!confirm('Importing data will remove all items currently stored. Are you sure?')) {
+            return;
+        }
+
+        await background.db.items.clear();
+
+        event.target.setAttribute('disabled', 'disabled');
+        let file = document.querySelector('#ndjson-file').files[0];
+        let reader = new FileReader();
+        reader.readAsText(file);
+        reader.addEventListener('load', async function (e) {
+            let imported_items = 0;
+            let skipped = 0;
+            let jsons = reader.result.split("\n");
+            for(let index in jsons) {
+                let raw_json = jsons[index];
+                if (!raw_json) {
+                    continue;
+                }
+
+                try {
+                    let imported = JSON.parse(raw_json);
+                    await background.db.items.add(imported);
+                    imported_items += 1;
+                } catch (e) {
+                    skipped += 1;
+                    console.log('Skipping invalid JSON string: (' + e + ') ' + raw_json);
+                }
+            }
+
+            if(skipped) {
+                alert('Imported ' + imported_items + ' item(s), ' + skipped + ' skipped.');
+            } else {
+                alert('Imported ' + imported_items + ' item(s).');
+            }
+        });
+
+        reader.addEventListener('loadend', function(e) {
+            event.target.removeAttribute('disabled');
+        });
+
+    } else if (event.target.matches('#toggle-advanced-mode')) {
+        let section = document.querySelector('#advanced-mode');
+        let is_hidden = section.getAttribute('aria-hidden') == 'true';
+        if(is_hidden) {
+            section.setAttribute('aria-hidden', 'false');
+            event.target.innerText = 'Hide advanced options';
+        } else {
+            section.setAttribute('aria-hidden', 'true');
+            event.target.innerText = 'Show advanced options';
+        }
     }
 
     get_stats();