Merge pull request #36 from chrros95/issue_34

Changed Indexing for Paths

.github/workflows/integration-tests.yml

@@ -6,13 +6,11 @@ on:
     paths: '**.php'
   pull_request:
     paths: '**.php'
+  workflow_dispatch:
 env:
-  POSTGRES_PASSWORD: nc_test_db
-  MYSQL_USER: nc_test
-  MYSQL_PASSWORD: nc_test_db
-  MYSQL_DATABASE: nc_test
-  MYSQL_PORT: 3800
-
+  DB_USER: root
+  DB_PASSWORD: nc_test
+  DATABASE: nc_test
 jobs:
   integration:
     runs-on: ubuntu-latest
@@ -22,19 +20,40 @@ jobs:
       postgres:
         image: postgres
         env:
-          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.DATABASE }}
+          POSTGRES_USER: ${{ env.DB_USER }}
+          POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }}
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
         ports:
-          - 5432:5432 # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+      mariadb:
+         # Nextcloud Requires compressed tables which are not supported in 10.6 (nextcloud/server #25436)
+        image: mariadb:10.5
+        ports:
+          - 3306:3306
+        env:
+          MYSQL_DATABASE: ${{ env.DATABASE }}
+          MYSQL_ROOT_PASSWORD: ${{ env.DB_PASSWORD }}
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+      mysql:
+         # Nextcloud Requires compressed tables which are not supported in 10.6 (nextcloud/server #25436)
+        image: chrros95/mysql
+        ports:
+          - 3307:3306
+        env:
+          MYSQL_DATABASE: ${{ env.DATABASE }}
+          MYSQL_ROOT_PASSWORD: ${{ env.DB_PASSWORD }}
+          AUTH_PLUGIN: mysql_native_password
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
     strategy:
       matrix:
         php-versions: ['7.3', '7.4']
-        nextcloud: ['stable20', 'stable21']
-        database: ['sqlite', 'pgsql', 'mysql']
+        nextcloud: ['stable20', 'stable21', 'stable22']
+        database: ['sqlite', 'pgsql', 'mysql','mariadb']
         experimental: [false]
         include:
           - php-versions: 7.4
@@ -45,76 +64,47 @@ jobs:
             nextcloud: pre-release
             database: sqlite
             experimental: true
+          - database: pgsql
+            dbport: 5432
+          - database: mariadb
+            dbtype: mysql
+          - database: mysql
+            dbport: 3307
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
       - name: Setup PHP
         uses: shivammathur/setup-php@v2
         with:
           php-version: ${{ matrix.php-versions }}
           extensions: pdo_sqlite,pdo_mysql,pdo_pgsql,gd,zip
           coverage: none
-
       - name: Setup BATS
         uses: mig4/setup-bats@v1
-
-      ### MySQL specific setup
-      - name: Setup mysql
-        if: matrix.database == 'mysql'
-        uses: getong/[email protected]
-        with:
-          host port: ${{ env.MYSQL_PORT }}
-          mysql database: ${{ env.MYSQL_DATABASE }}
-          mysql root password: ${{ env.MYSQL_PASSWORD }}
-          mysql user: ${{ env.MYSQL_USER }}
-          mysql password: ${{ env.MYSQL_PASSWORD }}
-
-      - name: Set up server MySQL
-        if: matrix.database == 'mysql'
+      - name: Set up server
         uses: SMillerDev/nextcloud-actions/setup-nextcloud@main
         with:
           version: ${{ matrix.nextcloud }}
           cron: true
-          database-type: ${{ matrix.database }}
+          database-type: ${{ matrix.dbtype || matrix.database }}
           database-host: 127.0.0.1
-          database-port: ${{ env.MYSQL_PORT }}
-          database-name: ${{ env.MYSQL_DATABASE }}
-          database-user: root
-          database-password: ${{ env.MYSQL_PASSWORD }}
-
-      ### Back to normal setup
-      - name: Set up server non MySQL
-        if: matrix.database != 'mysql'
-        uses: SMillerDev/nextcloud-actions/setup-nextcloud@main
-        with:
-          version: ${{ matrix.nextcloud }}
-          cron: true
-          database-type: ${{ matrix.database }}
-          database-host: localhost
-          database-port: 5432
-          database-name: postgres
-          database-user: postgres
-          database-password: ${{ env.POSTGRES_PASSWORD }}
-
+          database-port: ${{ matrix.dbport || 3306 }}
+          database-name: ${{ env.DATABASE }}
+          database-user: ${{ env.DB_USER }}
+          database-password: ${{ env.DB_PASSWORD }}
       - name: Prime app build
         run: make
-
       - name: Configure server with app
         uses: SMillerDev/nextcloud-actions/setup-nextcloud-app@main
         with:
           app: 'duplicatefinder'
           check-code: true
           force: ${{ matrix.experimental }}
-
       - name: Setup problem matchers for PHP
         run: echo "::add-matcher::${{ runner.tool_cache }}/php.json"
-
       - name: Functional tests maintenance
         working-directory: ../server
-        run: |
-          ./occ maintenance:repair
-
+        run: ./occ maintenance:repair
       - name: Functional tests
         working-directory: ../server
         run: bats apps/duplicatefinder/tests/bats

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.0.10
+- Added Support for MySQL as database
+
 ## 0.0.9
 
 - Restored 0.0.7 with fixes

appinfo/info.xml

@@ -50,7 +50,7 @@
     ![Preview of the GUI](https://raw.githubusercontent.com/PaulLereverend/NextcloudDuplicateFinder/master/img/preview.png)
 ]]>
   </description>
-  <version>0.0.9</version>
+  <version>0.0.10</version>
   <licence>agpl</licence>
   <author mail="[email protected]" >Paul Lereverend</author>
   <namespace>DuplicateFinder</namespace>

lib/Db/FileDuplicate.php

@@ -1,6 +1,14 @@
 <?php
 namespace OCA\DuplicateFinder\Db;
 
+/**
+ * @method void setType(string $s)
+ * @method void setHash(string $s)
+ * @method void setFiles(array<string|FileInfo> $a)
+ * @method string getType()
+ * @method string getHash()
+ * @method array<string|FileInfo> getFiles()
+ */
 class FileDuplicate extends EEntity
 {
 

lib/Db/FileInfo.php

@@ -1,6 +1,26 @@
 <?php
 namespace OCA\DuplicateFinder\Db;
 
+/**
+ * @method void setOwner(string $s)
+ * @method void setPath(string $s)
+ * @method void setPathHash(string $s)
+ * @method void setFileHash(string $s)
+ * @method void setImageHash(string $s)
+ * @method void setUpdatedAt(int|\DateTime $d)
+ * @method void setNodeId(int $i)
+ * @method void setMimetype(string $s)
+ * @method void setSize(int $i)
+ * @method string getOwner()
+ * @method string getPath()
+ * @method string getPathHash()
+ * @method string getFileHash()
+ * @method string getImageHash()
+ * @method \DateTime getUpdatedAt()
+ * @method int getNodeId()
+ * @method string getMimetype()
+ * @method int getSize()
+ */
 class FileInfo extends EEntity
 {
 
@@ -9,6 +29,8 @@ class FileInfo extends EEntity
     /** @var string */
     protected $path;
     /** @var string */
+    protected $pathHash;
+    /** @var string */
     protected $fileHash;
     /** @var string */
     protected $imageHash;
@@ -34,4 +56,11 @@ public function __construct(?string $path = null, ?string $owner = null)
             $this->setOwner($owner);
         }
     }
+
+    public function setPath(string $path):void
+    {
+        // SHA1 because we need a function to short the path and not be cryptographically secure
+        $this->setPathHash(sha1($path));
+        parent::setPath($path);
+    }
 }

lib/Db/FileInfoMapper.php

@@ -24,7 +24,7 @@ public function find(string $path):FileInfo
         $qb->select('*')
         ->from($this->getTableName())
         ->where(
-            $qb->expr()->eq('path', $qb->createNamedParameter($path))
+            $qb->expr()->eq('path_hash', $qb->createNamedParameter(sha1($path)))
         );
         return $this->findEntity($qb);
     }

lib/Migration/Version0001Date20210508222200.php

@@ -30,9 +30,6 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
                 $pathColumn->setType(Type::getType(Types::STRING));
                 $pathColumn->setOptions(['length' => 4000]);
             }
-            if (!$table->hasIndex('duplicatefinder_path_idx')) {
-                $table->addIndex(['path'], 'duplicatefinder_path_idx');
-            }
             return $schema;
         }
         return null;

lib/Migration/Version0003Date20210715132400.php

@@ -0,0 +1,44 @@
+<?php
+namespace OCA\DuplicateFinder\Migration;
+
+use Closure;
+use Doctrine\DBAL\Types\Type;
+use Doctrine\DBAL\Types\Types;
+// This is OCP\DB\Types is support on NC 21 but not on NC 20
+//use OCP\DB\Types;
+use OCP\DB\ISchemaWrapper;
+use OCP\Migration\SimpleMigrationStep;
+use OCP\Migration\IOutput;
+
+class Version0003Date20210715132400 extends SimpleMigrationStep
+{
+
+  /**
+  * @param IOutput $output
+  * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+  * @param array<mixed> $options
+  * @return null|ISchemaWrapper
+  */
+    public function changeSchema(IOutput $output, Closure $schemaClosure, array $options)
+    {
+      /** @var ISchemaWrapper $schema */
+        $schema = $schemaClosure();
+        if ($schema->hasTable('duplicatefinder_finfo')) {
+            $table = $schema->getTable('duplicatefinder_finfo');
+            if ($table->hasIndex('duplicatefinder_path_idx')) {
+                $table->dropIndex('duplicatefinder_path_idx');
+            }
+            if (!$table->hasColumn('path_hash')) {
+                $table->addColumn('path_hash', 'string', [
+                  'notnull' => true,
+                  'length' => 40,
+                ]);
+            }
+            if (!$table->hasIndex('duplicatefinder_ph_idx')) {
+                $table->addIndex(['path_hash'], 'duplicatefinder_ph_idx');
+            }
+            return $schema;
+        }
+        return null;
+    }
+}

lib/Service/FileInfoService.php

@@ -178,10 +178,15 @@ public function calculateHashes(FileInfo $fileInfo):FileInfo
           $fileInfo->getUpdatedAt()->getTimestamp()
         || $file->getUploadTime() >
           $fileInfo->getUpdatedAt()->getTimestamp())) {
-            $fileInfo->setFileHash($file->getStorage()->hash('sha256', $file->getInternalPath()));
-            $fileInfo->setUpdatedAt(new \DateTime());
-            $this->update($fileInfo);
-            $this->eventDispatcher->dispatchTyped(new CalculatedHashEvent($fileInfo, $oldHash));
+             $hash = $file->getStorage()->hash('sha256', $file->getInternalPath());
+            if (!is_bool($hash)) {
+                $fileInfo->setFileHash($hash);
+                $fileInfo->setUpdatedAt(new \DateTime());
+                $this->update($fileInfo);
+                $this->eventDispatcher->dispatchTyped(new CalculatedHashEvent($fileInfo, $oldHash));
+            } else {
+                throw new \Exception('Unable to calculate hash for '.$file->getInternalPath());
+            }
         }
         return $fileInfo;
     }

phpstan.neon.dist

@@ -5,4 +5,4 @@ parameters:
   bootstrapFiles:
     - %currentWorkingDirectory%/../../lib/base.php
   ignoreErrors:
-    - '#Call to an undefined method OCA\\DuplicateFinder\\Db\\((?!EEntity).)+::(s|g)et[A-Z].+\(\)#'
+    - '#Call to an undefined static method OCA\\DuplicateFinder\\Db\\EEntity::(s|g)et[A-Z].+\(\).#'