DB backups

.gitignore

@@ -29,9 +29,13 @@ ressources
 scripts/*.zip
 scripts/votes_individual/*.zip
 scripts/architecture
+scripts/update_dataset/backup/votes_datan/*.sql
+scripts/update_dataset/backup/users/*.sql
+scripts/update_dataset/backup/newsletter/*.sql
 
 #assets folder
 assets/opendata
+assets/dataset_backup/general/*.sql
 assets/imgs/deputes/*.png
 assets/imgs/deputes_original/*.png
 assets/imgs/deputes_ogp/*.png

README.md

@@ -29,7 +29,7 @@ grunt watch
 ```
 
 ## 2. Base de données
-* Importer le fichier SQL principal. Le fichier contient la structure de la base de données, ainsi que les données essentielles. Fichier : *scripts/update_dataset/backup/datan_database_[date].sql*
+* Importer le fichier SQL principal. Le fichier contient la structure de la base de données, ainsi que les données essentielles. Importer le fichier de backup le plus récent se trouvant dans le dossier suivant : *https://datan.fr/assets/dataset_backup/general/*
 
 ## 3. Variables environnement
 * Dupliquer *.htaccess.dist* et renomer le nouveau fichier en *.htaccess*  

scripts/admin.php

@@ -209,11 +209,6 @@
               <div class="list-group">
                 <a href="code_photos_ogp.php" class="list-group-item list-group-item-action">Code photos OGP</a>
               </div>
-              <h3 class="mt-3">Database backup</h2>
-              <div class="list-group">
-                <a href="exportSql.php" class="list-group-item list-group-item-action">All the database</a>
-                <a href="exportSql-votes_datan.php" class="list-group-item list-group-item-action">Only votes_datan</a>
-              </div>
               <h3 class="mt-3">Unused scripts</h2>
               <div class="list-group">
                 <a href="#" class="list-group-item list-group-item-action">Code communes_slug (code_communes_slug.php)</a>

scripts/exportSql-newsletter.php

@@ -0,0 +1,112 @@
+<?php
+
+define("MAX_EXECUTION_TIME", -1); // seconds
+ini_set('memory_limit', '-1');
+
+$timeline = time() + MAX_EXECUTION_TIME;
+
+EXPORT_TABLES($_SERVER['DATABASE_HOST'], $_SERVER['DATABASE_USERNAME'], $_SERVER['DATABASE_PASSWORD'], $_SERVER['DATABASE_NAME'], array("newsletter"));
+
+function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name = false)
+{
+    $tablesData = array(
+      'newsletter'
+    );
+    $mysqli = new mysqli($host, $user, $pass, $name);
+    $mysqli->select_db($name);
+    $mysqli->query("SET NAMES 'utf8'");
+    $queryTables = $mysqli->query('SHOW TABLES');
+    while ($row = $queryTables->fetch_row())
+    {
+        $target_tables[] = $row[0];
+    }
+    if ($tables !== false)
+    {
+        $target_tables = array_intersect($target_tables, $tables);
+    }
+    try
+    {
+        foreach ($target_tables as $table)
+        {
+          if (in_array($table, $tablesData)) {
+            $result = $mysqli->query('SELECT * FROM ' . $table);
+            $fields_amount = $result->field_count;
+            $rows_num = $mysqli->affected_rows;
+            $res = $mysqli->query('SHOW CREATE TABLE ' . $table);
+            $TableMLine = $res->fetch_row();
+            $content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
+            for ($i = 0, $st_counter = 0; $i < $fields_amount; $i ++, $st_counter = 0)
+            {
+                while ($row = $result->fetch_row())
+                { //when started (and every after 100 command cycle):
+                    if ($st_counter % 100 == 0 || $st_counter == 0)
+                    {
+                        $content .= "\nINSERT INTO " . $table . " VALUES";
+                    }
+                    $content .= "\n(";
+                    for ($j = 0; $j < $fields_amount; $j ++)
+                    {
+                        $row[$j] = str_replace("\n", "\\n", addslashes($row[$j]));
+                        if (isset($row[$j]))
+                        {
+                            $content .= '"' . $row[$j] . '"';
+                        } else
+                        {
+                            $content .= '""';
+                        }
+                        if ($j < ($fields_amount - 1))
+                        {
+                            $content .= ',';
+                        }
+                    }
+                    $content .= ")";
+                    //every after 100 command cycle [or at last line] ....p.s. but should be inserted 1 cycle eariler
+                    if ((($st_counter + 1) % 100 == 0 && $st_counter != 0) || $st_counter + 1 == $rows_num)
+                    {
+                        $content .= ";";
+                    } else
+                    {
+                        $content .= ",";
+                    }
+                    $st_counter = $st_counter + 1;
+                }
+            }
+            $content .= "\n\n\n";
+          } else {
+            $res = $mysqli->query('SHOW CREATE TABLE ' . $table);
+            $TableMLine = $res->fetch_row();
+            $content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
+            $content .= "\n\n\n";
+          }
+        }
+    } catch (Exception $e)
+    {
+        echo 'Caught exception: ', $e->getMessage(), "\n";
+    }
+    $backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
+    //header('Content-Type: application/octet-stream');
+    //header("Content-Transfer-Encoding: Binary");
+    //header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
+    //echo $content;
+
+    // NEW SYSTEM ==> Save the file in the update_dataset/backup folder
+
+    $date = date("Y-m-d");
+    $file = 'update_dataset/backup/newsletter/' . date("Ymd") .'.sql';
+
+    if(!is_file($file)){
+      file_put_contents($file, "");
+
+      $contentFinal = " /* DATABASE BACKUP FOR THE newsletter TABLE \n";
+      $contentFinal .= " Date of the backup: " . $date . " */ \n";
+      $contentFinal .= $content;
+
+      file_put_contents($file, $contentFinal);
+    } else {
+      die("File alreay exists!");
+    }
+
+    exit;
+}
+
+?>

scripts/exportSql-users.php

@@ -0,0 +1,112 @@
+<?php
+
+define("MAX_EXECUTION_TIME", -1); // seconds
+ini_set('memory_limit', '-1');
+
+$timeline = time() + MAX_EXECUTION_TIME;
+
+EXPORT_TABLES($_SERVER['DATABASE_HOST'], $_SERVER['DATABASE_USERNAME'], $_SERVER['DATABASE_PASSWORD'], $_SERVER['DATABASE_NAME'], array("users"));
+
+function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name = false)
+{
+    $tablesData = array(
+      'users'
+    );
+    $mysqli = new mysqli($host, $user, $pass, $name);
+    $mysqli->select_db($name);
+    $mysqli->query("SET NAMES 'utf8'");
+    $queryTables = $mysqli->query('SHOW TABLES');
+    while ($row = $queryTables->fetch_row())
+    {
+        $target_tables[] = $row[0];
+    }
+    if ($tables !== false)
+    {
+        $target_tables = array_intersect($target_tables, $tables);
+    }
+    try
+    {
+        foreach ($target_tables as $table)
+        {
+          if (in_array($table, $tablesData)) {
+            $result = $mysqli->query('SELECT * FROM ' . $table);
+            $fields_amount = $result->field_count;
+            $rows_num = $mysqli->affected_rows;
+            $res = $mysqli->query('SHOW CREATE TABLE ' . $table);
+            $TableMLine = $res->fetch_row();
+            $content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
+            for ($i = 0, $st_counter = 0; $i < $fields_amount; $i ++, $st_counter = 0)
+            {
+                while ($row = $result->fetch_row())
+                { //when started (and every after 100 command cycle):
+                    if ($st_counter % 100 == 0 || $st_counter == 0)
+                    {
+                        $content .= "\nINSERT INTO " . $table . " VALUES";
+                    }
+                    $content .= "\n(";
+                    for ($j = 0; $j < $fields_amount; $j ++)
+                    {
+                        $row[$j] = str_replace("\n", "\\n", addslashes($row[$j]));
+                        if (isset($row[$j]))
+                        {
+                            $content .= '"' . $row[$j] . '"';
+                        } else
+                        {
+                            $content .= '""';
+                        }
+                        if ($j < ($fields_amount - 1))
+                        {
+                            $content .= ',';
+                        }
+                    }
+                    $content .= ")";
+                    //every after 100 command cycle [or at last line] ....p.s. but should be inserted 1 cycle eariler
+                    if ((($st_counter + 1) % 100 == 0 && $st_counter != 0) || $st_counter + 1 == $rows_num)
+                    {
+                        $content .= ";";
+                    } else
+                    {
+                        $content .= ",";
+                    }
+                    $st_counter = $st_counter + 1;
+                }
+            }
+            $content .= "\n\n\n";
+          } else {
+            $res = $mysqli->query('SHOW CREATE TABLE ' . $table);
+            $TableMLine = $res->fetch_row();
+            $content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
+            $content .= "\n\n\n";
+          }
+        }
+    } catch (Exception $e)
+    {
+        echo 'Caught exception: ', $e->getMessage(), "\n";
+    }
+    $backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
+    //header('Content-Type: application/octet-stream');
+    //header("Content-Transfer-Encoding: Binary");
+    //header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
+    //echo $content;
+
+    // NEW SYSTEM ==> Save the file in the update_dataset/backup folder
+
+    $date = date("Y-m-d");
+    $file = 'update_dataset/backup/users/' . date("Ymd") .'.sql';
+
+    if(!is_file($file)){
+      file_put_contents($file, "");
+
+      $contentFinal = " /* DATABASE BACKUP FOR THE users TABLE \n";
+      $contentFinal .= " Date of the backup: " . $date . " */ \n";
+      $contentFinal .= $content;
+
+      file_put_contents($file, $contentFinal);
+    } else {
+      die("File alreay exists!");
+    }
+
+    exit;
+}
+
+?>

scripts/exportSql-votes_datan.php

@@ -10,20 +10,6 @@
 function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name = false)
 {
     $tablesData = array(
-      'categories',
-      'circos',
-      'cities_infos',
-      'cities_mayors',
-      'departement',
-      'election2017_results',
-      'elect_2017_pres_2',
-      'elect_2019_europe',
-      'elect_2019_europe_clean',
-      'elect_2019_europe_listes',
-      'fields',
-      'insee',
-      'posts',
-      'regions',
       'votes_datan'
     );
     $mysqli = new mysqli($host, $user, $pass, $name);
@@ -97,11 +83,32 @@ function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name
     {
         echo 'Caught exception: ', $e->getMessage(), "\n";
     }
-    $backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
-    header('Content-Type: application/octet-stream');
-    header("Content-Transfer-Encoding: Binary");
-    header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
-    echo $content;
+    //$backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
+    //header('Content-Type: application/octet-stream');
+    //header("Content-Transfer-Encoding: Binary");
+    //header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
+    //echo $content;
+
+    // NEW SYSTEM ==> Save the file in the update_dataset/backup folder
+
+    $date = date("Y-m-d");
+    $file = 'update_dataset/backup/votes_datan/' . date("Ymd") .'.sql';
+
+    if(!is_file($file)){
+      file_put_contents($file, "");
+
+      $contentFinal = " /* DATABASE BACKUP FOR THE votes_datan TABLE \n";
+      $contentFinal .= " Date of the backup: " . $date . " \n";
+      $contentFinal .= " Information: This table gathers all votes from the French National Assembly that have been decrypted by the Datan team \n";
+      $contentFinal .= " https://datan.fr \n";
+      $contentFinal .= " For additional requests, contact [email protected] */ \n";
+      $contentFinal .= $content;
+
+      file_put_contents($file, $contentFinal);
+    } else {
+      die("File alreay exists!");
+    }
+
     exit;
 }
 

scripts/exportSql.php

@@ -97,11 +97,31 @@ function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name
     {
         echo 'Caught exception: ', $e->getMessage(), "\n";
     }
-    $backup_name = $backup_name ? $backup_name : $name . "_database_" . date('Y-m-d') . ".sql";
-    header('Content-Type: application/octet-stream');
-    header("Content-Transfer-Encoding: Binary");
-    header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
-    echo $content;
+    //$backup_name = $backup_name ? $backup_name : $name . "_database_" . date('Y-m-d') . ".sql";
+    //header('Content-Type: application/octet-stream');
+    //header("Content-Transfer-Encoding: Binary");
+    //header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
+    //echo $content;
+
+    // NEW SYSTEM ==> Save the file in the update_dataset/backup folder
+
+    $date = date("Y-m-d");
+    $file = '../assets/dataset_backup/general/' . date("Ymd") .'.sql';
+
+    if(!is_file($file)){
+      file_put_contents($file, "");
+
+      $contentFinal = " /* DATABASE BACKUP FOR ALL IMPORTANT DATAN TABLES \n";
+      $contentFinal .= " Date of the backup: " . $date . " \n";
+      $contentFinal .= " Information: This document is a backup of all important tables of the website https://datan.fr \n";
+      $contentFinal .= " For additional requests, contact [email protected] */ \n";
+      $contentFinal .= $content;
+
+      file_put_contents($file, $contentFinal);
+    } else {
+      die("File alreay exists!");
+    }
+
     exit;
 }