Skip to content

Commit

Permalink
Merge pull request #291 from datanfr/database-backup
Browse files Browse the repository at this point in the history
DB backups
  • Loading branch information
awenig authored Sep 30, 2021
2 parents a760a2a + d112d82 commit a666650
Show file tree
Hide file tree
Showing 13 changed files with 280 additions and 1,428,479 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,13 @@ ressources
scripts/*.zip
scripts/votes_individual/*.zip
scripts/architecture
scripts/update_dataset/backup/votes_datan/*.sql
scripts/update_dataset/backup/users/*.sql
scripts/update_dataset/backup/newsletter/*.sql

#assets folder
assets/opendata
assets/dataset_backup/general/*.sql
assets/imgs/deputes/*.png
assets/imgs/deputes_original/*.png
assets/imgs/deputes_ogp/*.png
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ grunt watch
```

## 2. Base de données
* Importer le fichier SQL principal. Le fichier contient la structure de la base de données, ainsi que les données essentielles. Fichier : *scripts/update_dataset/backup/datan_database_[date].sql*
* Importer le fichier SQL principal. Le fichier contient la structure de la base de données, ainsi que les données essentielles. Importer le fichier de backup le plus récent se trouvant dans le dossier suivant : *https://datan.fr/assets/dataset_backup/general/*

## 3. Variables environnement
* Dupliquer *.htaccess.dist* et renomer le nouveau fichier en *.htaccess*
Expand Down
Empty file.
5 changes: 0 additions & 5 deletions scripts/admin.php
Original file line number Diff line number Diff line change
Expand Up @@ -209,11 +209,6 @@
<div class="list-group">
<a href="code_photos_ogp.php" class="list-group-item list-group-item-action">Code photos OGP</a>
</div>
<h3 class="mt-3">Database backup</h2>
<div class="list-group">
<a href="exportSql.php" class="list-group-item list-group-item-action">All the database</a>
<a href="exportSql-votes_datan.php" class="list-group-item list-group-item-action">Only votes_datan</a>
</div>
<h3 class="mt-3">Unused scripts</h2>
<div class="list-group">
<a href="#" class="list-group-item list-group-item-action">Code communes_slug (code_communes_slug.php)</a>
Expand Down
112 changes: 112 additions & 0 deletions scripts/exportSql-newsletter.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
<?php

define("MAX_EXECUTION_TIME", -1); // seconds
ini_set('memory_limit', '-1');

$timeline = time() + MAX_EXECUTION_TIME;

EXPORT_TABLES($_SERVER['DATABASE_HOST'], $_SERVER['DATABASE_USERNAME'], $_SERVER['DATABASE_PASSWORD'], $_SERVER['DATABASE_NAME'], array("newsletter"));

function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name = false)
{
$tablesData = array(
'newsletter'
);
$mysqli = new mysqli($host, $user, $pass, $name);
$mysqli->select_db($name);
$mysqli->query("SET NAMES 'utf8'");
$queryTables = $mysqli->query('SHOW TABLES');
while ($row = $queryTables->fetch_row())
{
$target_tables[] = $row[0];
}
if ($tables !== false)
{
$target_tables = array_intersect($target_tables, $tables);
}
try
{
foreach ($target_tables as $table)
{
if (in_array($table, $tablesData)) {
$result = $mysqli->query('SELECT * FROM ' . $table);
$fields_amount = $result->field_count;
$rows_num = $mysqli->affected_rows;
$res = $mysqli->query('SHOW CREATE TABLE ' . $table);
$TableMLine = $res->fetch_row();
$content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
for ($i = 0, $st_counter = 0; $i < $fields_amount; $i ++, $st_counter = 0)
{
while ($row = $result->fetch_row())
{ //when started (and every after 100 command cycle):
if ($st_counter % 100 == 0 || $st_counter == 0)
{
$content .= "\nINSERT INTO " . $table . " VALUES";
}
$content .= "\n(";
for ($j = 0; $j < $fields_amount; $j ++)
{
$row[$j] = str_replace("\n", "\\n", addslashes($row[$j]));
if (isset($row[$j]))
{
$content .= '"' . $row[$j] . '"';
} else
{
$content .= '""';
}
if ($j < ($fields_amount - 1))
{
$content .= ',';
}
}
$content .= ")";
//every after 100 command cycle [or at last line] ....p.s. but should be inserted 1 cycle eariler
if ((($st_counter + 1) % 100 == 0 && $st_counter != 0) || $st_counter + 1 == $rows_num)
{
$content .= ";";
} else
{
$content .= ",";
}
$st_counter = $st_counter + 1;
}
}
$content .= "\n\n\n";
} else {
$res = $mysqli->query('SHOW CREATE TABLE ' . $table);
$TableMLine = $res->fetch_row();
$content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
$content .= "\n\n\n";
}
}
} catch (Exception $e)
{
echo 'Caught exception: ', $e->getMessage(), "\n";
}
$backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
//header('Content-Type: application/octet-stream');
//header("Content-Transfer-Encoding: Binary");
//header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
//echo $content;

// NEW SYSTEM ==> Save the file in the update_dataset/backup folder

$date = date("Y-m-d");
$file = 'update_dataset/backup/newsletter/' . date("Ymd") .'.sql';

if(!is_file($file)){
file_put_contents($file, "");

$contentFinal = " /* DATABASE BACKUP FOR THE newsletter TABLE \n";
$contentFinal .= " Date of the backup: " . $date . " */ \n";
$contentFinal .= $content;

file_put_contents($file, $contentFinal);
} else {
die("File alreay exists!");
}

exit;
}

?>
112 changes: 112 additions & 0 deletions scripts/exportSql-users.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
<?php

define("MAX_EXECUTION_TIME", -1); // seconds
ini_set('memory_limit', '-1');

$timeline = time() + MAX_EXECUTION_TIME;

EXPORT_TABLES($_SERVER['DATABASE_HOST'], $_SERVER['DATABASE_USERNAME'], $_SERVER['DATABASE_PASSWORD'], $_SERVER['DATABASE_NAME'], array("users"));

function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name = false)
{
$tablesData = array(
'users'
);
$mysqli = new mysqli($host, $user, $pass, $name);
$mysqli->select_db($name);
$mysqli->query("SET NAMES 'utf8'");
$queryTables = $mysqli->query('SHOW TABLES');
while ($row = $queryTables->fetch_row())
{
$target_tables[] = $row[0];
}
if ($tables !== false)
{
$target_tables = array_intersect($target_tables, $tables);
}
try
{
foreach ($target_tables as $table)
{
if (in_array($table, $tablesData)) {
$result = $mysqli->query('SELECT * FROM ' . $table);
$fields_amount = $result->field_count;
$rows_num = $mysqli->affected_rows;
$res = $mysqli->query('SHOW CREATE TABLE ' . $table);
$TableMLine = $res->fetch_row();
$content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
for ($i = 0, $st_counter = 0; $i < $fields_amount; $i ++, $st_counter = 0)
{
while ($row = $result->fetch_row())
{ //when started (and every after 100 command cycle):
if ($st_counter % 100 == 0 || $st_counter == 0)
{
$content .= "\nINSERT INTO " . $table . " VALUES";
}
$content .= "\n(";
for ($j = 0; $j < $fields_amount; $j ++)
{
$row[$j] = str_replace("\n", "\\n", addslashes($row[$j]));
if (isset($row[$j]))
{
$content .= '"' . $row[$j] . '"';
} else
{
$content .= '""';
}
if ($j < ($fields_amount - 1))
{
$content .= ',';
}
}
$content .= ")";
//every after 100 command cycle [or at last line] ....p.s. but should be inserted 1 cycle eariler
if ((($st_counter + 1) % 100 == 0 && $st_counter != 0) || $st_counter + 1 == $rows_num)
{
$content .= ";";
} else
{
$content .= ",";
}
$st_counter = $st_counter + 1;
}
}
$content .= "\n\n\n";
} else {
$res = $mysqli->query('SHOW CREATE TABLE ' . $table);
$TableMLine = $res->fetch_row();
$content = (!isset($content) ? '' : $content) . "\n\n" . $TableMLine[1] . ";\n\n";
$content .= "\n\n\n";
}
}
} catch (Exception $e)
{
echo 'Caught exception: ', $e->getMessage(), "\n";
}
$backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
//header('Content-Type: application/octet-stream');
//header("Content-Transfer-Encoding: Binary");
//header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
//echo $content;

// NEW SYSTEM ==> Save the file in the update_dataset/backup folder

$date = date("Y-m-d");
$file = 'update_dataset/backup/users/' . date("Ymd") .'.sql';

if(!is_file($file)){
file_put_contents($file, "");

$contentFinal = " /* DATABASE BACKUP FOR THE users TABLE \n";
$contentFinal .= " Date of the backup: " . $date . " */ \n";
$contentFinal .= $content;

file_put_contents($file, $contentFinal);
} else {
die("File alreay exists!");
}

exit;
}

?>
45 changes: 26 additions & 19 deletions scripts/exportSql-votes_datan.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,6 @@
function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name = false)
{
$tablesData = array(
'categories',
'circos',
'cities_infos',
'cities_mayors',
'departement',
'election2017_results',
'elect_2017_pres_2',
'elect_2019_europe',
'elect_2019_europe_clean',
'elect_2019_europe_listes',
'fields',
'insee',
'posts',
'regions',
'votes_datan'
);
$mysqli = new mysqli($host, $user, $pass, $name);
Expand Down Expand Up @@ -97,11 +83,32 @@ function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name
{
echo 'Caught exception: ', $e->getMessage(), "\n";
}
$backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
header('Content-Type: application/octet-stream');
header("Content-Transfer-Encoding: Binary");
header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
echo $content;
//$backup_name = $backup_name ? $backup_name : $name . "_votes_datan_backup_" . date('Y-m-d') . ".sql";
//header('Content-Type: application/octet-stream');
//header("Content-Transfer-Encoding: Binary");
//header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
//echo $content;

// NEW SYSTEM ==> Save the file in the update_dataset/backup folder

$date = date("Y-m-d");
$file = 'update_dataset/backup/votes_datan/' . date("Ymd") .'.sql';

if(!is_file($file)){
file_put_contents($file, "");

$contentFinal = " /* DATABASE BACKUP FOR THE votes_datan TABLE \n";
$contentFinal .= " Date of the backup: " . $date . " \n";
$contentFinal .= " Information: This table gathers all votes from the French National Assembly that have been decrypted by the Datan team \n";
$contentFinal .= " https://datan.fr \n";
$contentFinal .= " For additional requests, contact [email protected] */ \n";
$contentFinal .= $content;

file_put_contents($file, $contentFinal);
} else {
die("File alreay exists!");
}

exit;
}

Expand Down
30 changes: 25 additions & 5 deletions scripts/exportSql.php
Original file line number Diff line number Diff line change
Expand Up @@ -97,11 +97,31 @@ function EXPORT_TABLES($host, $user, $pass, $name, $tables = false, $backup_name
{
echo 'Caught exception: ', $e->getMessage(), "\n";
}
$backup_name = $backup_name ? $backup_name : $name . "_database_" . date('Y-m-d') . ".sql";
header('Content-Type: application/octet-stream');
header("Content-Transfer-Encoding: Binary");
header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
echo $content;
//$backup_name = $backup_name ? $backup_name : $name . "_database_" . date('Y-m-d') . ".sql";
//header('Content-Type: application/octet-stream');
//header("Content-Transfer-Encoding: Binary");
//header("Content-disposition: attachment; filename=\"" . $backup_name . "\"");
//echo $content;

// NEW SYSTEM ==> Save the file in the update_dataset/backup folder

$date = date("Y-m-d");
$file = '../assets/dataset_backup/general/' . date("Ymd") .'.sql';

if(!is_file($file)){
file_put_contents($file, "");

$contentFinal = " /* DATABASE BACKUP FOR ALL IMPORTANT DATAN TABLES \n";
$contentFinal .= " Date of the backup: " . $date . " \n";
$contentFinal .= " Information: This document is a backup of all important tables of the website https://datan.fr \n";
$contentFinal .= " For additional requests, contact [email protected] */ \n";
$contentFinal .= $content;

file_put_contents($file, $contentFinal);
} else {
die("File alreay exists!");
}

exit;
}

Expand Down
Loading

0 comments on commit a666650

Please sign in to comment.