Skip to content

Commit

Permalink
Dev and Prod Load synced again
Browse files Browse the repository at this point in the history
  • Loading branch information
@martinheppner
martinheppner committed Nov 3, 2024
1 parent 5349aa8 commit f9cee1e
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 330 deletions.
213 changes: 0 additions & 213 deletions src/jobs/sync.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -685,201 +685,6 @@ async function createFileFromGpx(data, filePath, title, fieldLat = "lat", fieldL
}


export async function syncGPXdata(mode='dev'){
if(mode=='prod'){
// On production the table gpx has been already loaded
console.log('Skipping this step, as we are on production.');
return true;
}


// As we do a full load of the table "gpx" here, we empty it completely and fill it up afterwards
try {
await knex.raw(`TRUNCATE gpx;`);
} catch(err){
console.log('error: ', err)
}

let limit = 5000;
const query_count = await knexTourenDb('vw_gpx_to_search').count('* as anzahl');
let count_gpx = query_count[0]["anzahl"];
let count_chunks = Math.ceil(count_gpx / limit, 0);
let counter = 0;

console.log('Info: Handling', count_gpx.toLocaleString("de-de"), 'rows with gpx datapoints via ', count_chunks, ' chunks.');

while(counter < count_chunks){
const result_query = knexTourenDb('vw_gpx_to_search').select('provider', 'hashed_url', 'typ', 'waypoint', 'lat', 'lon', 'ele').whereRaw(`ROUND(lat*lon*10000) % ${count_chunks} = ${counter}`);
const result = await result_query;

try {
await knex('gpx').insert([...result]);
} catch(err){
console.log('error syncGPXdata: ', err)
}
counter++;
}
}


export async function syncFahrplan(mode='dev'){
if(mode=='prod'){
// On production the table fahrplan has been already loaded
console.log('Skipping this step, as we are on production.');
return true;
}

try {
await knex.raw(`TRUNCATE fahrplan;`);
} catch(err){
console.log('error: ', err)
}

// Now add new lines
let limit = 2500; // not more than 5000;
let counter = 0;

let chunksizer = 0;
let count_tours = 0;

const query_count = await knexTourenDb('vw_fplan_to_search').count('* as anzahl');
count_tours = query_count[0]["anzahl"];
chunksizer = Math.ceil( count_tours / limit, 0 );
if (isNaN(chunksizer) || chunksizer < 1) {
chunksizer = 1;
}

console.log('Info: Handling ', count_tours.toLocaleString("de-de"), ' rows fplan data.');
while (counter < chunksizer) {
await readAndInsertFahrplan(counter, chunksizer)
counter++;
}
}




const readAndInsertFahrplan = async (counter, chunksizer) => {
let insert_sql = '';
let mysql_sql = '';

mysql_sql = `select
trigger_id,
provider,
hashed_url,
CONCAT(DATE_FORMAT(calendar_date, '%Y-%m-%d'), ' 00:00:00') as calendar_date,
weekday, date_any_connection,
city_slug,
city_name,
city_any_connection,
best_connection_duration,
connection_rank,
DATE_FORMAT(connection_departure_datetime, '%Y-%m-%d %H:%i:%s') as connection_departure_datetime,
connection_duration,
connection_no_of_transfers,
DATE_FORMAT(connection_arrival_datetime, '%Y-%m-%d %H:%i:%s') as connection_arrival_datetime,
connection_returns_trips_back,
connection_returns_min_waiting_duration, connection_returns_max_waiting_duration,
connection_returns_warning_level,
connection_returns_warning,
return_row,
return_waiting_duration,
DATE_FORMAT(return_departure_datetime, '%Y-%m-%d %H:%i:%s') as return_departure_datetime,
return_duration,
return_no_of_transfers,
DATE_FORMAT(return_arrival_datetime, '%Y-%m-%d %H:%i:%s') as return_arrival_datetime,
totour_track_key, totour_track_duration,
fromtour_track_key,
fromtour_track_duration,
REPLACE(REPLACE(connection_description_json, '\n', ''), "'", "´") as connection_description_json,
DATE_FORMAT(connection_lastregular_arrival_datetime, '%Y-%m-%d %H:%i:%s') as connection_lastregular_arrival_datetime,
REPLACE(REPLACE(return_description_json, '\n', ''), "'", "´") as return_description_json,
DATE_FORMAT(return_firstregular_departure_datetime, '%Y-%m-%d %H:%i:%s') as return_firstregular_departure_datetime
FROM vw_fplan_to_search
WHERE ABS(trigger_id) % ${chunksizer} = ${counter};`

const result = await knexTourenDb.raw(mysql_sql);

let data = result[0].map(row => ({ ...row }));

if (!!data && Array.isArray(data) && data.length > 0) {
insert_sql = `INSERT INTO fahrplan (id,
tour_provider,
hashed_url,
calendar_date,
weekday,
date_any_connection,
city_slug,
city_name,
city_any_connection,
best_connection_duration,
connection_rank,
connection_departure_datetime,
connection_duration,
connection_no_of_transfers,
connection_arrival_datetime,
connection_returns_trips_back,
connection_returns_min_waiting_duration,
connection_returns_max_waiting_duration,
connection_returns_warning_level,
connection_returns_warning,
return_row,
return_waiting_duration,
return_departure_datetime,
return_duration,
return_no_of_transfers,
return_arrival_datetime,
totour_track_key,
totour_track_duration,
fromtour_track_key,
fromtour_track_duration,
connection_description_json,
connection_lastregular_arrival_datetime,
return_description_json,
return_firstregular_departure_datetime) VALUES `;

for (let i = 0; i < data.length; i++) {
insert_sql += '(';

Object.keys(data[i]).forEach(column => {
//check the type of each column
const col_value = data[i][column];

if (col_value === null || col_value === undefined) {
// case of null or undefined
insert_sql += 'NULL';
} else if (column == 'connection_description_json' || column == 'return_description_json') {
insert_sql += "'";
insert_sql += col_value.replaceAll("'", '"');
insert_sql += "'";
} else {
insert_sql += "'"+col_value+"'";
}

if (column !== 'return_firstregular_departure_datetime') {
insert_sql += ', ';
}
}
);
insert_sql += ')';
if (i < data.length - 1) {
insert_sql += ', ';
}
}

try {
await knex.raw(insert_sql);
} catch (err) {
logger('############### Error with this SQL ###############');
logger(`Insert sql into fahrplan table: ${insert_sql}`);
logger('############### End of error with this SQL ###############');
}
}
return true;
};



export async function syncTours(){
// Set Maintenance mode for Zuugle (webpage is disabled)
await knex.raw(`UPDATE kpi SET VALUE=0 WHERE name='total_tours';`);
Expand Down Expand Up @@ -1024,23 +829,6 @@ const bulk_insert_tours = async (entries) => {
for (let i=0; i<entries.length; i++) {
let entry = entries[i];

let gpxData = [];
if(entry.lat_start && entry.lon_start){
gpxData.push({
lat: entry.lat_start,
lon: entry.lon_start,
typ: "first"
});
}
if(entry.lat_end && entry.lon_end){
gpxData.push({
lat: entry.lat_end,
lon: entry.lon_end,
typ: "last"
});
}
// entry.gpx_data = JSON.stringify(gpxData);

queries.push({
id: entry.id,
url: entry.url,
Expand Down Expand Up @@ -1078,7 +866,6 @@ const bulk_insert_tours = async (entries) => {
traverse: entry.traverse,
quality_rating: entry.quality_rating,
full_text: entry.full_text,
// gpx_data: entry.gpx_data,
text_lang: entry.text_lang,
max_ele: entry.maxele
});
Expand Down
38 changes: 14 additions & 24 deletions src/jobs/syncData.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,37 +4,27 @@ import {
writeKPIs,
fixTours,
syncCities,
syncFahrplan,
syncGPXdata,
syncTours
} from "./sync";
import moment from "moment";

console.log('DEVELOPMENT MODE: ', moment().format('HH:mm:ss'));
console.log('START SYNC TOURS: ', moment().format('HH:mm:ss'));
syncTours().then(res => {
console.log('DONE SYNC TOURS 1: ', moment().format('HH:mm:ss'));
console.log('START SYNC FAHRPLAN 1: ', moment().format('HH:mm:ss'));
syncFahrplan('dev').then(res1 => {
console.log('DONE SYNC FAHRPLAN 1: ', moment().format('HH:mm:ss'));
console.log('START SYNC CITIES 1: ', moment().format('HH:mm:ss'));
syncCities().then(res2 => {
console.log('DONE SYNC CITIES: ', moment().format('HH:mm:ss'));
console.log('START FIX TOURS: ', moment().format('HH:mm:ss'));
fixTours().then(res4 => {
console.log('DONE FIX TOURS: ', moment().format('HH:mm:ss'));
console.log('START WRITE KPIs: ', moment().format('HH:mm:ss'));
writeKPIs().then(res5 => {
console.log('DONE WRITING KPIs: ', moment().format('HH:mm:ss'));
console.log('START FETCH PROVIDER: ', moment().format('HH:mm:ss'));
getProvider().then(res6 => {
console.log('FETCHED PROVIDER: ', moment().format('HH:mm:ss'));
console.log('START FETCH GPX DATA: ', moment().format('HH:mm:ss'));
syncGPXdata('dev').then(res7 => {
console.log('FETCHED GPX DATA: ', moment().format('HH:mm:ss'));
process.exit();
});
});
console.log('DONE SYNC TOURS 1: ', moment().format('HH:mm:ss'));
console.log('START SYNC CITIES 1: ', moment().format('HH:mm:ss'));
syncCities().then(res2 => {
console.log('DONE SYNC CITIES: ', moment().format('HH:mm:ss'));
console.log('START FIX TOURS: ', moment().format('HH:mm:ss'));
fixTours().then(res4 => {
console.log('DONE FIX TOURS: ', moment().format('HH:mm:ss'));
console.log('START WRITE KPIs: ', moment().format('HH:mm:ss'));
writeKPIs().then(res5 => {
console.log('DONE WRITING KPIs: ', moment().format('HH:mm:ss'));
console.log('START FETCH PROVIDER: ', moment().format('HH:mm:ss'));
getProvider().then(res6 => {
console.log('FETCHED PROVIDER: ', moment().format('HH:mm:ss'));
process.exit();
});
});
});
Expand Down
28 changes: 0 additions & 28 deletions src/jobs/syncDataApplicationSide.js
View file

This file was deleted.

42 changes: 0 additions & 42 deletions src/jobs/syncDataFull.js
View file

This file was deleted.

Loading

0 comments on commit f9cee1e

Please sign in to comment.