Skip to content

Commit

Permalink
migration script has been updated to target the sinharaja data source…
Browse files Browse the repository at this point in the history 
… (no exposed data).

The connection closed state issue is persisting, despite debugging attempt. To resolve this:
1. a manual inactivity timer has been implemented to automatically close the pool after 1 hour of inactivity.
2. A new user, `deployment`, has been created to act as a dedicated user for the live site. This will ensure that my connections to the database do not cause issues for any active site users.
3. For added redundancy, the connectTimeout parameter has been added to the sqlConfig object.
  • Loading branch information
@siddheshraze
siddheshraze committed Jul 8, 2024
1 parent 5715810 commit e99b46b
Show file tree
Hide file tree
Showing 4 changed files with 1,578 additions and 243 deletions.
11 changes: 5 additions & 6 deletions frontend/components/processors/processormacros.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,20 +114,19 @@ export const fileMappings: Record<string, FileMapping> = {
},
};
const sqlConfig: PoolOptions = {
user: process.env.AZURE_SQL_USER, // better stored in an app setting such as process.env.DB_USER
password: process.env.AZURE_SQL_PASSWORD, // better stored in an app setting such as process.env.DB_PASSWORD
host: process.env.AZURE_SQL_SERVER, // better stored in an app setting such as process.env.DB_SERVER
port: parseInt(process.env.AZURE_SQL_PORT!), // optional, defaults to 1433, better stored in an app setting such as process.env.DB_PORT
user: process.env.AZURE_SQL_USER,
password: process.env.AZURE_SQL_PASSWORD,
host: process.env.AZURE_SQL_SERVER,
port: parseInt(process.env.AZURE_SQL_PORT!),
database: process.env.AZURE_SQL_CATALOG_SCHEMA,
waitForConnections: true,
connectionLimit: 10,
queueLimit: 0,
keepAliveInitialDelay: 10000, // 0 by default.
enableKeepAlive: true, // false by default.
connectTimeout: 10000, // 10 seconds by default.
};
// database: process.env.AZURE_SQL_SCHEMA!, // better stored in an app setting such as process.env.DB_NAME
export const poolMonitor = new PoolMonitor(sqlConfig);
// const pool = createPool(sqlConfig);

// Function to get a connection from the pool
export async function getSqlConnection(tries: number): Promise<PoolConnection> {
Expand Down
28 changes: 26 additions & 2 deletions frontend/config/poolmonitor.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { Pool, PoolConnection, PoolOptions, createPool } from 'mysql2/promise';

export class PoolMonitor {
public pool: Pool; // Make pool public to access it in shutdown script
public pool: Pool;
private activeConnections = 0;
private totalConnectionsCreated = 0;
private waitingForConnection = 0;
private inactivityTimer: NodeJS.Timeout | null = null;

constructor(config: PoolOptions) {
this.pool = createPool(config);
Expand All @@ -14,6 +15,7 @@ export class PoolMonitor {
this.totalConnectionsCreated++;
console.log(`Acquired: ${connection.threadId}`);
console.log('Connection state:', this.getPoolStatus());
this.resetInactivityTimer();
});

this.pool.on('release', (connection) => {
Expand All @@ -22,26 +24,33 @@ export class PoolMonitor {
}
console.log(`Released: ${connection.threadId}`);
console.log('Connection state:', this.getPoolStatus());
this.resetInactivityTimer();
});

this.pool.on('connection', (connection) => {
this.totalConnectionsCreated++;
console.log(`New: ${connection.threadId}`);
console.log('Connection state:', this.getPoolStatus());
this.resetInactivityTimer();
});

this.pool.on('enqueue', () => {
this.waitingForConnection++;
console.log(`Enqueued.`);
console.log('Connection state:', this.getPoolStatus());
this.resetInactivityTimer();
});

// Initialize inactivity timer
this.resetInactivityTimer();
}

async getConnection(): Promise<PoolConnection> {
try {
console.log('Requesting new connection...');
const connection = await this.pool.getConnection();
console.log('Connection acquired');
this.resetInactivityTimer();
return connection;
} catch (error) {
console.error('Error getting connection from pool:', error);
Expand All @@ -55,11 +64,26 @@ export class PoolMonitor {

async closeAllConnections(): Promise<void> {
try {
console.log('Ending pool connections...');
await this.pool.end();
console.log('All connections closed.');
console.log('Pool connections ended.');
} catch (error) {
console.error('Error closing connections:', error);
throw error;
}
}

private resetInactivityTimer() {
if (this.inactivityTimer) {
clearTimeout(this.inactivityTimer);
}

this.inactivityTimer = setTimeout(async () => {
if (this.activeConnections === 0) {
console.log('Inactivity period exceeded. Initiating graceful shutdown...');
await this.closeAllConnections();
console.log('Graceful shutdown complete.');
}
}, 3600000); // 1 hour in milliseconds
}
}
68 changes: 36 additions & 32 deletions frontend/sqlscripting/migration_no_mapping.sql
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
SET foreign_key_checks = 0;

-- stable_mpala: old ctfsweb schema
-- stable_sinharaja: old ctfsweb schema
-- forestgeo_scbi: new schema.
-- make sure you replace this for each new schema you pull/push from/to.

Expand Down Expand Up @@ -109,9 +109,9 @@ INSERT INTO plots (PlotID, PlotName, LocationName, CountryName, DimensionX, Dime
SELECT s.PlotID,LEFT(s.PlotName, 65535),LEFT(s.LocationName, 65535),c.CountryName,s.QDimX,s.QDimY,
IF(s.PUOM IN ('km', 'hm', 'dam', 'm', 'dm', 'cm', 'mm'), s.PUOM, 'm'),s.Area,'m2',co.GX,co.GY,co.GZ,
IF(s.GUOM IN ('km', 'hm', 'dam', 'm', 'dm', 'cm', 'mm'), s.GUOM, 'm'),s.ShapeOfSite,LEFT(s.DescriptionOfSite, 65535)
FROM stable_mpala.Site s
LEFT JOIN stable_mpala.Country c ON s.CountryID = c.CountryID
LEFT JOIN stable_mpala.Coordinates co ON s.PlotID = co.PlotID
FROM stable_sinharaja.Site s
LEFT JOIN stable_sinharaja.Country c ON s.CountryID = c.CountryID
LEFT JOIN stable_sinharaja.Coordinates co ON s.PlotID = co.PlotID
GROUP BY s.PlotID, s.PlotName, s.LocationName, c.CountryName, s.QDimX, s.QDimY, s.PUOM, s.Area, s.GUOM, co.GX, co.GY, co.GZ, s.ShapeOfSite, s.DescriptionOfSite
ON DUPLICATE KEY UPDATE
PlotName = IF(VALUES(PlotName) != '', VALUES(PlotName), plots.PlotName),
Expand All @@ -134,7 +134,7 @@ INSERT INTO reference (ReferenceID, PublicationTitle, FullReference, DateOfPubli
SELECT r.ReferenceID, r.PublicationTitle, r.FullReference,
IF(CAST(r.DateofPublication AS CHAR) = '0000-00-00', NULL, r.DateofPublication) AS DateOfPublication,
NULL
FROM stable_mpala.reference r
FROM stable_sinharaja.reference r
ON DUPLICATE KEY UPDATE
PublicationTitle = IF(VALUES(PublicationTitle) != '', VALUES(PublicationTitle), reference.PublicationTitle),
FullReference = IF(VALUES(FullReference) != '', VALUES(FullReference), reference.FullReference),
Expand All @@ -143,15 +143,15 @@ ON DUPLICATE KEY UPDATE
-- Insert into family with ON DUPLICATE KEY UPDATE
INSERT INTO family (FamilyID, Family, ReferenceID)
SELECT f.FamilyID, f.Family, f.ReferenceID
FROM stable_mpala.family f
FROM stable_sinharaja.family f
ON DUPLICATE KEY UPDATE
Family = IF(VALUES(Family) != '', VALUES(Family), family.Family),
ReferenceID = VALUES(ReferenceID);

-- Insert into genus with ON DUPLICATE KEY UPDATE
INSERT INTO genus (GenusID, FamilyID, Genus, ReferenceID, GenusAuthority)
SELECT g.GenusID, g.FamilyID, g.Genus, g.ReferenceID, g.Authority
FROM stable_mpala.genus g
FROM stable_sinharaja.genus g
ON DUPLICATE KEY UPDATE
FamilyID = VALUES(FamilyID),
Genus = IF(VALUES(Genus) != '', VALUES(Genus), genus.Genus),
Expand All @@ -161,9 +161,9 @@ ON DUPLICATE KEY UPDATE
-- Insert into species with ON DUPLICATE KEY UPDATE
INSERT INTO species (SpeciesID, GenusID, SpeciesCode, SpeciesName, SubspeciesName, IDLevel, SpeciesAuthority, SubspeciesAuthority, FieldFamily, Description, ValidCode, ReferenceID)
SELECT sp.SpeciesID, sp.GenusID, sp.Mnemonic, sp.SpeciesName, MIN(subs.SubSpeciesName), sp.IDLevel, sp.Authority, MIN(subs.Authority), sp.FieldFamily, LEFT(sp.Description, 65535), NULL, sp.ReferenceID
FROM stable_mpala.species sp
LEFT JOIN stable_mpala.subspecies subs ON sp.SpeciesID = subs.SpeciesID
LEFT JOIN stable_mpala.reference ref ON sp.ReferenceID = ref.ReferenceID
FROM stable_sinharaja.species sp
LEFT JOIN stable_sinharaja.subspecies subs ON sp.SpeciesID = subs.SpeciesID
LEFT JOIN stable_sinharaja.reference ref ON sp.ReferenceID = ref.ReferenceID
GROUP BY sp.SpeciesID, sp.GenusID, sp.Mnemonic, sp.IDLevel, sp.Authority, sp.FieldFamily, sp.Description, sp.ReferenceID
ON DUPLICATE KEY UPDATE
GenusID = VALUES(GenusID),
Expand All @@ -181,25 +181,29 @@ ON DUPLICATE KEY UPDATE
-- Insert into roles table
INSERT INTO roles (RoleID, RoleName, RoleDescription)
SELECT RoleID, Description, NULL
FROM stable_mpala.rolereference
FROM stable_sinharaja.rolereference
ON DUPLICATE KEY UPDATE
RoleName = VALUES(RoleName),
RoleDescription = VALUES(RoleDescription);

-- Insert into personnel with ON DUPLICATE KEY UPDATE and handling RoleID
INSERT INTO personnel (PersonnelID, CensusID, FirstName, LastName, RoleID)
SELECT p.PersonnelID, NULL, p.FirstName, p.LastName, pr.RoleID
FROM stable_mpala.personnel p
JOIN stable_mpala.personnelrole pr ON p.PersonnelID = pr.PersonnelID
FROM stable_sinharaja.personnel p
JOIN stable_sinharaja.personnelrole pr ON p.PersonnelID = pr.PersonnelID
ON DUPLICATE KEY UPDATE
FirstName = IF(VALUES(FirstName) != '', VALUES(FirstName), personnel.FirstName),
LastName = IF(VALUES(LastName) != '', VALUES(LastName), personnel.LastName),
RoleID = VALUES(RoleID);

UPDATE stable_sinharaja.census
SET StartDate = NULL
WHERE CAST(StartDate AS CHAR(10)) = '0000-00-00';

-- Insert into census with ON DUPLICATE KEY UPDATE
INSERT INTO census (CensusID, PlotID, StartDate, EndDate, Description, PlotCensusNumber)
SELECT c.CensusID, c.PlotID, NULLIF(c.StartDate, '0000-00-00'), c.EndDate, LEFT(c.Description, 65535), c.PlotCensusNumber
FROM stable_mpala.census c
SELECT c.CensusID, c.PlotID, c.StartDate, c.EndDate, LEFT(c.Description, 65535), c.PlotCensusNumber
FROM stable_sinharaja.census c
ON DUPLICATE KEY UPDATE
PlotID = VALUES(PlotID),
StartDate = VALUES(StartDate),
Expand All @@ -214,10 +218,10 @@ SELECT q.QuadratID,q.PlotID,cq.CensusID,LEFT(q.QuadratName, 65535),MIN(co.PX),MI
IF(s.QUOM IN ('km2', 'hm2', 'dam2', 'm2', 'dm2', 'cm2', 'mm2'), s.QUOM, 'm2'),
IF(q.IsStandardShape = 'Y', 'standard', 'not standard'),
IF(s.GUOM IN ('km', 'hm', 'dam', 'm', 'dm', 'cm', 'mm'), s.GUOM, 'm')
FROM stable_mpala.quadrat q
LEFT JOIN stable_mpala.censusquadrat cq ON q.QuadratID = cq.QuadratID
LEFT JOIN stable_mpala.Coordinates co ON q.QuadratID = co.QuadratID
LEFT JOIN stable_mpala.Site s ON q.PlotID = s.PlotID
FROM stable_sinharaja.quadrat q
LEFT JOIN stable_sinharaja.censusquadrat cq ON q.QuadratID = cq.QuadratID
LEFT JOIN stable_sinharaja.Coordinates co ON q.QuadratID = co.QuadratID
LEFT JOIN stable_sinharaja.Site s ON q.PlotID = s.PlotID
GROUP BY q.QuadratID, q.PlotID, cq.CensusID, q.QuadratName, s.QDimX, s.QDimY, s.QUOM, q.Area, q.IsStandardShape, s.GUOM
ON DUPLICATE KEY UPDATE
PlotID = VALUES(PlotID),
Expand All @@ -236,7 +240,7 @@ ON DUPLICATE KEY UPDATE
-- Insert into trees with ON DUPLICATE KEY UPDATE
INSERT INTO trees (TreeID, TreeTag, SpeciesID)
SELECT t.TreeID, t.Tag, t.SpeciesID
FROM stable_mpala.tree t
FROM stable_sinharaja.tree t
ON DUPLICATE KEY UPDATE
TreeTag = IF(VALUES(TreeTag) != '', VALUES(TreeTag), trees.TreeTag),
SpeciesID = VALUES(SpeciesID);
Expand All @@ -246,9 +250,9 @@ INSERT INTO stems (StemID, TreeID, QuadratID, StemNumber, StemTag, LocalX, Local
SELECT s.StemID,s.TreeID,s.QuadratID,s.StemNumber,s.StemTag,MIN(s.QX),MIN(s.QY),
IF(si.QUOM IN ('km', 'hm', 'dam', 'm', 'dm', 'cm', 'mm'), si.QUOM, 'm') AS CoordinateUnits,
IF(s.Moved = 'Y', 1, 0) AS Moved,LEFT(s.StemDescription, 65535)
FROM stable_mpala.stem s
LEFT JOIN stable_mpala.quadrat q ON q.QuadratID = s.QuadratID
LEFT JOIN stable_mpala.Site si ON q.PlotID = si.PlotID
FROM stable_sinharaja.stem s
LEFT JOIN stable_sinharaja.quadrat q ON q.QuadratID = s.QuadratID
LEFT JOIN stable_sinharaja.Site si ON q.PlotID = si.PlotID
GROUP BY s.StemID, s.TreeID, s.QuadratID, s.StemNumber, s.StemTag, s.Moved, s.StemDescription, si.QUOM
ON DUPLICATE KEY UPDATE
TreeID = VALUES(TreeID),
Expand All @@ -264,7 +268,7 @@ ON DUPLICATE KEY UPDATE
-- Insert into coremeasurements with ON DUPLICATE KEY UPDATE
INSERT INTO coremeasurements (CoreMeasurementID, StemID, IsValidated, MeasurementDate, MeasuredDBH, DBHUnit, MeasuredHOM, HOMUnit, Description, UserDefinedFields)
SELECT dbh.DBHID, dbh.StemID, NULL, dbh.ExactDate, dbh.DBH, 'cm', dbh.HOM, 'm', LEFT(dbh.Comments, 65535), NULL
FROM stable_mpala.dbh dbh
FROM stable_sinharaja.dbh dbh
ON DUPLICATE KEY UPDATE
StemID = VALUES(StemID),
IsValidated = VALUES(IsValidated),
Expand All @@ -279,8 +283,8 @@ ON DUPLICATE KEY UPDATE
-- Insert into quadratpersonnel with ON DUPLICATE KEY UPDATE
INSERT INTO quadratpersonnel (QuadratPersonnelID, QuadratID, PersonnelID, CensusID)
SELECT dc.DataCollectionID, dc.QuadratID, pr.PersonnelID, dc.CensusID
FROM stable_mpala.datacollection dc
JOIN stable_mpala.personnelrole pr ON dc.PersonnelRoleID = pr.PersonnelRoleID
FROM stable_sinharaja.datacollection dc
JOIN stable_sinharaja.personnelrole pr ON dc.PersonnelRoleID = pr.PersonnelRoleID
ON DUPLICATE KEY UPDATE
QuadratID = VALUES(QuadratID),
PersonnelID = VALUES(PersonnelID),
Expand All @@ -291,7 +295,7 @@ INSERT INTO attributes (Code, Description, Status)
SELECT ta.TSMCode, LEFT(ta.Description, 65535),
IF(ta.Status IN ('alive', 'alive-not measured', 'dead', 'stem dead', 'broken below', 'omitted', 'missing'),
ta.Status, NULL)
FROM stable_mpala.tsmattributes ta
FROM stable_sinharaja.tsmattributes ta
GROUP BY ta.TSMCode, ta.Description, ta.Status
ON DUPLICATE KEY UPDATE
Description = IF(VALUES(Description) != '', VALUES(Description), attributes.Description),
Expand All @@ -300,18 +304,18 @@ ON DUPLICATE KEY UPDATE
-- Insert into cmattributes with ON DUPLICATE KEY UPDATE
INSERT INTO cmattributes (CMAID, CoreMeasurementID, Code)
SELECT dbha.DBHAttID, dbha.DBHID, ta.TSMCode
FROM stable_mpala.dbhattributes dbha
JOIN stable_mpala.tsmattributes ta ON dbha.TSMID = ta.TSMID
FROM stable_sinharaja.dbhattributes dbha
JOIN stable_sinharaja.tsmattributes ta ON dbha.TSMID = ta.TSMID
ON DUPLICATE KEY UPDATE
CoreMeasurementID = VALUES(CoreMeasurementID),
Code = VALUES(Code);

-- Insert into specimens with ON DUPLICATE KEY UPDATE
INSERT INTO specimens (SpecimenID, StemID, PersonnelID, SpecimenNumber, SpeciesID, Herbarium, Voucher, CollectionDate, DeterminedBy, Description)
SELECT sp.SpecimenID, st.StemID, pr.PersonnelID, sp.SpecimenNumber, sp.SpeciesID, sp.Herbarium, sp.Voucher, sp.CollectionDate, sp.DeterminedBy, LEFT(sp.Description, 65535)
FROM stable_mpala.specimen sp
LEFT JOIN stable_mpala.stem st ON st.TreeID = sp.TreeID
LEFT JOIN stable_mpala.personnel pr ON sp.Collector = CONCAT(pr.FirstName, ' ', pr.LastName)
FROM stable_sinharaja.specimen sp
LEFT JOIN stable_sinharaja.stem st ON st.TreeID = sp.TreeID
LEFT JOIN stable_sinharaja.personnel pr ON sp.Collector = CONCAT(pr.FirstName, ' ', pr.LastName)
ON DUPLICATE KEY UPDATE
StemID = VALUES(StemID),
PersonnelID = VALUES(PersonnelID),
Expand Down
Loading

0 comments on commit e99b46b

Please sign in to comment.