Skip to content

Commit

Permalink
chore: clean up drift between parsers after proven success (#759)
Browse files Browse the repository at this point in the history
  • Loading branch information
asalem1 authored Jun 8, 2022
1 parent 13cabd6 commit 56fef60
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 84 deletions.
2 changes: 1 addition & 1 deletion giraffe/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@influxdata/giraffe",
"version": "2.26.2",
"version": "2.27.0",
"main": "dist/index.js",
"module": "dist/index.js",
"license": "MIT",
Expand Down
174 changes: 92 additions & 82 deletions giraffe/src/utils/fromFlux.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,49 @@ export const fromFlux = (fluxCSV: string): FromFluxResult => {
let tableLength = 0

try {
const chunks = splitChunks(fluxCSV)
/*
A Flux CSV response can contain multiple CSV files each joined by a newline.
This function splits up a CSV response into these individual CSV files.
See https://github.com/influxdata/flux/blob/master/docs/SPEC.md#multiple-tables.
*/
// finds the first non-whitespace character
let currentIndex = fluxCSV.search(/\S/)

if (currentIndex === -1) {
return {
table: newTable(0),
fluxGroupKeyUnion: [],
resultColumnNames: [],
}
}

// Split the response into separate chunks whenever we encounter:
//
// 1. A newline
// 2. Followed by any amount of whitespace
// 3. Followed by a newline
// 4. Followed by a `#` character
//
// The last condition is [necessary][0] for handling CSV responses with
// values containing newlines.
//
// [0]: https://github.com/influxdata/influxdb/issues/15017

const chunks = []
while (currentIndex !== -1) {
const prevIndex = currentIndex
const nextIndex = fluxCSV
.substring(currentIndex, fluxCSV.length)
.search(/\n\s*\n#/)
if (nextIndex === -1) {
chunks.push([prevIndex, fluxCSV.length])
currentIndex = -1
break
} else {
chunks.push([prevIndex, prevIndex + nextIndex])
currentIndex = prevIndex + nextIndex + 2
}
}

// declaring all nested variables here to reduce memory drain
let tableText = ''
Expand All @@ -86,8 +128,10 @@ export const fromFlux = (fluxCSV: string): FromFluxResult => {
let columnType: any = ''
let columnKey = ''
let columnDefault: any = ''
let chunk = ''

for (const chunk of chunks) {
for (const [start, end] of chunks) {
chunk = fluxCSV.substring(start, end)
const splittedChunk = chunk.split('\n')

const tableTexts = []
Expand Down Expand Up @@ -148,10 +192,39 @@ export const fromFlux = (fluxCSV: string): FromFluxResult => {
resultColumnNames.add(tableData[i][columnName])
}
}
columns[columnKey].data[tableLength + i] = parseValue(
tableData[i][columnName] || columnDefault,
columnType
)
const value = tableData[i][columnName] || columnDefault
let result = null

if (value === undefined) {
result = undefined
} else if (value === 'null') {
result = null
} else if (value === 'NaN') {
result = NaN
} else if (columnType === 'boolean' && value === 'true') {
result = true
} else if (columnType === 'boolean' && value === 'false') {
result = false
} else if (columnType === 'string') {
result = value
} else if (columnType === 'time') {
if (/\s/.test(value)) {
result = Date.parse(value.trim())
} else {
result = Date.parse(value)
}
} else if (columnType === 'number') {
if (value === '') {
result = null
} else {
const parsedValue = Number(value)
result = parsedValue === parsedValue ? parsedValue : value
}
} else {
result = null
}

columns[columnKey].data[tableLength + i] = result
}

if (annotationData.groupKey.includes(columnName)) {
Expand Down Expand Up @@ -196,9 +269,15 @@ export const fastFromFlux = (fluxCSV: string): FromFluxResult => {
let tableLength = 0

try {
fluxCSV = fluxCSV.trimEnd()
/*
A Flux CSV response can contain multiple CSV files each joined by a newline.
This function splits up a CSV response into these individual CSV files.
See https://github.com/influxdata/flux/blob/master/docs/SPEC.md#multiple-tables.
*/
// finds the first non-whitespace character
let curr = fluxCSV.search(/\S/)

if (fluxCSV === '') {
if (curr === -1) {
return {
table: newTable(0),
fluxGroupKeyUnion: [],
Expand All @@ -218,9 +297,6 @@ export const fastFromFlux = (fluxCSV: string): FromFluxResult => {
//
// [0]: https://github.com/influxdata/influxdb/issues/15017

// finds the first non-whitespace character
let curr = fluxCSV.search(/\S/)

const chunks = []
while (curr !== -1) {
const oldVal = curr
Expand Down Expand Up @@ -323,7 +399,11 @@ export const fastFromFlux = (fluxCSV: string): FromFluxResult => {
} else if (columnType === 'string') {
result = value
} else if (columnType === 'time') {
result = Date.parse(value.trim())
if (/\s/.test(value)) {
result = Date.parse(value.trim())
} else {
result = Date.parse(value)
}
} else if (columnType === 'number') {
if (value === '') {
result = null
Expand Down Expand Up @@ -373,36 +453,6 @@ export const fastFromFlux = (fluxCSV: string): FromFluxResult => {
}
}

/*
A Flux CSV response can contain multiple CSV files each joined by a newline.
This function splits up a CSV response into these individual CSV files.
See https://github.com/influxdata/flux/blob/master/docs/SPEC.md#multiple-tables.
*/
const splitChunks = (fluxCSV: string): string[] => {
const trimmedResponse = fluxCSV.trim()

if (trimmedResponse === '') {
return []
}

// Split the response into separate chunks whenever we encounter:
//
// 1. A newline
// 2. Followed by any amount of whitespace
// 3. Followed by a newline
// 4. Followed by a `#` character
//
// The last condition is [necessary][0] for handling CSV responses with
// values containing newlines.
//
// [0]: https://github.com/influxdata/influxdb/issues/15017
const chunks = trimmedResponse
.split(/\n\s*\n#/)
.map((s, i) => (i === 0 ? s : `#${s}`)) // Add back the `#` characters that were removed by splitting

return chunks
}

const parseAnnotations = (
annotationData: string,
headerRow: string[]
Expand Down Expand Up @@ -446,46 +496,6 @@ const TO_COLUMN_TYPE: {[fluxDatatype: string]: ColumnType} = {
'dateTime:RFC3339': 'time',
}

const parseValue = (value: string | undefined, columnType: ColumnType): any => {
if (value === undefined) {
return undefined
}

if (value === 'null') {
return null
}

if (value === 'NaN') {
return NaN
}

if (columnType === 'boolean' && value === 'true') {
return true
}

if (columnType === 'boolean' && value === 'false') {
return false
}

if (columnType === 'string') {
return value
}

if (columnType === 'time') {
return Date.parse(value.trim())
}

if (columnType === 'number') {
if (value === '') {
return null
}
const parsedValue = Number(value)
return parsedValue === parsedValue ? parsedValue : value
}

return null
}

/*
Each column in a parsed `Table` can only have a single type, but because we
combine columns from multiple Flux tables into a single table, we may
Expand Down
2 changes: 1 addition & 1 deletion stories/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@influxdata/giraffe-stories",
"version": "2.26.2",
"version": "2.27.0",
"license": "MIT",
"repository": {
"type": "git",
Expand Down

0 comments on commit 56fef60

Please sign in to comment.