Skip to content

Commit

Permalink
handle stdin
Browse files Browse the repository at this point in the history
  • Loading branch information
do-i-need-a-username committed Dec 11, 2023
1 parent 43041cd commit e98e155
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 49 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@
# Go workspace file
go.work
bin/**
*.ndjson
39 changes: 28 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
[![codecov](https://codecov.io/gh/do-i-need-a-username/j2ndj/branch/main/graph/badge.svg?token=ZQZQZQZQZQ)](https://codecov.io/gh/do-i-need-a-username/j2ndj)
[![Go Reference](https://pkg.go.dev/badge/github.com/do-i-need-a-username/j2ndj.svg)](https://pkg.go.dev/github.com/do-i-need-a-username/j2ndj)

Converts a json file to [ndjson](https://ndjson.org/) file. (New Line Delimited JSON)
Converts json to [ndjson](https://ndjson.org/), New Line Delimited JSON.

## Installation ##

```bash
# Install go
# Install with go
go install github.com/do-i-need-a-username/j2ndj@latest
```

Expand All @@ -20,13 +20,30 @@ go install github.com/do-i-need-a-username/j2ndj@latest
# find install path
which j2ndj
/Users/myuser/go/bin/j2ndj
# Run the command with output file
/Users/myuser/go/bin/j2ndj -input /tmp/logs.json -output /tmp/nd_logs.ndjson
# ls
# logs.json nd_logs.ndjson

# Run the command with output file inferred from input file
/Users/myuser/go/bin/j2ndj -input /tmp/logs.json
# ls
# logs.json logs.ndjson

# stdin t ostdout
cat input.json | ./bin/j2ndj
{"age":30,"city":"New York","name":"John Doe"}
{"age":28,"city":"Los Angeles","name":"Jane Doe"}
{"age":35,"city":"Chicago","name":"Bob Smith"}

# stdin to output file
cat input.json | ./bin/j2ndj -output=./output.ndjson

# input file to stdout
./bin/j2ndj -input=./input.json
{"age":30,"city":"New York","name":"John Doe"}
{"age":28,"city":"Los Angeles","name":"Jane Doe"}
{"age":35,"city":"Chicago","name":"Bob Smith"}

# input file to output file
./bin/j2ndj -input=./input.json -output=./output.ndjson
```

## Alternative ##

Use jq to convert json to ndjson

```bash
jq -c '.[]' input.json > output.ndjson
```
17 changes: 17 additions & 0 deletions input.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
[
{
"name": "John Doe",
"age": 30,
"city": "New York"
},
{
"name": "Jane Doe",
"age": 28,
"city": "Los Angeles"
},
{
"name": "Bob Smith",
"age": 35,
"city": "Chicago"
}
]
67 changes: 29 additions & 38 deletions main.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
// run converts JSON to NDJSON format.
// It reads the input JSON file or standard input if no input file is provided.
// It writes the output to the specified file or standard output if no output file is provided.
// The function returns an error if any error occurs during the conversion process.
package main

import (
Expand All @@ -6,8 +10,6 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
)

var (
Expand All @@ -17,61 +19,50 @@ var (

// Convert json to ndjson
func run() error {
flag.Usage = func() {
fmt.Printf("Usage of %s:\n", os.Args[0])
fmt.Println("Converts a json file to a New Line Delimited JSON (ndjson) file.")
fmt.Println("Flags:")
flag.PrintDefaults()
}

flag.Parse()
var reader io.Reader
var writer io.Writer

if *input == "" {
flag.Usage()
os.Exit(1)
reader = os.Stdin
} else {
file, err := os.Open(*input)
if err != nil {
return fmt.Errorf("Error opening input file: %w", err)
}
defer file.Close()
reader = file
}

jsonFile, err := os.Open(*input)
if err != nil {
fmt.Println("Error opening file:", err)
os.Exit(1)
if *output == "" {
writer = os.Stdout
} else {
file, err := os.Create(*output)
if err != nil {
return fmt.Errorf("Error creating output file: %w", err)
}
defer file.Close()
writer = file
}
defer jsonFile.Close()

byteValue, err := io.ReadAll(jsonFile)
byteValue, err := io.ReadAll(reader)
if err != nil {
fmt.Println("Error reading file:", err)
os.Exit(1)
return fmt.Errorf("Error reading from input: %w", err)
}

var result []map[string]interface{}
err = json.Unmarshal(byteValue, &result)
if err != nil {
fmt.Println("Error parsing JSON:", err)
os.Exit(1)
}

outputFileName := *output
if outputFileName == "" {
outputFileName = strings.TrimSuffix(*input, filepath.Ext(*input)) + ".ndjson"
}
outputFile, err := os.Create(outputFileName)
if err != nil {
fmt.Println("Error creating output file:", err)
os.Exit(1)
return fmt.Errorf("Error decoding JSON from input: %w", err)
}
defer outputFile.Close()

for _, item := range result {
jsonStr, err := json.Marshal(item)
if err != nil {
fmt.Println("Error converting to JSON:", err)
os.Exit(1)
return fmt.Errorf("Error converting to JSON: %w", err)
}
_, err = outputFile.WriteString(string(jsonStr) + "\n")
_, err = writer.Write([]byte(string(jsonStr) + "\n"))
if err != nil {
fmt.Println("Error writing to output file:", err)
os.Exit(1)
return fmt.Errorf("Error writing to output: %w", err)
}
}

Expand Down

0 comments on commit e98e155

Please sign in to comment.