Fix import misses some rows due to quotes

README.md

@@ -31,8 +31,8 @@ Steps:
    - Example: `imdb2meta-import -tsvPath "/home/john/Downloads/data.tsv" -badgerPath "/home/john/imdb2meta/badger"`
 
 > Note: The import takes a while (and much longer with bbolt than with BadgerDB), the process requires a lot of memory and the final DB size is fairly big.  
-> With a 6-core, 12-thread CPU and a mid-range SSD, an import of all data (5820284 rows as of 2020-11-21) into BadgerDB takes 2.5 minutes, up to 760 MB memory and the final DB size is 1.14 GB.  
-> When skipping TV episodes and storing only the minimal metadata it takes 45 seconds, up to 510 MB memory and the final DB size is 255 MB.
+> With a 6-core, 12-thread CPU and a mid-range SSD, an import of all data (7351639 rows as of 2020-11-21) into BadgerDB takes 4 minutes, up to 1.03 GB memory and the final DB size is 1.29 GB.  
+> When skipping TV episodes and storing only the minimal metadata it takes 1 minute and 5 seconds, up to 530 MB memory and the final DB size is 314 MB.
 
 CLI reference:
 

cmd/imdb2meta-import/main.go

@@ -1,11 +1,10 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
-	"encoding/csv"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"os"
 	"strconv"
@@ -33,8 +32,9 @@ var (
 )
 
 var (
-	tabRune, _ = utf8.DecodeRuneInString("\t")
-	imdbBytes  = []byte("imdb") // Bucket name for bbolt
+	tabRune, _      = utf8.DecodeRuneInString("\t")
+	imdbBytes       = []byte("imdb") // Bucket name for bbolt
+	expectedColumns = 9
 )
 
 func main() {
@@ -61,23 +61,13 @@ func main() {
 		log.Fatalf("Couldn't open TSV file: %v\n", err)
 	}
 
-	r := csv.NewReader(f)
-	r.Comma = tabRune
-	// ReuseRecord for better performance.
-	// Note: In this case the slices of the returned records when reading are backed by a reused array!
-	r.ReuseRecord = true
-	// Required for example for row 32542
-	r.LazyQuotes = true
+	s := bufio.NewScanner(f)
 
 	i := 1
 	// The first row is just the headers
-	_, err = r.Read()
-	if err == io.EOF {
+	if !s.Scan() || len(strings.Split(s.Text(), "\t")) != expectedColumns {
 		log.Fatalf("The TSV file doesn't seem to contain any data: %v\n", err)
 	}
-	if err != nil {
-		log.Fatalf("Couldn't read TSV row %v: %v\n", i, err)
-	}
 
 	var badgerDB *badger.DB
 	var boltDB *bbolt.DB
@@ -114,16 +104,19 @@ func main() {
 	storedCount := 0
 	start := time.Now()
 	for ; *limit == 0 || i <= *limit; i++ {
-		record, err := r.Read()
-		if err == io.EOF {
-			// No need to decrement i here
+		if !s.Scan() {
 			break
 		}
 		if err != nil {
 			log.Printf("Couldn't read TSV row %v: %v\n", i, err)
 			return
 		}
 
+		record := strings.Split(s.Text(), "\t")
+		if len(record) != expectedColumns {
+			log.Printf("The row didn't have the expected number of columns (row %v): %#v\n", i, record)
+			return
+		}
 		m, err := toMeta(record, *minimal)
 		if err != nil {
 			log.Printf("Couldn't create Meta from record at row %v: %#v: %v\n", i, record, err)