From 1c63947ba0ed23f7d63aedefcf7b0239755ce000 Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 20 Nov 2019 14:12:17 +0100 Subject: [PATCH 01/10] [FIX] Log fatal errors and migrate to urfave/cli Errors are properly logged to stderr and github.com/codegangsta/cli was replaced by github.com/urfave/cli. This includes the open pull request at https://github.com/lukasmartinelli/pgfutter/pull/67 --- pgfutter.go | 75 +++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/pgfutter.go b/pgfutter.go index 5e1c918..6d4ad69 100644 --- a/pgfutter.go +++ b/pgfutter.go @@ -6,19 +6,12 @@ import ( "path/filepath" "strings" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) -func exitOnError(err error) { - log.SetFlags(0) - if err != nil { - log.Fatalln(err) - } -} - //Parse table to copy to from given filename or passed flags func parseTableName(c *cli.Context, filename string) string { - tableName := c.GlobalString("table") + tableName := c.String("table") if tableName == "" { if filename == "" { // if no filename is not set, we reading stdin @@ -33,7 +26,7 @@ func parseTableName(c *cli.Context, filename string) string { func getDataType(c *cli.Context) string { dataType := "json" - if c.GlobalBool("jsonb") { + if c.Bool("jsonb") { dataType = "jsonb" } @@ -46,62 +39,62 @@ func main() { app.Version = "1.2" app.Usage = "Import JSON and CSV into PostgreSQL the easy way" app.Flags = []cli.Flag{ - cli.StringFlag{ + &cli.StringFlag{ Name: "dbname, db", Value: "postgres", Usage: "database to connect to", - EnvVar: "DB_NAME", + EnvVars: []string{"DB_NAME"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "host", Value: "localhost", Usage: "host name", - EnvVar: "DB_HOST", + EnvVars: []string{"DB_HOST"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "port", Value: "5432", Usage: "port", - EnvVar: "DB_PORT", + EnvVars: []string{"DB_PORT"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "username, user", Value: "postgres", Usage: "username", - EnvVar: "DB_USER", + EnvVars: []string{"DB_USER"}, }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "ssl", Usage: "require ssl mode", }, - cli.StringFlag{ + &cli.StringFlag{ Name: "pass, pw", Value: "", Usage: "password", - EnvVar: "DB_PASS", + EnvVars: []string{"DB_PASS"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "schema", Value: "import", Usage: "database schema", - EnvVar: "DB_SCHEMA", + EnvVars: []string{"DB_SCHEMA"}, }, - cli.StringFlag{ + &cli.StringFlag{ Name: "table", Usage: "destination table", - EnvVar: "DB_TABLE", + EnvVars: []string{"DB_TABLE"}, }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "jsonb", Usage: "use JSONB data type", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "ignore-errors", Usage: "halt transaction on inconsistencies", }, } - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ { Name: "json", Usage: "Import newline-delimited JSON objects into database", @@ -110,8 +103,8 @@ func main() { filename := c.Args().First() - ignoreErrors := c.GlobalBool("ignore-errors") - schema := c.GlobalString("schema") + ignoreErrors := c.Bool("ignore-errors") + schema := c.String("schema") tableName := parseTableName(c, filename) dataType := getDataType(c) @@ -124,29 +117,29 @@ func main() { Name: "csv", Usage: "Import CSV into database", Flags: []cli.Flag{ - cli.BoolFlag{ + &cli.BoolFlag{ Name: "excel", Usage: "support problematic Excel 2008 and Excel 2011 csv line endings", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "skip-header", Usage: "skip header row", }, - cli.StringFlag{ + &cli.StringFlag{ Name: "fields", Usage: "comma separated field names if no header row", }, - cli.StringFlag{ + &cli.StringFlag{ Name: "delimiter, d", Value: ",", Usage: "field delimiter", }, - cli.StringFlag{ + &cli.StringFlag{ Name: "null-delimiter, nd", Value: "\\N", Usage: "null delimiter", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "skip-parse-delimiter", Usage: "skip parsing escape sequences in the given delimiter", }, @@ -156,8 +149,8 @@ func main() { filename := c.Args().First() - ignoreErrors := c.GlobalBool("ignore-errors") - schema := c.GlobalString("schema") + ignoreErrors := c.Bool("ignore-errors") + schema := c.String("schema") tableName := parseTableName(c, filename) skipHeader := c.Bool("skip-header") @@ -173,5 +166,9 @@ func main() { }, } - app.Run(os.Args) + err := app.Run(os.Args) + if err != nil { + log.SetFlags(0) + log.Fatal(err) + } } From 8d59a60bdc5e29c4fb6e90e539a74ea67281b4df Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 20 Nov 2019 14:17:45 +0100 Subject: [PATCH 02/10] Update travis ci build badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 841c71d..08b5286 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# pgfutter [![Build Status](https://travis-ci.org/lukasmartinelli/pgfutter.svg?branch=master)](https://travis-ci.org/lukasmartinelli/pgfutter) [![Go Report Card](https://goreportcard.com/badge/github.com/lukasmartinelli/pgfutter)](https://goreportcard.com/report/github.com/lukasmartinelli/pgfutter) ![License](https://img.shields.io/badge/license-MIT%20License-blue.svg) +# pgfutter [![Build Status](https://travis-ci.com/romnnn/pgfutter.svg?branch=master)](https://travis-ci.org/romnnn/pgfutter) elephant From 11a40b6560bd9689441a6dc0ae3b025254e41579 Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 20 Nov 2019 14:18:15 +0100 Subject: [PATCH 03/10] Migrate to urfave/cli Replaces github.com/codegangsta/cli with github.com/urfave/cli --- .travis.yml | 2 +- postgres.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3777367..786ddfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ cache: directories: - samples install: - - go get github.com/codegangsta/cli + - go get github.com/urfave/cli - go get github.com/lib/pq - go get github.com/kennygrant/sanitize - go get github.com/cheggaaa/pb diff --git a/postgres.go b/postgres.go index 4b53b1b..78a7ffb 100644 --- a/postgres.go +++ b/postgres.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/codegangsta/cli" + "github.com/urfave/cli" "github.com/kennygrant/sanitize" ) @@ -81,15 +81,15 @@ func postgresify(identifier string) string { //parse sql connection string from cli flags func parseConnStr(c *cli.Context) string { otherParams := "sslmode=disable connect_timeout=5" - if c.GlobalBool("ssl") { + if c.Bool("ssl") { otherParams = "sslmode=require connect_timeout=5" } return fmt.Sprintf("user=%s dbname=%s password='%s' host=%s port=%s %s", - c.GlobalString("username"), - c.GlobalString("dbname"), - c.GlobalString("pass"), - c.GlobalString("host"), - c.GlobalString("port"), + c.String("username"), + c.String("dbname"), + c.String("pass"), + c.String("host"), + c.String("port"), otherParams, ) } From 72a6ca5f70e29151ddf4ab44ce31c0568a96b0b8 Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 20 Nov 2019 14:48:56 +0100 Subject: [PATCH 04/10] [FIX] Bump go version to 1.10 Additionally, obsolete sudo parameter was removed and linux was set explicitly --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 786ddfb..d953681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,9 @@ language: go -sudo: false +os: linux addons: - postgresql: 9.4 + postgresql: "9.4" go: - - 1.8 - - 1.9 + - "1.10" - tip cache: directories: From 53644110ff89064e231d5955ffe2d88e7348e48b Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 20 Nov 2019 14:51:34 +0100 Subject: [PATCH 05/10] Ignore IDEA artifacts --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5c1baca..108f63a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.idea/ + # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a From 60c349c36b28e8a8d904c5fb009790bb48f98f8a Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 20 Nov 2019 14:51:47 +0100 Subject: [PATCH 06/10] [FIX] Use aliases for cli flags --- pgfutter.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/pgfutter.go b/pgfutter.go index 6d4ad69..4c44eab 100644 --- a/pgfutter.go +++ b/pgfutter.go @@ -40,7 +40,8 @@ func main() { app.Usage = "Import JSON and CSV into PostgreSQL the easy way" app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "dbname, db", + Name: "dbname", + Aliases:[]string{"db"}, Value: "postgres", Usage: "database to connect to", EnvVars: []string{"DB_NAME"}, @@ -58,7 +59,8 @@ func main() { EnvVars: []string{"DB_PORT"}, }, &cli.StringFlag{ - Name: "username, user", + Name: "username", + Aliases:[]string{"user"}, Value: "postgres", Usage: "username", EnvVars: []string{"DB_USER"}, @@ -68,7 +70,8 @@ func main() { Usage: "require ssl mode", }, &cli.StringFlag{ - Name: "pass, pw", + Name: "pass", + Aliases:[]string{"pw"}, Value: "", Usage: "password", EnvVars: []string{"DB_PASS"}, @@ -130,14 +133,16 @@ func main() { Usage: "comma separated field names if no header row", }, &cli.StringFlag{ - Name: "delimiter, d", - Value: ",", - Usage: "field delimiter", + Name: "delimiter", + Aliases: []string{"d"}, + Value: ",", + Usage: "field delimiter", }, &cli.StringFlag{ - Name: "null-delimiter, nd", - Value: "\\N", - Usage: "null delimiter", + Name: "null-delimiter", + Aliases: []string{"nd"}, + Value: "\\N", + Usage: "null delimiter", }, &cli.BoolFlag{ Name: "skip-parse-delimiter", From aaf01d481b8ca591fe587f75eccd4f980ffc2935 Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 2 Sep 2020 13:53:26 +0200 Subject: [PATCH 07/10] revert changes to the badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08b5286..841c71d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# pgfutter [![Build Status](https://travis-ci.com/romnnn/pgfutter.svg?branch=master)](https://travis-ci.org/romnnn/pgfutter) +# pgfutter [![Build Status](https://travis-ci.org/lukasmartinelli/pgfutter.svg?branch=master)](https://travis-ci.org/lukasmartinelli/pgfutter) [![Go Report Card](https://goreportcard.com/badge/github.com/lukasmartinelli/pgfutter)](https://goreportcard.com/report/github.com/lukasmartinelli/pgfutter) ![License](https://img.shields.io/badge/license-MIT%20License-blue.svg) elephant From 78e85dc32b5cad6a8e5855550136855bf98a33b6 Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 2 Sep 2020 13:53:38 +0200 Subject: [PATCH 08/10] switch to go modules --- .travis.yml | 9 ++------- go.mod | 13 +++++++++++++ go.sum | 41 +++++++++++++++++++++++++++++++++++++++++ pgfutter.go | 2 +- postgres.go | 2 +- 5 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/.travis.yml b/.travis.yml index d953681..201534d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,7 @@ cache: directories: - samples install: - - go get github.com/urfave/cli - - go get github.com/lib/pq - - go get github.com/kennygrant/sanitize - - go get github.com/cheggaaa/pb - - go get github.com/JensRantil/go-csv - ./download_samples.sh script: - - go install && ./test.sh - - go test + - go install github.com/lukasmartinelli/pgfutter && ./test.sh + - go test -v github.com/lukasmartinelli/pgfutter diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f929a9c --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/lukasmartinelli/pgfutter + +go 1.14 + +require ( + github.com/JensRantil/go-csv v0.0.0-20191126162552-aff1990e884a + github.com/cheggaaa/pb v1.0.29 + github.com/kennygrant/sanitize v1.2.4 + github.com/lib/pq v1.8.0 + github.com/urfave/cli v1.22.4 + github.com/urfave/cli/v2 v2.2.0 + golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..af17494 --- /dev/null +++ b/go.sum @@ -0,0 +1,41 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/JensRantil/go-csv v0.0.0-20191126162552-aff1990e884a h1:fywL1sHHCOgztiCwVQTonfyM1IR6B2lciAu3rthl3Y4= +github.com/JensRantil/go-csv v0.0.0-20191126162552-aff1990e884a/go.mod h1:Ym6iV3U8nw0LeBLakDn42VdNggbAxPSlZH1G6dtkAO0= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pgfutter.go b/pgfutter.go index 4c44eab..4b12862 100644 --- a/pgfutter.go +++ b/pgfutter.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" - "github.com/urfave/cli" + "github.com/urfave/cli/v2" ) //Parse table to copy to from given filename or passed flags diff --git a/postgres.go b/postgres.go index 78a7ffb..b8bcf22 100644 --- a/postgres.go +++ b/postgres.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/urfave/cli" + "github.com/urfave/cli/v2" "github.com/kennygrant/sanitize" ) From 14736a81526a63eadb5c872dd9816f204399ba3e Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 2 Sep 2020 17:27:20 +0200 Subject: [PATCH 09/10] add a docker compose deployment for testing and fix travis errors --- .travis.yml | 2 +- Dockerfile | 17 +++++++++++++++++ csv.go | 16 ++++++++++------ docker-compose.yaml | 19 +++++++++++++++++++ download_samples.sh | 2 +- pgfutter.go | 40 +++++++++++++++++++++++++++++++++------- test.sh | 24 +++++++++++++----------- 7 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yaml diff --git a/.travis.yml b/.travis.yml index 201534d..091da5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ os: linux addons: postgresql: "9.4" go: - - "1.10" + - "1.11" - tip cache: directories: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..280ddcc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:alpine AS DOWNLOAD + +WORKDIR /app +COPY ./download_samples.sh /app/download_samples.sh +RUN sh /app/download_samples.sh + +FROM golang:alpine + +WORKDIR /app +COPY ./ /app +ENV GO111MODULE=on + +COPY --from=DOWNLOAD /app/samples /app/samples +RUN apk update && apk add postgresql-client +RUN go install github.com/lukasmartinelli/pgfutter + +CMD ["/bin/sh", "/app/test.sh"] \ No newline at end of file diff --git a/csv.go b/csv.go index 1b05a51..14969a2 100644 --- a/csv.go +++ b/csv.go @@ -125,7 +125,7 @@ func copyCSVRows(i *Import, reader *csv.Reader, ignoreErrors bool, } func importCSV(filename string, connStr string, schema string, tableName string, ignoreErrors bool, - skipHeader bool, fields string, delimiter string, excel bool, nullDelimiter string) error { + skipHeader bool, fields string, delimiter string, excel bool, nullDelimiter string, lineTerminator string) error { db, err := connect(connStr, schema) if err != nil { @@ -138,12 +138,16 @@ func importCSV(filename string, connStr string, schema string, tableName string, // Excel 2008 and 2011 and possibly other versions uses a carriage return \r // rather than a line feed \n as a newline - if excel { - dialect.LineTerminator = "\r" - } else { - dialect.LineTerminator = "\n" - } + dialect.LineTerminator = lineTerminator + if dialect.LineTerminator == "" { + if excel { + dialect.LineTerminator = "\r" + } else { + dialect.LineTerminator = "\n" + } + } + var reader *csv.Reader var bar *pb.ProgressBar if filename != "" { diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..eb821b3 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,19 @@ +version: "3" +services: + postgres: + image: postgres:latest + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + logging: + driver: none + + test: + build: + context: . + environment: + DB_USER: postgres + PGPASSWORD: secret + DB_HOST: postgres + depends_on: + - postgres \ No newline at end of file diff --git a/download_samples.sh b/download_samples.sh index cdc20d1..bac2877 100755 --- a/download_samples.sh +++ b/download_samples.sh @@ -5,7 +5,7 @@ SAMPLES_DIR="$CWD/samples" function download_json_samples() { mkdir -p $SAMPLES_DIR cd $SAMPLES_DIR - wget -nc wget -nc https://github.com/lukasmartinelli/pgfutter/releases/download/v0.1-alpha/json_sample_2015-01-01-15.json + wget -nc https://github.com/lukasmartinelli/pgfutter/releases/download/v0.1-alpha/json_sample_2015-01-01-15.json cd $CWD } diff --git a/pgfutter.go b/pgfutter.go index 4b12862..142c508 100644 --- a/pgfutter.go +++ b/pgfutter.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "log" "os" "path/filepath" @@ -33,6 +34,17 @@ func getDataType(c *cli.Context) string { return dataType } +func getInputFile(c *cli.Context, typ string) (string, error) { + filenames := c.Args().Slice() + if len(filenames) < 1 { + return "", fmt.Errorf("missing %s input file", typ) + } + if len(filenames) > 1 { + return "", fmt.Errorf("need exactly one %s input file, got %d. note that any flags must come before the filename", typ, len(filenames)) + } + return filenames[0], nil +} + func main() { app := cli.NewApp() app.Name = "pgfutter" @@ -89,6 +101,7 @@ func main() { }, &cli.BoolFlag{ Name: "jsonb", + Value: false, Usage: "use JSONB data type", }, &cli.BoolFlag{ @@ -104,16 +117,18 @@ func main() { Action: func(c *cli.Context) error { cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "", -1) - filename := c.Args().First() - + filename, err := getInputFile(c, "json") + if err != nil { + return err + } + ignoreErrors := c.Bool("ignore-errors") schema := c.String("schema") tableName := parseTableName(c, filename) dataType := getDataType(c) connStr := parseConnStr(c) - err := importJSON(filename, connStr, schema, tableName, ignoreErrors, dataType) - return err + return importJSON(filename, connStr, schema, tableName, ignoreErrors, dataType) }, }, { @@ -122,6 +137,7 @@ func main() { Flags: []cli.Flag{ &cli.BoolFlag{ Name: "excel", + Value: false, Usage: "support problematic Excel 2008 and Excel 2011 csv line endings", }, &cli.BoolFlag{ @@ -138,6 +154,12 @@ func main() { Value: ",", Usage: "field delimiter", }, + &cli.StringFlag{ + Name: "line-terminator", + Aliases: []string{"lb", "line-break", "terminator"}, + Value: "", + Usage: "line terminator (default is newline or carriage return for excel)", + }, &cli.StringFlag{ Name: "null-delimiter", Aliases: []string{"nd"}, @@ -152,7 +174,10 @@ func main() { Action: func(c *cli.Context) error { cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "", -1) - filename := c.Args().First() + filename, err := getInputFile(c, "csv") + if err != nil { + return err + } ignoreErrors := c.Bool("ignore-errors") schema := c.String("schema") @@ -164,9 +189,10 @@ func main() { skipParseheader := c.Bool("skip-parse-delimiter") excel := c.Bool("excel") delimiter := parseDelimiter(c.String("delimiter"), skipParseheader) + lineTerminator := c.String("line-terminator") connStr := parseConnStr(c) - err := importCSV(filename, connStr, schema, tableName, ignoreErrors, skipHeader, fields, delimiter, excel, nullDelimiter) - return err + + return importCSV(filename, connStr, schema, tableName, ignoreErrors, skipHeader, fields, delimiter, excel, nullDelimiter, lineTerminator) }, }, } diff --git a/test.sh b/test.sh index a78a1c7..cb0b34e 100755 --- a/test.sh +++ b/test.sh @@ -1,31 +1,32 @@ #!/bin/bash readonly CWD=$(pwd) readonly SAMPLES_DIR="$CWD/samples" +readonly DB_HOST="${DB_HOST:-localhost}" readonly DB_USER="${DB_USER:-postgres}" readonly DB_NAME="integration_test" readonly DB_SCHEMA="import" # Use public schema instead of import because of permissions function recreate_db() { - psql -U "${DB_USER}" -c "drop database if exists ${DB_NAME};" - psql -U "${DB_USER}" -c "create database ${DB_NAME};" + psql -h "${DB_HOST}" -U "${DB_USER}" -c "drop database if exists ${DB_NAME};" + psql -h "${DB_HOST}" -U "${DB_USER}" -c "create database ${DB_NAME};" } function query_counts() { local table="$1" - local counts=$(psql -U "${DB_USER}" -d "${DB_NAME}" -t -c "select count(*) from ${DB_SCHEMA}.${table}") + local counts=$(psql -h "${DB_HOST}" -U "${DB_USER}" -d "${DB_NAME}" -t -c "select count(*) from ${DB_SCHEMA}.${table}") echo "$counts" } function query_field_type() { local table="$1" - local data_type=$(psql -U "${DB_USER}" -d "${DB_NAME}" -t -c "SELECT data_type FROM information_schema.columns WHERE table_schema='${DB_SCHEMA}' AND table_name='${table}'") + local data_type=$(psql -h "${DB_HOST}" -U "${DB_USER}" -d "${DB_NAME}" -t -c "SELECT data_type FROM information_schema.columns WHERE table_schema='${DB_SCHEMA}' AND table_name='${table}'") echo "$data_type" } function import_csv_with_special_delimiter_and_trailing() { local table="csv_sample_qip12_tabdaten" local filename="$SAMPLES_DIR/csv_sample_qip12_tabdaten.csv" - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv "$filename" --delimiter=";" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter=";" "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -38,13 +39,13 @@ function import_csv_with_special_delimiter_and_trailing() { function import_csv_and_skip_header_row_with_custom_fields() { local table="csv_sample_qip12_tabdaten" local filename="$SAMPLES_DIR/csv_sample_qip12_tabdaten.csv" - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv --delimiter ";" "$filename" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter ";" "$filename" } function test_json() { local table=$1 local filename=$2 - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" json "$filename" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" json "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -58,7 +59,7 @@ function test_json() { function test_json_as_jsonb() { local table=$1 local filename=$2 - pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --jsonb json "$filename" + pgfutter --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" --jsonb json "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -74,8 +75,8 @@ function test_excel_csv() { local filename=$2 local delimiter=${3:-,} local general_args=${4:-} - - pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv "$filename" --delimiter "$delimiter" --excel + + pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter "$delimiter" --excel "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -91,7 +92,7 @@ function test_csv() { local delimiter=${3:-,} local general_args=${4:-} - pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" csv "$filename" --delimiter "$delimiter" + pgfutter $general_args --table "$table" --schema "$DB_SCHEMA" --db "$DB_NAME" --user "$DB_USER" --pass "$PGPASSWORD" csv --delimiter "$delimiter" "$filename" if [ $? -ne 0 ]; then echo "pgfutter could not import $filename" exit 300 @@ -111,6 +112,7 @@ recreate_db # test_csv "local_severe_wheather_warning_systems" "$SAMPLES_DIR/csv_sample_local_severe_wheather_warning_systems.csv" #TODO does not work cause quoted multiline char # test_csv "residential_permits" "$SAMPLES_DIR/csv_sample_residential_permits.csv" + test_csv "distribution_of_wealth_switzerland" "$SAMPLES_DIR/csv_sample_distribution_of_wealth_switzerland.csv" test_excel_csv "techcrunch_continental_usa" "$SAMPLES_DIR/csv_sample_techcrunch_continental_usa.csv" test_csv "employee_salaries" "$SAMPLES_DIR/csv_sample_employee_salaries.csv" From c476bd1160925b2005e44280e0585012bc4d80ae Mon Sep 17 00:00:00 2001 From: romnnn Date: Wed, 2 Sep 2020 17:35:43 +0200 Subject: [PATCH 10/10] enable go modules --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 091da5c..d98a6b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,5 +11,5 @@ cache: install: - ./download_samples.sh script: - - go install github.com/lukasmartinelli/pgfutter && ./test.sh - - go test -v github.com/lukasmartinelli/pgfutter + - env GO111MODULE=on go install github.com/lukasmartinelli/pgfutter && ./test.sh + - env GO111MODULE=on go test -v github.com/lukasmartinelli/pgfutter