Skip to content
This repository has been archived by the owner on Jun 12, 2018. It is now read-only.

Features: configurable database-hostname and allow insecure connection to database and minio #35

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Features: configurable database-hostname and allow insecure connection to database and minio #35

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f21da56
Add DatabaseHostname to replica options
Jun 27, 2017
34b5a4d
Add allow insecure ssl connection flags for mongo and minio
Jun 29, 2017
e121d56
Change naming of DatabaseHostname to LocalHostname
Gitzoz Feb 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions strata/cmd/mongo/lreplica_drivers/lrazureblobdriver/lrazureblobdriver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,20 @@ import (

"github.com/facebookgo/rocks-strata/strata"
"github.com/facebookgo/rocks-strata/strata/azureblobstorage"
"github.com/facebookgo/rocks-strata/strata/cmd/mongo/lreplica_drivers/lrs3driver"
"github.com/facebookgo/rocks-strata/strata/mongo/lreplica"
)

// AzureBlobOptions define basic options of your azure blob storage
type AzureBlobOptions struct {
Container string `short:"C" long:"container" description:"Azure Blob Storage container name" required:"true"`
BlobPrefix string `short:"p" long:"blob-prefix" description:"Prefix used when storing and retrieving files. Optional" optional:"true"`
}

// ReplicaOptions are used for commands like backup and restore
type ReplicaOptions struct {
MaxBackgroundCopies int `long:"max-background-copies" default:"16" description:"Backup and restore actions will use up to this many goroutines to copy files"`
Port int `long:"port" default:"27017" description:"Backup should look for a mongod instance that is listening on this port"`
Username string `long:"username" description:"If auth is configured, specify the username with admin privileges here"`
Password string `long:"password" description:"Password for the specified user."`
}

// Options define the common options needed by this strata command
type Options struct {
AzureBlobOptions AzureBlobOptions `group:"Azure Blob Options"`
Replica ReplicaOptions `group:"Replica Options"`
AzureBlobOptions AzureBlobOptions `group:"Azure Blob Options"`
Replica lrs3driver.ReplicaOptions `group:"Replica Options"`
}

// DriverFactory implements strata.DriverFactory
Expand Down Expand Up @@ -60,9 +54,11 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) {

replica, err := lreplica.NewLocalReplica(
options.Replica.MaxBackgroundCopies,
options.Replica.DatabaseHostname,
strconv.Itoa(options.Replica.Port),
options.Replica.Username,
options.Replica.Password,
options.Replica.SslAllowInvalidCertificates,
)
if err != nil {
return nil, err
Expand Down
10 changes: 6 additions & 4 deletions strata/cmd/mongo/lreplica_drivers/lrldriver/lrldriver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,19 @@ import (

"github.com/facebookgo/rocks-strata/strata"
"github.com/facebookgo/rocks-strata/strata/cmd/mongo/lreplica_drivers/lrs3driver"
"github.com/facebookgo/rocks-strata/strata/mongo/lreplica"
"github.com/facebookgo/rocks-strata/strata/lstorage"
"github.com/facebookgo/rocks-strata/strata/mongo/lreplica"
)

// FsOptions are common to all commands
type FsOptions struct {
Mountpoint string `short:"m" long:"mpoint" description:"Mount point name, such as \"~/sbackup\"" default:"~/sbackup"`
Mountpoint string `short:"m" long:"mpoint" description:"Mount point name, such as \"~/sbackup\"" default:"~/sbackup"`
}

// Options define the common options needed by this strata command
type Options struct {
LocalStorage FsOptions `group:"Storage Options"`
Replica lrs3driver.ReplicaOptions `group:"Replica Options"`
LocalStorage FsOptions `group:"Storage Options"`
Replica lrs3driver.ReplicaOptions `group:"Replica Options"`
}

// DriverFactory implements strata.DriverFactory
Expand All @@ -46,9 +46,11 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) {

replica, err := lreplica.NewLocalReplica(
options.Replica.MaxBackgroundCopies,
options.Replica.DatabaseHostname,
strconv.Itoa(options.Replica.Port),
options.Replica.Username,
options.Replica.Password,
options.Replica.SslAllowInvalidCertificates,
)
if err != nil {
return nil, err
Expand Down
15 changes: 14 additions & 1 deletion strata/cmd/mongo/lreplica_drivers/lrminiodriver/lrminiodriver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) {
secure := os.Getenv("MINIO_SECURE")
accessKey := os.Getenv("MINIO_ACCESS_KEY_ID")
secretKey := os.Getenv("MINIO_SECRET_ACCESS_KEY")
allowInsecureHTTPS := os.Getenv("MINIO_ALLOW_INSECURE_HTTPS")
if endPoint == "" || accessKey == "" || secretKey == "" {
return nil, errors.New("Environment variables MINIO_ENDPOINT, MINIO_ACCESS_KEY_ID and MINIO_SECRET_ACCESS_KEY must be set")
}
Expand All @@ -55,23 +56,35 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) {
return nil, errors.New("Valid values for environment variable MINIO_SECURE are 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False")
}

if allowInsecureHTTPS == "" {
allowInsecureHTTPS = "false"
}

allowInsecureHTTPSBool, err := strconv.ParseBool(allowInsecureHTTPS)
if err != nil {
return nil, errors.New("Valid values for environment variable MINIO_ALLOW_INSECURE_HTTPS are 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False")
}

minio, err := miniostorage.NewMinioStorage(
endPoint,
accessKey, secretKey,
options.Minio.BucketName,
options.Minio.BucketPrefix,
options.Minio.Region,
secureBool)
secureBool,
allowInsecureHTTPSBool)

if err != nil {
return nil, err
}

replica, err := lreplica.NewLocalReplica(
options.Replica.MaxBackgroundCopies,
options.Replica.DatabaseHostname,
strconv.Itoa(options.Replica.Port),
options.Replica.Username,
options.Replica.Password,
options.Replica.SslAllowInvalidCertificates,
)

if err != nil {
Expand Down
12 changes: 8 additions & 4 deletions strata/cmd/mongo/lreplica_drivers/lrs3driver/lrs3driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,12 @@ type AWSOptions struct {

// ReplicaOptions are used for commands like backup and restore
type ReplicaOptions struct {
MaxBackgroundCopies int `long:"max-background-copies" default:"16" description:"Backup and restore actions will use up to this many goroutines to copy files"`
Port int `long:"port" default:"27017" description:"Backup should look for a mongod instance that is listening on this port"`
Username string `long:"username" description:"If auth is configured, specify the username with admin privileges here"`
Password string `long:"password" description:"Password for the specified user."`
DatabaseHostname string `long:"database-hostname" default:"localhost" description:"Database hostname can be override with a specific hostname in most cases localhost is sufficient"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think a name like LocalHostname would be better than DatabaseHostname.

Do you think the following would description would make sense? 'localhost' or a hostname that is accessible on the local machine via e.g. kubernetes network.

By the way, another reason to avoid the name DatabaseHostname is that the MongoDB database might be distributed over multiple hosts.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I have totally forgotten this PR, because I changed the company and moved to a new city. :(
I've implemented your requested changes.

MaxBackgroundCopies int `long:"max-background-copies" default:"16" description:"Backup and restore actions will use up to this many goroutines to copy files"`
Port int `long:"port" default:"27017" description:"Backup should look for a mongod instance that is listening on this port"`
Username string `long:"username" description:"If auth is configured, specify the username with admin privileges here"`
Password string `long:"password" description:"Password for the specified user."`
SslAllowInvalidCertificates bool `long:"sslAllowInvalidCertificates" description:"Allows to connect to a insecure mongo instance"`
}

// Options define the common options needed by this strata command
Expand Down Expand Up @@ -70,9 +72,11 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) {
}
replica, err := lreplica.NewLocalReplica(
options.Replica.MaxBackgroundCopies,
options.Replica.DatabaseHostname,
strconv.Itoa(options.Replica.Port),
options.Replica.Username,
options.Replica.Password,
options.Replica.SslAllowInvalidCertificates,
)
if err != nil {
return nil, err
Expand Down
13 changes: 10 additions & 3 deletions strata/miniostorage/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package miniostorage

import (
"bytes"
"crypto/tls"
"io"
"io/ioutil"
"net/http"

minio "github.com/minio/minio-go"
)
Expand All @@ -25,14 +27,19 @@ func (m *MinioStorage) removePrefix(name string) string {
}

// NewMinioStorage initializes the MinioStorage with Minio arguments
func NewMinioStorage(endPoint, accessKeyID, secretAccessKey, bucket, prefix, region string, secure bool) (*MinioStorage, error) {

func NewMinioStorage(endPoint, accessKeyID, secretAccessKey, bucket, prefix, region string, secure bool, allowInsecureHTTPS bool) (*MinioStorage, error) {
mc, err := minio.New(endPoint, accessKeyID, secretAccessKey, secure)

if err != nil {
return nil, err
}

if allowInsecureHTTPS {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
mc.SetCustomTransport(tr)
}

if region == "" {
region = "us-east-1"
}
Expand Down
2 changes: 1 addition & 1 deletion strata/mongo/lreplica/mock_replica.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ type mockLocalSessionGetter struct {
mongo *mgotest.Server
}

func (mlsg *mockLocalSessionGetter) get(string, string, string) (*mgo.Session, error) {
func (mlsg *mockLocalSessionGetter) get(bool, string, string, string, string) (*mgo.Session, error) {
return mlsg.mongo.Session(), nil
}

Expand Down
58 changes: 43 additions & 15 deletions strata/mongo/lreplica/replica.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@
package lreplica

import (
"crypto/tls"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"os"
"strings"
"syscall"
Expand All @@ -22,17 +25,38 @@ import (
)

type sessionGetter interface {
get(port, username, password string) (*mgo.Session, error)
get(sslAllowInvalidCertificates bool, databaseHostname, port, username, password string) (*mgo.Session, error)
}

type localSessionGetter struct{}

// port could be the empty string
func (l *localSessionGetter) get(port, username, password string) (*mgo.Session, error) {
addr := "localhost"
func (l *localSessionGetter) get(sslAllowInvalidCertificates bool, databaseHostname, port, username, password string) (*mgo.Session, error) {
addr := databaseHostname
if port != "" {
addr += ":" + port
}

if sslAllowInvalidCertificates {
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
}

return mgo.DialWithInfo(&mgo.DialInfo{
Direct: true,
Addrs: []string{addr},
Timeout: 5 * time.Minute,
Username: username,
Password: password,
DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) {
conn, err := tls.Dial("tcp", addr.String(), tlsConfig)
if err != nil {
log.Println(err)
}
return conn, err
}})
}

return mgo.DialWithInfo(&mgo.DialInfo{
Direct: true,
Addrs: []string{addr},
Expand All @@ -44,21 +68,25 @@ func (l *localSessionGetter) get(port, username, password string) (*mgo.Session,
// LocalReplica is a replica where all methods that take a ReplicaID must be
// run on the host corresponding to ReplicaID
type LocalReplica struct {
port string
username string
password string
sessionGetter sessionGetter
maxBackgroundCopies int
databaseHostname string
port string
username string
password string
sslAllowInvalidCertificates bool
sessionGetter sessionGetter
maxBackgroundCopies int
}

// NewLocalReplica constructs a LocalReplica
func NewLocalReplica(maxBackgroundCopies int, port, username, password string) (*LocalReplica, error) {
func NewLocalReplica(maxBackgroundCopies int, databaseHostname, port, username, password string, sslAllowInvalidCertificates bool) (*LocalReplica, error) {
return &LocalReplica{
sessionGetter: &localSessionGetter{},
maxBackgroundCopies: maxBackgroundCopies,
port: port,
username: username,
password: password,
sessionGetter: &localSessionGetter{},
maxBackgroundCopies: maxBackgroundCopies,
databaseHostname: databaseHostname,
port: port,
username: username,
password: password,
sslAllowInvalidCertificates: sslAllowInvalidCertificates,
}, nil

}
Expand Down Expand Up @@ -170,7 +198,7 @@ func nestedBsonMapGet(m bson.M, arg string, moreArgs ...string) (interface{}, er
// TODO(agf): Have a way to pass in tags
func (r *LocalReplica) CreateSnapshot(replicaID, snapshotID string) (*strata.Snapshot, error) {
strata.Log("Getting session for CreateSnapshot()")
session, err := r.sessionGetter.get(r.port, r.username, r.password)
session, err := r.sessionGetter.get(r.sslAllowInvalidCertificates, r.databaseHostname, r.port, r.username, r.password)
if err != nil {
return nil, err
}
Expand Down