Commit 9a2fa6cd authored by Kevin Lyda's avatar Kevin Lyda 💬
Browse files

Merge branch 'master' into save-public-keys

parents b800292d a3587188
Pipeline #1178 passed with stage
in 3 minutes and 5 seconds
......@@ -186,7 +186,7 @@ server {
```
## auth
- `provider` : string. Name of the oauth provider. Valid providers are currently "google" and "github".
- `provider` : string. Name of the oauth provider. Valid providers are currently "google", "github" and "gitlab".
- `oauth_client_id` : string. Oauth Client ID. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/oauth_client_id`.
- `oauth_client_secret` : string. Oauth secret. This can be a secret stored in a [vault](https://www.vaultproject.io/) using the form `/vault/path/key` e.g. `/vault/secret/cashier/oauth_client_secret`.
- `oauth_callback_url` : string. URL that the Oauth provider will redirect to after user authorisation. The path is hardcoded to `"/auth/callback"` in the source.
......@@ -216,6 +216,9 @@ Supported options:
|---------:|-------------:|----------------------------------------------------------------------------------------------------------------------------------------|
| Google | domain | If this is unset then you must whitelist individual email addresses using `users_whitelist`. |
| Github | organization | If this is unset then you must whitelist individual users using `users_whitelist`. The oauth client and secrets should be issued by the specified organization. |
| Gitlab | siteurl | Optional. The url of the Gitlab site. Default: `https://gitlab.com/api/v3/` |
| Gitlab | allusers | Allow all valid users to get signed keys. Only allowed if siteurl set. |
| Gitlab | group | If `allusers` and this are unset then you must whitelist individual users using `users_whitelist`. Otherwise the user must be a member of this group. |
## ssh
- `signing_key`: string. Path to the signing ssh private key you created earlier. See the [note](#a-note-on-files) on files above.
......
......@@ -30,6 +30,7 @@ import (
"github.com/nsheridan/cashier/lib"
"github.com/nsheridan/cashier/server/auth"
"github.com/nsheridan/cashier/server/auth/github"
"github.com/nsheridan/cashier/server/auth/gitlab"
"github.com/nsheridan/cashier/server/auth/google"
"github.com/nsheridan/cashier/server/config"
"github.com/nsheridan/cashier/server/signer"
......@@ -292,15 +293,6 @@ func newState() string {
return hex.EncodeToString(k)
}
func readConfig(filename string) (*config.Config, error) {
f, err := os.Open(filename)
if err != nil {
return nil, errors.Wrap(err, "failed to parse config file")
}
defer f.Close()
return config.ReadConfig(f)
}
func loadCerts(certFile, keyFile string) (tls.Certificate, error) {
key, err := wkfs.ReadFile(keyFile)
if err != nil {
......@@ -316,7 +308,7 @@ func loadCerts(certFile, keyFile string) (tls.Certificate, error) {
func main() {
// Privileged section
flag.Parse()
conf, err := readConfig(*cfg)
conf, err := config.ReadConfig(*cfg)
if err != nil {
log.Fatal(err)
}
......@@ -388,6 +380,8 @@ func main() {
authprovider, err = google.New(conf.Auth)
case "github":
authprovider, err = github.New(conf.Auth)
case "gitlab":
authprovider, err = gitlab.New(conf.Auth)
default:
log.Fatalf("Unknown provider %s\n", conf.Auth.Provider)
}
......
package gitlab
import (
"errors"
"strconv"
"github.com/nsheridan/cashier/server/auth"
"github.com/nsheridan/cashier/server/config"
gitlabapi "github.com/xanzy/go-gitlab"
"golang.org/x/oauth2"
)
const (
name = "gitlab"
)
// Config is an implementation of `auth.Provider` for authenticating using a
// Gitlab account.
type Config struct {
config *oauth2.Config
baseurl string
group string
whitelist map[string]bool
allusers bool
}
// New creates a new Gitlab provider from a configuration.
func New(c *config.Auth) (auth.Provider, error) {
uw := make(map[string]bool)
for _, u := range c.UsersWhitelist {
uw[u] = true
}
allUsers, _ := strconv.ParseBool(c.ProviderOpts["allusers"])
if !allUsers && c.ProviderOpts["group"] == "" && len(uw) == 0 {
return nil, errors.New("gitlab_opts group and the users whitelist must not be both empty if allusers isn't true")
}
siteURL := "https://gitlab.com/"
if c.ProviderOpts["siteurl"] != "" {
siteURL = c.ProviderOpts["siteurl"]
if siteURL[len(siteURL)-1] != '/' {
return nil, errors.New("gitlab_opts siteurl must end in /")
}
} else {
if allUsers {
return nil, errors.New("gitlab_opts if allusers is set, siteurl must be set")
}
}
return &Config{
config: &oauth2.Config{
ClientID: c.OauthClientID,
ClientSecret: c.OauthClientSecret,
RedirectURL: c.OauthCallbackURL,
Endpoint: oauth2.Endpoint{
AuthURL: siteURL + "oauth/authorize",
TokenURL: siteURL + "oauth/token",
},
Scopes: []string{
"api",
},
},
group: c.ProviderOpts["group"],
whitelist: uw,
allusers: allUsers,
baseurl: siteURL + "api/v3/",
}, nil
}
// Name returns the name of the provider.
func (c *Config) Name() string {
return name
}
// Valid validates the oauth token.
func (c *Config) Valid(token *oauth2.Token) bool {
if !token.Valid() {
return false
}
if c.allusers {
return true
}
if len(c.whitelist) > 0 && !c.whitelist[c.Username(token)] {
return false
}
if c.group == "" {
// There's no group and token is valid. Can only reach
// here if user whitelist is set and user is in whitelist.
return true
}
client := gitlabapi.NewOAuthClient(nil, token.AccessToken)
client.SetBaseURL(c.baseurl)
groups, _, err := client.Groups.SearchGroup(c.group)
if err != nil {
return false
}
for _, g := range groups {
if g.Path == c.group {
return true
}
}
return false
}
// Revoke is a no-op revoke method. Gitlab doesn't allow token
// revocation - tokens live for an hour.
// Returns nil to satisfy the Provider interface.
func (c *Config) Revoke(token *oauth2.Token) error {
return nil
}
// StartSession retrieves an authentication endpoint from Gitlab.
func (c *Config) StartSession(state string) *auth.Session {
return &auth.Session{
AuthURL: c.config.AuthCodeURL(state),
}
}
// Exchange authorizes the session and returns an access token.
func (c *Config) Exchange(code string) (*oauth2.Token, error) {
return c.config.Exchange(oauth2.NoContext, code)
}
// Username retrieves the username of the Gitlab user.
func (c *Config) Username(token *oauth2.Token) string {
client := gitlabapi.NewOAuthClient(nil, token.AccessToken)
client.SetBaseURL(c.baseurl)
u, _, err := client.Users.CurrentUser()
if err != nil {
return ""
}
return u.Username
}
package gitlab
import (
"fmt"
"testing"
"github.com/nsheridan/cashier/server/auth"
"github.com/nsheridan/cashier/server/config"
"github.com/stretchr/testify/assert"
)
var (
oauthClientID = "id"
oauthClientSecret = "secret"
oauthCallbackURL = "url"
allusers = ""
siteurl = "https://exampleorg/"
group = "exampleorg"
)
func TestNew(t *testing.T) {
a := assert.New(t)
p, _ := newGitlab()
g := p.(*Config)
a.Equal(g.config.ClientID, oauthClientID)
a.Equal(g.config.ClientSecret, oauthClientSecret)
a.Equal(g.config.RedirectURL, oauthCallbackURL)
}
func TestNewBrokenSiteURL(t *testing.T) {
siteurl = "https://exampleorg"
a := assert.New(t)
_, err := newGitlab()
a.EqualError(err, "gitlab_opts siteurl must end in /")
siteurl = "https://exampleorg/"
}
func TestBadAllUsers(t *testing.T) {
allusers = "true"
siteurl = ""
a := assert.New(t)
_, err := newGitlab()
a.EqualError(err, "gitlab_opts if allusers is set, siteurl must be set")
allusers = ""
siteurl = "https://exampleorg/"
}
func TestGoodAllUsers(t *testing.T) {
allusers = "true"
a := assert.New(t)
p, _ := newGitlab()
s := p.StartSession("test_state")
a.Contains(s.AuthURL, "exampleorg/oauth/authorize")
a.Contains(s.AuthURL, "state=test_state")
a.Contains(s.AuthURL, fmt.Sprintf("client_id=%s", oauthClientID))
allusers = ""
}
func TestNewEmptyGroupList(t *testing.T) {
group = ""
a := assert.New(t)
_, err := newGitlab()
a.EqualError(err, "gitlab_opts group and the users whitelist must not be both empty if allusers isn't true")
group = "exampleorg"
}
func TestStartSession(t *testing.T) {
a := assert.New(t)
p, _ := newGitlab()
s := p.StartSession("test_state")
a.Contains(s.AuthURL, "exampleorg/oauth/authorize")
a.Contains(s.AuthURL, "state=test_state")
a.Contains(s.AuthURL, fmt.Sprintf("client_id=%s", oauthClientID))
}
func newGitlab() (auth.Provider, error) {
c := &config.Auth{
OauthClientID: oauthClientID,
OauthClientSecret: oauthClientSecret,
OauthCallbackURL: oauthCallbackURL,
ProviderOpts: map[string]string{
"group": group,
"siteurl": siteurl,
"allusers": allusers,
},
}
return New(c)
}
package config
import (
"bytes"
"fmt"
"io"
"log"
"os"
"strconv"
"strings"
"github.com/hashicorp/go-multierror"
"github.com/mitchellh/mapstructure"
"github.com/homemade/scl"
"github.com/nsheridan/cashier/server/helpers/vault"
"github.com/pkg/errors"
"github.com/spf13/viper"
)
// Config holds the final server configuration.
type Config struct {
Server *Server `mapstructure:"server"`
Auth *Auth `mapstructure:"auth"`
SSH *SSH `mapstructure:"ssh"`
AWS *AWS `mapstructure:"aws"`
Vault *Vault `mapstructure:"vault"`
Server *Server `hcl:"server"`
Auth *Auth `hcl:"auth"`
SSH *SSH `hcl:"ssh"`
AWS *AWS `hcl:"aws"`
Vault *Vault `hcl:"vault"`
}
// Database holds database configuration.
......@@ -29,51 +28,51 @@ type Database map[string]string
// Server holds the configuration specific to the web server and sessions.
type Server struct {
UseTLS bool `mapstructure:"use_tls"`
TLSKey string `mapstructure:"tls_key"`
TLSCert string `mapstructure:"tls_cert"`
LetsEncryptServername string `mapstructure:"letsencrypt_servername"`
LetsEncryptCache string `mapstructure:"letsencrypt_cachedir"`
Addr string `mapstructure:"address"`
Port int `mapstructure:"port"`
User string `mapstructure:"user"`
CookieSecret string `mapstructure:"cookie_secret"`
CSRFSecret string `mapstructure:"csrf_secret"`
HTTPLogFile string `mapstructure:"http_logfile"`
Database Database `mapstructure:"database"`
Datastore string `mapstructure:"datastore"` // Deprecated. TODO: remove.
UseTLS bool `hcl:"use_tls"`
TLSKey string `hcl:"tls_key"`
TLSCert string `hcl:"tls_cert"`
LetsEncryptServername string `hcl:"letsencrypt_servername"`
LetsEncryptCache string `hcl:"letsencrypt_cachedir"`
Addr string `hcl:"address"`
Port int `hcl:"port"`
User string `hcl:"user"`
CookieSecret string `hcl:"cookie_secret"`
CSRFSecret string `hcl:"csrf_secret"`
HTTPLogFile string `hcl:"http_logfile"`
Database Database `hcl:"database"`
Datastore string `hcl:"datastore"` // Deprecated. TODO: remove.
}
// Auth holds the configuration specific to the OAuth provider.
type Auth struct {
OauthClientID string `mapstructure:"oauth_client_id"`
OauthClientSecret string `mapstructure:"oauth_client_secret"`
OauthCallbackURL string `mapstructure:"oauth_callback_url"`
Provider string `mapstructure:"provider"`
ProviderOpts map[string]string `mapstructure:"provider_opts"`
UsersWhitelist []string `mapstructure:"users_whitelist"`
OauthClientID string `hcl:"oauth_client_id"`
OauthClientSecret string `hcl:"oauth_client_secret"`
OauthCallbackURL string `hcl:"oauth_callback_url"`
Provider string `hcl:"provider"`
ProviderOpts map[string]string `hcl:"provider_opts"`
UsersWhitelist []string `hcl:"users_whitelist"`
}
// SSH holds the configuration specific to signing ssh keys.
type SSH struct {
SigningKey string `mapstructure:"signing_key"`
AdditionalPrincipals []string `mapstructure:"additional_principals"`
MaxAge string `mapstructure:"max_age"`
Permissions []string `mapstructure:"permissions"`
SigningKey string `hcl:"signing_key"`
AdditionalPrincipals []string `hcl:"additional_principals"`
MaxAge string `hcl:"max_age"`
Permissions []string `hcl:"permissions"`
}
// AWS holds Amazon AWS configuration.
// AWS can also be configured using SDK methods.
type AWS struct {
Region string `mapstructure:"region"`
AccessKey string `mapstructure:"access_key"`
SecretKey string `mapstructure:"secret_key"`
Region string `hcl:"region"`
AccessKey string `hcl:"access_key"`
SecretKey string `hcl:"secret_key"`
}
// Vault holds Hashicorp Vault configuration.
type Vault struct {
Address string `mapstructure:"address"`
Token string `mapstructure:"token"`
Address string `hcl:"address"`
Token string `hcl:"token"`
}
func verifyConfig(c *Config) error {
......@@ -111,20 +110,22 @@ func convertDatastoreConfig(c *Config) {
case "mem":
c.Server.Database = map[string]string{"type": "mem"}
}
log.Println("The `datastore` option has been deprecated in favour of the `database` option. You should update your config.")
log.Println("The new config (passwords have been redacted) should look something like:")
fmt.Printf("server {\n database {\n")
var out bytes.Buffer
out.WriteString("The `datastore` option has been deprecated in favour of the `database` option. You should update your config.\n")
out.WriteString("The new config (passwords have been redacted) should look something like:\n")
out.WriteString("server {\n database {\n")
for k, v := range c.Server.Database {
if v == "" {
continue
}
if k == "password" {
fmt.Printf(" password = \"[ REDACTED ]\"\n")
out.WriteString(" password = \"[ REDACTED ]\"\n")
continue
}
fmt.Printf(" %s = \"%s\"\n", k, v)
out.WriteString(fmt.Sprintf(" %s = \"%s\"\n", k, v))
}
fmt.Printf(" }\n}\n")
out.WriteString(" }\n}")
log.Println(out.String())
}
}
......@@ -183,38 +184,11 @@ func setFromVault(c *Config) error {
return errors.Wrap(errs, "errors reading from vault")
}
// Unmarshal the config into a *Config
func decode() (*Config, error) {
var errs error
config := &Config{}
configPieces := map[string]interface{}{
"auth": &config.Auth,
"aws": &config.AWS,
"server": &config.Server,
"ssh": &config.SSH,
"vault": &config.Vault,
}
for key, val := range configPieces {
conf, ok := viper.Get(key).([]map[string]interface{})
if !ok {
continue
}
if err := mapstructure.WeakDecode(conf[0], val); err != nil {
errs = multierror.Append(errs, err)
}
}
return config, errs
}
// ReadConfig parses a hcl configuration file into a Config struct.
func ReadConfig(r io.Reader) (*Config, error) {
viper.SetConfigType("hcl")
if err := viper.ReadConfig(r); err != nil {
return nil, errors.Wrap(err, "unable to read config")
}
config, err := decode()
if err != nil {
return nil, errors.Wrap(err, "unable to parse config")
func ReadConfig(f string) (*Config, error) {
config := &Config{}
if err := scl.DecodeFile(config, f); err != nil {
return nil, errors.Wrapf(err, "unable to load config from file %s", f)
}
if err := setFromVault(config); err != nil {
return nil, err
......
package config
import (
"bytes"
"testing"
"github.com/nsheridan/cashier/server/config/testdata"
"github.com/stretchr/testify/assert"
)
......@@ -21,7 +19,6 @@ var (
CSRFSecret: "supersecret",
HTTPLogFile: "cashierd.log",
Database: Database{"type": "mysql", "username": "user", "password": "passwd", "address": "localhost:3306"},
Datastore: "mysql:user:passwd:localhost:3306",
},
Auth: &Auth{
OauthClientID: "client_id",
......@@ -50,7 +47,7 @@ var (
)
func TestConfigParser(t *testing.T) {
c, err := ReadConfig(bytes.NewBuffer(testdata.Config))
c, err := ReadConfig("testdata/test.config")
if err != nil {
t.Error(err)
}
......@@ -58,8 +55,7 @@ func TestConfigParser(t *testing.T) {
}
func TestConfigVerify(t *testing.T) {
bad := bytes.NewBuffer([]byte(""))
_, err := ReadConfig(bad)
_, err := ReadConfig("testdata/empty.config")
assert.Contains(t, err.Error(), "missing ssh config section", "missing server config section", "missing auth config section")
}
......
package testdata
var Config = []byte(`
server {
use_tls = true
tls_key = "server.key"
tls_cert = "server.crt"
address = "127.0.0.1"
port = 443
user = "nobody"
cookie_secret = "supersecret"
csrf_secret = "supersecret"
http_logfile = "cashierd.log"
datastore = "mysql:user:passwd:localhost:3306"
database {
type = "mysql"
username = "user"
password = "passwd"
address = "localhost:3306"
}
datastore = "mysql:user:passwd:localhost:3306"
}
auth {
provider = "google"
oauth_client_id = "client_id"
oauth_client_secret = "secret"
oauth_callback_url = "https://sshca.example.com/auth/callback"
provider_opts {
domain = "example.com"
}
users_whitelist = ["a_user"]
}
ssh {
signing_key = "signing_key"
additional_principals = ["ec2-user", "ubuntu"]
max_age = "720h"
permissions = ["permit-pty", "permit-X11-forwarding", "permit-port-forwarding", "permit-user-rc"]
}
aws {
region = "us-east-1"
access_key = "abcdef"
secret_key = "omg123"
}
vault {
address = "https://vault:8200"
token = "abc-def-456-789"
}
`)
server {
use_tls = true
tls_key = "server.key"
tls_cert = "server.crt"
address = "127.0.0.1"
port = 443
user = "nobody"
cookie_secret = "supersecret"
csrf_secret = "supersecret"
http_logfile = "cashierd.log"
database {
type = "mysql"
username = "user"
password = "passwd"
address = "localhost:3306"
}
}
auth {
provider = "google"
oauth_client_id = "client_id"
oauth_client_secret = "secret"
oauth_callback_url = "https://sshca.example.com/auth/callback"
provider_opts {
domain = "example.com"
}
users_whitelist = ["a_user"]
}
ssh {
signing_key = "signing_key"
additional_principals = ["ec2-user", "ubuntu"]
max_age = "720h"
permissions = ["permit-pty", "permit-X11-forwarding", "permit-port-forwarding", "permit-user-rc"]
}
aws {
region = "us-east-1"
access_key = "abcdef"
secret_key = "omg123"
}
vault {
address = "https://vault:8200"
token = "abc-def-456-789"
}
......@@ -346,7 +346,7 @@ func (p *Parser) listType() (*ast.ListType, error) {
}
}
switch tok.Type {
case token.NUMBER, token.FLOAT, token.STRING, token.HEREDOC:
case token.BOOL, token.NUMBER, token.FLOAT, token.STRING, token.HEREDOC:
node, err := p.literalType()
if err != nil {
return nil, err
......@@ -388,8 +388,6 @@ func (p *Parser) listType() (*ast.ListType, error) {
}
l.Add(node)
needComma = true
case token.BOOL:
// TODO(arslan) should we support? not supported by HCL yet
case token.LBRACK:
// TODO(arslan) should we support nested lists? Even though it's
// written in README of HCL, it's not a part of the grammar
......
MIT License
Copyright (c) 2016 HomeMade
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
package scl
import "github.com/hashicorp/hcl"
/*
DecodeFile reads the given input file and decodes it into the structure given by `out`.
*/
func DecodeFile(out interface{}, path string) error {
parser, err := NewParser(NewDiskSystem())
if err != nil {
return err
}
if err := parser.Parse(path); err != nil {
return err
}
return hcl.Decode(out, parser.String())
}
package scl
import (
"io"
"os"
"path/filepath"
"strings"
"time"
)
type diskFileSystem struct {
basePath string
}
/*
NewDiskSystem creates a filesystem that uses the local disk, at an optional
base path. The default base path is the current working directory.
*/
func NewDiskSystem(basePath ...string) FileSystem {
base := ""
if len(basePath) > 0 {
base = basePath[0]
}
return &diskFileSystem{base}
}
func (d *diskFileSystem) path(path string) string {
return filepath.Join(d.basePath, strings.TrimPrefix(path, d.basePath))
}
func (d *diskFileSystem) Glob(pattern string) (out []string, err error) {
return filepath.Glob(d.path(pattern))
}
func (d *diskFileSystem) ReadCloser(path string) (data io.ReadCloser, lastModified time.Time, err error) {
reader, err := os.Open(d.path(path))
if err != nil {
return nil, time.Time{}, err
}
stat, err := reader.Stat()
if err != nil {
return nil, time.Time{}, err
}
return reader, stat.ModTime(), nil
}
/*
Package scl is an implementation of a parser for the Sepia Configuration
Language.
SCL is a simple, declarative, self-documenting, semi-functional language that
extends HCL (as in https://github.com/hashicorp/hcl) in the same way that Sass
extends CSS. What that means is, any properly formatted HCL is valid SCL. If
you really enjoy HCL, you can keep using it exclusively: under the hood, SCL
‘compiles’ to HCL. The difference is that now you can explicitly include
files, use ‘mixins’ to quickly inject boilerplate code, and use properly
scoped, natural variables. The language is designed to accompany Sepia (and,
specifically, Sepia plugins) but it's a general purpose language, and can be
used for pretty much any configurational purpose.
Full documenation for the language itself, including a language specification,
tutorials and examples, is available at https://github.com/homemade/scl/wiki.
*/
package scl
/*
MixinDoc documents a mixin from a particular SCL file. Since mixins can be nested, it
also includes a tree of all child mixins.
*/
type MixinDoc struct {
Name string
File string
Line int
Reference string
Signature string
Docs string
Children MixinDocs
}
/*
MixinDocs is a slice of MixinDocs, for convenience.
*/
type MixinDocs []MixinDoc
package scl
import (
"io"
"time"
)
/*
A FileSystem is a representation of entities with names and content that can be
listed using stangard glob syntax and read by name. The typical implementation
for this is a local disk filesystem, but it could be anything – records in a
database, objects on AWS S3, the contents of a zip file, virtual files stored
inside a binary, and so forth. A FileSystem is required to instantiate the
standard Parser implementation.
*/
type FileSystem interface {
Glob(pattern string) ([]string, error)
ReadCloser(path string) (content io.ReadCloser, lastModified time.Time, err error)
}
hash: a63f3be588fdde1c135bba818644df041f3b39f773997e405f297237e78f1663
updated: 2016-11-08T15:18:15.308059681Z
imports:
- name: github.com/aryann/difflib
version: 035af7c09b120b0909dd998c92745b82f61e0b1c
- name: github.com/hashicorp/hcl
version: 6f5bfed9a0a22222fbe4e731ae3481730ba41e93
subpackages:
- hcl/ast
- hcl/parser
- hcl/scanner
- hcl/strconv
- hcl/token
- json/parser
- json/scanner
- json/token
- name: github.com/Masterminds/vcs
version: cff893e7f9fc3999fe4f1f50f5b504beb67e1164
- name: github.com/tucnak/climax
version: 4c021a579ddac03b8a085bebcb87d66c072341ef
testImports:
- name: github.com/davecgh/go-spew
version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9
subpackages:
- spew
- name: github.com/pmezard/go-difflib
version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
subpackages:
- difflib
- name: github.com/stretchr/testify
version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
subpackages:
- assert
- require
package: github.com/homemade/scl
import:
- package: github.com/hashicorp/hcl
subpackages:
- hcl/parser
testImport:
- package: github.com/stretchr/testify
version: ~1.1.3
subpackages:
- assert
- require
package scl
import (
"fmt"
"path/filepath"
"strings"
"github.com/hashicorp/hcl"
hclparser "github.com/hashicorp/hcl/hcl/parser"
)
const (
builtinMixinBody = "__body__"
builtinMixinInclude = "include"
hclIndentSize = 2
noMixinParamValue = "_"
)
/*
A Parser takes input in the form of filenames, variables values and include
paths, and transforms any SCL into HCL. Generally, a program will only call
Parse() for one file (the configuration file for that project) but it can be
called on any number of files, each of which will add to the Parser's HCL
output.
Variables and includes paths are global for all files parsed; that is, if you
Parse() multiple files, each of them will have access to the same set of
variables and use the same set of include paths. The parser variables are part
of the top-level scope: if a file changes them while it's being parsed, the
next file will have the same variable available with the changed value.
Similarly, if a file declares a new variable or mixin on the root scope, then
the next file will be able to access it. This can become confusing quickly,
so it's usually best to parse only one file and let it explicitly include
and other files at the SCL level.
SCL is an auto-documenting language, and the documentation is obtained using
the Parser's Documentation() function. Only mixins are currently documented.
Unlike the String() function, the documentation returned for Documentation()
only includes the nominated file.
*/
type Parser interface {
Parse(fileName string) error
Documentation(fileName string) (MixinDocs, error)
SetParam(name, value string)
AddIncludePath(name string)
String() string
}
type parser struct {
fs FileSystem
rootScope *scope
output []string
indent int
includePaths []string
}
/*
NewParser creates a new, standard Parser given a FileSystem. The most common FileSystem is
the DiskFileSystem, but any will do. The parser opens all files and reads all
includes using the FileSystem provided.
*/
func NewParser(fs FileSystem) (Parser, error) {
p := &parser{
fs: fs,
rootScope: newScope(),
}
return p, nil
}
func (p *parser) SetParam(name, value string) {
p.rootScope.setVariable(name, value)
}
func (p *parser) AddIncludePath(name string) {
p.includePaths = append(p.includePaths, name)
}
func (p *parser) String() string {
return strings.Join(p.output, "\n")
}
func (p *parser) Parse(fileName string) error {
lines, err := p.scanFile(fileName)
if err != nil {
return err
}
if err := p.parseTree(lines, newTokeniser(), p.rootScope); err != nil {
return err
}
return nil
}
func (p *parser) Documentation(fileName string) (MixinDocs, error) {
docs := MixinDocs{}
lines, err := p.scanFile(fileName)
if err != nil {
return docs, err
}
if err := p.parseTreeForDocumentation(lines, newTokeniser(), &docs); err != nil {
return docs, err
}
return docs, nil
}
func (p *parser) scanFile(fileName string) (lines scannerTree, err error) {
f, _, err := p.fs.ReadCloser(fileName)
if err != nil {
return lines, fmt.Errorf("Can't read %s: %s", fileName, err)
}
defer f.Close()
lines, err = newScanner(f, fileName).scan()
if err != nil {
return lines, fmt.Errorf("Can't scan %s: %s", fileName, err)
}
return
}
func (p *parser) isValid(hclString string) error {
e := hcl.Decode(&struct{}{}, hclString)
if pe, ok := e.(*hclparser.PosError); ok {
return pe.Err
} else if pe != nil {
return pe
}
return nil
}
func (p *parser) indentedValue(literal string) string {
return fmt.Sprintf("%s%s", strings.Repeat(" ", p.indent*hclIndentSize), literal)
}
func (p *parser) writeLiteralToOutput(scope *scope, literal string, block bool) error {
literal, err := scope.interpolateLiteral(literal)
if err != nil {
return err
}
line := p.indentedValue(literal)
if block {
if err := p.isValid(line + "{}"); err != nil {
return err
}
line += " {"
p.indent++
} else {
if hashCommentMatcher.MatchString(line) {
// Comments are passed through directly
} else if err := p.isValid(line + "{}"); err == nil {
line = line + "{}"
} else if err := p.isValid(line); err != nil {
return err
}
}
p.output = append(p.output, line)
return nil
}
func (p *parser) endBlock() {
p.indent--
p.output = append(p.output, p.indentedValue("}"))
}
func (p *parser) err(branch *scannerLine, e string, args ...interface{}) error {
return fmt.Errorf("[%s] %s", branch.String(), fmt.Sprintf(e, args...))
}
func (p *parser) parseTree(tree scannerTree, tkn *tokeniser, scope *scope) error {
for _, branch := range tree {
tokens, err := tkn.tokenise(branch)
if err != nil {
return p.err(branch, err.Error())
}
if len(tokens) > 0 {
token := tokens[0]
switch token.kind {
case tokenLiteral:
if err := p.parseLiteral(branch, tkn, token, scope); err != nil {
return err
}
case tokenVariableAssignment:
value, err := scope.interpolateLiteral(tokens[1].content)
if err != nil {
return err
}
scope.setVariable(token.content, value)
case tokenVariableDeclaration:
value, err := scope.interpolateLiteral(tokens[1].content)
if err != nil {
return err
}
scope.setArgumentVariable(token.content, value)
case tokenConditionalVariableAssignment:
value, err := scope.interpolateLiteral(tokens[1].content)
if err != nil {
return err
}
if v := scope.variable(token.content); v == "" {
scope.setArgumentVariable(token.content, value)
}
case tokenMixinDeclaration:
if err := p.parseMixinDeclaration(branch, tokens, scope); err != nil {
return err
}
case tokenFunctionCall:
if err := p.parseFunctionCall(branch, tkn, tokens, scope.clone()); err != nil {
return err
}
case tokenCommentStart, tokenCommentEnd, tokenLineComment:
// Do nothing
default:
return p.err(branch, "Unexpected token: %s (%s)", token.kind, branch.content)
}
}
}
return nil
}
func (p *parser) parseTreeForDocumentation(tree scannerTree, tkn *tokeniser, docs *MixinDocs) error {
comments := []string{}
resetComments := func() {
comments = []string{}
}
for _, branch := range tree {
tokens, err := tkn.tokenise(branch)
if err != nil {
return p.err(branch, err.Error())
}
if len(tokens) > 0 {
token := tokens[0]
switch token.kind {
case tokenLineComment, tokenCommentEnd:
// Do nothing
case tokenCommentStart:
p.parseBlockComment(branch.children, &comments, branch.line, 0)
case tokenMixinDeclaration:
if token.content[0] == '_' {
resetComments()
continue
}
doc := MixinDoc{
Name: token.content,
File: branch.file,
Line: branch.line,
Reference: branch.String(),
Signature: string(branch.content),
Docs: strings.Join(comments, "\n"),
}
// Clear comments
resetComments()
// Store the mixin docs and empty the running comment
if err := p.parseTreeForDocumentation(branch.children, tkn, &doc.Children); err != nil {
return err
}
*docs = append(*docs, doc)
default:
resetComments()
if err := p.parseTreeForDocumentation(branch.children, tkn, docs); err != nil {
return err
}
}
}
}
return nil
}
func (p *parser) parseBlockComment(tree scannerTree, comments *[]string, line, indentation int) error {
for _, branch := range tree {
// Re-add missing blank lines
if line == 0 {
line = branch.line
} else {
if line != branch.line-1 {
*comments = append(*comments, "")
}
line = branch.line
}
*comments = append(*comments, strings.Repeat(" ", indentation*4)+string(branch.content))
if err := p.parseBlockComment(branch.children, comments, line, indentation+1); err != nil {
return nil
}
}
return nil
}
func (p *parser) parseLiteral(branch *scannerLine, tkn *tokeniser, token token, scope *scope) error {
children := len(branch.children) > 0
if err := p.writeLiteralToOutput(scope, token.content, children); err != nil {
return p.err(branch, err.Error())
}
if children {
if err := p.parseTree(branch.children, tkn, scope.clone()); err != nil {
return err
}
p.endBlock()
}
return nil
}
func (p *parser) parseMixinDeclaration(branch *scannerLine, tokens []token, scope *scope) error {
i := 0
literalExpected := false
optionalArgStart := false
var (
arguments []token
defaults []string
current token
)
// Make sure that only variables are given as arguments
for _, v := range tokens[1:] {
switch v.kind {
case tokenLiteral:
if !literalExpected {
return p.err(branch, "Argument declaration %d [%s]: Unexpected literal", i, v.content)
}
value := v.content
// Underscore literals are 'no values' in mixin
// declarations
if value == noMixinParamValue {
value = ""
}
arguments = append(arguments, current)
defaults = append(defaults, value)
literalExpected = false
case tokenVariableAssignment:
optionalArgStart = true
literalExpected = true
current = token{
kind: tokenVariable,
content: v.content,
line: v.line,
}
i++
case tokenVariable:
if optionalArgStart {
return p.err(branch, "Argument declaration %d [%s]: A required argument can't follow an optional argument", i, v.content)
}
arguments = append(arguments, v)
defaults = append(defaults, "")
i++
default:
return p.err(branch, "Argument declaration %d [%s] is not a variable or a variable assignment", i, v.content)
}
}
if literalExpected {
return p.err(branch, "Expected a literal in mixin signature")
}
if a, d := len(arguments), len(defaults); a != d {
return p.err(branch, "Expected eqaual numbers of arguments and defaults (a:%d,d:%d)", a, d)
}
scope.setMixin(tokens[0].content, branch, arguments, defaults)
return nil
}
func (p *parser) parseFunctionCall(branch *scannerLine, tkn *tokeniser, tokens []token, scope *scope) error {
// Handle built-ins
if tokens[0].content == builtinMixinBody {
return p.parseBodyCall(branch, tkn, scope)
} else if tokens[0].content == builtinMixinInclude {
return p.parseIncludeCall(branch, tokens, scope)
}
// Make sure the mixin exists in the scope
mx, err := scope.mixin(tokens[0].content)
if err != nil {
return p.err(branch, err.Error())
}
args, err := p.extractValuesFromArgTokens(branch, tokens[1:], scope)
if err != nil {
return p.err(branch, err.Error())
}
// Add in the defaults
if l := len(args); l < len(mx.defaults) {
args = append(args, mx.defaults[l:]...)
}
// Check the argument counts
if r, g := len(mx.arguments), len(args); r != g {
return p.err(branch, "Wrong number of arguments for %s (required %d, got %d)", tokens[0].content, r, g)
}
// Set the argument values
for i := 0; i < len(mx.arguments); i++ {
scope.setArgumentVariable(mx.arguments[i].name, args[i])
}
// Set an anchor branch for the __body__ built-in
scope.branch = branch
scope.branchScope = scope.parent
// Call the function!
return p.parseTree(mx.declaration.children, tkn, scope)
}
func (p *parser) parseBodyCall(branch *scannerLine, tkn *tokeniser, scope *scope) error {
if scope.branchScope == nil {
return p.err(branch, "Unexpected error: No parent scope somehow!")
}
if scope.branch == nil {
return p.err(branch, "Unexpected error: No anchor branch!")
}
s := scope.branchScope.clone()
s.mixins = scope.mixins
s.variables = scope.variables // FIXME Merge?
return p.parseTree(scope.branch.children, tkn, s)
}
func (p *parser) includeGlob(name string, branch *scannerLine) error {
name = strings.TrimSuffix(strings.Trim(name, `"'`), ".scl") + ".scl"
vendorPath := []string{filepath.Join(filepath.Dir(branch.file), "vendor")}
vendorPath = append(vendorPath, p.includePaths...)
var paths []string
for _, ip := range vendorPath {
ipaths, err := p.fs.Glob(ip + "/" + name)
if err != nil {
return err
}
if len(ipaths) > 0 {
paths = ipaths
break
}
}
if len(paths) == 0 {
var err error
paths, err = p.fs.Glob(name)
if err != nil {
return err
}
}
if len(paths) == 0 {
return fmt.Errorf("Can't read %s: no files found", name)
}
for _, path := range paths {
if err := p.Parse(path); err != nil {
return fmt.Errorf(err.Error())
}
}
return nil
}
func (p *parser) parseIncludeCall(branch *scannerLine, tokens []token, scope *scope) error {
args, err := p.extractValuesFromArgTokens(branch, tokens[1:], scope)
if err != nil {
return p.err(branch, err.Error())
}
for _, v := range args {
if err := p.includeGlob(v, branch); err != nil {
return p.err(branch, err.Error())
}
}
return nil
}
func (p *parser) extractValuesFromArgTokens(branch *scannerLine, tokens []token, scope *scope) ([]string, error) {
var args []string
for _, v := range tokens {
switch v.kind {
case tokenLiteral:
value, err := scope.interpolateLiteral(v.content)
if err != nil {
return args, err
}
args = append(args, value)
case tokenVariable:
value := scope.variable(v.content)
if value == "" {
return args, fmt.Errorf("Variable $%s is not declared in this scope", v.content)
}
args = append(args, value)
default:
return args, fmt.Errorf("Invalid token type for function argument: %s (%s)", v.kind, branch.content)
}
}
return args, nil
}
[![Build Status](https://travis-ci.org/homemade/scl.svg?branch=master)](https://travis-ci.org/homemade/scl) [![Coverage Status](https://coveralls.io/repos/github/homemade/scl/badge.svg?branch=master)](https://coveralls.io/github/homemade/scl?branch=master) [![GoDoc](https://godoc.org/github.com/homemade/scl?status.svg)](https://godoc.org/github.com/homemade/scl) [![Language reference](https://img.shields.io/badge/language-reference-736caf.svg)](https://github.com/homemade/scl/wiki)
## Sepia Configuration Language
The Sepia Configuration Language is a simple, declarative, semi-functional, self-documenting language that extends HashiCorp's [HCL](https://github.com/hashicorp/hcl) in the same sort of way that Sass extends CSS. The syntax of SCL is concise, intuitive and flexible. Critically, it also validates much of your configuration by design, so it's harder to configure an application that seems like it should work &mdash; but doesn't.
SCL transpiles to HCL and, like CSS and Sass, any [properly formatted](https://github.com/fatih/hclfmt) HCL is valid SCL. If you have an existing HCL setup, you can transplant it to SCL directly and then start making use of the code organisation, mixins, and properly scoped variables that SCL offers.
In addition to the language itself, there is a useful [command-line tool](https://github.com/homemade/scl/tree/master/cmd/scl) than can compile your .scl files and write the output to the terminal, run gold standard tests against you code, and even fetch libraries of code from public version control systems.
This readme is concerned with the technical implementation of the Go package and the CLI tool. For a full language specification complete with examples and diagrams, see the [wiki](https://github.com/homemade/scl/wiki).
## Installation
Assuming you have Go installed, the package and CLI tool can be fetched in the usual way:
```
$ go get -u github.com/homemade/scl/...
```
## Contributions
This is fairly new software that has been tested intensively over a fairly narrow range of functions. Minor bugs are expected! If you have any suggestions or feature requests [please open an issue](https://github.com/homemade/scl/issues/new). Pull requests for bug fixes or uncontroversial improvements are appreciated.
We're currently working on standard libraries for Terraform and Hugo. If you build an SCL library for anything else, please let us know!
## Using SCL in your application
SCL is built on top of HCL, and the fundamental procedure for using it is the more or less the same: SCL code is decoded into a Go struct, informed by `hcl` tags on the struct's fields. A trivially simple example is as follows:
``` go
myConfigObject := struct {
SomeVariable int `hcl:"some_variable"`
}{}
if err := scl.DecodeFile(&myConfigObject, "/path/to/a/config/file.scl"); err != nil {
// handle error
}
// myConfigObject is now populated!
```
There are many more options&mdash;like include paths, predefined variables and documentation generation&mdash;available in the [API](https://godoc.org/github.com/homemade/scl). If you have an existing HCL set up in your application, you can easily swap out your HCL loading function for an SCL loading function to try it out!
## CLI tool
The tool, which is installed with the package, is named `scl`. With it, you can transpile .scl files to stdout, run gold standard tests that compare .scl files to .hcl files, and fetch external libraries from version control.
### Usage
Run `scl` for a command syntax.
### Examples
Basic example:
```
$ scl run $GOPATH/src/bitbucket.org/homemade/scl/fixtures/valid/basic.scl
/* .../bitbucket.org/homemade/scl/fixtures/valid/basic.scl */
wrapper {
inner = "yes"
another = "1" {
yet_another = "123"
}
}
```
Adding includes:
```
$ scl run -include $GOPATH/src/bitbucket.org/homemade/scl $GOPATH/src/bitbucket.org/homemade/scl/fixtures/valid/import.scl
/* .../bitbucket.org/homemade/scl/fixtures/valid/import.scl */
wrapper {
inner = "yes"
another = "1" {
yet_another = "123"
}
}
output = "this is from simpleMixin"
```
Adding params via cli flags:
```
$ scl run -param myVar=1 $GOPATH/src/bitbucket.org/homemade/scl/fixtures/valid/variables.scl
/* .../bitbucket.org/homemade/scl/fixtures/valid/variables.scl */
outer {
inner = 1
}
```
Adding params via environmental variables:
```
$ myVar=1 scl run $GOPATH/src/bitbucket.org/homemade/scl/fixtures/valid/variables.scl
/* .../bitbucket.org/homemade/scl/fixtures/valid/variables.scl */
outer {
inner = 1
}
```
Skipping environmental variable slurping:
```
$ myVar=1 scl run -no-env -param myVar=2 $GOPATH/src/bitbucket.org/homemade/scl/fixtures/valid/variables.scl
/* .../src/bitbucket.org/homemade/scl/fixtures/valid/variables.scl */
outer {
inner = 2
}
```
package scl
import (
"bufio"
"fmt"
"io"
"strings"
)
type scannerTree []*scannerLine
type scanner struct {
file string
reader io.Reader
lines scannerTree
}
func newScanner(reader io.Reader, filename ...string) *scanner {
file := "<no file>"
if len(filename) > 0 {
file = filename[0]
}
s := scanner{
file: file,
reader: reader,
lines: make(scannerTree, 0),
}
return &s
}
func (s *scanner) scan() (lines scannerTree, err error) {
// Split to lines
scanner := bufio.NewScanner(s.reader)
scanner.Split(bufio.ScanLines)
lineNumber := 0
rawLines := make(scannerTree, 0)
heredoc := ""
heredocContent := ""
heredocLine := 0
for scanner.Scan() {
lineNumber++
if heredoc != "" {
heredocContent += "\n" + scanner.Text()
if strings.TrimSpace(scanner.Text()) == heredoc {
// HCL requires heredocs to be terminated with a newline
rawLines = append(rawLines, newLine(s.file, lineNumber, 0, heredocContent+"\n"))
heredoc = ""
heredocContent = ""
}
continue
}
text := strings.TrimRight(scanner.Text(), " \t{}")
if text == "" {
continue
}
if matches := heredocMatcher.FindAllStringSubmatch(text, -1); matches != nil {
heredoc = matches[0][1]
heredocContent = text
heredocLine = lineNumber
continue
}
rawLines = append(rawLines, newLine(s.file, lineNumber, 0, text))
}
if heredoc != "" {
return lines, fmt.Errorf("Heredoc '%s' (started line %d) not terminated", heredoc, heredocLine)
}
// Make sure the first line has no indent
if len(rawLines) > 0 {
index := 0
s.indentLines(&index, rawLines, &lines, rawLines[0].content.indent())
}
return
}
func (s *scanner) indentLines(index *int, input scannerTree, output *scannerTree, indent int) {
// Ends when there are no more lines
if *index >= len(input) {
return
}
var lineToAdd *scannerLine
for ; *index < len(input); *index++ {
lineIndent := input[*index].content.indent()
if lineIndent == indent {
lineToAdd = input[*index].branch()
*output = append(*output, lineToAdd)
} else if lineIndent > indent {
s.indentLines(index, input, &lineToAdd.children, lineIndent)
} else if lineIndent < indent {
*index--
return
}
}
return
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment