[MISC] Storage Schema Versioning Model (#1057)

* [MISC] Storage Schema Versioning Model 

* fixup go.sum
* remove pq
* fix int to text issue
* fix incorrect SQL text
* use key_name vs key
* use transactions for all queries during upgrades
* fix missing parenthesis
* move upgrades to their own file
* add provider name for future usage in upgrades
* fix missing create config table values
* fix using the const instead of the provider SQL
* import logging once and reuse
* update docs
* remove db at suite teardown
* apply suggestions from code review
* fix mysql
* make errors more uniform
* style changes
* remove commented code sections
* remove commented code sections
* add schema version type
* add sql mock unit tests
* go mod tidy
* test blank row situations

docs/configuration/storage/postgres.md

@@ -25,7 +25,7 @@ storage:
 SSL mode configures how to handle SSL connections with Postgres. 
 Valid options are 'disable', 'require', 'verify-ca', or 'verify-full'.
 See the [PostgreSQL Documentation](https://www.postgresql.org/docs/12/libpq-ssl.html)
-or [Pure Go Postgres driver Documentation](https://godoc.org/github.com/lib/pq) 
+or [pgx - PostgreSQL Driver and Toolkit Documentation](https://pkg.go.dev/github.com/jackc/pgx?tab=doc) 
 for more information.
 
 ## Loading a password from a secret instead of inside the configuration

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	aletheia.icu/broccoli/fs v0.0.0-20200420200651-c5ac961a357a
+	github.com/DATA-DOG/go-sqlmock v1.4.1
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
 	github.com/Workiva/go-datastructures v1.0.52
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
@@ -16,7 +17,8 @@ require (
 	github.com/go-ldap/ldap/v3 v3.2.2
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang/mock v1.4.3
-	github.com/lib/pq v1.7.0
+	github.com/jackc/pgx/v4 v4.7.1
+	github.com/lib/pq v1.7.0 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/onsi/ginkgo v1.10.3 // indirect
 	github.com/onsi/gomega v1.7.1 // indirect

go.sum

@@ -23,6 +23,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOC
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA=
 github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k=
+github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM=
+github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7 h1:qELHH0AWCvf98Yf+CNIJx9vOZOfHFDDzgDRYsnNk/vs=
 github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
 github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4 h1:vdT7QwBhJJEVNFMBNhRSFDRCB6O16T28VhvqRgqFyn8=
@@ -61,14 +63,18 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
+github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -101,10 +107,6 @@ github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us
 github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-ldap/ldap/v3 v3.1.11 h1:EojIR9zHvfQS8LEz+EjvnPSvsfPYS3UioBezeOOskIA=
-github.com/go-ldap/ldap/v3 v3.1.11/go.mod h1:dtLsnBXnSLIsMRbCBuRpHflCGaYzZ5jn+x1q7XqMTKU=
-github.com/go-ldap/ldap/v3 v3.2.0 h1:fkS0nXg43MZvU0UNTOGyQv60WdwHRXa1eX0CSzuKLvY=
-github.com/go-ldap/ldap/v3 v3.2.0/go.mod h1:dtLsnBXnSLIsMRbCBuRpHflCGaYzZ5jn+x1q7XqMTKU=
 github.com/go-ldap/ldap/v3 v3.2.1 h1:mbP3BPfsULz5DuI3ejHuAypAbcg38Xv5T7eEHp3+XAE=
 github.com/go-ldap/ldap/v3 v3.2.1/go.mod h1:phWI+JSJ/eGvABjJxU7bT7CBv03KfS0e16+bQxLtjMw=
 github.com/go-ldap/ldap/v3 v3.2.2 h1:XIXsu/Z2SbIMrh51WMAf0t7zWftlCKoZiLU6MS8KWm8=
@@ -117,6 +119,8 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
@@ -193,6 +197,56 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
+github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
+github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
+github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
+github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
+github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
+github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
+github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
+github.com/jackc/pgconn v1.6.1 h1:lwofaXKPbIx6qEaK8mNm7uZuOwxHw+PnAFGDsDFpkRI=
+github.com/jackc/pgconn v1.6.1/go.mod h1:g8mKMqmSUO6AzAvha7vy07g1rbGOlc7iF0nU0ei83hc=
+github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
+github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
+github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.0.2 h1:q1Hsy66zh4vuNsajBUF2PNqfAMMfxU5mk594lPE9vjY=
+github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 h1:Q3tB+ExeflWUW7AFcAhXqk40s9mnNYLk1nOkKNZ5GnU=
+github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
+github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
+github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
+github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
+github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
+github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
+github.com/jackc/pgtype v1.4.0 h1:pHQfb4jh9iKqHyxPthq1fr+0HwSNIl3btYPbw2m2lbM=
+github.com/jackc/pgtype v1.4.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
+github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
+github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
+github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
+github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
+github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
+github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
+github.com/jackc/pgx/v4 v4.7.1 h1:aqUSOcStk6fik+lSE+tqfFhvt/EwT8q/oMtJbP9CjXI=
+github.com/jackc/pgx/v4 v4.7.1/go.mod h1:nu42q3aPjuC1M0Nak4bnoprKlXPINqopEKqbq5AZSC4=
+github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@@ -219,6 +273,7 @@ github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqy
 github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -226,8 +281,13 @@ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc=
 github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g=
 github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
@@ -237,7 +297,15 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -302,17 +370,26 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
+github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/savsgio/dictpool v0.0.0-20200608150529-6a3c1a8f6ab2 h1:V+VG/pzeMdwBlS21mJmNkBnQQmZWyuBgYRoz0SVxaVk=
 github.com/savsgio/dictpool v0.0.0-20200608150529-6a3c1a8f6ab2/go.mod h1:LTEdLD+Y+KR4yx9eRMIgciXZo4Od0doGWP/hjgfOlE0=
 github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c h1:2nF5+FZ4/qp7pZVL7fR6DEaSTzuDmNaFTyqp92/hwF8=
 github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY=
+github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/simia-tech/crypt v0.4.3 h1:aljHxrQWZFUuTWGhLsCwr+0fwCBqDjEaRVyq69PfltY=
 github.com/simia-tech/crypt v0.4.3/go.mod h1:DMwvjPTzsiHrjqHVW5HvIbF4vUUzMCYDKVLsPWmLdTo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -338,10 +415,12 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
@@ -363,28 +442,40 @@ github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opentelemetry.io/otel v0.5.0 h1:tdIR1veg/z+VRJaw/6SIxz+QX3l+m+BDleYLTs+GC1g=
 go.opentelemetry.io/otel v0.5.0/go.mod h1:jzBIgIzK43Iu1BpDAXwqOd6UPsSAk+ewVZ5ofSXw4Ek=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA=
 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -419,6 +510,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
@@ -443,7 +535,9 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116161606-93218def8b18/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -452,8 +546,12 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -463,6 +561,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -474,6 +574,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -482,9 +583,14 @@ golang.org/x/tools v0.0.0-20190624190245-7f2218787638 h1:uIfBkD8gLczr4XDgYpt/qJY
 golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -531,6 +637,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=

internal/storage/const.go

@@ -1,44 +1,39 @@
 package storage
 
-import "fmt"
+import (
+	"fmt"
+)
+
+const storageSchemaCurrentVersion = SchemaVersion(1)
+const storageSchemaUpgradeMessage = "Storage schema upgraded to v"
+const storageSchemaUpgradeErrorText = "storage schema upgrade failed at v"
 
 // Keep table names in lower case because some DB does not support upper case.
-const preferencesTableName = "user_preferences"
+const userPreferencesTableName = "user_preferences"
 const identityVerificationTokensTableName = "identity_verification_tokens"
 const totpSecretsTableName = "totp_secrets"
 const u2fDeviceHandlesTableName = "u2f_devices"
 const authenticationLogsTableName = "authentication_logs"
+const configTableName = "config"
 
-// SQLCreateUserPreferencesTable common SQL query to create user_preferences table.
+// sqlUpgradeCreateTableStatements is a map of the schema version number, plus a map of the table name and the statement used to create it.
-var SQLCreateUserPreferencesTable = fmt.Sprintf(`
+// The statement is fmt.Sprintf'd with the table name as the first argument.
-CREATE TABLE IF NOT EXISTS %s (
+var sqlUpgradeCreateTableStatements = map[SchemaVersion]map[string]string{
-	username VARCHAR(100) PRIMARY KEY,
+	SchemaVersion(1): {
-	second_factor_method VARCHAR(11)
+		userPreferencesTableName:            "CREATE TABLE %s (username VARCHAR(100) PRIMARY KEY, second_factor_method VARCHAR(11))",
-)`, preferencesTableName)
+		identityVerificationTokensTableName: "CREATE TABLE %s (token VARCHAR(512))",
+		totpSecretsTableName:                "CREATE TABLE %s (username VARCHAR(100) PRIMARY KEY, secret VARCHAR(64))",
+		u2fDeviceHandlesTableName:           "CREATE TABLE %s (username VARCHAR(100) PRIMARY KEY, keyHandle TEXT, publicKey TEXT)",
+		authenticationLogsTableName:         "CREATE TABLE %s (username VARCHAR(100), successful BOOL, time INTEGER)",
+		configTableName:                     "CREATE TABLE %s (category VARCHAR(32) NOT NULL, key_name VARCHAR(32) NOT NULL, value TEXT, PRIMARY KEY (category, key_name))",
+	},
+}
 
-// SQLCreateIdentityVerificationTokensTable common SQL query to create identity_verification_tokens table.
+// sqlUpgradesCreateTableIndexesStatements is a map of t he schema version number, plus a slice of statements to create all of the indexes.
-var SQLCreateIdentityVerificationTokensTable = fmt.Sprintf(`
+var sqlUpgradesCreateTableIndexesStatements = map[SchemaVersion][]string{
-CREATE TABLE IF NOT EXISTS %s (token VARCHAR(512))
+	SchemaVersion(1): {
-`, identityVerificationTokensTableName)
+		fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s (username, time)", authenticationLogsTableName),
+	},
+}
 
-// SQLCreateTOTPSecretsTable common SQL query to create totp_secrets table.
+const unitTestUser = "john"
-var SQLCreateTOTPSecretsTable = fmt.Sprintf(`
-CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, secret VARCHAR(64))
-`, totpSecretsTableName)
-
-// SQLCreateU2FDeviceHandlesTable common SQL query to create u2f_device_handles table.
-var SQLCreateU2FDeviceHandlesTable = fmt.Sprintf(`
-CREATE TABLE IF NOT EXISTS %s (
-	username VARCHAR(100) PRIMARY KEY,
-	keyHandle TEXT,
-	publicKey TEXT
-)`, u2fDeviceHandlesTableName)
-
-// SQLCreateAuthenticationLogsTable common SQL query to create authentication_logs table.
-var SQLCreateAuthenticationLogsTable = fmt.Sprintf(`
-CREATE TABLE IF NOT EXISTS %s (
-	username VARCHAR(100),
-	successful BOOL,
-	time INTEGER,
-	INDEX usr_time_idx (username, time)
-)`, authenticationLogsTableName)

internal/storage/mysql_provider.go

@@ -7,7 +7,6 @@ import (
 	_ "github.com/go-sql-driver/mysql" // Load the MySQL Driver used in the connection string.
 
 	"github.com/authelia/authelia/internal/configuration/schema"
-	"github.com/authelia/authelia/internal/logging"
 )
 
 // MySQLProvider is a MySQL provider.
@@ -17,6 +16,38 @@ type MySQLProvider struct {
 
 // NewMySQLProvider a MySQL provider.
 func NewMySQLProvider(configuration schema.MySQLStorageConfiguration) *MySQLProvider {
+	provider := MySQLProvider{
+		SQLProvider{
+			name: "mysql",
+
+			sqlUpgradesCreateTableStatements: sqlUpgradeCreateTableStatements,
+
+			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", userPreferencesTableName),
+			sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", userPreferencesTableName),
+
+			sqlTestIdentityVerificationTokenExistence: fmt.Sprintf("SELECT EXISTS (SELECT * FROM %s WHERE token=?)", identityVerificationTokensTableName),
+			sqlInsertIdentityVerificationToken:        fmt.Sprintf("INSERT INTO %s (token) VALUES (?)", identityVerificationTokensTableName),
+			sqlDeleteIdentityVerificationToken:        fmt.Sprintf("DELETE FROM %s WHERE token=?", identityVerificationTokensTableName),
+
+			sqlGetTOTPSecretByUsername: fmt.Sprintf("SELECT secret FROM %s WHERE username=?", totpSecretsTableName),
+			sqlUpsertTOTPSecret:        fmt.Sprintf("REPLACE INTO %s (username, secret) VALUES (?, ?)", totpSecretsTableName),
+			sqlDeleteTOTPSecret:        fmt.Sprintf("DELETE FROM %s WHERE username=?", totpSecretsTableName),
+
+			sqlGetU2FDeviceHandleByUsername: fmt.Sprintf("SELECT keyHandle, publicKey FROM %s WHERE username=?", u2fDeviceHandlesTableName),
+			sqlUpsertU2FDeviceHandle:        fmt.Sprintf("REPLACE INTO %s (username, keyHandle, publicKey) VALUES (?, ?, ?)", u2fDeviceHandlesTableName),
+
+			sqlInsertAuthenticationLog:     fmt.Sprintf("INSERT INTO %s (username, successful, time) VALUES (?, ?, ?)", authenticationLogsTableName),
+			sqlGetLatestAuthenticationLogs: fmt.Sprintf("SELECT successful, time FROM %s WHERE time>? AND username=? ORDER BY time DESC", authenticationLogsTableName),
+
+			sqlGetExistingTables: "SELECT table_name FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema=database()",
+
+			sqlConfigSetValue: fmt.Sprintf("REPLACE INTO %s (category, key_name, value) VALUES (?, ?, ?)", configTableName),
+			sqlConfigGetValue: fmt.Sprintf("SELECT value FROM %s WHERE category=? AND key_name=?", configTableName),
+		},
+	}
+
+	provider.sqlUpgradesCreateTableStatements[SchemaVersion(1)][authenticationLogsTableName] = "CREATE TABLE %s (username VARCHAR(100), successful BOOL, time INTEGER, INDEX usr_time_idx (username, time))"
+
 	connectionString := configuration.Username
 
 	if configuration.Password != "" {
@@ -39,37 +70,11 @@ func NewMySQLProvider(configuration schema.MySQLStorageConfiguration) *MySQLProv
 
 	db, err := sql.Open("mysql", connectionString)
 	if err != nil {
-		logging.Logger().Fatalf("Unable to connect to SQL database: %v", err)
+		provider.log.Fatalf("Unable to connect to SQL database: %v", err)
 	}
 
-	provider := MySQLProvider{
-		SQLProvider{
-			sqlCreateUserPreferencesTable:            SQLCreateUserPreferencesTable,
-			sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable,
-			sqlCreateTOTPSecretsTable:                SQLCreateTOTPSecretsTable,
-			sqlCreateU2FDeviceHandlesTable:           SQLCreateU2FDeviceHandlesTable,
-			sqlCreateAuthenticationLogsTable:         SQLCreateAuthenticationLogsTable,
-
-			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", preferencesTableName),
-			sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", preferencesTableName),
-
-			sqlTestIdentityVerificationTokenExistence: fmt.Sprintf("SELECT EXISTS (SELECT * FROM %s WHERE token=?)", identityVerificationTokensTableName),
-			sqlInsertIdentityVerificationToken:        fmt.Sprintf("INSERT INTO %s (token) VALUES (?)", identityVerificationTokensTableName),
-			sqlDeleteIdentityVerificationToken:        fmt.Sprintf("DELETE FROM %s WHERE token=?", identityVerificationTokensTableName),
-
-			sqlGetTOTPSecretByUsername: fmt.Sprintf("SELECT secret FROM %s WHERE username=?", totpSecretsTableName),
-			sqlUpsertTOTPSecret:        fmt.Sprintf("REPLACE INTO %s (username, secret) VALUES (?, ?)", totpSecretsTableName),
-			sqlDeleteTOTPSecret:        fmt.Sprintf("DELETE FROM %s WHERE username=?", totpSecretsTableName),
-
-			sqlGetU2FDeviceHandleByUsername: fmt.Sprintf("SELECT keyHandle, publicKey FROM %s WHERE username=?", u2fDeviceHandlesTableName),
-			sqlUpsertU2FDeviceHandle:        fmt.Sprintf("REPLACE INTO %s (username, keyHandle, publicKey) VALUES (?, ?, ?)", u2fDeviceHandlesTableName),
-
-			sqlInsertAuthenticationLog:     fmt.Sprintf("INSERT INTO %s (username, successful, time) VALUES (?, ?, ?)", authenticationLogsTableName),
-			sqlGetLatestAuthenticationLogs: fmt.Sprintf("SELECT successful, time FROM %s WHERE time>? AND username=? ORDER BY time DESC", authenticationLogsTableName),
-		},
-	}
 	if err := provider.initialize(db); err != nil {
-		logging.Logger().Fatalf("Unable to initialize SQL database: %v", err)
+		provider.log.Fatalf("Unable to initialize SQL database: %v", err)
 	}
 
 	return &provider

internal/storage/postgres_provider.go

@@ -5,10 +5,9 @@ import (
 	"fmt"
 	"strings"
 
-	_ "github.com/lib/pq" // Load the PostgreSQL Driver used in the connection string.
+	_ "github.com/jackc/pgx/v4/stdlib" // Load the PostgreSQL Driver used in the connection string.
 
 	"github.com/authelia/authelia/internal/configuration/schema"
-	"github.com/authelia/authelia/internal/logging"
 )
 
 // PostgreSQLProvider is a PostgreSQL provider.
@@ -18,6 +17,37 @@ type PostgreSQLProvider struct {
 
 // NewPostgreSQLProvider a PostgreSQL provider.
 func NewPostgreSQLProvider(configuration schema.PostgreSQLStorageConfiguration) *PostgreSQLProvider {
+	provider := PostgreSQLProvider{
+		SQLProvider{
+			name: "postgres",
+
+			sqlUpgradesCreateTableStatements:        sqlUpgradeCreateTableStatements,
+			sqlUpgradesCreateTableIndexesStatements: sqlUpgradesCreateTableIndexesStatements,
+
+			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=$1", userPreferencesTableName),
+			sqlUpsertSecondFactorPreference: fmt.Sprintf("INSERT INTO %s (username, second_factor_method) VALUES ($1, $2) ON CONFLICT (username) DO UPDATE SET second_factor_method=$2", userPreferencesTableName),
+
+			sqlTestIdentityVerificationTokenExistence: fmt.Sprintf("SELECT EXISTS (SELECT * FROM %s WHERE token=$1)", identityVerificationTokensTableName),
+			sqlInsertIdentityVerificationToken:        fmt.Sprintf("INSERT INTO %s (token) VALUES ($1)", identityVerificationTokensTableName),
+			sqlDeleteIdentityVerificationToken:        fmt.Sprintf("DELETE FROM %s WHERE token=$1", identityVerificationTokensTableName),
+
+			sqlGetTOTPSecretByUsername: fmt.Sprintf("SELECT secret FROM %s WHERE username=$1", totpSecretsTableName),
+			sqlUpsertTOTPSecret:        fmt.Sprintf("INSERT INTO %s (username, secret) VALUES ($1, $2) ON CONFLICT (username) DO UPDATE SET secret=$2", totpSecretsTableName),
+			sqlDeleteTOTPSecret:        fmt.Sprintf("DELETE FROM %s WHERE username=$1", totpSecretsTableName),
+
+			sqlGetU2FDeviceHandleByUsername: fmt.Sprintf("SELECT keyHandle, publicKey FROM %s WHERE username=$1", u2fDeviceHandlesTableName),
+			sqlUpsertU2FDeviceHandle:        fmt.Sprintf("INSERT INTO %s (username, keyHandle, publicKey) VALUES ($1, $2, $3) ON CONFLICT (username) DO UPDATE SET keyHandle=$2, publicKey=$3", u2fDeviceHandlesTableName),
+
+			sqlInsertAuthenticationLog:     fmt.Sprintf("INSERT INTO %s (username, successful, time) VALUES ($1, $2, $3)", authenticationLogsTableName),
+			sqlGetLatestAuthenticationLogs: fmt.Sprintf("SELECT successful, time FROM %s WHERE time>$1 AND username=$2 ORDER BY time DESC", authenticationLogsTableName),
+
+			sqlGetExistingTables: "SELECT table_name FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema='public'",
+
+			sqlConfigSetValue: fmt.Sprintf("INSERT INTO %s (category, key_name, value) VALUES ($1, $2, $3) ON CONFLICT (category, key_name) DO UPDATE SET value=$3", configTableName),
+			sqlConfigGetValue: fmt.Sprintf("SELECT value FROM %s WHERE category=$1 AND key_name=$2", configTableName),
+		},
+	}
+
 	args := make([]string, 0)
 	if configuration.Username != "" {
 		args = append(args, fmt.Sprintf("user='%s'", configuration.Username))
@@ -45,40 +75,13 @@ func NewPostgreSQLProvider(configuration schema.PostgreSQLStorageConfiguration)
 
 	connectionString := strings.Join(args, " ")
 
-	db, err := sql.Open("postgres", connectionString)
+	db, err := sql.Open("pgx", connectionString)
 	if err != nil {
-		logging.Logger().Fatalf("Unable to connect to SQL database: %v", err)
+		provider.log.Fatalf("Unable to connect to SQL database: %v", err)
 	}
 
-	provider := PostgreSQLProvider{
-		SQLProvider{
-			sqlCreateUserPreferencesTable:            SQLCreateUserPreferencesTable,
-			sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable,
-			sqlCreateTOTPSecretsTable:                SQLCreateTOTPSecretsTable,
-			sqlCreateU2FDeviceHandlesTable:           SQLCreateU2FDeviceHandlesTable,
-			sqlCreateAuthenticationLogsTable:         fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName),
-			sqlCreateAuthenticationLogsUserTimeIndex: fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s (username, time)", authenticationLogsTableName),
-
-			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=$1", preferencesTableName),
-			sqlUpsertSecondFactorPreference: fmt.Sprintf("INSERT INTO %s (username, second_factor_method) VALUES ($1, $2) ON CONFLICT (username) DO UPDATE SET second_factor_method=$2", preferencesTableName),
-
-			sqlTestIdentityVerificationTokenExistence: fmt.Sprintf("SELECT EXISTS (SELECT * FROM %s WHERE token=$1)", identityVerificationTokensTableName),
-			sqlInsertIdentityVerificationToken:        fmt.Sprintf("INSERT INTO %s (token) VALUES ($1)", identityVerificationTokensTableName),
-			sqlDeleteIdentityVerificationToken:        fmt.Sprintf("DELETE FROM %s WHERE token=$1", identityVerificationTokensTableName),
-
-			sqlGetTOTPSecretByUsername: fmt.Sprintf("SELECT secret FROM %s WHERE username=$1", totpSecretsTableName),
-			sqlUpsertTOTPSecret:        fmt.Sprintf("INSERT INTO %s (username, secret) VALUES ($1, $2) ON CONFLICT (username) DO UPDATE SET secret=$2", totpSecretsTableName),
-			sqlDeleteTOTPSecret:        fmt.Sprintf("DELETE FROM %s WHERE username=$1", totpSecretsTableName),
-
-			sqlGetU2FDeviceHandleByUsername: fmt.Sprintf("SELECT keyHandle, publicKey FROM %s WHERE username=$1", u2fDeviceHandlesTableName),
-			sqlUpsertU2FDeviceHandle:        fmt.Sprintf("INSERT INTO %s (username, keyHandle, publicKey) VALUES ($1, $2, $3) ON CONFLICT (username) DO UPDATE SET keyHandle=$2, publicKey=$3", u2fDeviceHandlesTableName),
-
-			sqlInsertAuthenticationLog:     fmt.Sprintf("INSERT INTO %s (username, successful, time) VALUES ($1, $2, $3)", authenticationLogsTableName),
-			sqlGetLatestAuthenticationLogs: fmt.Sprintf("SELECT successful, time FROM %s WHERE time>$1 AND username=$2 ORDER BY time DESC", authenticationLogsTableName),
-		},
-	}
 	if err := provider.initialize(db); err != nil {
-		logging.Logger().Fatalf("Unable to initialize SQL database: %v", err)
+		provider.log.Fatalf("Unable to initialize SQL database: %v", err)
 	}
 
 	return &provider

internal/storage/sql_provider.go

@@ -6,19 +6,21 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
+	"github.com/authelia/authelia/internal/logging"
 	"github.com/authelia/authelia/internal/models"
+	"github.com/authelia/authelia/internal/utils"
 )
 
 // SQLProvider is a storage provider persisting data in a SQL database.
 type SQLProvider struct {
-	db *sql.DB
+	db   *sql.DB
+	log  *logrus.Logger
+	name string
 
-	sqlCreateUserPreferencesTable            string
+	sqlUpgradesCreateTableStatements        map[SchemaVersion]map[string]string
-	sqlCreateIdentityVerificationTokensTable string
+	sqlUpgradesCreateTableIndexesStatements map[SchemaVersion][]string
-	sqlCreateTOTPSecretsTable                string
-	sqlCreateU2FDeviceHandlesTable           string
-	sqlCreateAuthenticationLogsTable         string
-	sqlCreateAuthenticationLogsUserTimeIndex string
 
 	sqlGetPreferencesByUsername     string
 	sqlUpsertSecondFactorPreference string
@@ -36,50 +38,107 @@ type SQLProvider struct {
 
 	sqlInsertAuthenticationLog     string
 	sqlGetLatestAuthenticationLogs string
+
+	sqlGetExistingTables string
+
+	sqlConfigSetValue string
+	sqlConfigGetValue string
 }
 
 func (p *SQLProvider) initialize(db *sql.DB) error {
 	p.db = db
+	p.log = logging.Logger()
 
-	_, err := db.Exec(p.sqlCreateUserPreferencesTable)
+	return p.upgrade()
+}
+
+func (p *SQLProvider) getSchemaBasicDetails() (version SchemaVersion, tables []string, err error) {
+	rows, err := p.db.Query(p.sqlGetExistingTables)
 	if err != nil {
-		return fmt.Errorf("Unable to create table %s: %v", preferencesTableName, err)
+		return version, tables, err
 	}
 
-	_, err = db.Exec(p.sqlCreateIdentityVerificationTokensTable)
+	defer rows.Close()
-	if err != nil {
-		return fmt.Errorf("Unable to create table %s: %v", identityVerificationTokensTableName, err)
-	}
 
-	_, err = db.Exec(p.sqlCreateTOTPSecretsTable)
+	var table string
-	if err != nil {
-		return fmt.Errorf("Unable to create table %s: %v", totpSecretsTableName, err)
-	}
 
-	// keyHandle and publicKey are stored in base64 format
+	for rows.Next() {
-	_, err = db.Exec(p.sqlCreateU2FDeviceHandlesTable)
+		err := rows.Scan(&table)
-	if err != nil {
-		return fmt.Errorf("Unable to create table %s: %v", u2fDeviceHandlesTableName, err)
-	}
-
-	_, err = db.Exec(p.sqlCreateAuthenticationLogsTable)
-	if err != nil {
-		return fmt.Errorf("Unable to create table %s: %v", authenticationLogsTableName, err)
-	}
-
-	// Create an index on (username, time) because this couple is highly used by the regulation module
-	// to check whether a user is banned.
-	if p.sqlCreateAuthenticationLogsUserTimeIndex != "" {
-		_, err = db.Exec(p.sqlCreateAuthenticationLogsUserTimeIndex)
 		if err != nil {
-			return fmt.Errorf("Unable to create table %s: %v", authenticationLogsTableName, err)
+			return version, tables, err
 		}
+
+		tables = append(tables, table)
+	}
+
+	if utils.IsStringInSlice(configTableName, tables) {
+		rows, err := p.db.Query(p.sqlConfigGetValue, "schema", "version")
+		if err != nil {
+			return version, tables, err
+		}
+
+		for rows.Next() {
+			err := rows.Scan(&version)
+			if err != nil {
+				return version, tables, err
+			}
+		}
+	}
+
+	return version, tables, nil
+}
+
+func (p *SQLProvider) upgrade() error {
+	p.log.Debug("Storage schema is being checked to verify it is up to date")
+
+	version, tables, err := p.getSchemaBasicDetails()
+	if err != nil {
+		return err
+	}
+
+	if version < storageSchemaCurrentVersion {
+		p.log.Debugf("Storage schema is v%d, latest is v%d", version, storageSchemaCurrentVersion)
+
+		tx, err := p.db.Begin()
+		if err != nil {
+			return err
+		}
+
+		switch version {
+		case 0:
+			err := p.upgradeSchemaToVersion001(tx, tables)
+			if err != nil {
+				return p.handleUpgradeFailure(tx, 1, err)
+			}
+
+			fallthrough
+		default:
+			err := tx.Commit()
+			if err != nil {
+				return err
+			}
+
+			p.log.Infof("Storage schema upgrade to v%d completed", storageSchemaCurrentVersion)
+		}
+	} else {
+		p.log.Debug("Storage schema is up to date")
 	}
 
 	return nil
 }
 
-// LoadPreferred2FAMethod load the preferred method for 2FA from sqlite db.
+func (p *SQLProvider) handleUpgradeFailure(tx *sql.Tx, version SchemaVersion, err error) error {
+	rollbackErr := tx.Rollback()
+	formattedErr := fmt.Errorf("%s%d: %v", storageSchemaUpgradeErrorText, version, err)
+
+	if rollbackErr != nil {
+		return fmt.Errorf("rollback error occurred: %v (inner error %v)", rollbackErr, formattedErr)
+	}
+
+	return formattedErr
+}
+
+// LoadPreferred2FAMethod load the preferred method for 2FA from the database.
 func (p *SQLProvider) LoadPreferred2FAMethod(username string) (string, error) {
 	var method string
 
@@ -98,13 +157,13 @@ func (p *SQLProvider) LoadPreferred2FAMethod(username string) (string, error) {
 	return method, err
 }
 
-// SavePreferred2FAMethod save the preferred method for 2FA in sqlite db.
+// SavePreferred2FAMethod save the preferred method for 2FA to the database.
 func (p *SQLProvider) SavePreferred2FAMethod(username string, method string) error {
 	_, err := p.db.Exec(p.sqlUpsertSecondFactorPreference, username, method)
 	return err
 }
 
-// FindIdentityVerificationToken look for an identity verification token in DB.
+// FindIdentityVerificationToken look for an identity verification token in the database.
 func (p *SQLProvider) FindIdentityVerificationToken(token string) (bool, error) {
 	var found bool
 
@@ -116,25 +175,25 @@ func (p *SQLProvider) FindIdentityVerificationToken(token string) (bool, error)
 	return found, nil
 }
 
-// SaveIdentityVerificationToken save an identity verification token in DB.
+// SaveIdentityVerificationToken save an identity verification token in the database.
 func (p *SQLProvider) SaveIdentityVerificationToken(token string) error {
 	_, err := p.db.Exec(p.sqlInsertIdentityVerificationToken, token)
 	return err
 }
 
-// RemoveIdentityVerificationToken remove an identity verification token from the DB.
+// RemoveIdentityVerificationToken remove an identity verification token from the database.
 func (p *SQLProvider) RemoveIdentityVerificationToken(token string) error {
 	_, err := p.db.Exec(p.sqlDeleteIdentityVerificationToken, token)
 	return err
 }
 
-// SaveTOTPSecret save a TOTP secret of a given user.
+// SaveTOTPSecret save a TOTP secret of a given user in the database.
 func (p *SQLProvider) SaveTOTPSecret(username string, secret string) error {
 	_, err := p.db.Exec(p.sqlUpsertTOTPSecret, username, secret)
 	return err
 }
 
-// LoadTOTPSecret load a TOTP secret given a username.
+// LoadTOTPSecret load a TOTP secret given a username from the database.
 func (p *SQLProvider) LoadTOTPSecret(username string) (string, error) {
 	var secret string
 	if err := p.db.QueryRow(p.sqlGetTOTPSecretByUsername, username).Scan(&secret); err != nil {
@@ -148,7 +207,7 @@ func (p *SQLProvider) LoadTOTPSecret(username string) (string, error) {
 	return secret, nil
 }
 
-// DeleteTOTPSecret delete a TOTP secret given a username.
+// DeleteTOTPSecret delete a TOTP secret from the database given a username.
 func (p *SQLProvider) DeleteTOTPSecret(username string) error {
 	_, err := p.db.Exec(p.sqlDeleteTOTPSecret, username)
 	return err

internal/storage/sql_provider_test.go

@@ -0,0 +1,400 @@
+package storage
+
+import (
+	"database/sql/driver"
+	"encoding/base64"
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/authelia/authelia/internal/authentication"
+	"github.com/authelia/authelia/internal/models"
+)
+
+const currentSchemaMockSchemaVersion = "1"
+
+func TestSQLInitializeDatabase(t *testing.T) {
+	provider, mock := NewSQLMockProvider()
+
+	rows := sqlmock.NewRows([]string{"name"})
+	mock.ExpectQuery(
+		"SELECT name FROM sqlite_master WHERE type='table'").
+		WillReturnRows(rows)
+
+	mock.ExpectBegin()
+
+	keys := make([]string, 0, len(sqlUpgradeCreateTableStatements[1]))
+	for k := range sqlUpgradeCreateTableStatements[1] {
+		keys = append(keys, k)
+	}
+
+	sort.Strings(keys)
+
+	for _, table := range keys {
+		mock.ExpectExec(
+			fmt.Sprintf("CREATE TABLE %s .*", table)).
+			WillReturnResult(sqlmock.NewResult(0, 0))
+	}
+
+	mock.ExpectExec(
+		fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s .*", authenticationLogsTableName)).
+		WillReturnResult(sqlmock.NewResult(0, 0))
+
+	mock.ExpectExec(
+		fmt.Sprintf("REPLACE INTO %s \\(category, key_name, value\\) VALUES \\(\\?, \\?, \\?\\)", configTableName)).
+		WithArgs("schema", "version", "1").
+		WillReturnResult(sqlmock.NewResult(1, 1))
+
+	mock.ExpectCommit()
+
+	err := provider.initialize(provider.db)
+	assert.NoError(t, err)
+}
+
+func TestSQLUpgradeDatabase(t *testing.T) {
+	provider, mock := NewSQLMockProvider()
+
+	mock.ExpectQuery(
+		"SELECT name FROM sqlite_master WHERE type='table'").
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).
+			AddRow(userPreferencesTableName).
+			AddRow(identityVerificationTokensTableName).
+			AddRow(totpSecretsTableName).
+			AddRow(u2fDeviceHandlesTableName).
+			AddRow(authenticationLogsTableName))
+
+	mock.ExpectBegin()
+
+	mock.ExpectExec(
+		fmt.Sprintf("CREATE TABLE %s .*", configTableName)).
+		WillReturnResult(sqlmock.NewResult(0, 0))
+
+	mock.ExpectExec(
+		fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s .*", authenticationLogsTableName)).
+		WillReturnResult(sqlmock.NewResult(0, 0))
+
+	mock.ExpectExec(
+		fmt.Sprintf("REPLACE INTO %s \\(category, key_name, value\\) VALUES \\(\\?, \\?, \\?\\)", configTableName)).
+		WithArgs("schema", "version", "1").
+		WillReturnResult(sqlmock.NewResult(1, 1))
+
+	mock.ExpectCommit()
+
+	err := provider.initialize(provider.db)
+	assert.NoError(t, err)
+}
+
+func TestSQLProviderMethodsAuthenticationLogs(t *testing.T) {
+	provider, mock := NewSQLMockProvider()
+
+	mock.ExpectQuery(
+		"SELECT name FROM sqlite_master WHERE type='table'").
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).
+			AddRow(userPreferencesTableName).
+			AddRow(identityVerificationTokensTableName).
+			AddRow(totpSecretsTableName).
+			AddRow(u2fDeviceHandlesTableName).
+			AddRow(authenticationLogsTableName).
+			AddRow(configTableName))
+
+	args := []driver.Value{"schema", "version"}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT value FROM %s WHERE category=\\? AND key_name=\\?", configTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"value"}).
+			AddRow("1"))
+
+	err := provider.initialize(provider.db)
+	assert.NoError(t, err)
+
+	attempts := []models.AuthenticationAttempt{
+		{Username: unitTestUser, Successful: true, Time: time.Unix(1577880001, 0)},
+		{Username: unitTestUser, Successful: true, Time: time.Unix(1577880002, 0)},
+		{Username: unitTestUser, Successful: false, Time: time.Unix(1577880003, 0)},
+	}
+
+	rows := sqlmock.NewRows([]string{"successful", "time"})
+
+	for id, attempt := range attempts {
+		args = []driver.Value{attempt.Username, attempt.Successful, attempt.Time.Unix()}
+		mock.ExpectExec(
+			fmt.Sprintf("INSERT INTO %s \\(username, successful, time\\) VALUES \\(\\?, \\?, \\?\\)", authenticationLogsTableName)).
+			WithArgs(args...).
+			WillReturnResult(sqlmock.NewResult(int64(id), 1))
+
+		err := provider.AppendAuthenticationLog(attempt)
+		assert.NoError(t, err)
+		rows.AddRow(attempt.Successful, attempt.Time.Unix())
+	}
+
+	args = []driver.Value{1577880000, unitTestUser}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT successful, time FROM %s WHERE time>\\? AND username=\\? ORDER BY time DESC", authenticationLogsTableName)).
+		WithArgs(args...).
+		WillReturnRows(rows)
+
+	after := time.Unix(1577880000, 0)
+	results, err := provider.LoadLatestAuthenticationLogs(unitTestUser, after)
+	assert.NoError(t, err)
+	require.Len(t, results, 3)
+	assert.Equal(t, unitTestUser, results[0].Username)
+	assert.Equal(t, true, results[0].Successful)
+	assert.Equal(t, time.Unix(1577880001, 0), results[0].Time)
+	assert.Equal(t, unitTestUser, results[1].Username)
+	assert.Equal(t, true, results[1].Successful)
+	assert.Equal(t, time.Unix(1577880002, 0), results[1].Time)
+	assert.Equal(t, unitTestUser, results[2].Username)
+	assert.Equal(t, false, results[2].Successful)
+	assert.Equal(t, time.Unix(1577880003, 0), results[2].Time)
+
+	// Test Blank Rows.
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT successful, time FROM %s WHERE time>\\? AND username=\\? ORDER BY time DESC", authenticationLogsTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"successful", "time"}))
+
+	results, err = provider.LoadLatestAuthenticationLogs(unitTestUser, after)
+	assert.NoError(t, err)
+	assert.Len(t, results, 0)
+}
+
+func TestSQLProviderMethodsPreferred(t *testing.T) {
+	provider, mock := NewSQLMockProvider()
+
+	mock.ExpectQuery(
+		"SELECT name FROM sqlite_master WHERE type='table'").
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).
+			AddRow(userPreferencesTableName).
+			AddRow(identityVerificationTokensTableName).
+			AddRow(totpSecretsTableName).
+			AddRow(u2fDeviceHandlesTableName).
+			AddRow(authenticationLogsTableName).
+			AddRow(configTableName))
+
+	args := []driver.Value{"schema", "version"}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT value FROM %s WHERE category=\\? AND key_name=\\?", configTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"value"}).
+			AddRow(currentSchemaMockSchemaVersion))
+
+	err := provider.initialize(provider.db)
+	assert.NoError(t, err)
+
+	mock.ExpectExec(
+		fmt.Sprintf("REPLACE INTO %s \\(username, second_factor_method\\) VALUES \\(\\?, \\?\\)", userPreferencesTableName)).
+		WithArgs(unitTestUser, authentication.TOTP).
+		WillReturnResult(sqlmock.NewResult(0, 1))
+
+	err = provider.SavePreferred2FAMethod(unitTestUser, authentication.TOTP)
+	assert.NoError(t, err)
+
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=\\?", userPreferencesTableName)).
+		WithArgs(unitTestUser).
+		WillReturnRows(sqlmock.NewRows([]string{"second_factor_method"}).AddRow(authentication.TOTP))
+
+	method, err := provider.LoadPreferred2FAMethod(unitTestUser)
+	assert.NoError(t, err)
+	assert.Equal(t, authentication.TOTP, method)
+
+	// Test Blank Rows.
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=\\?", userPreferencesTableName)).
+		WithArgs(unitTestUser).
+		WillReturnRows(sqlmock.NewRows([]string{"second_factor_method"}))
+
+	method, err = provider.LoadPreferred2FAMethod(unitTestUser)
+	assert.NoError(t, err)
+	assert.Equal(t, "", method)
+}
+
+func TestSQLProviderMethodsTOTP(t *testing.T) {
+	provider, mock := NewSQLMockProvider()
+
+	mock.ExpectQuery(
+		"SELECT name FROM sqlite_master WHERE type='table'").
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).
+			AddRow(userPreferencesTableName).
+			AddRow(identityVerificationTokensTableName).
+			AddRow(totpSecretsTableName).
+			AddRow(u2fDeviceHandlesTableName).
+			AddRow(authenticationLogsTableName).
+			AddRow(configTableName))
+
+	args := []driver.Value{"schema", "version"}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT value FROM %s WHERE category=\\? AND key_name=\\?", configTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"value"}).
+			AddRow(currentSchemaMockSchemaVersion))
+
+	err := provider.initialize(provider.db)
+	assert.NoError(t, err)
+
+	pretendSecret := "abc123"
+	args = []driver.Value{unitTestUser, pretendSecret}
+	mock.ExpectExec(
+		fmt.Sprintf("REPLACE INTO %s \\(username, secret\\) VALUES \\(\\?, \\?\\)", totpSecretsTableName)).
+		WithArgs(args...).
+		WillReturnResult(sqlmock.NewResult(0, 1))
+
+	err = provider.SaveTOTPSecret(unitTestUser, pretendSecret)
+	assert.NoError(t, err)
+
+	args = []driver.Value{unitTestUser}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT secret FROM %s WHERE username=\\?", totpSecretsTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"secret"}).AddRow(pretendSecret))
+
+	secret, err := provider.LoadTOTPSecret(unitTestUser)
+	assert.NoError(t, err)
+	assert.Equal(t, pretendSecret, secret)
+
+	mock.ExpectExec(
+		fmt.Sprintf("DELETE FROM %s WHERE username=\\?", totpSecretsTableName)).
+		WithArgs(unitTestUser).
+		WillReturnResult(sqlmock.NewResult(0, 1))
+
+	err = provider.DeleteTOTPSecret(unitTestUser)
+	assert.NoError(t, err)
+
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT secret FROM %s WHERE username=\\?", totpSecretsTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"secret"}))
+
+	//Test Blank Rows
+	secret, err = provider.LoadTOTPSecret(unitTestUser)
+	assert.EqualError(t, err, "No TOTP secret registered")
+	assert.Equal(t, "", secret)
+}
+
+func TestSQLProviderMethodsU2F(t *testing.T) {
+	provider, mock := NewSQLMockProvider()
+
+	mock.ExpectQuery(
+		"SELECT name FROM sqlite_master WHERE type='table'").
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).
+			AddRow(userPreferencesTableName).
+			AddRow(identityVerificationTokensTableName).
+			AddRow(totpSecretsTableName).
+			AddRow(u2fDeviceHandlesTableName).
+			AddRow(authenticationLogsTableName).
+			AddRow(configTableName))
+
+	args := []driver.Value{"schema", "version"}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT value FROM %s WHERE category=\\? AND key_name=\\?", configTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"value"}).
+			AddRow(currentSchemaMockSchemaVersion))
+
+	err := provider.initialize(provider.db)
+	assert.NoError(t, err)
+
+	pretendKeyHandle := []byte("abc")
+	pretendPublicKey := []byte("123")
+	pretendKeyHandleB64 := base64.StdEncoding.EncodeToString(pretendKeyHandle)
+	pretendPublicKeyB64 := base64.StdEncoding.EncodeToString(pretendPublicKey)
+
+	args = []driver.Value{unitTestUser, pretendKeyHandleB64, pretendPublicKeyB64}
+	mock.ExpectExec(
+		fmt.Sprintf("REPLACE INTO %s \\(username, keyHandle, publicKey\\) VALUES \\(\\?, \\?, \\?\\)", u2fDeviceHandlesTableName)).
+		WithArgs(args...).
+		WillReturnResult(sqlmock.NewResult(0, 1))
+
+	err = provider.SaveU2FDeviceHandle(unitTestUser, pretendKeyHandle, pretendPublicKey)
+	assert.NoError(t, err)
+
+	args = []driver.Value{unitTestUser}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT keyHandle, publicKey FROM %s WHERE username=\\?", u2fDeviceHandlesTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"keyHandle", "publicKey"}).
+			AddRow(pretendKeyHandleB64, pretendPublicKeyB64))
+
+	keyHandle, publicKey, err := provider.LoadU2FDeviceHandle(unitTestUser)
+	assert.NoError(t, err)
+	assert.Equal(t, pretendKeyHandle, keyHandle)
+	assert.Equal(t, pretendPublicKey, publicKey)
+
+	// Test Blank Rows.
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT keyHandle, publicKey FROM %s WHERE username=\\?", u2fDeviceHandlesTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"keyHandle", "publicKey"}))
+
+	keyHandle, publicKey, err = provider.LoadU2FDeviceHandle(unitTestUser)
+	assert.EqualError(t, err, "No U2F device handle found")
+	assert.Equal(t, []byte(nil), keyHandle)
+	assert.Equal(t, []byte(nil), publicKey)
+}
+
+func TestSQLProviderMethodsIdentityVerificationTokens(t *testing.T) {
+	provider, mock := NewSQLMockProvider()
+
+	mock.ExpectQuery(
+		"SELECT name FROM sqlite_master WHERE type='table'").
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).
+			AddRow(userPreferencesTableName).
+			AddRow(identityVerificationTokensTableName).
+			AddRow(totpSecretsTableName).
+			AddRow(u2fDeviceHandlesTableName).
+			AddRow(authenticationLogsTableName).
+			AddRow(configTableName))
+
+	args := []driver.Value{"schema", "version"}
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT value FROM %s WHERE category=\\? AND key_name=\\?", configTableName)).
+		WithArgs(args...).
+		WillReturnRows(sqlmock.NewRows([]string{"value"}).
+			AddRow(currentSchemaMockSchemaVersion))
+
+	err := provider.initialize(provider.db)
+	assert.NoError(t, err)
+
+	fakeIdentityVerificationToken := "abc"
+
+	mock.ExpectExec(
+		fmt.Sprintf("INSERT INTO %s \\(token\\) VALUES \\(\\?\\)", identityVerificationTokensTableName)).
+		WithArgs(fakeIdentityVerificationToken).
+		WillReturnResult(sqlmock.NewResult(1, 1))
+
+	err = provider.SaveIdentityVerificationToken(fakeIdentityVerificationToken)
+	assert.NoError(t, err)
+
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT EXISTS \\(SELECT \\* FROM %s WHERE token=\\?\\)", identityVerificationTokensTableName)).
+		WithArgs(fakeIdentityVerificationToken).
+		WillReturnRows(sqlmock.NewRows([]string{"EXISTS"}).
+			AddRow(true))
+
+	valid, err := provider.FindIdentityVerificationToken(fakeIdentityVerificationToken)
+	assert.NoError(t, err)
+	assert.True(t, valid)
+
+	mock.ExpectExec(
+		fmt.Sprintf("DELETE FROM %s WHERE token=\\?", identityVerificationTokensTableName)).
+		WithArgs(fakeIdentityVerificationToken).
+		WillReturnResult(sqlmock.NewResult(0, 1))
+
+	err = provider.RemoveIdentityVerificationToken(fakeIdentityVerificationToken)
+	assert.NoError(t, err)
+
+	mock.ExpectQuery(
+		fmt.Sprintf("SELECT EXISTS \\(SELECT \\* FROM %s WHERE token=\\?\\)", identityVerificationTokensTableName)).
+		WithArgs(fakeIdentityVerificationToken).
+		WillReturnRows(sqlmock.NewRows([]string{"EXISTS"}).
+			AddRow(false))
+
+	valid, err = provider.FindIdentityVerificationToken(fakeIdentityVerificationToken)
+	assert.NoError(t, err)
+	assert.False(t, valid)
+}

internal/storage/sqlite_provider.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 
 	_ "github.com/mattn/go-sqlite3" // Load the SQLite Driver used in the connection string.
-
-	"github.com/authelia/authelia/internal/logging"
 )
 
 // SQLiteProvider is a SQLite3 provider.
@@ -16,22 +14,15 @@ type SQLiteProvider struct {
 
 // NewSQLiteProvider constructs a SQLite provider.
 func NewSQLiteProvider(path string) *SQLiteProvider {
-	db, err := sql.Open("sqlite3", path)
-	if err != nil {
-		logging.Logger().Fatalf("Unable to create SQLite database %s: %s", path, err)
-	}
-
 	provider := SQLiteProvider{
 		SQLProvider{
-			sqlCreateUserPreferencesTable:            SQLCreateUserPreferencesTable,
+			name: "sqlite",
-			sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable,
-			sqlCreateTOTPSecretsTable:                SQLCreateTOTPSecretsTable,
-			sqlCreateU2FDeviceHandlesTable:           SQLCreateU2FDeviceHandlesTable,
-			sqlCreateAuthenticationLogsTable:         fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName),
-			sqlCreateAuthenticationLogsUserTimeIndex: fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s (username, time)", authenticationLogsTableName),
 
-			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", preferencesTableName),
+			sqlUpgradesCreateTableStatements:        sqlUpgradeCreateTableStatements,
-			sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", preferencesTableName),
+			sqlUpgradesCreateTableIndexesStatements: sqlUpgradesCreateTableIndexesStatements,
+
+			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", userPreferencesTableName),
+			sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", userPreferencesTableName),
 
 			sqlTestIdentityVerificationTokenExistence: fmt.Sprintf("SELECT EXISTS (SELECT * FROM %s WHERE token=?)", identityVerificationTokensTableName),
 			sqlInsertIdentityVerificationToken:        fmt.Sprintf("INSERT INTO %s (token) VALUES (?)", identityVerificationTokensTableName),
@@ -46,10 +37,21 @@ func NewSQLiteProvider(path string) *SQLiteProvider {
 
 			sqlInsertAuthenticationLog:     fmt.Sprintf("INSERT INTO %s (username, successful, time) VALUES (?, ?, ?)", authenticationLogsTableName),
 			sqlGetLatestAuthenticationLogs: fmt.Sprintf("SELECT successful, time FROM %s WHERE time>? AND username=? ORDER BY time DESC", authenticationLogsTableName),
+
+			sqlGetExistingTables: "SELECT name FROM sqlite_master WHERE type='table'",
+
+			sqlConfigSetValue: fmt.Sprintf("REPLACE INTO %s (category, key_name, value) VALUES (?, ?, ?)", configTableName),
+			sqlConfigGetValue: fmt.Sprintf("SELECT value FROM %s WHERE category=? AND key_name=?", configTableName),
 		},
 	}
+
+	db, err := sql.Open("sqlite3", path)
+	if err != nil {
+		provider.log.Fatalf("Unable to create SQL database %s: %s", path, err)
+	}
+
 	if err := provider.initialize(db); err != nil {
-		logging.Logger().Fatalf("Unable to initialize SQLite database %s: %s", path, err)
+		provider.log.Fatalf("Unable to initialize SQL database %s: %s", path, err)
 	}
 
 	return &provider

internal/storage/sqlmock_provider.go

@@ -0,0 +1,60 @@
+package storage
+
+import (
+	"fmt"
+
+	"github.com/DATA-DOG/go-sqlmock"
+)
+
+// SQLMockProvider is a SQLMock provider.
+type SQLMockProvider struct {
+	SQLProvider
+}
+
+// NewSQLMockProvider constructs a SQLMock provider.
+func NewSQLMockProvider() (*SQLMockProvider, sqlmock.Sqlmock) {
+	provider := SQLMockProvider{
+		SQLProvider{
+			name: "sqlmock",
+
+			sqlUpgradesCreateTableStatements:        sqlUpgradeCreateTableStatements,
+			sqlUpgradesCreateTableIndexesStatements: sqlUpgradesCreateTableIndexesStatements,
+
+			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", userPreferencesTableName),
+			sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", userPreferencesTableName),
+
+			sqlTestIdentityVerificationTokenExistence: fmt.Sprintf("SELECT EXISTS (SELECT * FROM %s WHERE token=?)", identityVerificationTokensTableName),
+			sqlInsertIdentityVerificationToken:        fmt.Sprintf("INSERT INTO %s (token) VALUES (?)", identityVerificationTokensTableName),
+			sqlDeleteIdentityVerificationToken:        fmt.Sprintf("DELETE FROM %s WHERE token=?", identityVerificationTokensTableName),
+
+			sqlGetTOTPSecretByUsername: fmt.Sprintf("SELECT secret FROM %s WHERE username=?", totpSecretsTableName),
+			sqlUpsertTOTPSecret:        fmt.Sprintf("REPLACE INTO %s (username, secret) VALUES (?, ?)", totpSecretsTableName),
+			sqlDeleteTOTPSecret:        fmt.Sprintf("DELETE FROM %s WHERE username=?", totpSecretsTableName),
+
+			sqlGetU2FDeviceHandleByUsername: fmt.Sprintf("SELECT keyHandle, publicKey FROM %s WHERE username=?", u2fDeviceHandlesTableName),
+			sqlUpsertU2FDeviceHandle:        fmt.Sprintf("REPLACE INTO %s (username, keyHandle, publicKey) VALUES (?, ?, ?)", u2fDeviceHandlesTableName),
+
+			sqlInsertAuthenticationLog:     fmt.Sprintf("INSERT INTO %s (username, successful, time) VALUES (?, ?, ?)", authenticationLogsTableName),
+			sqlGetLatestAuthenticationLogs: fmt.Sprintf("SELECT successful, time FROM %s WHERE time>? AND username=? ORDER BY time DESC", authenticationLogsTableName),
+
+			sqlGetExistingTables: "SELECT name FROM sqlite_master WHERE type='table'",
+
+			sqlConfigSetValue: fmt.Sprintf("REPLACE INTO %s (category, key_name, value) VALUES (?, ?, ?)", configTableName),
+			sqlConfigGetValue: fmt.Sprintf("SELECT value FROM %s WHERE category=? AND key_name=?", configTableName),
+		},
+	}
+
+	db, mock, err := sqlmock.New()
+
+	if err != nil {
+		provider.log.Fatalf("Unable to create SQL database: %s", err)
+	}
+
+	provider.db = db
+
+	/*
+		We do initialize in the tests rather than in the new up.
+	*/
+
+	return &provider, mock
+}

internal/storage/types.go

@@ -0,0 +1,18 @@
+package storage
+
+import (
+	"database/sql"
+	"strconv"
+)
+
+// SchemaVersion is a simple int representation of the schema version.
+type SchemaVersion int
+
+// ToString converts the schema version into a string and returns that converted value.
+func (s SchemaVersion) ToString() string {
+	return strconv.Itoa(int(s))
+}
+
+type transaction interface {
+	Exec(query string, args ...interface{}) (sql.Result, error)
+}

internal/storage/upgrades.go

@@ -0,0 +1,76 @@
+package storage
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/authelia/authelia/internal/utils"
+)
+
+func (p *SQLProvider) upgradeCreateTableStatements(tx transaction, statements map[string]string, existingTables []string) error {
+	keys := make([]string, 0, len(statements))
+	for k := range statements {
+		keys = append(keys, k)
+	}
+
+	sort.Strings(keys)
+
+	for _, table := range keys {
+		if !utils.IsStringInSlice(table, existingTables) {
+			_, err := tx.Exec(fmt.Sprintf(statements[table], table))
+			if err != nil {
+				return fmt.Errorf("Unable to create table %s: %v", table, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (p *SQLProvider) upgradeRunMultipleStatements(tx transaction, statements []string) error {
+	for _, statement := range statements {
+		_, err := tx.Exec(statement)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// upgradeFinalize sets the schema version and logs a message, as well as any other future finalization tasks.
+func (p *SQLProvider) upgradeFinalize(tx transaction, version SchemaVersion) error {
+	_, err := tx.Exec(p.sqlConfigSetValue, "schema", "version", version.ToString())
+	if err != nil {
+		return err
+	}
+
+	p.log.Debugf("%s%d", storageSchemaUpgradeMessage, version)
+
+	return nil
+}
+
+// upgradeSchemaToVersion001 upgrades the schema to version 1.
+func (p *SQLProvider) upgradeSchemaToVersion001(tx transaction, tables []string) error {
+	version := SchemaVersion(1)
+
+	err := p.upgradeCreateTableStatements(tx, p.sqlUpgradesCreateTableStatements[version], tables)
+	if err != nil {
+		return err
+	}
+
+	// Skip mysql create index statements. It doesn't support CREATE INDEX IF NOT EXIST. May be able to work around this with an Index struct.
+	if p.name != "mysql" {
+		err = p.upgradeRunMultipleStatements(tx, p.sqlUpgradesCreateTableIndexesStatements[1])
+		if err != nil {
+			return fmt.Errorf("Unable to create index: %v", err)
+		}
+	}
+
+	err = p.upgradeFinalize(tx, version)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/suites/suite_standalone.go

@@ -54,6 +54,8 @@ func init() {
 
 	teardown := func(suitePath string) error {
 		err := dockerEnvironment.Down()
+		_ = os.Remove("/tmp/db.sqlite3")
+
 		return err
 	}