2016-05-13 06:49:40 +03:00
// Copyright 2015 The etcd Authors
2015-08-08 15:58:29 +03:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package etcdserver
import (
2016-06-09 03:11:26 +03:00
"strconv"
"strings"
2016-01-20 08:09:09 +03:00
"time"
2015-08-14 21:45:31 +03:00
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
"github.com/coreos/etcd/auth"
2015-08-08 15:58:29 +03:00
pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
2016-09-09 02:14:14 +03:00
"github.com/coreos/etcd/etcdserver/membership"
2016-01-05 21:16:50 +03:00
"github.com/coreos/etcd/lease"
2016-01-25 01:09:54 +03:00
"github.com/coreos/etcd/lease/leasehttp"
2016-09-09 02:14:14 +03:00
"github.com/coreos/etcd/lease/leasepb"
2016-04-25 22:32:58 +03:00
"github.com/coreos/etcd/mvcc"
2016-03-23 03:10:28 +03:00
"golang.org/x/net/context"
2016-05-17 09:20:50 +03:00
"google.golang.org/grpc/metadata"
2015-08-08 15:58:29 +03:00
)
2016-02-08 23:44:25 +03:00
const (
// the max request size that raft accepts.
// TODO: make this a flag? But we probably do not want to
// accept large request which might block raft stream. User
// specify a large value might end up with shooting in the foot.
maxRequestBytes = 1.5 * 1024 * 1024
2016-04-28 07:36:01 +03:00
2016-07-12 23:16:35 +03:00
// In the health case, there might be a small gap (10s of entries) between
2016-08-14 06:54:48 +03:00
// the applied index and committed index.
2016-07-12 23:16:35 +03:00
// However, if the committed entries are very heavy to apply, the gap might grow.
// We should stop accepting new proposals if the gap growing to a certain point.
maxGapBetweenApplyAndCommitIndex = 1000
2016-02-08 23:44:25 +03:00
)
2015-11-04 22:23:33 +03:00
type RaftKV interface {
Range ( ctx context . Context , r * pb . RangeRequest ) ( * pb . RangeResponse , error )
Put ( ctx context . Context , r * pb . PutRequest ) ( * pb . PutResponse , error )
DeleteRange ( ctx context . Context , r * pb . DeleteRangeRequest ) ( * pb . DeleteRangeResponse , error )
Txn ( ctx context . Context , r * pb . TxnRequest ) ( * pb . TxnResponse , error )
Compact ( ctx context . Context , r * pb . CompactionRequest ) ( * pb . CompactionResponse , error )
}
2016-01-06 00:49:25 +03:00
type Lessor interface {
2016-04-07 20:27:36 +03:00
// LeaseGrant sends LeaseGrant request to raft and apply it after committed.
LeaseGrant ( ctx context . Context , r * pb . LeaseGrantRequest ) ( * pb . LeaseGrantResponse , error )
2016-01-10 00:26:45 +03:00
// LeaseRevoke sends LeaseRevoke request to raft and apply it after committed.
2016-01-06 00:49:25 +03:00
LeaseRevoke ( ctx context . Context , r * pb . LeaseRevokeRequest ) ( * pb . LeaseRevokeResponse , error )
2016-01-10 00:26:45 +03:00
// LeaseRenew renews the lease with given ID. The renewed TTL is returned. Or an error
// is returned.
LeaseRenew ( id lease . LeaseID ) ( int64 , error )
2016-09-09 02:14:14 +03:00
// LeaseTimeToLive retrieves lease information.
LeaseTimeToLive ( ctx context . Context , r * pb . LeaseTimeToLiveRequest ) ( * pb . LeaseTimeToLiveResponse , error )
2016-01-06 00:49:25 +03:00
}
2016-03-02 08:56:42 +03:00
type Authenticator interface {
AuthEnable ( ctx context . Context , r * pb . AuthEnableRequest ) ( * pb . AuthEnableResponse , error )
2016-05-07 21:24:43 +03:00
AuthDisable ( ctx context . Context , r * pb . AuthDisableRequest ) ( * pb . AuthDisableResponse , error )
2016-04-13 07:44:53 +03:00
Authenticate ( ctx context . Context , r * pb . AuthenticateRequest ) ( * pb . AuthenticateResponse , error )
2016-03-29 08:32:19 +03:00
UserAdd ( ctx context . Context , r * pb . AuthUserAddRequest ) ( * pb . AuthUserAddResponse , error )
2016-03-31 05:29:47 +03:00
UserDelete ( ctx context . Context , r * pb . AuthUserDeleteRequest ) ( * pb . AuthUserDeleteResponse , error )
2016-03-31 08:31:07 +03:00
UserChangePassword ( ctx context . Context , r * pb . AuthUserChangePasswordRequest ) ( * pb . AuthUserChangePasswordResponse , error )
2016-06-07 06:17:28 +03:00
UserGrantRole ( ctx context . Context , r * pb . AuthUserGrantRoleRequest ) ( * pb . AuthUserGrantRoleResponse , error )
2016-05-30 11:21:11 +03:00
UserGet ( ctx context . Context , r * pb . AuthUserGetRequest ) ( * pb . AuthUserGetResponse , error )
2016-06-05 10:57:23 +03:00
UserRevokeRole ( ctx context . Context , r * pb . AuthUserRevokeRoleRequest ) ( * pb . AuthUserRevokeRoleResponse , error )
2016-04-01 05:07:43 +03:00
RoleAdd ( ctx context . Context , r * pb . AuthRoleAddRequest ) ( * pb . AuthRoleAddResponse , error )
2016-06-07 06:17:28 +03:00
RoleGrantPermission ( ctx context . Context , r * pb . AuthRoleGrantPermissionRequest ) ( * pb . AuthRoleGrantPermissionResponse , error )
2016-06-02 17:25:15 +03:00
RoleGet ( ctx context . Context , r * pb . AuthRoleGetRequest ) ( * pb . AuthRoleGetResponse , error )
2016-06-05 10:57:23 +03:00
RoleRevokePermission ( ctx context . Context , r * pb . AuthRoleRevokePermissionRequest ) ( * pb . AuthRoleRevokePermissionResponse , error )
2016-06-03 11:33:05 +03:00
RoleDelete ( ctx context . Context , r * pb . AuthRoleDeleteRequest ) ( * pb . AuthRoleDeleteResponse , error )
2016-06-15 09:11:27 +03:00
UserList ( ctx context . Context , r * pb . AuthUserListRequest ) ( * pb . AuthUserListResponse , error )
RoleList ( ctx context . Context , r * pb . AuthRoleListRequest ) ( * pb . AuthRoleListResponse , error )
2016-03-02 08:56:42 +03:00
}
2015-11-04 22:23:33 +03:00
func ( s * EtcdServer ) Range ( ctx context . Context , r * pb . RangeRequest ) ( * pb . RangeResponse , error ) {
2016-02-12 23:04:06 +03:00
if r . Serializable {
2016-07-27 21:49:58 +03:00
var resp * pb . RangeResponse
var err error
chk := func ( ai * auth . AuthInfo ) error {
return s . authStore . IsRangePermitted ( ai , r . Key , r . RangeEnd )
2016-06-10 20:53:40 +03:00
}
2016-07-27 21:49:58 +03:00
get := func ( ) { resp , err = s . applyV3Base . Range ( noTxn , r ) }
if serr := s . doSerialize ( ctx , chk , get ) ; serr != nil {
return nil , serr
}
return resp , err
2016-02-12 23:04:06 +03:00
}
2016-07-27 21:49:58 +03:00
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { Range : r } )
2015-11-04 22:23:33 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . RangeResponse ) , nil
2015-11-04 22:23:33 +03:00
}
func ( s * EtcdServer ) Put ( ctx context . Context , r * pb . PutRequest ) ( * pb . PutResponse , error ) {
2015-11-10 00:37:15 +03:00
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { Put : r } )
2015-11-04 22:23:33 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . PutResponse ) , nil
2015-11-04 22:23:33 +03:00
}
func ( s * EtcdServer ) DeleteRange ( ctx context . Context , r * pb . DeleteRangeRequest ) ( * pb . DeleteRangeResponse , error ) {
2015-11-10 00:37:15 +03:00
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { DeleteRange : r } )
2015-11-04 22:23:33 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . DeleteRangeResponse ) , nil
2015-11-04 22:23:33 +03:00
}
func ( s * EtcdServer ) Txn ( ctx context . Context , r * pb . TxnRequest ) ( * pb . TxnResponse , error ) {
2016-04-04 14:21:42 +03:00
if isTxnSerializable ( r ) {
2016-07-27 21:49:58 +03:00
var resp * pb . TxnResponse
var err error
chk := func ( ai * auth . AuthInfo ) error {
return checkTxnAuth ( s . authStore , ai , r )
2016-06-22 20:16:13 +03:00
}
2016-07-27 21:49:58 +03:00
get := func ( ) { resp , err = s . applyV3Base . Txn ( r ) }
if serr := s . doSerialize ( ctx , chk , get ) ; serr != nil {
return nil , serr
}
return resp , err
2016-04-04 14:21:42 +03:00
}
2016-07-27 21:49:58 +03:00
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { Txn : r } )
2015-11-04 22:23:33 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . TxnResponse ) , nil
2015-11-04 22:23:33 +03:00
}
2016-04-04 14:21:42 +03:00
func isTxnSerializable ( r * pb . TxnRequest ) bool {
for _ , u := range r . Success {
if r := u . GetRequestRange ( ) ; r == nil || ! r . Serializable {
return false
}
}
for _ , u := range r . Failure {
if r := u . GetRequestRange ( ) ; r == nil || ! r . Serializable {
return false
}
}
return true
}
2015-11-04 22:23:33 +03:00
func ( s * EtcdServer ) Compact ( ctx context . Context , r * pb . CompactionRequest ) ( * pb . CompactionResponse , error ) {
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
result , err := s . processInternalRaftRequestOnce ( ctx , pb . InternalRaftRequest { Compaction : r } )
2016-05-07 01:26:05 +03:00
if r . Physical && result != nil && result . physc != nil {
2016-03-29 02:07:56 +03:00
<- result . physc
2016-03-31 10:03:34 +03:00
// The compaction is done deleting keys; the hash is now settled
// but the data is not necessarily committed. If there's a crash,
// the hash may revert to a hash prior to compaction completing
// if the compaction resumes. Force the finished compaction to
// commit so it won't resume following a crash.
s . be . ForceCommit ( )
2016-03-29 02:07:56 +03:00
}
2016-03-30 23:58:10 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
2016-02-12 23:25:26 +03:00
resp := result . resp . ( * pb . CompactionResponse )
if resp == nil {
resp = & pb . CompactionResponse { }
}
if resp . Header == nil {
resp . Header = & pb . ResponseHeader { }
}
resp . Header . Revision = s . kv . Rev ( )
2016-05-24 05:10:01 +03:00
return resp , nil
2015-08-08 20:14:42 +03:00
}
2016-04-07 20:27:36 +03:00
func ( s * EtcdServer ) LeaseGrant ( ctx context . Context , r * pb . LeaseGrantRequest ) ( * pb . LeaseGrantResponse , error ) {
2016-01-20 08:09:09 +03:00
// no id given? choose one
for r . ID == int64 ( lease . NoLease ) {
// only use positive int64 id's
r . ID = int64 ( s . reqIDGen . Next ( ) & ( ( 1 << 63 ) - 1 ) )
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
result , err := s . processInternalRaftRequestOnce ( ctx , pb . InternalRaftRequest { LeaseGrant : r } )
2016-01-06 00:49:25 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . LeaseGrantResponse ) , nil
2016-01-06 00:49:25 +03:00
}
func ( s * EtcdServer ) LeaseRevoke ( ctx context . Context , r * pb . LeaseRevokeRequest ) ( * pb . LeaseRevokeResponse , error ) {
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
result , err := s . processInternalRaftRequestOnce ( ctx , pb . InternalRaftRequest { LeaseRevoke : r } )
2016-01-06 00:49:25 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . LeaseRevokeResponse ) , nil
2016-01-06 00:49:25 +03:00
}
2016-01-10 00:26:45 +03:00
func ( s * EtcdServer ) LeaseRenew ( id lease . LeaseID ) ( int64 , error ) {
2016-01-20 08:09:09 +03:00
ttl , err := s . lessor . Renew ( id )
2016-09-09 02:14:14 +03:00
if err == nil { // already requested to primary lessor(leader)
2016-01-20 08:09:09 +03:00
return ttl , nil
}
if err != lease . ErrNotPrimary {
return - 1 , err
}
// renewals don't go through raft; forward to leader manually
2016-09-09 02:14:14 +03:00
leader , err := s . waitLeader ( )
if err != nil {
return - 1 , err
}
for _ , url := range leader . PeerURLs {
lurl := url + leasehttp . LeasePrefix
ttl , err = leasehttp . RenewHTTP ( id , lurl , s . peerRt , s . Cfg . peerDialTimeout ( ) )
if err == nil {
break
}
}
return ttl , err
}
func ( s * EtcdServer ) LeaseTimeToLive ( ctx context . Context , r * pb . LeaseTimeToLiveRequest ) ( * pb . LeaseTimeToLiveResponse , error ) {
if s . Leader ( ) == s . ID ( ) {
// primary; timetolive directly from leader
le := s . lessor . Lookup ( lease . LeaseID ( r . ID ) )
if le == nil {
return nil , lease . ErrLeaseNotFound
}
// TODO: fill out ResponseHeader
resp := & pb . LeaseTimeToLiveResponse { Header : & pb . ResponseHeader { } , ID : r . ID , TTL : int64 ( le . Remaining ( ) . Seconds ( ) ) , GrantedTTL : le . TTL }
if r . Keys {
ks := le . Keys ( )
kbs := make ( [ ] [ ] byte , len ( ks ) )
for i := range ks {
kbs [ i ] = [ ] byte ( ks [ i ] )
}
resp . Keys = kbs
}
return resp , nil
}
// manually request to leader
leader , err := s . waitLeader ( )
if err != nil {
return nil , err
}
var lresp * pb . LeaseTimeToLiveResponse
for _ , url := range leader . PeerURLs {
lurl := url + leasehttp . LeaseInternalPrefix
var iresp * leasepb . LeaseInternalResponse
iresp , err = leasehttp . TimeToLiveHTTP ( ctx , lease . LeaseID ( r . ID ) , r . Keys , lurl , s . peerRt )
if err == nil {
lresp = iresp . LeaseTimeToLiveResponse
break
}
}
return lresp , nil
}
func ( s * EtcdServer ) waitLeader ( ) ( * membership . Member , error ) {
2016-01-20 08:09:09 +03:00
leader := s . cluster . Member ( s . Leader ( ) )
for i := 0 ; i < 5 && leader == nil ; i ++ {
// wait an election
2016-05-12 22:34:03 +03:00
dur := time . Duration ( s . Cfg . ElectionTicks ) * time . Duration ( s . Cfg . TickMs ) * time . Millisecond
2016-01-20 08:09:09 +03:00
select {
case <- time . After ( dur ) :
leader = s . cluster . Member ( s . Leader ( ) )
2016-09-14 01:46:05 +03:00
case <- s . stopping :
2016-09-09 02:14:14 +03:00
return nil , ErrStopped
2016-01-20 08:09:09 +03:00
}
}
if leader == nil || len ( leader . PeerURLs ) == 0 {
2016-09-09 02:14:14 +03:00
return nil , ErrNoLeader
2016-01-20 08:09:09 +03:00
}
2016-09-09 02:14:14 +03:00
return leader , nil
2016-01-10 00:26:45 +03:00
}
2016-03-24 03:20:52 +03:00
func ( s * EtcdServer ) Alarm ( ctx context . Context , r * pb . AlarmRequest ) ( * pb . AlarmResponse , error ) {
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
result , err := s . processInternalRaftRequestOnce ( ctx , pb . InternalRaftRequest { Alarm : r } )
2016-03-24 03:20:52 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AlarmResponse ) , nil
2016-03-24 03:20:52 +03:00
}
2016-03-02 08:56:42 +03:00
func ( s * EtcdServer ) AuthEnable ( ctx context . Context , r * pb . AuthEnableRequest ) ( * pb . AuthEnableResponse , error ) {
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
result , err := s . processInternalRaftRequestOnce ( ctx , pb . InternalRaftRequest { AuthEnable : r } )
2016-03-08 11:31:48 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthEnableResponse ) , nil
2016-03-02 08:56:42 +03:00
}
2016-05-07 21:24:43 +03:00
func ( s * EtcdServer ) AuthDisable ( ctx context . Context , r * pb . AuthDisableRequest ) ( * pb . AuthDisableResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthDisable : r } )
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthDisableResponse ) , nil
2016-05-07 21:24:43 +03:00
}
2016-04-13 07:44:53 +03:00
func ( s * EtcdServer ) Authenticate ( ctx context . Context , r * pb . AuthenticateRequest ) ( * pb . AuthenticateResponse , error ) {
2016-06-09 03:11:26 +03:00
st , err := s . AuthStore ( ) . GenSimpleToken ( )
if err != nil {
return nil , err
}
internalReq := & pb . InternalAuthenticateRequest {
Name : r . Name ,
Password : r . Password ,
SimpleToken : st ,
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
result , err := s . processInternalRaftRequestOnce ( ctx , pb . InternalRaftRequest { Authenticate : internalReq } )
2016-04-13 07:44:53 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthenticateResponse ) , nil
2016-04-13 07:44:53 +03:00
}
2016-03-29 08:32:19 +03:00
func ( s * EtcdServer ) UserAdd ( ctx context . Context , r * pb . AuthUserAddRequest ) ( * pb . AuthUserAddResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthUserAdd : r } )
2016-03-23 07:25:48 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthUserAddResponse ) , nil
2016-03-23 07:25:48 +03:00
}
2016-03-31 05:29:47 +03:00
func ( s * EtcdServer ) UserDelete ( ctx context . Context , r * pb . AuthUserDeleteRequest ) ( * pb . AuthUserDeleteResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthUserDelete : r } )
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthUserDeleteResponse ) , nil
2016-03-31 05:29:47 +03:00
}
2016-03-31 08:31:07 +03:00
func ( s * EtcdServer ) UserChangePassword ( ctx context . Context , r * pb . AuthUserChangePasswordRequest ) ( * pb . AuthUserChangePasswordResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthUserChangePassword : r } )
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthUserChangePasswordResponse ) , nil
2016-03-31 08:31:07 +03:00
}
2016-06-07 06:17:28 +03:00
func ( s * EtcdServer ) UserGrantRole ( ctx context . Context , r * pb . AuthUserGrantRoleRequest ) ( * pb . AuthUserGrantRoleResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthUserGrantRole : r } )
2016-04-11 08:34:13 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
2016-06-07 06:17:28 +03:00
return result . resp . ( * pb . AuthUserGrantRoleResponse ) , nil
2016-04-11 08:34:13 +03:00
}
2016-05-30 11:21:11 +03:00
func ( s * EtcdServer ) UserGet ( ctx context . Context , r * pb . AuthUserGetRequest ) ( * pb . AuthUserGetResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthUserGet : r } )
if err != nil {
return nil , err
}
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthUserGetResponse ) , nil
}
2016-06-15 09:11:27 +03:00
func ( s * EtcdServer ) UserList ( ctx context . Context , r * pb . AuthUserListRequest ) ( * pb . AuthUserListResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthUserList : r } )
if err != nil {
return nil , err
}
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthUserListResponse ) , nil
}
2016-06-05 10:57:23 +03:00
func ( s * EtcdServer ) UserRevokeRole ( ctx context . Context , r * pb . AuthUserRevokeRoleRequest ) ( * pb . AuthUserRevokeRoleResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthUserRevokeRole : r } )
2016-06-03 10:38:59 +03:00
if err != nil {
return nil , err
}
if result . err != nil {
return nil , result . err
}
2016-06-05 10:57:23 +03:00
return result . resp . ( * pb . AuthUserRevokeRoleResponse ) , nil
2016-06-03 10:38:59 +03:00
}
2016-04-01 05:07:43 +03:00
func ( s * EtcdServer ) RoleAdd ( ctx context . Context , r * pb . AuthRoleAddRequest ) ( * pb . AuthRoleAddResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthRoleAdd : r } )
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthRoleAddResponse ) , nil
2016-04-01 05:07:43 +03:00
}
2016-06-07 06:17:28 +03:00
func ( s * EtcdServer ) RoleGrantPermission ( ctx context . Context , r * pb . AuthRoleGrantPermissionRequest ) ( * pb . AuthRoleGrantPermissionResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthRoleGrantPermission : r } )
2016-04-05 09:42:38 +03:00
if err != nil {
return nil , err
}
2016-05-24 05:10:01 +03:00
if result . err != nil {
return nil , result . err
}
2016-06-07 06:17:28 +03:00
return result . resp . ( * pb . AuthRoleGrantPermissionResponse ) , nil
2016-04-05 09:42:38 +03:00
}
2016-06-02 17:25:15 +03:00
func ( s * EtcdServer ) RoleGet ( ctx context . Context , r * pb . AuthRoleGetRequest ) ( * pb . AuthRoleGetResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthRoleGet : r } )
if err != nil {
return nil , err
}
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthRoleGetResponse ) , nil
}
2016-06-15 09:11:27 +03:00
func ( s * EtcdServer ) RoleList ( ctx context . Context , r * pb . AuthRoleListRequest ) ( * pb . AuthRoleListResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthRoleList : r } )
if err != nil {
return nil , err
}
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthRoleListResponse ) , nil
}
2016-06-05 10:57:23 +03:00
func ( s * EtcdServer ) RoleRevokePermission ( ctx context . Context , r * pb . AuthRoleRevokePermissionRequest ) ( * pb . AuthRoleRevokePermissionResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthRoleRevokePermission : r } )
2016-06-03 11:10:08 +03:00
if err != nil {
return nil , err
}
if result . err != nil {
return nil , result . err
}
2016-06-05 10:57:23 +03:00
return result . resp . ( * pb . AuthRoleRevokePermissionResponse ) , nil
2016-06-03 11:10:08 +03:00
}
2016-06-03 11:33:05 +03:00
func ( s * EtcdServer ) RoleDelete ( ctx context . Context , r * pb . AuthRoleDeleteRequest ) ( * pb . AuthRoleDeleteResponse , error ) {
result , err := s . processInternalRaftRequest ( ctx , pb . InternalRaftRequest { AuthRoleDelete : r } )
if err != nil {
return nil , err
}
if result . err != nil {
return nil , result . err
}
return result . resp . ( * pb . AuthRoleDeleteResponse ) , nil
}
2016-06-09 03:11:26 +03:00
func ( s * EtcdServer ) isValidSimpleToken ( token string ) bool {
splitted := strings . Split ( token , "." )
if len ( splitted ) != 2 {
return false
}
index , err := strconv . Atoi ( splitted [ 1 ] )
if err != nil {
return false
}
2016-08-20 02:10:58 +03:00
select {
case <- s . applyWait . Wait ( uint64 ( index ) ) :
return true
case <- s . stop :
return true
2016-06-09 03:11:26 +03:00
}
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
func ( s * EtcdServer ) authInfoFromCtx ( ctx context . Context ) ( * auth . AuthInfo , error ) {
2016-06-08 07:17:32 +03:00
md , ok := metadata . FromContext ( ctx )
if ! ok {
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
return nil , nil
2016-06-08 07:17:32 +03:00
}
ts , tok := md [ "token" ]
if ! tok {
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
return nil , nil
2016-05-17 09:20:50 +03:00
}
2016-06-08 07:17:32 +03:00
token := ts [ 0 ]
2016-06-09 03:11:26 +03:00
if ! s . isValidSimpleToken ( token ) {
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
return nil , ErrInvalidAuthToken
2016-06-09 03:11:26 +03:00
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
authInfo , uok := s . AuthStore ( ) . AuthInfoFromToken ( token )
2016-06-08 07:17:32 +03:00
if ! uok {
plog . Warningf ( "invalid auth token: %s" , token )
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
return nil , ErrInvalidAuthToken
2016-06-08 07:17:32 +03:00
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
return authInfo , nil
2016-05-17 09:20:50 +03:00
}
2016-07-27 21:49:58 +03:00
// doSerialize handles the auth logic, with permissions checked by "chk", for a serialized request "get". Returns a non-nil error on authentication failure.
func ( s * EtcdServer ) doSerialize ( ctx context . Context , chk func ( * auth . AuthInfo ) error , get func ( ) ) error {
for {
ai , err := s . authInfoFromCtx ( ctx )
if err != nil {
return err
}
if ai == nil {
// chk expects non-nil AuthInfo; use empty credentials
ai = & auth . AuthInfo { }
}
if err = chk ( ai ) ; err != nil {
if err == auth . ErrAuthOldRevision {
continue
}
return err
}
// fetch response for serialized request
get ( )
// empty credentials or current auth info means no need to retry
if ai . Revision == 0 || ai . Revision == s . authStore . Revision ( ) {
return nil
}
// avoid TOCTOU error, retry of the request is required.
}
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
func ( s * EtcdServer ) processInternalRaftRequestOnce ( ctx context . Context , r pb . InternalRaftRequest ) ( * applyResult , error ) {
2016-07-12 23:16:35 +03:00
ai := s . getAppliedIndex ( )
ci := s . getCommittedIndex ( )
if ci > ai + maxGapBetweenApplyAndCommitIndex {
return nil , ErrTooManyRequests
}
2016-05-17 09:20:50 +03:00
r . Header = & pb . RequestHeader {
ID : s . reqIDGen . Next ( ) ,
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
authInfo , err := s . authInfoFromCtx ( ctx )
2016-05-17 09:20:50 +03:00
if err != nil {
return nil , err
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
if authInfo != nil {
r . Header . Username = authInfo . Username
r . Header . AuthRevision = authInfo . Revision
}
2015-09-13 18:32:01 +03:00
data , err := r . Marshal ( )
if err != nil {
2015-11-04 22:23:33 +03:00
return nil , err
2015-09-13 18:32:01 +03:00
}
2016-02-08 23:44:25 +03:00
if len ( data ) > maxRequestBytes {
return nil , ErrRequestTooLarge
}
2016-05-17 09:20:50 +03:00
id := r . ID
if id == 0 {
id = r . Header . ID
}
ch := s . w . Register ( id )
2015-09-13 18:32:01 +03:00
2016-08-26 04:34:13 +03:00
cctx , cancel := context . WithTimeout ( ctx , s . Cfg . ReqTimeout ( ) )
2016-04-28 07:36:01 +03:00
defer cancel ( )
2016-06-17 22:50:58 +03:00
start := time . Now ( )
2016-04-28 07:36:01 +03:00
s . r . Propose ( cctx , data )
2016-06-17 22:50:58 +03:00
proposalsPending . Inc ( )
defer proposalsPending . Dec ( )
2015-09-13 18:32:01 +03:00
select {
case x := <- ch :
2015-11-04 22:23:33 +03:00
return x . ( * applyResult ) , nil
2016-04-28 07:36:01 +03:00
case <- cctx . Done ( ) :
2016-06-17 22:50:58 +03:00
proposalsFailed . Inc ( )
2016-05-17 09:20:50 +03:00
s . w . Trigger ( id , nil ) // GC wait
2016-06-17 22:50:58 +03:00
return nil , s . parseProposeCtxErr ( cctx . Err ( ) , start )
2015-09-13 18:32:01 +03:00
case <- s . done :
2015-11-04 22:23:33 +03:00
return nil , ErrStopped
2015-09-13 18:32:01 +03:00
}
}
auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem
This commit introduces revision of authStore. The revision number
represents a version of authStore that is incremented by updating auth
related information.
The revision is required for avoiding TOCTOU problems. Currently there
are two types of the TOCTOU problems in v3 auth.
The first one is in ordinal linearizable requests with a sequence like
below ():
1. Request from client CA is processed in follower FA. FA looks up the
username (let it U) for the request from a token of the request. At
this time, the request is authorized correctly.
2. Another request from client CB is processed in follower FB. CB
is for changing U's password.
3. FB forwards the request from CB to the leader before FA. Now U's
password is updated and the request from CA should be rejected.
4. However, the request from CA is processed by the leader because
authentication is already done in FA.
For avoiding the above sequence, this commit lets
etcdserverpb.RequestHeader have a member revision. The member is
initialized during authentication by followers and checked in a
leader. If the revision in RequestHeader is lower than the leader's
authStore revision, it means a sequence like above happened. In such a
case, the state machine returns auth.ErrAuthRevisionObsolete. The
error code lets nodes retry their requests.
The second one, a case of serializable range and txn, is more
subtle. Because these requests are processed in follower directly. The
TOCTOU problem can be caused by a sequence like below:
1. Serializable request from client CA is processed in follower FA. At
first, FA looks up the username (let it U) and its permission
before actual access to KV.
2. Another request from client CB is processed in follower FB and
forwarded to the leader. The cluster including FA now commits a log
entry of the request from CB. Assume the request changed the
permission or password of U.
3. Now the serializable request from CA is accessing to KV. Even if
the access is allowed at the point of 1, now it can be invalid
because of the change introduced in 2.
For avoiding the above sequence, this commit lets the functions of
serializable requests (EtcdServer.Range() and EtcdServer.Txn())
compare the revision in the request header with the latest revision of
authStore after the actual access. If the saved revision is lower than
the latest one, it means the permission can be changed. Although it
would introduce false positives (e.g. changing other user's password),
it prevents the TOCTOU problem. This idea is an implementation of
Anthony's comment:
https://github.com/coreos/etcd/pull/5739#issuecomment-228128254
2016-06-23 12:31:12 +03:00
func ( s * EtcdServer ) processInternalRaftRequest ( ctx context . Context , r pb . InternalRaftRequest ) ( * applyResult , error ) {
var result * applyResult
var err error
for {
result , err = s . processInternalRaftRequestOnce ( ctx , r )
if err != auth . ErrAuthOldRevision {
break
}
}
return result , err
}
2016-01-08 11:21:19 +03:00
// Watchable returns a watchable interface attached to the etcdserver.
2016-07-02 05:17:01 +03:00
func ( s * EtcdServer ) Watchable ( ) mvcc . WatchableKV { return s . KV ( ) }