Skip to content

Commit 73614dd

Browse files
MikeSpreitzerMikeSpreitzer
committedFeb 11, 2020
Added API Priority and Fairness filter and config consumer
·
v1.34.0-alpha.3v1.18.0-beta.0
1 parent 6eba154 commit 73614dd

File tree

32 files changed

+3113
-312
lines changed

32 files changed

+3113
-312
lines changed
 

‎cmd/kube-apiserver/app/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ go_library(
5555
"//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
5656
"//staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library",
5757
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
58+
"//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:go_default_library",
5859
"//staging/src/k8s.io/apiserver/pkg/util/term:go_default_library",
5960
"//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library",
6061
"//staging/src/k8s.io/client-go/informers:go_default_library",

‎cmd/kube-apiserver/app/options/options_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ func TestAddFlags(t *testing.T) {
9898
"--contention-profiling=true",
9999
"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
100100
"--enable-aggregator-routing=true",
101+
"--enable-inflight-quota-handler=false",
101102
"--enable-logs-handler=false",
102103
"--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType),
103104
"--etcd-keyfile=/var/run/kubernetes/etcd.key",

‎cmd/kube-apiserver/app/server.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,15 @@ import (
4343
"k8s.io/apiserver/pkg/authentication/authenticator"
4444
"k8s.io/apiserver/pkg/authorization/authorizer"
4545
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
46+
genericfeatures "k8s.io/apiserver/pkg/features"
4647
genericapiserver "k8s.io/apiserver/pkg/server"
4748
"k8s.io/apiserver/pkg/server/filters"
4849
serveroptions "k8s.io/apiserver/pkg/server/options"
4950
serverstorage "k8s.io/apiserver/pkg/server/storage"
5051
"k8s.io/apiserver/pkg/storage/etcd3/preflight"
5152
"k8s.io/apiserver/pkg/util/feature"
5253
utilfeature "k8s.io/apiserver/pkg/util/feature"
54+
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
5355
"k8s.io/apiserver/pkg/util/term"
5456
"k8s.io/apiserver/pkg/util/webhook"
5557
clientgoinformers "k8s.io/client-go/informers"
@@ -523,6 +525,10 @@ func buildGenericConfig(
523525
lastErr = fmt.Errorf("failed to initialize admission: %v", err)
524526
}
525527

528+
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && s.GenericServerRunOptions.EnableInflightQuotaHandler {
529+
genericConfig.FlowControl = BuildPriorityAndFairness(s, clientgoExternalClient, versionedInformers)
530+
}
531+
526532
return
527533
}
528534

@@ -553,6 +559,16 @@ func BuildAuthorizer(s *options.ServerRunOptions, versionedInformers clientgoinf
553559
return authorizationConfig.New()
554560
}
555561

562+
// BuildPriorityAndFairness constructs the guts of the API Priority and Fairness filter
563+
func BuildPriorityAndFairness(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) utilflowcontrol.Interface {
564+
return utilflowcontrol.New(
565+
versionedInformer,
566+
extclient.FlowcontrolV1alpha1(),
567+
s.GenericServerRunOptions.MaxRequestsInFlight+s.GenericServerRunOptions.MaxMutatingRequestsInFlight,
568+
s.GenericServerRunOptions.RequestTimeout/4,
569+
)
570+
}
571+
556572
// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
557573
type completedServerRunOptions struct {
558574
*options.ServerRunOptions

‎pkg/kubeapiserver/server/insecure_handler.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,14 @@ import (
2727
// BuildInsecureHandlerChain sets up the server to listen to http. Should be removed.
2828
func BuildInsecureHandlerChain(apiHandler http.Handler, c *server.Config) http.Handler {
2929
handler := apiHandler
30-
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
30+
// Temporarily disable APIPriorityAndFairness during development
31+
// so that /debug/pprof works even while this feature is totally
32+
// hosed
33+
if c.FlowControl != nil && false {
34+
handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl)
35+
} else {
36+
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
37+
}
3138
handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyChecker, c.LongRunningFunc)
3239
handler = genericapifilters.WithAuthentication(handler, server.InsecureSuperuser{}, nil, nil)
3340
handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")

‎pkg/registry/flowcontrol/rest/storage_flowcontrol.go

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ func (p RESTStorageProvider) GroupName() string {
8989
return flowcontrol.GroupName
9090
}
9191

92+
// PostStartHook returns the hook func that launches the config provider
9293
func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStartHookFunc, error) {
9394
return PostStartHookName, func(hookContext genericapiserver.PostStartHookContext) error {
9495
flowcontrolClientSet := flowcontrolclient.NewForConfigOrDie(hookContext.LoopbackClientConfig)
@@ -152,28 +153,30 @@ func lastMandatoryExists(flowcontrolClientSet flowcontrolclient.FlowcontrolV1alp
152153
return false, nil
153154
}
154155

156+
const thisFieldManager = "api-priority-and-fairness-config-producer"
157+
155158
func ensure(flowcontrolClientSet flowcontrolclient.FlowcontrolV1alpha1Interface, flowSchemas []*flowcontrolv1alpha1.FlowSchema, priorityLevels []*flowcontrolv1alpha1.PriorityLevelConfiguration) error {
156159
for _, flowSchema := range flowSchemas {
157-
_, err := flowcontrolClientSet.FlowSchemas().Create(context.TODO(), flowSchema, metav1.CreateOptions{})
160+
_, err := flowcontrolClientSet.FlowSchemas().Create(context.TODO(), flowSchema, metav1.CreateOptions{FieldManager: thisFieldManager})
158161
if apierrors.IsAlreadyExists(err) {
159-
klog.V(3).Infof("system preset FlowSchema %s already exists, skipping creating", flowSchema.Name)
162+
klog.V(3).Infof("Suggested FlowSchema %s already exists, skipping creating", flowSchema.Name)
160163
continue
161164
}
162165
if err != nil {
163-
return fmt.Errorf("cannot create FlowSchema %s due to %v", flowSchema.Name, err)
166+
return fmt.Errorf("cannot create suggested FlowSchema %s due to %v", flowSchema.Name, err)
164167
}
165-
klog.V(3).Infof("created system preset FlowSchema %s", flowSchema.Name)
168+
klog.V(3).Infof("Created suggested FlowSchema %s", flowSchema.Name)
166169
}
167170
for _, priorityLevelConfiguration := range priorityLevels {
168-
_, err := flowcontrolClientSet.PriorityLevelConfigurations().Create(context.TODO(), priorityLevelConfiguration, metav1.CreateOptions{})
171+
_, err := flowcontrolClientSet.PriorityLevelConfigurations().Create(context.TODO(), priorityLevelConfiguration, metav1.CreateOptions{FieldManager: thisFieldManager})
169172
if apierrors.IsAlreadyExists(err) {
170-
klog.V(3).Infof("system preset PriorityLevelConfiguration %s already exists, skipping creating", priorityLevelConfiguration.Name)
173+
klog.V(3).Infof("Suggested PriorityLevelConfiguration %s already exists, skipping creating", priorityLevelConfiguration.Name)
171174
continue
172175
}
173176
if err != nil {
174-
return fmt.Errorf("cannot create PriorityLevelConfiguration %s due to %v", priorityLevelConfiguration.Name, err)
177+
return fmt.Errorf("cannot create suggested PriorityLevelConfiguration %s due to %v", priorityLevelConfiguration.Name, err)
175178
}
176-
klog.V(3).Infof("created system preset PriorityLevelConfiguration %s", priorityLevelConfiguration.Name)
179+
klog.V(3).Infof("Created suggested PriorityLevelConfiguration %s", priorityLevelConfiguration.Name)
177180
}
178181
return nil
179182
}
@@ -184,58 +187,60 @@ func upgrade(flowcontrolClientSet flowcontrolclient.FlowcontrolV1alpha1Interface
184187
if err == nil {
185188
// TODO(yue9944882): extract existing version from label and compare
186189
// TODO(yue9944882): create w/ version string attached
187-
identical, err := flowSchemaHasWrongSpec(expectedFlowSchema, actualFlowSchema)
190+
wrongSpec, err := flowSchemaHasWrongSpec(expectedFlowSchema, actualFlowSchema)
188191
if err != nil {
189192
return fmt.Errorf("failed checking if mandatory FlowSchema %s is up-to-date due to %v, will retry later", expectedFlowSchema.Name, err)
190193
}
191-
if !identical {
192-
if _, err := flowcontrolClientSet.FlowSchemas().Update(context.TODO(), expectedFlowSchema, metav1.UpdateOptions{}); err != nil {
194+
if wrongSpec {
195+
if _, err := flowcontrolClientSet.FlowSchemas().Update(context.TODO(), expectedFlowSchema, metav1.UpdateOptions{FieldManager: thisFieldManager}); err != nil {
193196
return fmt.Errorf("failed upgrading mandatory FlowSchema %s due to %v, will retry later", expectedFlowSchema.Name, err)
194197
}
198+
klog.V(3).Infof("Updated mandatory FlowSchema %s because its spec was %#+v but it must be %#+v", expectedFlowSchema.Name, actualFlowSchema.Spec, expectedFlowSchema.Spec)
195199
}
196200
continue
197201
}
198202
if !apierrors.IsNotFound(err) {
199-
return fmt.Errorf("failed getting FlowSchema %s due to %v, will retry later", expectedFlowSchema.Name, err)
203+
return fmt.Errorf("failed getting mandatory FlowSchema %s due to %v, will retry later", expectedFlowSchema.Name, err)
200204
}
201-
_, err = flowcontrolClientSet.FlowSchemas().Create(context.TODO(), expectedFlowSchema, metav1.CreateOptions{})
205+
_, err = flowcontrolClientSet.FlowSchemas().Create(context.TODO(), expectedFlowSchema, metav1.CreateOptions{FieldManager: thisFieldManager})
202206
if apierrors.IsAlreadyExists(err) {
203-
klog.V(3).Infof("system preset FlowSchema %s already exists, skipping creating", expectedFlowSchema.Name)
207+
klog.V(3).Infof("Mandatory FlowSchema %s already exists, skipping creating", expectedFlowSchema.Name)
204208
continue
205209
}
206210
if err != nil {
207-
return fmt.Errorf("cannot create FlowSchema %s due to %v", expectedFlowSchema.Name, err)
211+
return fmt.Errorf("cannot create mandatory FlowSchema %s due to %v", expectedFlowSchema.Name, err)
208212
}
209-
klog.V(3).Infof("created system preset FlowSchema %s", expectedFlowSchema.Name)
213+
klog.V(3).Infof("Created mandatory FlowSchema %s", expectedFlowSchema.Name)
210214
}
211215
for _, expectedPriorityLevelConfiguration := range priorityLevels {
212216
actualPriorityLevelConfiguration, err := flowcontrolClientSet.PriorityLevelConfigurations().Get(context.TODO(), expectedPriorityLevelConfiguration.Name, metav1.GetOptions{})
213217
if err == nil {
214218
// TODO(yue9944882): extract existing version from label and compare
215219
// TODO(yue9944882): create w/ version string attached
216-
identical, err := priorityLevelHasWrongSpec(expectedPriorityLevelConfiguration, actualPriorityLevelConfiguration)
220+
wrongSpec, err := priorityLevelHasWrongSpec(expectedPriorityLevelConfiguration, actualPriorityLevelConfiguration)
217221
if err != nil {
218222
return fmt.Errorf("failed checking if mandatory PriorityLevelConfiguration %s is up-to-date due to %v, will retry later", expectedPriorityLevelConfiguration.Name, err)
219223
}
220-
if !identical {
221-
if _, err := flowcontrolClientSet.PriorityLevelConfigurations().Update(context.TODO(), expectedPriorityLevelConfiguration, metav1.UpdateOptions{}); err != nil {
224+
if wrongSpec {
225+
if _, err := flowcontrolClientSet.PriorityLevelConfigurations().Update(context.TODO(), expectedPriorityLevelConfiguration, metav1.UpdateOptions{FieldManager: thisFieldManager}); err != nil {
222226
return fmt.Errorf("failed upgrading mandatory PriorityLevelConfiguration %s due to %v, will retry later", expectedPriorityLevelConfiguration.Name, err)
223227
}
228+
klog.V(3).Infof("Updated mandatory PriorityLevelConfiguration %s because its spec was %#+v but must be %#+v", expectedPriorityLevelConfiguration.Name, actualPriorityLevelConfiguration.Spec, expectedPriorityLevelConfiguration.Spec)
224229
}
225230
continue
226231
}
227232
if !apierrors.IsNotFound(err) {
228233
return fmt.Errorf("failed getting PriorityLevelConfiguration %s due to %v, will retry later", expectedPriorityLevelConfiguration.Name, err)
229234
}
230-
_, err = flowcontrolClientSet.PriorityLevelConfigurations().Create(context.TODO(), expectedPriorityLevelConfiguration, metav1.CreateOptions{})
235+
_, err = flowcontrolClientSet.PriorityLevelConfigurations().Create(context.TODO(), expectedPriorityLevelConfiguration, metav1.CreateOptions{FieldManager: thisFieldManager})
231236
if apierrors.IsAlreadyExists(err) {
232-
klog.V(3).Infof("system preset PriorityLevelConfiguration %s already exists, skipping creating", expectedPriorityLevelConfiguration.Name)
237+
klog.V(3).Infof("Mandatory PriorityLevelConfiguration %s already exists, skipping creating", expectedPriorityLevelConfiguration.Name)
233238
continue
234239
}
235240
if err != nil {
236-
return fmt.Errorf("cannot create PriorityLevelConfiguration %s due to %v", expectedPriorityLevelConfiguration.Name, err)
241+
return fmt.Errorf("cannot create mandatory PriorityLevelConfiguration %s due to %v", expectedPriorityLevelConfiguration.Name, err)
237242
}
238-
klog.V(3).Infof("created system preset PriorityLevelConfiguration %s", expectedPriorityLevelConfiguration.Name)
243+
klog.V(3).Infof("Created mandatory PriorityLevelConfiguration %s", expectedPriorityLevelConfiguration.Name)
239244
}
240245
return nil
241246
}

‎staging/src/k8s.io/apiserver/BUILD

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,7 @@ filegroup(
4444
"//staging/src/k8s.io/apiserver/pkg/util/cache:all-srcs",
4545
"//staging/src/k8s.io/apiserver/pkg/util/dryrun:all-srcs",
4646
"//staging/src/k8s.io/apiserver/pkg/util/feature:all-srcs",
47-
"//staging/src/k8s.io/apiserver/pkg/util/flowcontrol/counter:all-srcs",
48-
"//staging/src/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing:all-srcs",
49-
"//staging/src/k8s.io/apiserver/pkg/util/flowcontrol/metrics:all-srcs",
47+
"//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:all-srcs",
5048
"//staging/src/k8s.io/apiserver/pkg/util/flushwriter:all-srcs",
5149
"//staging/src/k8s.io/apiserver/pkg/util/openapi:all-srcs",
5250
"//staging/src/k8s.io/apiserver/pkg/util/proxy:all-srcs",

‎staging/src/k8s.io/apiserver/pkg/server/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ go_library(
105105
"//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library",
106106
"//staging/src/k8s.io/apiserver/pkg/server/routes:go_default_library",
107107
"//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
108+
"//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:go_default_library",
108109
"//staging/src/k8s.io/apiserver/pkg/util/openapi:go_default_library",
109110
"//staging/src/k8s.io/client-go/informers:go_default_library",
110111
"//staging/src/k8s.io/client-go/rest:go_default_library",

‎staging/src/k8s.io/apiserver/pkg/server/config.go

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ import (
6060
"k8s.io/apiserver/pkg/server/healthz"
6161
"k8s.io/apiserver/pkg/server/routes"
6262
serverstore "k8s.io/apiserver/pkg/server/storage"
63+
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
6364
"k8s.io/client-go/informers"
6465
restclient "k8s.io/client-go/rest"
6566
"k8s.io/component-base/logs"
@@ -107,6 +108,9 @@ type Config struct {
107108
AdmissionControl admission.Interface
108109
CorsAllowedOriginList []string
109110

111+
// FlowControl, if not nil, gives priority and fairness to request handling
112+
FlowControl utilflowcontrol.Interface
113+
110114
EnableIndex bool
111115
EnableProfiling bool
112116
EnableDiscovery bool
@@ -606,6 +610,21 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
606610
}
607611
}
608612

613+
const priorityAndFairnessConfigConsumerHookName = "priority-and-fairness-config-consumer"
614+
if s.isPostStartHookRegistered(priorityAndFairnessConfigConsumerHookName) {
615+
} else if c.FlowControl != nil {
616+
err := s.AddPostStartHook(priorityAndFairnessConfigConsumerHookName, func(context PostStartHookContext) error {
617+
go c.FlowControl.Run(context.StopCh)
618+
return nil
619+
})
620+
if err != nil {
621+
return nil, err
622+
}
623+
// TODO(yue9944882): plumb pre-shutdown-hook for request-management system?
624+
} else {
625+
klog.V(3).Infof("Not requested to run hook %s", priorityAndFairnessConfigConsumerHookName)
626+
}
627+
609628
for _, delegateCheck := range delegationTarget.HealthzChecks() {
610629
skip := false
611630
for _, existingCheck := range c.HealthzChecks {
@@ -638,7 +657,11 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
638657

639658
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
640659
handler := genericapifilters.WithAuthorization(apiHandler, c.Authorization.Authorizer, c.Serializer)
641-
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
660+
if c.FlowControl != nil {
661+
handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl)
662+
} else {
663+
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
664+
}
642665
handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
643666
handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyChecker, c.LongRunningFunc)
644667
failedHandler := genericapifilters.Unauthorized(c.Serializer, c.Authentication.SupportsBasicAuth)

‎staging/src/k8s.io/apiserver/pkg/server/filters/BUILD

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ go_library(
3636
"doc.go",
3737
"longrunning.go",
3838
"maxinflight.go",
39+
"priority-and-fairness.go",
3940
"timeout.go",
4041
"waitgroup.go",
4142
"wrap.go",
@@ -44,8 +45,10 @@ go_library(
4445
importpath = "k8s.io/apiserver/pkg/server/filters",
4546
deps = [
4647
"//staging/src/k8s.io/api/core/v1:go_default_library",
48+
"//staging/src/k8s.io/api/flowcontrol/v1alpha1:go_default_library",
4749
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
4850
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
51+
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
4952
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
5053
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
5154
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@@ -55,6 +58,7 @@ go_library(
5558
"//staging/src/k8s.io/apiserver/pkg/endpoints/metrics:go_default_library",
5659
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
5760
"//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library",
61+
"//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:go_default_library",
5862
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
5963
"//vendor/k8s.io/klog:go_default_library",
6064
],

0 commit comments

Comments
 (0)
Please sign in to comment.