kube-apiserver源码阅读分享

func createAPIExtensionsServer(apiextensionsConfig *apiextensionsapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget) (*apiextensionsapiserver.CustomResourceDefinitions, error) {  return apiextensionsConfig.Complete().New(delegateAPIServer)}
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*CustomResourceDefinitions, error) {  genericServer, err := c.GenericConfig.New("apiextensions-apiserver", delegationTarget)  if err != nil {    return nil, err  }
  ...  return s, nil}
func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {  ...
  handlerChainBuilder := func(handler http.Handler) http.Handler {    return c.BuildHandlerChainFunc(handler, c.Config)  }
    // 在构建对于的handler时，将访问控制相应的逻辑加进去  apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
  ...
  return s, nil}

func CreateKubeAPIServerConfig(  s completedServerRunOptions,  nodeTunneler tunneler.Tunneler,  proxyTransport *http.Transport,) (  *master.Config,  *genericapiserver.DeprecatedInsecureServingInfo,  aggregatorapiserver.ServiceResolver,  []admission.PluginInitializer,  error,) {  genericConfig, versionedInformers, insecureServingInfo, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport)
    ...
  return config, insecureServingInfo, serviceResolver, pluginInitializers, nil}
func buildGenericConfig(  s *options.ServerRunOptions,  proxyTransport *http.Transport,) (  genericConfig *genericapiserver.Config,  versionedInformers clientgoinformers.SharedInformerFactory,  insecureServingInfo *genericapiserver.DeprecatedInsecureServingInfo,  serviceResolver aggregatorapiserver.ServiceResolver,  pluginInitializers []admission.PluginInitializer,  admissionPostStartHook genericapiserver.PostStartHookFunc,  storageFactory *serverstorage.DefaultStorageFactory,  lastErr error,) {  genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)  ...
  return}
func NewConfig(codecs serializer.CodecFactory) *Config {  defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}  return &Config{    Serializer:                  codecs,        // 访问控制的逻辑链在这里被传入    BuildHandlerChainFunc:       DefaultBuildHandlerChain,    HandlerChainWaitGroup:       new(utilwaitgroup.SafeWaitGroup),    LegacyAPIGroupPrefixes:      sets.NewString(DefaultLegacyAPIPrefix),    DisabledPostStartHooks:      sets.NewString(),    PostStartHooks:              map[string]PostStartHookConfigEntry{},    HealthzChecks:               append([]healthz.HealthChecker{}, defaultHealthChecks...),    ReadyzChecks:                append([]healthz.HealthChecker{}, defaultHealthChecks...),    LivezChecks:                 append([]healthz.HealthChecker{}, defaultHealthChecks...),    EnableIndex:                 true,    EnableDiscovery:             true,    EnableProfiling:             true,    EnableMetrics:               true,    MaxRequestsInFlight:         400,    MaxMutatingRequestsInFlight: 200,    RequestTimeout:              time.Duration(60) * time.Second,    MinRequestTimeout:           1800,    LivezGracePeriod:            time.Duration(0),    ShutdownDelayDuration:       time.Duration(0),    JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),    MaxRequestBodyBytes: int64(3 * 1024 * 1024),    LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),  }}

func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {  handler := genericapifilters.WithAuthorization(apiHandler, c.Authorization.Authorizer, c.Serializer)  handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)  handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)  handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyChecker, c.LongRunningFunc)  failedHandler := genericapifilters.Unauthorized(c.Serializer, c.Authentication.SupportsBasicAuth)  failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyChecker)  handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)  handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")  handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc, c.RequestTimeout)  handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)  handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)  handler = genericfilters.WithPanicRecovery(handler)  return handler}

func buildGenericConfig(  s *options.ServerRunOptions,  proxyTransport *http.Transport,) (  genericConfig *genericapiserver.Config,  versionedInformers clientgoinformers.SharedInformerFactory,  insecureServingInfo *genericapiserver.DeprecatedInsecureServingInfo,  serviceResolver aggregatorapiserver.ServiceResolver,  pluginInitializers []admission.PluginInitializer,  admissionPostStartHook genericapiserver.PostStartHookFunc,  storageFactory *serverstorage.DefaultStorageFactory,  lastErr error,) {  ...  // 构建鉴权相关配置  genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, clientgoExternalClient, versionedInformers)  // 构建权限验证相关配置  genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, versionedInformers)  ...
  return}

func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.NegotiatedSerializer) http.Handler {  ...  return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {    ...        // 逻辑为对传入的对应访问控制的入参进行判断，通过则进行对应的请求处理，否则返回权限错误    authorized, reason, err := a.Authorize(ctx, attributes)    if authorized == authorizer.DecisionAllow {      audit.LogAnnotation(ae, decisionAnnotationKey, decisionAllow)      audit.LogAnnotation(ae, reasonAnnotationKey, reason)      handler.ServeHTTP(w, req)      return    }    if err != nil {      audit.LogAnnotation(ae, reasonAnnotationKey, reasonError)      responsewriters.InternalError(w, req, err)      return    }
    klog.V(4).Infof("Forbidden: %#v, Reason: %q", req.RequestURI, reason)    audit.LogAnnotation(ae, decisionAnnotationKey, decisionForbid)    audit.LogAnnotation(ae, reasonAnnotationKey, reason)    responsewriters.Forbidden(ctx, attributes, w, req, reason, s)  })}

func BuildAuthorizer(s *options.ServerRunOptions, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) {  authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers)  return authorizationConfig.New()}

func (config Config) New() (authorizer.Authorizer, authorizer.RuleResolver, error) {  ...
  for _, authorizationMode := range config.AuthorizationModes {    switch authorizationMode {        // Node模式    case modes.ModeNode:      graph := node.NewGraph()      node.AddGraphEventHandlers(        graph,        config.VersionedInformerFactory.Core().V1().Nodes(),        config.VersionedInformerFactory.Core().V1().Pods(),        config.VersionedInformerFactory.Core().V1().PersistentVolumes(),        config.VersionedInformerFactory.Storage().V1().VolumeAttachments(),      )      nodeAuthorizer := node.NewAuthorizer(graph, nodeidentifier.NewDefaultNodeIdentifier(), bootstrappolicy.NodeRules())      authorizers = append(authorizers, nodeAuthorizer)    // AlwaysAllow模式    case modes.ModeAlwaysAllow:      alwaysAllowAuthorizer := authorizerfactory.NewAlwaysAllowAuthorizer()      authorizers = append(authorizers, alwaysAllowAuthorizer)      ruleResolvers = append(ruleResolvers, alwaysAllowAuthorizer)        // AlwaysDeny模式    case modes.ModeAlwaysDeny:      alwaysDenyAuthorizer := authorizerfactory.NewAlwaysDenyAuthorizer()      authorizers = append(authorizers, alwaysDenyAuthorizer)      ruleResolvers = append(ruleResolvers, alwaysDenyAuthorizer)        // ABAC模式    case modes.ModeABAC:      abacAuthorizer, err := abac.NewFromFile(config.PolicyFile)      if err != nil {        return nil, nil, err      }      authorizers = append(authorizers, abacAuthorizer)      ruleResolvers = append(ruleResolvers, abacAuthorizer)        // Webhook模式    case modes.ModeWebhook:      webhookAuthorizer, err := webhook.New(config.WebhookConfigFile,        config.WebhookVersion,        config.WebhookCacheAuthorizedTTL,        config.WebhookCacheUnauthorizedTTL)      if err != nil {        return nil, nil, err      }      authorizers = append(authorizers, webhookAuthorizer)      ruleResolvers = append(ruleResolvers, webhookAuthorizer)        // RBAC模式    case modes.ModeRBAC:      rbacAuthorizer := rbac.New(        &rbac.RoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().Roles().Lister()},        &rbac.RoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().RoleBindings().Lister()},        &rbac.ClusterRoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoles().Lister()},        &rbac.ClusterRoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoleBindings().Lister()},      )      authorizers = append(authorizers, rbacAuthorizer)      ruleResolvers = append(ruleResolvers, rbacAuthorizer)    default:      return nil, nil, fmt.Errorf("unknown authorization mode %s specified", authorizationMode)    }  }
  return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil}

func (r *RBACAuthorizer) Authorize(ctx context.Context, requestAttributes authorizer.Attributes) (authorizer.Decision, string, error) {  ruleCheckingVisitor := &authorizingVisitor{requestAttributes: requestAttributes}
  r.authorizationRuleResolver.VisitRulesFor(requestAttributes.GetUser(), requestAttributes.GetNamespace(), ruleCheckingVisitor.visit)  if ruleCheckingVisitor.allowed {    return authorizer.DecisionAllow, ruleCheckingVisitor.reason, nil  }
  if klog.V(5) {    var operation string    if requestAttributes.IsResourceRequest() {      b := &bytes.Buffer{}      b.WriteString(`"`)      b.WriteString(requestAttributes.GetVerb())      b.WriteString(`" resource "`)      b.WriteString(requestAttributes.GetResource())      if len(requestAttributes.GetAPIGroup()) > 0 {        b.WriteString(`.`)        b.WriteString(requestAttributes.GetAPIGroup())      }      if len(requestAttributes.GetSubresource()) > 0 {        b.WriteString(`/`)        b.WriteString(requestAttributes.GetSubresource())      }      b.WriteString(`"`)      if len(requestAttributes.GetName()) > 0 {        b.WriteString(` named "`)        b.WriteString(requestAttributes.GetName())        b.WriteString(`"`)      }      operation = b.String()    } else {      operation = fmt.Sprintf("%q nonResourceURL %q", requestAttributes.GetVerb(), requestAttributes.GetPath())    }
    var scope string    if ns := requestAttributes.GetNamespace(); len(ns) > 0 {      scope = fmt.Sprintf("in namespace %q", ns)    } else {      scope = "cluster-wide"    }
    klog.Infof("RBAC DENY: user %q groups %q cannot %s %s", requestAttributes.GetUser().GetName(), requestAttributes.GetUser().GetGroups(), operation, scope)  }
  reason := ""  if len(ruleCheckingVisitor.errors) > 0 {    reason = fmt.Sprintf("RBAC: %v", utilerrors.NewAggregate(ruleCheckingVisitor.errors))  }  return authorizer.DecisionNoOpinion, reason, nil}

func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences) http.Handler {  ...  return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {    authenticationStart := time.Now()
    ...        // 主要逻辑在这里    resp, ok, err := auth.AuthenticateRequest(req)    ...
    handler.ServeHTTP(w, req)  })}

func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {  authenticatorConfig, err := s.Authentication.ToAuthenticationConfig()
  if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {    authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(      extclient,      versionedInformer.Core().V1().Secrets().Lister(),      versionedInformer.Core().V1().ServiceAccounts().Lister(),      versionedInformer.Core().V1().Pods().Lister(),    )  }    // Bootstrap模式  authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(    versionedInformer.Core().V1().Secrets().Lister().Secrets(v1.NamespaceSystem),  )
  return authenticatorConfig.New()}

func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) {  var authenticators []authenticator.Request  var tokenAuthenticators []authenticator.Token  securityDefinitions := spec.SecurityDefinitions{}
  // front proxy鉴权配置  if config.RequestHeaderConfig != nil {    requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(      config.RequestHeaderConfig.CAContentProvider.VerifyOptions,      config.RequestHeaderConfig.AllowedClientNames,      config.RequestHeaderConfig.UsernameHeaders,      config.RequestHeaderConfig.GroupHeaders,      config.RequestHeaderConfig.ExtraHeaderPrefixes,    )    authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, requestHeaderAuthenticator))  }
  // basic auth鉴权配置  if len(config.BasicAuthFile) > 0 {    basicAuth, err := newAuthenticatorFromBasicAuthFile(config.BasicAuthFile)    authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, basicAuth))
    securityDefinitions["HTTPBasic"] = &spec.SecurityScheme{      SecuritySchemeProps: spec.SecuritySchemeProps{        Type:        "basic",        Description: "HTTP Basic authentication",      },    }  }
  // X509 methods鉴权配置  if config.ClientCAContentProvider != nil {    certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)    authenticators = append(authenticators, certAuth)  }
  // Bearer token methods鉴权配置  if len(config.TokenAuthFile) > 0 {    tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
    tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))  }  if len(config.ServiceAccountKeyFiles) > 0 {    serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter)
    tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)  }  if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) && config.ServiceAccountIssuer != "" {    serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuer, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter)
    tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)  }  if config.BootstrapToken {    if config.BootstrapTokenAuthenticator != nil {      tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, config.BootstrapTokenAuthenticator))    }  }  if len(config.OIDCIssuerURL) > 0 && len(config.OIDCClientID) > 0 {    oidcAuth, err := newAuthenticatorFromOIDCIssuerURL(oidc.Options{      IssuerURL:            config.OIDCIssuerURL,      ClientID:             config.OIDCClientID,      APIAudiences:         config.APIAudiences,      CAFile:               config.OIDCCAFile,      UsernameClaim:        config.OIDCUsernameClaim,      UsernamePrefix:       config.OIDCUsernamePrefix,      GroupsClaim:          config.OIDCGroupsClaim,      GroupsPrefix:         config.OIDCGroupsPrefix,      SupportedSigningAlgs: config.OIDCSigningAlgs,      RequiredClaims:       config.OIDCRequiredClaims,    })
    tokenAuthenticators = append(tokenAuthenticators, oidcAuth)  }  if len(config.WebhookTokenAuthnConfigFile) > 0 {    webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnVersion, config.WebhookTokenAuthnCacheTTL, config.APIAudiences)
    tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)  }
  if len(tokenAuthenticators) > 0 {    tokenAuth := tokenunion.New(tokenAuthenticators...)    if config.TokenSuccessCacheTTL > 0 || config.TokenFailureCacheTTL > 0 {      tokenAuth = tokencache.New(tokenAuth, true, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL)    }    authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth))    securityDefinitions["BearerToken"] = &spec.SecurityScheme{      SecuritySchemeProps: spec.SecuritySchemeProps{        Type:        "apiKey",        Name:        "authorization",        In:          "header",        Description: "Bearer Token authentication",      },    }  }
  if len(authenticators) == 0 {    if config.Anonymous {      return anonymous.NewAuthenticator(), &securityDefinitions, nil    }    return nil, &securityDefinitions, nil  }
  authenticator := union.New(authenticators...)
  authenticator = group.NewAuthenticatedGroupAdder(authenticator)
  if config.Anonymous {    authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())  }
  return authenticator, &securityDefinitions, nil}

func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {  auth := strings.TrimSpace(req.Header.Get("Authorization"))  if auth == "" {    return nil, false, nil  }  parts := strings.Split(auth, " ")  if len(parts) < 2 || strings.ToLower(parts[0]) != "bearer" {    return nil, false, nil  }
  token := parts[1]
  if len(token) == 0 {    return nil, false, nil  }
  resp, ok, err := a.auth.AuthenticateToken(req.Context(), token)  if ok {    req.Header.Del("Authorization")  }
  if !ok && err == nil {    err = invalidToken  }
  return resp, ok, err}

func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, error) {  ...    switch action.Verb {    case "GET":       var handler restful.RouteFunction      if isGetterWithOptions {        handler = restfulGetResourceWithOptions(getterWithOptions, reqScope, isSubresource)      } else {        handler = restfulGetResource(getter, exporter, reqScope)      }      ...    }  }  ...
  return &apiResource, nil}

func restfulGetResourceWithOptions(r rest.GetterWithOptions, scope handlers.RequestScope, isSubresource bool) restful.RouteFunction {  return func(req *restful.Request, res *restful.Response) {    handlers.GetResourceWithOptions(r, &scope, isSubresource)(res.ResponseWriter, req.Request)  }}

func GetResourceWithOptions(r rest.GetterWithOptions, scope *RequestScope, isSubresource bool) http.HandlerFunc {  return getResourceHandler(scope,    func(ctx context.Context, name string, req *http.Request, trace *utiltrace.Trace) (runtime.Object, error) {      opts, subpath, subpathKey := r.NewGetOptions()      trace.Step("About to process Get options")            if err := getRequestOptions(req, scope, opts, subpath, subpathKey, isSubresource); err != nil {        err = errors.NewBadRequest(err.Error())        return nil, err      }
      if trace != nil {       trace.Step("About to Get from storage")      }
            // 逻辑实现为对应的rest.GetterWithOptions的相应操作。      return r.Get(ctx, name, opts)    })}

func createAPIExtensionsServer(apiextensionsConfig *apiextensionsapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget) (*apiextensionsapiserver.CustomResourceDefinitions, error) {  return apiextensionsConfig.Complete().New(delegateAPIServer)}
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*CustomResourceDefinitions, error) {  ...  apiResourceConfig := c.GenericConfig.MergedResourceConfig  apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiextensions.GroupName, Scheme, metav1.ParameterCodec, Codecs)
  if apiResourceConfig.VersionEnabled(v1beta1.SchemeGroupVersion) {    storage := map[string]rest.Storage{}        // 以customresourcedefinitions这种资源对应的操作为例。NewREST来构造对应的后端交互结构体    customResourceDefintionStorage := customresourcedefinition.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter)    storage["customresourcedefinitions"] = customResourceDefintionStorage    storage["customresourcedefinitions/status"] = customresourcedefinition.NewStatusREST(Scheme, customResourceDefintionStorage)    apiGroupInfo.VersionedResourcesStorageMap[v1beta1.SchemeGroupVersion.Version] = storage  }
  ...
  return s, nil}

func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) *REST {  strategy := NewStrategy(scheme)  // 相应数据结构在前面已经提到，这里不再赘述  store := &genericregistry.Store{    ...  }
  options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs}    // 完善结构体的action操作  if err := store.CompleteWithOptions(options); err != nil {    panic(err)   }  return &REST{store}
}

func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {  ...  opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource)
  ...  return nil}

func (t CRDRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {  ret := generic.RESTOptions{    StorageConfig:           &t.StorageConfig,    Decorator:               generic.UndecoratedStorage,    EnableGarbageCollection: t.EnableGarbageCollection,    DeleteCollectionWorkers: t.DeleteCollectionWorkers,    ResourcePrefix:          resource.Group + "/" + resource.Resource,    CountMetricPollPeriod:   t.CountMetricPollPeriod,  }
  if t.EnableWatchCache {    ret.Decorator = genericregistry.StorageWithCacher(t.DefaultWatchCacheSize)  }  return ret, nil}

func StorageWithCacher(capacity int) generic.StorageDecorator {  return func(    storageConfig *storagebackend.Config,    resourcePrefix string,    keyFunc func(obj runtime.Object) (string, error),    newFunc func() runtime.Object,    newListFunc func() runtime.Object,    getAttrsFunc storage.AttrFunc,    triggerFuncs storage.IndexerFuncs) (storage.Interface, factory.DestroyFunc, error) {    // 构造和后端etcd交互的结构体Storage    s, d, err := generic.NewRawStorage(storageConfig)    }    ...    cacher, err := cacherstorage.NewCacherFromConfig(cacherConfig)    destroyFunc := func() {      cacher.Stop()      d()    }    RegisterStorageCleanup(destroyFunc)
    return cacher, destroyFunc, nil  }}

func NewRawStorage(config *storagebackend.Config) (storage.Interface, factory.DestroyFunc, error) {  return factory.Create(*config)}// 根据etcd的版本来创建对应的后端交互结构体，1.17版本仅支持etcd v3版本func Create(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {  switch c.Type {  case "etcd2":    return nil, nil, fmt.Errorf("%v is no longer a supported storage backend", c.Type)  case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:    return newETCD3Storage(c)
  default:    return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type)  }}

func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {  stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
  client, err := newETCD3Client(c.Transport)
  var once sync.Once  destroyFunc := func() {    once.Do(func() {      stopCompactor()      client.Close()    })  }  transformer := c.Transformer  if transformer == nil {    transformer = value.IdentityTransformer  }  return etcd3.New(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil}

func New(c *clientv3.Client, codec runtime.Codec, prefix string, transformer value.Transformer, pagingEnabled bool) storage.Interface {  return newStore(c, pagingEnabled, codec, prefix, transformer)}
// 返回对应资源的完整的后端交互结构体storefunc newStore(c *clientv3.Client, pagingEnabled bool, codec runtime.Codec, prefix string, transformer value.Transformer) *store {  versioner := APIObjectVersioner{}
  result := &store{    client:        c,    codec:         codec,    versioner:     versioner,    transformer:   transformer,    pagingEnabled: pagingEnabled,    pathPrefix:   path.Join("/", prefix),    watcher:      newWatcher(c, codec, versioner, transformer),    leaseManager: newDefaultLeaseManager(c),  }  return result}

func (s *store) Get(ctx context.Context, key string, resourceVersion string, out runtime.Object, ignoreNotFound bool) error {  key = path.Join(s.pathPrefix, key)  startTime := time.Now()  getResp, err := s.client.KV.Get(ctx, key, s.getOps...)  metrics.RecordEtcdRequestLatency("get", getTypeName(out), startTime)  if err != nil {    return err  }  if err = s.ensureMinimumResourceVersion(resourceVersion, uint64(getResp.Header.Revision)); err != nil {    return err  }    if len(getResp.Kvs) == 0 {    if ignoreNotFound {      return runtime.SetZeroValue(out)    }    return storage.NewKeyNotFoundError(key, 0)  }  kv := getResp.Kvs[0]    data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(key))  if err != nil {    return storage.NewInternalError(err.Error())  }  return decode(s.codec, s.versioner, data, out, kv.ModRevision)}