This article is based on source code learning of Kubernetes v1.22.4 version
9、KubeAPIServer
GenericAPIServer provides some common functions. If other servers are expanded based on GenericAPIServer, the amount of code will be reduced a lot.
KubeAPIServer is responsible for processing REST requests for Kubernetes built-in resources, such as Pods, Deployments, etc.
// cmd/kube-apiserver/app/server.go
func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{
}) (*aggregatorapiserver.APIAggregator, error) {
// 创建KubeAPIServer所需配置
kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions)
if err != nil {
return nil, err
}
// 创建APIExtensionsServer所需配置,这里复用了kubeAPIServerConfig的部分通用配置
// If additional API servers are added, they should be gated.
apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount,
serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIServerConfig.ExtraConfig.ProxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.GenericConfig.TracerProvider))
if err != nil {
return nil, err
}
// 创建APIExtensionsServer实例
apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
// 创建KubeAPIServer实例
kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer)
if err != nil {
return nil, err
}
// aggregator comes last in the chain
// 创建AggregatorServer所需配置,这里复用了kubeAPIServerConfig的部分通用配置
aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, kubeAPIServerConfig.ExtraConfig.ProxyTransport, pluginInitializer)
if err != nil {
return nil, err
}
// 创建AggregatorServer实例
aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
if err != nil {
// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
return nil, err
}
return aggregatorServer, nil
}
CreateServerChain()The logic related to KubeAPIServer in the method is as follows:
- Call
CreateKubeAPIServerConfig()the method to create the configuration required for KubeAPIServer - Call
CreateKubeAPIServer()the method to create a KubeAPIServer instance
1). Prepare Config
CreateKubeAPIServerConfig()The method code is as follows:
// cmd/kube-apiserver/app/server.go
func CreateKubeAPIServerConfig(s completedServerRunOptions) (
*controlplane.Config,
aggregatorapiserver.ServiceResolver,
[]admission.PluginInitializer,
error,
) {
proxyTransport := CreateProxyTransport()
// 构建通用的配置
genericConfig, versionedInformers, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport)
if err != nil {
return nil, nil, nil, err
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{
},
HostPIDSources: []string{
},
HostIPCSources: []string{
},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
s.Metrics.Apply()
serviceaccount.RegisterMetrics()
s.Logs.Apply()
config := &controlplane.Config{
GenericConfig: genericConfig,
ExtraConfig: controlplane.ExtraConfig{
APIResourceConfigSource: storageFactory.APIResourceConfigSource,
StorageFactory: storageFactory,
EventTTL: s.EventTTL,
KubeletClientConfig: s.KubeletConfig,
EnableLogsSupport: s.EnableLogsHandler,
ProxyTransport: proxyTransport,
ServiceIPRange: s.PrimaryServiceClusterIPRange,
APIServerServiceIP: s.APIServerServiceIP,
SecondaryServiceIPRange: s.SecondaryServiceClusterIPRange,
APIServerServicePort: 443,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType),
MasterCount: s.MasterCount,
ServiceAccountIssuer: s.ServiceAccountIssuer,
ServiceAccountMaxExpiration: s.ServiceAccountTokenMaxExpiration,
ExtendExpiration: s.Authentication.ServiceAccounts.ExtendExpiration,
VersionedInformers: versionedInformers,
IdentityLeaseDurationSeconds: s.IdentityLeaseDurationSeconds,
IdentityLeaseRenewIntervalSeconds: s.IdentityLeaseRenewIntervalSeconds,
},
}
// 准备ClusterAuthenticationInfo
// ClientCA:设置Client证书的签发机构
// Aggregator API Server和Aggregator之间通信时,哪些Header中有用户信息
clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider()
if err != nil {
return nil, nil, nil, err
}
config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider
requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
if err != nil {
return nil, nil, nil, err
}
if requestHeaderConfig != nil {
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders
config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
}
// 注册admissionPostStartHook
// 作用是启动一个go routine,在server结束时清理一些cache的信息
if err := config.GenericConfig.AddPostStartHook("start-kube-apiserver-admission-initializer", admissionPostStartHook); err != nil {
return nil, nil, nil, err
}
if config.GenericConfig.EgressSelector != nil {
// Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet
config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup
// Use the config.GenericConfig.EgressSelector lookup as the transport used by the "proxy" subresources.
networkContext := egressselector.Cluster.AsNetworkContext()
dialer, err := config.GenericConfig.EgressSelector.Lookup(networkContext)
if err != nil {
return nil, nil, nil, err
}
c := proxyTransport.Clone()
c.DialContext = dialer
config.ExtraConfig.ProxyTransport = c
}
// 准备OpenID验证机构的信息
// 客户端向API Server发一个请求,可以带上OpenID机构提供的身份信息,API Server可以据此识别处用户.这里配置API Server所使用的的OpenID机构信息
// Load the public keys.
var pubKeys []interface{
}
for _, f := range s.Authentication.ServiceAccounts.KeyFiles {
keys, err := keyutil.PublicKeysFromFile(f)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err)
}
pubKeys = append(pubKeys, keys...)
}
// Plumb the required metadata through ExtraConfig.
config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuers[0]
config.ExtraConfig.ServiceAccountJWKSURI = s.Authentication.ServiceAccounts.JWKSURI
config.ExtraConfig.ServiceAccountPublicKeys = pubKeys
return config, serviceResolver, pluginInitializers, nil
}
CreateKubeAPIServerConfig()Call buildGenericConfig()the method to build a general configuration and convert the option into config. The code is as follows:
// cmd/kube-apiserver/app/server.go
func buildGenericConfig(
s *options.ServerRunOptions,
proxyTransport *http.Transport,
) (
genericConfig *genericapiserver.Config,
versionedInformers clientgoinformers.SharedInformerFactory,
serviceResolver aggregatorapiserver.ServiceResolver,
pluginInitializers []admission.PluginInitializer,
admissionPostStartHook genericapiserver.PostStartHookFunc,
storageFactory *serverstorage.DefaultStorageFactory,
lastErr error,
) {
// 从GenericAPIServer得到config
genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource()
// 应用option中GenericServerRunOptions
if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {
return
}
// 应用option中SecureServing
if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
return
}
// 应用option中Features
if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil {
return
}
// 应用option中APIEnablement
if lastErr = s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil {
return
}
// 应用option中EgressSelector
if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil {
return
}
// 应用option中Traces
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
if lastErr = s.Traces.ApplyTo(genericConfig.EgressSelector, genericConfig); lastErr != nil {
return
}
}
// 准备OpenAPIConfig
// 生成的GetOpenAPIDefinitions会被交给GenericAPIServer的OpenAPIConfig,用于后续生成open api spec
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme))
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
completedStorageFactoryConfig, err := storageFactoryConfig.Complete(s.Etcd)
if err != nil {
lastErr = err
return
}
// 准备StorageFactory,考虑option中的etcd设置
storageFactory, lastErr = completedStorageFactoryConfig.New()
if lastErr != nil {
return
}
if genericConfig.EgressSelector != nil {
storageFactory.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) && genericConfig.TracerProvider != nil {
storageFactory.StorageConfig.Transport.TracerProvider = genericConfig.TracerProvider
}
if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil {
return
}
// Use protobufs for self-communication.
// Since not every generic apiserver has to support protobufs, we
// cannot default to it in generic apiserver and need to explicitly
// set it in kube-apiserver.
genericConfig.LoopbackClientConfig.ContentConfig.ContentType = "application/vnd.kubernetes.protobuf"
// Disable compression for self-communication, since we are going to be
// on a fast local network
genericConfig.LoopbackClientConfig.DisableCompression = true
kubeClientConfig := genericConfig.LoopbackClientConfig
clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig)
if err != nil {
lastErr = fmt.Errorf("failed to create real external clientset: %v", err)
return
}
// 制作一个shared informer:versionedInformers
versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
// 应用option中Authentication(登录验证)到多处
// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil {
return
}
// 应用option中Authorization(鉴权)
genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, versionedInformers)
if err != nil {
lastErr = fmt.Errorf("invalid authorization config: %v", err)
return
}
if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) {
genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
}
// 应用option中audit设置
lastErr = s.Audit.ApplyTo(genericConfig)
if lastErr != nil {
return
}
admissionConfig := &kubeapiserveradmission.Config{
ExternalInformers: versionedInformers,
LoopbackClientConfig: genericConfig.LoopbackClientConfig,
CloudConfigFile: s.CloudProvider.CloudConfigFile,
}
serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
// 制作pluginInitializers、admissionPostStartHook
pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider)
if err != nil {
lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err)
return
}
// 应用option中admission到多处
err = s.Admission.ApplyTo(
genericConfig,
versionedInformers,
kubeClientConfig,
utilfeature.DefaultFeatureGate,
pluginInitializers...)
if err != nil {
lastErr = fmt.Errorf("failed to initialize admission: %v", err)
return
}
// 制作FlowControl,用于priority和fairness的控制
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && s.GenericServerRunOptions.EnablePriorityAndFairness {
genericConfig.FlowControl, lastErr = BuildPriorityAndFairness(s, clientgoExternalClient, versionedInformers)
}
return
}
CreateKubeAPIServerConfig()The main logic is as follows:
2). Create KubeAPIServer Instance
CreateKubeAPIServer()The method code is as follows:
// cmd/kube-apiserver/app/server.go
func CreateKubeAPIServer(kubeAPIServerConfig *controlplane.Config, delegateAPIServer genericapiserver.DelegationTarget) (*controlplane.Instance, error) {
// Complete()方法完善配置
// New()方法创建kubeAPIServer实例
kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer)
if err != nil {
return nil, err
}
return kubeAPIServer, nil
}
CreateKubeAPIServer()In the method, call the method first Complete(), and then call New()the method to create the kubeAPIServer instance.
Complete()The method code is as follows:
// pkg/controlplane/instance.go
func (c *Config) Complete() CompletedConfig {
cfg := completedConfig{
// 调用GenericConfig.Complete()
c.GenericConfig.Complete(c.ExtraConfig.VersionedInformers),
&c.ExtraConfig,
}
// 设置Service可用的IP Range
// 系统中定义的Service可以使用的IP地址以及API Server自己的IP
serviceIPRange, apiServerServiceIP, err := ServiceIPRange(cfg.ExtraConfig.ServiceIPRange)
if err != nil {
klog.Fatalf("Error determining service IP ranges: %v", err)
}
if cfg.ExtraConfig.ServiceIPRange.IP == nil {
cfg.ExtraConfig.ServiceIPRange = serviceIPRange
}
if cfg.ExtraConfig.APIServerServiceIP == nil {
cfg.ExtraConfig.APIServerServiceIP = apiServerServiceIP
}
// discoveryAddresses的设置
// 当Client希望同API Server通信时,通过这个对象可以找到最优的API Server地址
discoveryAddresses := discovery.DefaultAddresses{
DefaultAddress: cfg.GenericConfig.ExternalAddress}
discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules,
discovery.CIDRRule{
IPRange: cfg.ExtraConfig.ServiceIPRange, Address: net.JoinHostPort(cfg.ExtraConfig.APIServerServiceIP.String(), strconv.Itoa(cfg.ExtraConfig.APIServerServicePort))})
cfg.GenericConfig.DiscoveryAddresses = discoveryAddresses
// 设置NodePort Service可以port的范围
if cfg.ExtraConfig.ServiceNodePortRange.Size == 0 {
// TODO: Currently no way to specify an empty range (do we need to allow this?)
// We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE)
// but then that breaks the strict nestedness of ServiceType.
// Review post-v1
cfg.ExtraConfig.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange
klog.Infof("Node port range unspecified. Defaulting to %v.", cfg.ExtraConfig.ServiceNodePortRange)
}
// EndpointReconciler相关设置
if cfg.ExtraConfig.EndpointReconcilerConfig.Interval == 0 {
cfg.ExtraConfig.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval
}
if cfg.ExtraConfig.MasterEndpointReconcileTTL == 0 {
cfg.ExtraConfig.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL
}
if cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler == nil {
cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler = c.createEndpointReconciler()
}
return CompletedConfig{
&cfg}
}
Complete()The main logic of the method is as follows:
New()The method code is as follows:
// pkg/controlplane/instance.go
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) {
if reflect.DeepEqual(c.ExtraConfig.KubeletClientConfig, kubeletclient.KubeletClientConfig{
}) {
return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig")
}
// New一个GenericAPIServer
s, err := c.GenericConfig.New("kube-apiserver", delegationTarget)
if err != nil {
return nil, err
}
// 开关打开的话,注册logs相关的endpoint
if c.ExtraConfig.EnableLogsSupport {
routes.Logs{
}.Install(s.Handler.GoRestfulContainer)
}
// 制作OpenIDMetadata,并注册相关的endpoint
// Metadata and keys are expected to only change across restarts at present,
// so we just marshal immediately and serve the cached JSON bytes.
md, err := serviceaccount.NewOpenIDMetadata(
c.ExtraConfig.ServiceAccountIssuerURL,
c.ExtraConfig.ServiceAccountJWKSURI,
c.GenericConfig.ExternalAddress,
c.ExtraConfig.ServiceAccountPublicKeys,
)
if err != nil {
// If there was an error, skip installing the endpoints and log the
// error, but continue on. We don't return the error because the
// metadata responses require additional, backwards incompatible
// validation of command-line options.
msg := fmt.Sprintf("Could not construct pre-rendered responses for"+
" ServiceAccountIssuerDiscovery endpoints. Endpoints will not be"+
" enabled. Error: %v", err)
if c.ExtraConfig.ServiceAccountIssuerURL != "" {
// The user likely expects this feature to be enabled if issuer URL is
// set and the feature gate is enabled. In the future, if there is no
// longer a feature gate and issuer URL is not set, the user may not
// expect this feature to be enabled. We log the former case as an Error
// and the latter case as an Info.
klog.Error(msg)
} else {
klog.Info(msg)
}
} else {
routes.NewOpenIDMetadataServer(md.ConfigJSON, md.PublicKeysetJSON).
Install(s.Handler.GoRestfulContainer)
}
m := &Instance{
GenericAPIServer: s,
ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo,
}
// install legacy rest storage
if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) {
legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{
StorageFactory: c.ExtraConfig.StorageFactory,
ProxyTransport: c.ExtraConfig.ProxyTransport,
KubeletClientConfig: c.ExtraConfig.KubeletClientConfig,
EventTTL: c.ExtraConfig.EventTTL,
ServiceIPRange: c.ExtraConfig.ServiceIPRange,
SecondaryServiceIPRange: c.ExtraConfig.SecondaryServiceIPRange,
ServiceNodePortRange: c.ExtraConfig.ServiceNodePortRange,
LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig,
ServiceAccountIssuer: c.ExtraConfig.ServiceAccountIssuer,
ExtendExpiration: c.ExtraConfig.ExtendExpiration,
ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration,
APIAudiences: c.GenericConfig.Authentication.APIAudiences,
}
// 安装LegacyAPI:用于支持Pod之类core APIGroup的API Resource
if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider); err != nil {
return nil, err
}
}
// The order here is preserved in discovery.
// If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"),
// the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer.
// This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go
// with specific priorities.
// TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery
// handlers that we have.
// 所有内建的APIGroup(非core)的RESTStorageProvider
restStorageProviders := []RESTStorageProvider{
apiserverinternalrest.StorageProvider{
},
authenticationrest.RESTStorageProvider{
Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences},
authorizationrest.RESTStorageProvider{
Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver},
autoscalingrest.RESTStorageProvider{
},
batchrest.RESTStorageProvider{
},
certificatesrest.RESTStorageProvider{
},
coordinationrest.RESTStorageProvider{
},
discoveryrest.StorageProvider{
},
extensionsrest.RESTStorageProvider{
},
networkingrest.RESTStorageProvider{
},
noderest.RESTStorageProvider{
},
policyrest.RESTStorageProvider{
},
rbacrest.RESTStorageProvider{
Authorizer: c.GenericConfig.Authorization.Authorizer},
schedulingrest.RESTStorageProvider{
},
storagerest.RESTStorageProvider{
},
flowcontrolrest.RESTStorageProvider{
},
// keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
// See https://github.com/kubernetes/kubernetes/issues/42392
appsrest.StorageProvider{
},
admissionregistrationrest.RESTStorageProvider{
},
eventsrest.RESTStorageProvider{
TTL: c.ExtraConfig.EventTTL},
}
// 安装内建的API:用于支持那些内建的带有APIGroup的API Resource
if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil {
return nil, err
}
// Post start hook:start-cluster-authentication-info-controller
// 当用户自定义了API Server(Custom API Server)并通过APIService Object加入到API Server中时,需要合并它和API Service各自的Authentication信息并回写它的authentication设置(在kube-system的configMap/extension-apiserver-authentication)
m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error {
kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
if err != nil {
return err
}
controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, kubeClient)
// prime values and start listeners
if m.ClusterAuthenticationInfo.ClientCA != nil {
m.ClusterAuthenticationInfo.ClientCA.AddListener(controller)
if controller, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.ControllerRunner); ok {
// runonce to be sure that we have a value.
if err := controller.RunOnce(); err != nil {
runtime.HandleError(err)
}
go controller.Run(1, hookContext.StopCh)
}
}
if m.ClusterAuthenticationInfo.RequestHeaderCA != nil {
m.ClusterAuthenticationInfo.RequestHeaderCA.AddListener(controller)
if controller, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.ControllerRunner); ok {
// runonce to be sure that we have a value.
if err := controller.RunOnce(); err != nil {
runtime.HandleError(err)
}
go controller.Run(1, hookContext.StopCh)
}
}
go controller.Run(1, hookContext.StopCh)
return nil
})
// Post start hook:identity-lease相关
// 为每一个APIServer创建一个lease(租约),并在超过租期没有续约后清理这些lease.引入identity的原因是在HA集群中,需要用到可用的APIServer列表
if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.APIServerIdentity) {
m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-controller", func(hookContext genericapiserver.PostStartHookContext) error {
kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
if err != nil {
return err
}
controller := lease.NewController(
clock.RealClock{
},
kubeClient,
m.GenericAPIServer.APIServerID,
int32(c.ExtraConfig.IdentityLeaseDurationSeconds),
nil,
time.Duration(c.ExtraConfig.IdentityLeaseRenewIntervalSeconds)*time.Second,
metav1.NamespaceSystem,
labelAPIServerHeartbeat)
go controller.Run(wait.NeverStop)
return nil
})
m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-garbage-collector", func(hookContext genericapiserver.PostStartHookContext) error {
kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
if err != nil {
return err
}
go apiserverleasegc.NewAPIServerLeaseGC(
kubeClient,
time.Duration(c.ExtraConfig.IdentityLeaseDurationSeconds)*time.Second,
metav1.NamespaceSystem,
KubeAPIServerIdentityLeaseLabelSelector,
).Run(wait.NeverStop)
return nil
})
}
return m, nil
}
New()The main logic of the method is as follows:
10、APIExtensionsServer
APIExtensionsServer is responsible for processing REST requests of CR and registering a CR through CRD to expand the native resource objects of APIServer.
Each instance of an API Object like CRD defines another API Object——Custom Resource
The method provided by GenericAPIServer InstallAPIGroups()can load static API Objects so that the Server can handle requests for them; however, this loading method can only overwrite CRD, and Custom Resources may be dynamically created, so other methods are needed to load them into Server to respond to requests for them
// cmd/kube-apiserver/app/server.go
func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{
}) (*aggregatorapiserver.APIAggregator, error) {
// 创建KubeAPIServer所需配置
kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions)
if err != nil {
return nil, err
}
// 创建APIExtensionsServer所需配置,这里复用了kubeAPIServerConfig的部分通用配置
// If additional API servers are added, they should be gated.
apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount,
serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIServerConfig.ExtraConfig.ProxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.GenericConfig.TracerProvider))
if err != nil {
return nil, err
}
// 创建APIExtensionsServer实例
apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
// 创建KubeAPIServer实例
kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer)
if err != nil {
return nil, err
}
// aggregator comes last in the chain
// 创建AggregatorServer所需配置,这里复用了kubeAPIServerConfig的部分通用配置
aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, kubeAPIServerConfig.ExtraConfig.ProxyTransport, pluginInitializer)
if err != nil {
return nil, err
}
// 创建AggregatorServer实例
aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
if err != nil {
// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
return nil, err
}
return aggregatorServer, nil
}
CreateServerChain()The logic related to APIExtensionsServer in the method is as follows:
- Call
createAPIExtensionsConfig()the method to create the configuration required for APIExtensionsServer. Some common configurations of kubeAPIServerConfig are reused here. - Call
createAPIExtensionsServer()the method to create an APIExtensionsServer instance
1). Prepare Config
createAPIExtensionsConfig()The method code is as follows:
// cmd/kube-apiserver/app/apiextensions.go
func createAPIExtensionsConfig(
kubeAPIServerConfig genericapiserver.Config,
externalInformers kubeexternalinformers.SharedInformerFactory,
pluginInitializers []admission.PluginInitializer,
commandOptions *options.ServerRunOptions,
masterCount int,
serviceResolver webhook.ServiceResolver,
authResolverWrapper webhook.AuthenticationInfoResolverWrapper,
) (*apiextensionsapiserver.Config, error) {
// 复制kubeAPIServerConfig形成自己的GenericConfig
// 大部分都是直接复用的kubeAPIServer的,只需清除PostStartHooks和RESTOptionsGetter
// make a shallow copy to let us twiddle a few things
// most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions
genericConfig := kubeAPIServerConfig
genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{
}
genericConfig.RESTOptionsGetter = nil
// 用Option中信息和APIExtensionsServer自身的信息设置Admission
// 制作的流程和kubeAPIServer中是一样的:*options.ServerRunOptions.Admission.ApplyTo(),不同的是参数用APIExtensionsServer相关的.主要作用在GenericConfig
// override genericConfig.AdmissionControl with apiextensions' scheme,
// because apiextensions apiserver should use its own scheme to convert resources.
err := commandOptions.Admission.ApplyTo(
&genericConfig,
externalInformers,
genericConfig.LoopbackClientConfig,
feature.DefaultFeatureGate,
pluginInitializers...)
if err != nil {
return nil, err
}
// 制作RESTOptionsGetter
// 复制的时候这个信息被清除了,用APIExtensionsServer自己的EtcdOptions做一个新的,放入GenericConfig
// copy the etcd options so we don't mutate originals.
etcdOptions := *commandOptions.Etcd
etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
// this is where the true decodable levels come from.
etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion)
// prefer the more compact serialization (v1beta1) for storage until http://issue.k8s.io/82292 is resolved for objects whose v1 serialization is too big but whose v1beta1 serialization can be stored
etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{
Group: v1beta1.GroupName})
genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{
Options: etcdOptions}
// 用Option中信息和APIExtensionsServer自身的信息设置APIEnablement
// 制作的流程和kubeAPIServer中是一样的:*options.ServerRunOptions.APIEnablement.ApplyTo(),主要作用在GenericConfig
// override MergedResourceConfig with apiextensions defaults and registry
if err := commandOptions.APIEnablement.ApplyTo(
&genericConfig,
apiextensionsapiserver.DefaultAPIResourceConfigSource(),
apiextensionsapiserver.Scheme); err != nil {
return nil, err
}
// 制作APIExtensionsConfig并返回
// 生成APIExtensionsConfig结构体实例,填入之前做出的GenericConfig.除此之外,另一个字段CRDRESTOptionsGetter是基于前序的EtcdOptions做出来的
apiextensionsConfig := &apiextensionsapiserver.Config{
GenericConfig: &genericapiserver.RecommendedConfig{
Config: genericConfig,
SharedInformerFactory: externalInformers,
},
ExtraConfig: apiextensionsapiserver.ExtraConfig{
CRDRESTOptionsGetter: apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions),
MasterCount: masterCount,
AuthResolverWrapper: authResolverWrapper,
ServiceResolver: serviceResolver,
},
}
// we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails)
apiextensionsConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{
}
return apiextensionsConfig, nil
}
createAPIExtensionsConfig()The main logic of the method is as follows:
2) Create APIExtensionsServer Instance
createAPIExtensionsServer()The method code is as follows:
// cmd/kube-apiserver/app/apiextensions.go
func createAPIExtensionsServer(apiextensionsConfig *apiextensionsapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget) (*apiextensionsapiserver.CustomResourceDefinitions, error) {
return apiextensionsConfig.Complete().New(delegateAPIServer)
}
createAPIExtensionsServer()In the method, call the method first Complete(), and then call New()the method to create an APIExtensionsServer instance.
New()The method code is as follows:
// vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*CustomResourceDefinitions, error) {
// New一个GenericAPIServer
genericServer, err := c.GenericConfig.New("apiextensions-apiserver", delegationTarget)
if err != nil {
return nil, err
}
s := &CustomResourceDefinitions{
GenericAPIServer: genericServer,
}
// used later to filter the served resource by those that have expired.
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version)
if err != nil {
return nil, err
}
apiResourceConfig := c.GenericConfig.MergedResourceConfig
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiextensions.GroupName, Scheme, metav1.ParameterCodec, Codecs)
if resourceExpirationEvaluator.ShouldServeForVersion(1, 22) && apiResourceConfig.VersionEnabled(v1beta1.SchemeGroupVersion) {
storage := map[string]rest.Storage{
}
// customresourcedefinitions
customResourceDefinitionStorage, err := customresourcedefinition.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter)
if err != nil {
return nil, err
}
storage["customresourcedefinitions"] = customResourceDefinitionStorage
storage["customresourcedefinitions/status"] = customresourcedefinition.NewStatusREST(Scheme, customResourceDefinitionStorage)
apiGroupInfo.VersionedResourcesStorageMap[v1beta1.SchemeGroupVersion.Version] = storage
}
if apiResourceConfig.VersionEnabled(v1.SchemeGroupVersion) {
storage := map[string]rest.Storage{
}
// customresourcedefinitions
customResourceDefinitionStorage, err := customresourcedefinition.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter)
if err != nil {
return nil, err
}
storage["customresourcedefinitions"] = customResourceDefinitionStorage
storage["customresourcedefinitions/status"] = customresourcedefinition.NewStatusREST(Scheme, customResourceDefinitionStorage)
apiGroupInfo.VersionedResourcesStorageMap[v1.SchemeGroupVersion.Version] = storage
}
// 安装APIGroup下Object到GenericAPIServer
// apiextensions.k8s.io APIGroup下只有CustomResourceDefinition一个API Object
// 准备好APIGroupInfo后,调用GenericAPIServer的InstallAPIGroup()方法
if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
return nil, err
}
// 制作一个Informer:用于后续监控CRD变更
crdClient, err := clientset.NewForConfig(s.GenericAPIServer.LoopbackClientConfig)
if err != nil {
// it's really bad that this is leaking here, but until we can fix the test (which I'm pretty sure isn't even testing what it wants to test),
// we need to be able to move forward
return nil, fmt.Errorf("failed to create clientset: %v", err)
}
s.Informers = externalinformers.NewSharedInformerFactory(crdClient, 5*time.Minute)
delegateHandler := delegationTarget.UnprotectedHandler()
if delegateHandler == nil {
delegateHandler = http.NotFoundHandler()
}
versionDiscoveryHandler := &versionDiscoveryHandler{
discovery: map[schema.GroupVersion]*discovery.APIVersionHandler{
},
delegate: delegateHandler,
}
groupDiscoveryHandler := &groupDiscoveryHandler{
discovery: map[string]*discovery.APIGroupHandler{
},
delegate: delegateHandler,
}
establishingController := establish.NewEstablishingController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())
// 制作Custom Resource的http request handler
// 这类API Object是通过CRD动态创建出来的,不能利用GenericAPIServer的InstallAPIGroup()方法去装载它们
// 那么针对/apis、/apis/、/apis/<group>、/apis/<group>/<version>、/apis/<group>/<version>/<resource>的请求就需要有handler
crdHandler, err := NewCustomResourceDefinitionHandler(
versionDiscoveryHandler,
groupDiscoveryHandler,
s.Informers.Apiextensions().V1().CustomResourceDefinitions(),
delegateHandler,
c.ExtraConfig.CRDRESTOptionsGetter,
c.GenericConfig.AdmissionControl,
establishingController,
c.ExtraConfig.ServiceResolver,
c.ExtraConfig.AuthResolverWrapper,
c.ExtraConfig.MasterCount,
s.GenericAPIServer.Authorizer,
c.GenericConfig.RequestTimeout,
time.Duration(c.GenericConfig.MinRequestTimeout)*time.Second,
apiGroupInfo.StaticOpenAPISpec,
c.GenericConfig.MaxRequestBodyBytes,
)
if err != nil {
return nil, err
}
s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler)
s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler)
// 制作controller,放入Post Start Hook
// 通过Post Start Hook来启动一系列controller,它们都是去监控Custom Resource的变更状况(也就是CRD实例的变更)来做出必要调整
// discoveryController:调整以便正确处理针对CR的/apis/<group>、/apis/<group>/<version>请求
discoveryController := NewDiscoveryController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler)
// namingController:检测CR命名上的违规,更新status
namingController := status.NewNamingConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())
// nonStructuralSchemaController:validate数据是否合乎openapi v3的schema,有错的话更新到status
nonStructuralSchemaController := nonstructuralschema.NewConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())
// apiApprovalController:创建APIGroup为k8s.io或kubernetes.io下的CR时,需要设置api-approved.kubernetes.io的annotation
apiApprovalController := apiapproval.NewKubernetesAPIApprovalPolicyConformantConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1())
// finalizingController:删除一个CRD时,清除它所有的CR
finalizingController := finalizer.NewCRDFinalizer(
s.Informers.Apiextensions().V1().CustomResourceDefinitions(),
crdClient.ApiextensionsV1(),
crdHandler,
)
// openapiController:监控CRD的变更,更新OpenAPI的Spec
openapiController := openapicontroller.NewController(s.Informers.Apiextensions().V1().CustomResourceDefinitions())
s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error {
s.Informers.Start(context.StopCh)
return nil
})
s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error {
// OpenAPIVersionedService and StaticOpenAPISpec are populated in generic apiserver PrepareRun().
// Together they serve the /openapi/v2 endpoint on a generic apiserver. A generic apiserver may
// choose to not enable OpenAPI by having null openAPIConfig, and thus OpenAPIVersionedService
// and StaticOpenAPISpec are both null. In that case we don't run the CRD OpenAPI controller.
if s.GenericAPIServer.OpenAPIVersionedService != nil && s.GenericAPIServer.StaticOpenAPISpec != nil {
go openapiController.Run(s.GenericAPIServer.StaticOpenAPISpec, s.GenericAPIServer.OpenAPIVersionedService, context.StopCh)
}
go namingController.Run(context.StopCh)
go establishingController.Run(context.StopCh)
go nonStructuralSchemaController.Run(5, context.StopCh)
go apiApprovalController.Run(5, context.StopCh)
go finalizingController.Run(5, context.StopCh)
discoverySyncedCh := make(chan struct{
})
go discoveryController.Run(context.StopCh, discoverySyncedCh)
select {
case <-context.StopCh:
case <-discoverySyncedCh:
}
return nil
})
// we don't want to report healthy until we can handle all CRDs that have already been registered. Waiting for the informer
// to sync makes sure that the lister will be valid before we begin. There may still be races for CRDs added after startup,
// but we won't go healthy until we can handle the ones already present.
s.GenericAPIServer.AddPostStartHookOrDie("crd-informer-synced", func(context genericapiserver.PostStartHookContext) error {
return wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
return s.Informers.Apiextensions().V1().CustomResourceDefinitions().Informer().HasSynced(), nil
}, context.StopCh)
})
return s, nil
}
New()The main logic of the method is as follows:
3）、Custom Resource的API Handler
New()The method is called NewCustomResourceDefinitionHandler()to make the http request handler of Custom Resource. In this method, a crdHandler structure instance is created. The core of this structure is ServeHTTP()the method. The code is as follows:
// vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go
func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
requestInfo, ok := apirequest.RequestInfoFrom(ctx)
if !ok {
responsewriters.ErrorNegotiated(
apierrors.NewInternalError(fmt.Errorf("no RequestInfo found in the context")),
Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req,
)
return
}
if !requestInfo.IsResourceRequest {
pathParts := splitPath(requestInfo.Path)
// 处理/apis/<group>/<version>
// 这属于列举version下API Resource的请求,交给version discovery去处理
// only match /apis/<group>/<version>
// only registered under /apis
if len(pathParts) == 3 {
if !r.hasSynced() {
responsewriters.ErrorNegotiated(serverStartingError(), Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req)
return
}
r.versionDiscoveryHandler.ServeHTTP(w, req)
return
}
// 处理/apis/<group>
// 这属于列举group下所有version下API Resource的请求,交给group discovery去处理
// only match /apis/<group>
if len(pathParts) == 2 {
if !r.hasSynced() {
responsewriters.ErrorNegotiated(serverStartingError(), Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req)
return
}
r.groupDiscoveryHandler.ServeHTTP(w, req)
return
}
r.delegate.ServeHTTP(w, req)
return
}
crdName := requestInfo.Resource + "." + requestInfo.APIGroup
crd, err := r.crdLister.Get(crdName)
if apierrors.IsNotFound(err) {
if !r.hasSynced() {
responsewriters.ErrorNegotiated(serverStartingError(), Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req)
return
}
r.delegate.ServeHTTP(w, req)
return
}
if err != nil {
utilruntime.HandleError(err)
responsewriters.ErrorNegotiated(
apierrors.NewInternalError(fmt.Errorf("error resolving resource")),
Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req,
)
return
}
// 校验crd和请求中的namespace是否相符,condition是否为true
// 如果namespace不相符,返回没有;如果condition不是true,也返回没有,condition就是由那些controller的计算结果决定的
// if the scope in the CRD and the scope in request differ (with exception of the verbs in possiblyAcrossAllNamespacesVerbs
// for namespaced resources), pass request to the delegate, which is supposed to lead to a 404.
namespacedCRD, namespacedReq := crd.Spec.Scope == apiextensionsv1.NamespaceScoped, len(requestInfo.Namespace) > 0
if !namespacedCRD && namespacedReq {
r.delegate.ServeHTTP(w, req)
return
}
if namespacedCRD && !namespacedReq && !possiblyAcrossAllNamespacesVerbs.Has(requestInfo.Verb) {
r.delegate.ServeHTTP(w, req)
return
}
if !apiextensionshelpers.HasServedCRDVersion(crd, requestInfo.APIVersion) {
r.delegate.ServeHTTP(w, req)
return
}
// There is a small chance that a CRD is being served because NamesAccepted condition is true,
// but it becomes "unserved" because another names update leads to a conflict
// and EstablishingController wasn't fast enough to put the CRD into the Established condition.
// We accept this as the problem is small and self-healing.
if !apiextensionshelpers.IsCRDConditionTrue(crd, apiextensionsv1.NamesAccepted) &&
!apiextensionshelpers.IsCRDConditionTrue(crd, apiextensionsv1.Established) {
r.delegate.ServeHTTP(w, req)
return
}
terminating := apiextensionshelpers.IsCRDConditionTrue(crd, apiextensionsv1.Terminating)
crdInfo, err := r.getOrCreateServingInfoFor(crd.UID, crd.Name)
if apierrors.IsNotFound(err) {
r.delegate.ServeHTTP(w, req)
return
}
if err != nil {
utilruntime.HandleError(err)
responsewriters.ErrorNegotiated(
apierrors.NewInternalError(fmt.Errorf("error resolving resource")),
Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req,
)
return
}
if !hasServedCRDVersion(crdInfo.spec, requestInfo.APIVersion) {
r.delegate.ServeHTTP(w, req)
return
}
deprecated := crdInfo.deprecated[requestInfo.APIVersion]
for _, w := range crdInfo.warnings[requestInfo.APIVersion] {
warning.AddWarning(req.Context(), "", w)
}
verb := strings.ToUpper(requestInfo.Verb)
resource := requestInfo.Resource
subresource := requestInfo.Subresource
scope := metrics.CleanScope(requestInfo)
supportedTypes := []string{
string(types.JSONPatchType),
string(types.MergePatchType),
}
if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
supportedTypes = append(supportedTypes, string(types.ApplyPatchType))
}
// 获取http request的handler function
// 根据请求目标resource、/resource的status、/resource的scale,调用serveResource、serveStatus、serveScale,来得到一个http request handler function
var handlerFunc http.HandlerFunc
subresources, err := apiextensionshelpers.GetSubresourcesForVersion(crd, requestInfo.APIVersion)
if err != nil {
utilruntime.HandleError(err)
responsewriters.ErrorNegotiated(
apierrors.NewInternalError(fmt.Errorf("could not properly serve the subresource")),
Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req,
)
return
}
switch {
case subresource == "status" && subresources != nil && subresources.Status != nil:
handlerFunc = r.serveStatus(w, req, requestInfo, crdInfo, terminating, supportedTypes)
case subresource == "scale" && subresources != nil && subresources.Scale != nil:
handlerFunc = r.serveScale(w, req, requestInfo, crdInfo, terminating, supportedTypes)
case len(subresource) == 0:
handlerFunc = r.serveResource(w, req, requestInfo, crdInfo, crd, terminating, supportedTypes)
default:
responsewriters.ErrorNegotiated(
apierrors.NewNotFound(schema.GroupResource{
Group: requestInfo.APIGroup, Resource: requestInfo.Resource}, requestInfo.Name),
Codecs, schema.GroupVersion{
Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, req,
)
}
if handlerFunc != nil {
handlerFunc = metrics.InstrumentHandlerFunc(verb, requestInfo.APIGroup, requestInfo.APIVersion, resource, subresource, scope, metrics.APIServerComponent, deprecated, "", handlerFunc)
handler := genericfilters.WithWaitGroup(handlerFunc, longRunningFilter, crdInfo.waitGroup)
// 调用handler
handler.ServeHTTP(w, req)
return
}
}
ServeHTTP()The main logic of the method is as follows:
11、AggregatorServer
AggregatorServer is at the top of the entire control plane Server. When an HTTP request comes in, it first enters AggregatorServer.
AggregatorServer faces two different types of sub-servers: the first type is KubeAPIServer and APIExtensionsServer, which are in the same web application as AggregatorServer; the second type is CustomServer, users can write and deploy their own API Server to provide customized resources Special implementation, it is deployed on Pods in the cluster and exposes services through Service. For AggregatorServer, CustomServer is remote
When an HTTP request reaches AggregatorServer, AggregatorServer determines that the local LocalServer wants to respond, and will hand the request to KubeAPIServer; if it judges that the CustomServer wants to respond, it will hand the request to the corresponding CustomServer in a proxy manner.
1), APIService: the core API Object of AggregatorServer
The structure code of the spec attribute of APIService is as follows:
// vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/types.go
type APIServiceSpec struct {
// Service is a reference to the service for this API server. It must communicate
// on port 443.
// If the Service is nil, that means the handling for the API groupversion is handled locally on this server.
// The call will simply delegate to the normal handler chain to be fulfilled.
// +optional
// Service指向这个APIService背后的Server,为空时代表这是由本地APIserver支持的
Service *ServiceReference `json:"service,omitempty" protobuf:"bytes,1,opt,name=service"`
// Group is the API group name this server hosts
Group string `json:"group,omitempty" protobuf:"bytes,2,opt,name=group"`
// Version is the API version this server hosts. For example, "v1"
Version string `json:"version,omitempty" protobuf:"bytes,3,opt,name=version"`
// InsecureSkipTLSVerify disables TLS certificate verification when communicating with this server.
// This is strongly discouraged. You should use the CABundle instead.
InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty" protobuf:"varint,4,opt,name=insecureSkipTLSVerify"`
// CABundle is a PEM encoded CA bundle which will be used to validate an API server's serving certificate.
// If unspecified, system trust roots on the apiserver are used.
// +listType=atomic
// +optional
CABundle []byte `json:"caBundle,omitempty" protobuf:"bytes,5,opt,name=caBundle"`
// GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is preferred by clients over lower priority ones.
// Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority.
// The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10).
// The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo)
// We'd recommend something like: *.k8s.io (except extensions) at 18000 and
// PaaSes (OpenShift, Deis) are recommended to be in the 2000s
GroupPriorityMinimum int32 `json:"groupPriorityMinimum" protobuf:"varint,7,opt,name=groupPriorityMinimum"`
// VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero.
// The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10).
// Since it's inside of a group, the number can be small, probably in the 10s.
// In case of equal version priorities, the version string will be used to compute the order inside a group.
// If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
// lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
// then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
// by GA > beta > alpha (where GA is a version with no suffix such as beta or alpha), and then by comparing major
// version, then minor version. An example sorted list of versions:
// v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
VersionPriority int32 `json:"versionPriority" protobuf:"varint,8,opt,name=versionPriority"`
// leaving this here so everyone remembers why proto index 6 is skipped
// Priority int64 `json:"priority" protobuf:"varint,6,opt,name=priority"`
}
There is a very important Service attribute in APIServiceSpec: Service points to the Server behind this APIService. When it is empty, it means that it is supported by the local APIserver. If the Service attribute of the APIService corresponding to the HTTP request is empty, the request will be handed over to KubeAPIServer; if the Service attribute is not empty, the request will be handed over to the corresponding CustomServer in the form of proxy.
2). Prepare Config
// cmd/kube-apiserver/app/server.go
func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{
}) (*aggregatorapiserver.APIAggregator, error) {
// 创建KubeAPIServer所需配置
kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions)
if err != nil {
return nil, err
}
// 创建APIExtensionsServer所需配置,这里复用了kubeAPIServerConfig的部分通用配置
// If additional API servers are added, they should be gated.
apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount,
serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIServerConfig.ExtraConfig.ProxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.GenericConfig.TracerProvider))
if err != nil {
return nil, err
}
// 创建APIExtensionsServer实例
apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
// 创建KubeAPIServer实例
kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer)
if err != nil {
return nil, err
}
// aggregator comes last in the chain
// 创建AggregatorServer所需配置,这里复用了kubeAPIServerConfig的部分通用配置
aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, kubeAPIServerConfig.ExtraConfig.ProxyTransport, pluginInitializer)
if err != nil {
return nil, err
}
// 创建AggregatorServer实例
aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
if err != nil {
// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
return nil, err
}
return aggregatorServer, nil
}
CreateServerChain()The logic related to AggregatorServer in the method is as follows:
- Call
createAggregatorConfig()the method to create the configuration required for AggregatorServer. Some common configurations of kubeAPIServerConfig are reused here. - Call
createAggregatorServer()the method to create an AggregatorServer instance
createAggregatorConfig()The method code is as follows:
// cmd/kube-apiserver/app/aggregator.go
func createAggregatorConfig(
kubeAPIServerConfig genericapiserver.Config,
commandOptions *options.ServerRunOptions,
externalInformers kubeexternalinformers.SharedInformerFactory,
serviceResolver aggregatorapiserver.ServiceResolver,
proxyTransport *http.Transport,
pluginInitializers []admission.PluginInitializer,
) (*aggregatorapiserver.Config, error) {
// 复制kubeAPIServerConfig形成自己的GenericConfig
// 大部分都是直接复用的kubeAPIServer的,只需清除PostStartHooks、RESTOptionsGetter和设置SkipOpenAPIInstallation为true
// make a shallow copy to let us twiddle a few things
// most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the aggregator
genericConfig := kubeAPIServerConfig
genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{
}
genericConfig.RESTOptionsGetter = nil
// prevent generic API server from installing the OpenAPI handler. Aggregator server
// has its own customized OpenAPI handler.
genericConfig.SkipOpenAPIInstallation = true
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) &&
utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
// Add StorageVersionPrecondition handler to aggregator-apiserver.
// The handler will block write requests to built-in resources until the
// target resources' storage versions are up-to-date.
genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition
}
// 用Option中信息和AggregatorServer自身的信息设置Admission
// 制作的流程和kubeAPIServer中是一样的:*options.ServerRunOptions.Admission.ApplyTo(),不同的是参数用AggregatorServer相关的.主要作用在GenericConfig
// override genericConfig.AdmissionControl with kube-aggregator's scheme,
// because aggregator apiserver should use its own scheme to convert its own resources.
err := commandOptions.Admission.ApplyTo(
&genericConfig,
externalInformers,
genericConfig.LoopbackClientConfig,
utilfeature.DefaultFeatureGate,
pluginInitializers...)
if err != nil {
return nil, err
}
// 制作RESTOptionsGetter
// 复制的时候这个信息被清除了,用AggregatorServer自己的EtcdOptions做一个新的,放入GenericConfig
// copy the etcd options so we don't mutate originals.
etcdOptions := *commandOptions.Etcd
etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIListChunking)
etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion, v1beta1.SchemeGroupVersion)
etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1.SchemeGroupVersion, schema.GroupKind{
Group: v1beta1.GroupName})
genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{
Options: etcdOptions}
// 用Option中信息和AggregatorServer自身的信息设置APIEnablement
// 制作的流程和kubeAPIServer中是一样的:*options.ServerRunOptions.APIEnablement.ApplyTo(),主要作用在GenericConfig
// override MergedResourceConfig with aggregator defaults and registry
if err := commandOptions.APIEnablement.ApplyTo(
&genericConfig,
aggregatorapiserver.DefaultAPIResourceConfigSource(),
aggregatorscheme.Scheme); err != nil {
return nil, err
}
// 制作AggregatorConfig并返回
// 生成AggregatorConfig结构体实例,填入之前做的GenericConfig.除此之外,在ExtraConfig中给出客户端证书和私钥,用于AggregatorServer与CustomServer交互做mTLS
aggregatorConfig := &aggregatorapiserver.Config{
GenericConfig: &genericapiserver.RecommendedConfig{
Config: genericConfig,
SharedInformerFactory: externalInformers,
},
ExtraConfig: aggregatorapiserver.ExtraConfig{
ProxyClientCertFile: commandOptions.ProxyClientCertFile,
ProxyClientKeyFile: commandOptions.ProxyClientKeyFile,
ServiceResolver: serviceResolver,
ProxyTransport: proxyTransport,
},
}
// we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails)
aggregatorConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{
}
return aggregatorConfig, nil
}
createAggregatorConfig()The main logic of the method is as follows:
3), create AggregatorServer Instance
createAggregatorServer()The method code is as follows:
// cmd/kube-apiserver/app/aggregator.go
func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, apiExtensionInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) {
// 创建aggregatorServer实例
aggregatorServer, err := aggregatorConfig.Complete().NewWithDelegate(delegateAPIServer)
if err != nil {
return nil, err
}
// 制作一个创建、修改APIService的Controller
// 来把对这类API Resource的创建、修改、删除落实到ETCD中
// create controllers for auto-registration
apiRegistrationClient, err := apiregistrationclient.NewForConfig(aggregatorConfig.GenericConfig.LoopbackClientConfig)
if err != nil {
return nil, err
}
autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient)
// 为所有内建APIGroup的各个Version生成APIService
// 内建的APIGroup的各个Version都会有一个对应的APIService resource被生成出来,交给上述Controller去存到etcd
apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController)
// 为由CRD创建出的APIGroup的各个Version生成APIService
// 类似内建APIGroup,CR的APIGroup各个Version同样要有APIService resource去对应,建好后同样交给上述Controller去存到etcd
// 这还需要用到Controller去监控CR的创建,因为CR随时会被用户创建无法提前全部拿到
crdRegistrationController := crdregistration.NewCRDRegistrationController(
apiExtensionInformers.Apiextensions().V1().CustomResourceDefinitions(),
autoRegistrationController)
// post start hook
// 在Post start hook中启动以上两个Controller:一个是保管APIService Resource的,一个是监控CR创建的
err = aggregatorServer.GenericAPIServer.AddPostStartHook("kube-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error {
go crdRegistrationController.Run(5, context.StopCh)
go func() {
// let the CRD controller process the initial set of CRDs before starting the autoregistration controller.
// this prevents the autoregistration controller's initial sync from deleting APIServices for CRDs that still exist.
// we only need to do this if CRDs are enabled on this server. We can't use discovery because we are the source for discovery.
if aggregatorConfig.GenericConfig.MergedResourceConfig.AnyVersionForGroupEnabled("apiextensions.k8s.io") {
crdRegistrationController.WaitForInitialSync()
}
autoRegistrationController.Run(5, context.StopCh)
}()
return nil
})
if err != nil {
return nil, err
}
err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks(
makeAPIServiceAvailableHealthCheck(
"autoregister-completion",
apiServices,
aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(),
),
)
if err != nil {
return nil, err
}
return aggregatorServer, nil
}
createAggregatorServer()The main logic of the method is as follows:
createAggregatorServer()Call the method NewWithDelegate()to create an aggregatorServer instance
NewWithDelegate()The method code is as follows:
// vendor/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go
func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.DelegationTarget) (*APIAggregator, error) {
// New一个GenericAPIServer
genericServer, err := c.GenericConfig.New("kube-aggregator", delegationTarget)
if err != nil {
return nil, err
}
// 制作一个Informerr:用于后续监控APIService的变更
apiregistrationClient, err := clientset.NewForConfig(c.GenericConfig.LoopbackClientConfig)
if err != nil {
return nil, err
}
informerFactory := informers.NewSharedInformerFactory(
apiregistrationClient,
5*time.Minute, // this is effectively used as a refresh interval right now. Might want to do something nicer later on.
)
s := &APIAggregator{
GenericAPIServer: genericServer,
delegateHandler: delegationTarget.UnprotectedHandler(),
proxyTransport: c.ExtraConfig.ProxyTransport,
proxyHandlers: map[string]*proxyHandler{
},
handledGroups: sets.String{
},
lister: informerFactory.Apiregistration().V1().APIServices().Lister(),
APIRegistrationInformers: informerFactory,
serviceResolver: c.ExtraConfig.ServiceResolver,
openAPIConfig: c.GenericConfig.OpenAPIConfig,
egressSelector: c.GenericConfig.EgressSelector,
proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) {
return nil, nil },
}
// used later to filter the served resource by those that have expired.
resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*c.GenericConfig.Version)
if err != nil {
return nil, err
}
apiGroupInfo := apiservicerest.NewRESTStorage(c.GenericConfig.MergedResourceConfig, c.GenericConfig.RESTOptionsGetter, resourceExpirationEvaluator.ShouldServeForVersion(1, 22))
// 安装APIGroup下Object到GenericAPIServer
// apiregistration.k8s.io APIGroup下只有APIService这个APIObject和它的子Object Status
// 准备好APIGroupInfo后,调用GenericAPIServer的InstallAPIGroup()方法
if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
return nil, err
}
enabledVersions := sets.NewString()
for v := range apiGroupInfo.VersionedResourcesStorageMap {
enabledVersions.Insert(v)
}
if !enabledVersions.Has(v1.SchemeGroupVersion.Version) {
return nil, fmt.Errorf("API group/version %s must be enabled", v1.SchemeGroupVersion.String())
}
// 制作Group Discovery Request的handler
// 用于响应对/apis的请求和对/apis/开头但没有命中其他handler的请求,返回结果是当前AggregatorServer所支持的APIGroup集合
apisHandler := &apisHandler{
codecs: aggregatorscheme.Codecs,
lister: s.lister,
discoveryGroup: discoveryGroup(enabledVersions),
}
s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandler)
s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle("/apis/", apisHandler)
// 制作监控APIService更新的Controller
// 每一个APIService在ETCD中创建出来以后,都需要被AggregatorServer考虑进来从而能代理它所提供的服务.最后加入post start hook
// 该Controller最终会调用APIAggregator的AddAPIService()和RemoveAPIService(),这两个方法里会注册、取消注册针对特定url的响应函数,这样就可以去响应针对Aggregated APIServer支持的APIObject的请求了
apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().V1().APIServices(), s)
// 制作监控客户端证书变化的Controller
// 当发生证书变更时,所有APIService会被重新入队上一步建出的Controller去重建一下.将本Controller加入post start hook
if len(c.ExtraConfig.ProxyClientCertFile) > 0 && len(c.ExtraConfig.ProxyClientKeyFile) > 0 {
aggregatorProxyCerts, err := dynamiccertificates.NewDynamicServingContentFromFiles("aggregator-proxy-cert", c.ExtraConfig.ProxyClientCertFile, c.ExtraConfig.ProxyClientKeyFile)
if err != nil {
return nil, err
}
if err := aggregatorProxyCerts.RunOnce(); err != nil {
return nil, err
}
aggregatorProxyCerts.AddListener(apiserviceRegistrationController)
s.proxyCurrentCertKeyContent = aggregatorProxyCerts.CurrentCertKeyContent
s.GenericAPIServer.AddPostStartHookOrDie("aggregator-reload-proxy-client-cert", func(context genericapiserver.PostStartHookContext) error {
go aggregatorProxyCerts.Run(1, context.StopCh)
return nil
})
}
// 制作检查Server可用性的Controller
// APIService中会指定一个Custom Server的Service,需要检验该Service是否可用,使用这个Controller.最后也加入post start hook
availableController, err := statuscontrollers.NewAvailableConditionController(
informerFactory.Apiregistration().V1().APIServices(),
c.GenericConfig.SharedInformerFactory.Core().V1().Services(),
c.GenericConfig.SharedInformerFactory.Core().V1().Endpoints(),
apiregistrationClient.ApiregistrationV1(),
c.ExtraConfig.ProxyTransport,
(func() ([]byte, []byte))(s.proxyCurrentCertKeyContent),
s.serviceResolver,
c.GenericConfig.EgressSelector,
)
if err != nil {
return nil, err
}
s.GenericAPIServer.AddPostStartHookOrDie("start-kube-aggregator-informers", func(context genericapiserver.PostStartHookContext) error {
informerFactory.Start(context.StopCh)
c.GenericConfig.SharedInformerFactory.Start(context.StopCh)
return nil
})
s.GenericAPIServer.AddPostStartHookOrDie("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error {
handlerSyncedCh := make(chan struct{
})
go apiserviceRegistrationController.Run(context.StopCh, handlerSyncedCh)
select {
case <-context.StopCh:
case <-handlerSyncedCh:
}
return nil
})
s.GenericAPIServer.AddPostStartHookOrDie("apiservice-status-available-controller", func(context genericapiserver.PostStartHookContext) error {
// if we end up blocking for long periods of time, we may need to increase threadiness.
go availableController.Run(5, context.StopCh)
return nil
})
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) &&
utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
// Spawn a goroutine in aggregator apiserver to update storage version for
// all built-in resources
s.GenericAPIServer.AddPostStartHookOrDie(StorageVersionPostStartHookName, func(hookContext genericapiserver.PostStartHookContext) error {
// Wait for apiserver-identity to exist first before updating storage
// versions, to avoid storage version GC accidentally garbage-collecting
// storage versions.
kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig)
if err != nil {
return err
}
if err := wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
_, err := kubeClient.CoordinationV1().Leases(metav1.NamespaceSystem).Get(
context.TODO(), s.GenericAPIServer.APIServerID, metav1.GetOptions{
})
if apierrors.IsNotFound(err) {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
}, hookContext.StopCh); err != nil {
return fmt.Errorf("failed to wait for apiserver-identity lease %s to be created: %v",
s.GenericAPIServer.APIServerID, err)
}
// 启动goroutine监控并处理Storage Version的更新
// 每10分钟检查一次是否有更新需求
// Technically an apiserver only needs to update storage version once during bootstrap.
// Reconcile StorageVersion objects every 10 minutes will help in the case that the
// StorageVersion objects get accidentally modified/deleted by a different agent. In that
// case, the reconciliation ensures future storage migration still works. If nothing gets
// changed, the reconciliation update is a noop and gets short-circuited by the apiserver,
// therefore won't change the resource version and trigger storage migration.
go wait.PollImmediateUntil(10*time.Minute, func() (bool, error) {
// All apiservers (aggregator-apiserver, kube-apiserver, apiextensions-apiserver)
// share the same generic apiserver config. The same StorageVersion manager is used
// to register all built-in resources when the generic apiservers install APIs.
s.GenericAPIServer.StorageVersionManager.UpdateStorageVersions(hookContext.LoopbackClientConfig, s.GenericAPIServer.APIServerID)
return false, nil
}, hookContext.StopCh)
// Once the storage version updater finishes the first round of update,
// the PostStartHook will return to unblock /healthz. The handler chain
// won't block write requests anymore. Check every second since it's not
// expensive.
wait.PollImmediateUntil(1*time.Second, func() (bool, error) {
return s.GenericAPIServer.StorageVersionManager.Completed(), nil
}, hookContext.StopCh)
return nil
})
}
return s, nil
}
NewWithDelegate()The main logic of the method is as follows:
4), PrepareRun: Start the Controller of OpenAPI
The method is called in the main process of starting the API Server APIAggregator.PrepareRun(). The code is as follows:
// vendor/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go
func (s *APIAggregator) PrepareRun() (preparedAPIAggregator, error) {
// 在post start hook中去启动OpenAPI Controller
// add post start hook before generic PrepareRun in order to be before /healthz installation
if s.openAPIConfig != nil {
s.GenericAPIServer.AddPostStartHookOrDie("apiservice-openapi-controller", func(context genericapiserver.PostStartHookContext) error {
go s.openAPIAggregationController.Run(context.StopCh)
return nil
})
}
// 调用s.GenericAPIServer.PrepareRun()方法
prepared := s.GenericAPIServer.PrepareRun()
// 制作一个OpenAPI Controller,观测各个Aggregated APIServer所支持的API的变化,然后更新AggregatorServer自己的OpenAPI Spec
// delay OpenAPI setup until the delegate had a chance to setup their OpenAPI handlers
if s.openAPIConfig != nil {
specDownloader := openapiaggregator.NewDownloader()
openAPIAggregator, err := openapiaggregator.BuildAndRegisterAggregator(
&specDownloader,
s.GenericAPIServer.NextDelegate(),
s.GenericAPIServer.Handler.GoRestfulContainer.RegisteredWebServices(),
s.openAPIConfig,
s.GenericAPIServer.Handler.NonGoRestfulMux)
if err != nil {
return preparedAPIAggregator{
}, err
}
s.openAPIAggregationController = openapicontroller.NewAggregationController(&specDownloader, openAPIAggregator)
}
return preparedAPIAggregator{
APIAggregator: s, runnable: prepared}, nil
}
5), APIAggregator.AddAPIService()method
NewWithDelegate()In the method, create a Controller that monitors APIService updates. The Controller will eventually call the APIAggregator AddAPIService()and RemoveAPIService()methods. AddAPIService()The method code is as follows:
// vendor/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go
func (s *APIAggregator) AddAPIService(apiService *v1.APIService) error {
// if the proxyHandler already exists, it needs to be updated. The aggregation bits do not
// since they are wired against listers because they require multiple resources to respond
if proxyHandler, exists := s.proxyHandlers[apiService.Name]; exists {
proxyHandler.updateAPIService(apiService)
if s.openAPIAggregationController != nil {
s.openAPIAggregationController.UpdateAPIService(proxyHandler, apiService)
}
return nil
}
// 制作/apis/<group>/<version>开头的请求的响应函数
// 最终是由proxyHandler.ServeHTTP()来处理.所以这里会生成该结构体实例,并赋给响应url
proxyPath := "/apis/" + apiService.Spec.Group + "/" + apiService.Spec.Version
// v1. is a special case for the legacy API. It proxies to a wider set of endpoints.
if apiService.Name == legacyAPIServiceName {
proxyPath = "/api"
}
// register the proxy handler
proxyHandler := &proxyHandler{
localDelegate: s.delegateHandler,
proxyCurrentCertKeyContent: s.proxyCurrentCertKeyContent,
proxyTransport: s.proxyTransport,
serviceResolver: s.serviceResolver,
egressSelector: s.egressSelector,
}
proxyHandler.updateAPIService(apiService)
if s.openAPIAggregationController != nil {
s.openAPIAggregationController.AddAPIService(proxyHandler, apiService)
}
s.proxyHandlers[apiService.Name] = proxyHandler
s.GenericAPIServer.Handler.NonGoRestfulMux.Handle(proxyPath, proxyHandler)
s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandlePrefix(proxyPath+"/", proxyHandler)
// if we're dealing with the legacy group, we're done here
if apiService.Name == legacyAPIServiceName {
return nil
}
// if we've already registered the path with the handler, we don't want to do it again.
if s.handledGroups.Has(apiService.Spec.Group) {
return nil
}
// 制作Version Discovery Request的handler
// 用于响应对/apis/<group>的请求,返回结果是该group下所有version的集合
// it's time to register the group aggregation endpoint
groupPath := "/apis/" + apiService.Spec.Group
groupDiscoveryHandler := &apiGroupHandler{
codecs: aggregatorscheme.Codecs,
groupName: apiService.Spec.Group,
lister: s.lister,
delegate: s.delegateHandler,
}
// aggregation is protected
s.GenericAPIServer.Handler.NonGoRestfulMux.Handle(groupPath, groupDiscoveryHandler)
s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle(groupPath+"/", groupDiscoveryHandler)
s.handledGroups.Insert(apiService.Spec.Group)
return nil
}
/apis/<group>/<version>The request at the beginning is ultimately processed proxyHandler.ServeHTTP()by , the code is as follows:
// vendor/k8s.io/kube-aggregator/pkg/apiserver/handler_proxy.go
func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// 如果是本地Server(KubeAPIServer和APIExtensionsServer)支持的API Resource,交给delegationServer去处理,就是KubeAPIServer
value := r.handlingInfo.Load()
if value == nil {
r.localDelegate.ServeHTTP(w, req)
return
}
handlingInfo := value.(proxyHandlingInfo)
if handlingInfo.local {
if r.localDelegate == nil {
http.Error(w, "", http.StatusNotFound)
return
}
r.localDelegate.ServeHTTP(w, req)
return
}
if !handlingInfo.serviceAvailable {
proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
return
}
if handlingInfo.transportBuildingError != nil {
proxyError(w, req, handlingInfo.transportBuildingError.Error(), http.StatusInternalServerError)
return
}
user, ok := genericapirequest.UserFrom(req.Context())
if !ok {
proxyError(w, req, "missing user", http.StatusInternalServerError)
return
}
// 其余request需要Custom Server响应,制作http client去请求对应的Server.这里会涉及证书的问题
// write a new location based on the existing request pointed at the target service
location := &url.URL{
}
location.Scheme = "https"
rloc, err := r.serviceResolver.ResolveEndpoint(handlingInfo.serviceNamespace, handlingInfo.serviceName, handlingInfo.servicePort)
if err != nil {
klog.Errorf("error resolving %s/%s: %v", handlingInfo.serviceNamespace, handlingInfo.serviceName, err)
proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
return
}
location.Host = rloc.Host
location.Path = req.URL.Path
location.RawQuery = req.URL.Query().Encode()
newReq, cancelFn := newRequestForProxy(location, req)
defer cancelFn()
if handlingInfo.proxyRoundTripper == nil {
proxyError(w, req, "", http.StatusNotFound)
return
}
proxyRoundTripper := handlingInfo.proxyRoundTripper
upgrade := httpstream.IsUpgradeRequest(req)
proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)
// If we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does
// NOT use the proxyRoundTripper. It's a direct dial that bypasses the proxyRoundTripper. This means that we have to
// attach the "correct" user headers to the request ahead of time.
if upgrade {
transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
}
handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{
w: w})
handler.InterceptRedirects = utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StreamingProxyRedirects)
handler.RequireSameHostRedirects = utilfeature.DefaultFeatureGate.Enabled(genericfeatures.ValidateProxyRedirects)
utilflowcontrol.RequestDelegated(req.Context())
handler.ServeHTTP(w, newReq)
}
AddAPIService()The main logic of the method is as follows:
reference:
Kubernetes source code development journey three: API Server source code analysis