// daemon/cmd/daemon_main.go

func runDaemon() {
    enableIPForwarding()                                  // turn on ip forwarding in kernel
    k8s.Init(Config)                                      // init k8s utils

    d, restoredEndpoints := NewDaemon()
    gc.Enable(restoredEndpoints.restored)                 // Starting connection tracking garbage collector
    d.initKVStore()                                       // init cilium-etcd

    restoreComplete := d.initRestore(restoredEndpoints)

    d.initHealth()                                        // init cilium health-checking if enabled
    d.startStatusCollector()
    d.startAgentHealthHTTPService()

    d.SendNotification(monitorAPI.AgentNotifyStart, repr)

    go func() { errs <- srv.Serve() }()                   // start Cilium HTTP API server

    k8s.Client().MarkNodeReady(nodeTypes.GetName())
    d.launchHubble()
}

// Init initializes the Kubernetes package. It is required to call Configure() beforehand.
func Init(conf k8sconfig.Configuration) error {
    k8sRestClient := createDefaultClient()
    heartBeat := func() { k8sRestClient.Get().Resource("healthz").Do(ctx) }

    if Config.K8sHeartbeatTimeout != 0
        controller.NewManager().UpdateController("k8s-heartbeat", ControllerParams{
                DoFunc: ...
                RunInterval: Config.K8sHeartbeatTimeout,
            })

    k8sversion.Update(Client(), conf)
}

// daemon/cmd/daemon.go

func NewDaemon(ctx , cancel context.CancelFunc, epMgr *endpointmanager.EndpointManager, dp datapath.Datapath) (*Daemon, *endpointRestoreState, error) {
    if option.Config.ReadCNIConfiguration != "" {
        netConf = cnitypes.ReadNetConf(option.Config.ReadCNIConfiguration)
    }

    apiLimiterSet := rate.NewAPILimiterSet(option.Config.APIRateLimit, apiRateLimitDefaults, &apiRateLimitingMetrics{})

    ctmap.InitMapInfo()
    policymap.InitMapInfo(option.Config.PolicyMapEntries)
    lbmap.Init()

    authKeySize, encryptKeyID := setupIPSec()
    nodeMngr := nodemanager.NewManager("all", dp.Node(), ipcache.IPIdentityCache, option.Config, nil, nil)
    num := identity.InitWellKnownIdentities(option.Config)
    nd := nodediscovery.NewNodeDiscovery(nodeMngr, mtuConfig, netConf)

    d := Daemon{
        ctx:               ctx,
        cancel:            cancel,
        prefixLengths:     createPrefixLengthCounter(),
        buildEndpointSem:  semaphore.NewWeighted(int64(numWorkerThreads())),
        compilationMutex:  new(lock.RWMutex),
        netConf:           netConf,
        mtuConfig:         mtuConfig,
        datapath:          dp,
        deviceManager:     NewDeviceManager(),
        nodeDiscovery:     nd,
        endpointCreations: newEndpointCreationManager(),
        apiLimiterSet:     apiLimiterSet,
    }

    d.configModifyQueue = eventqueue.NewEventQueueBuffered("config-modify-queue", ConfigModifyQueueSize)
    d.configModifyQueue.Run()

    d.svc = service.NewService(&d)
    d.rec = recorder.NewRecorder(d.ctx, &d)

    d.identityAllocator = NewCachingIdentityAllocator(&d)
    d.initPolicy(epMgr)
    nodeMngr = nodeMngr.WithSelectorCacheUpdater(d.policy.GetSelectorCache()) // must be after initPolicy
    nodeMngr = nodeMngr.WithPolicyTriggerer(d.policyUpdater)                  // must be after initPolicy

    ipcache.IdentityAllocator = d.identityAllocator
    proxy.Allocator = d.identityAllocator

    restoredCIDRidentities := make(map[string]*identity.Identity)

    d.endpointManager = epMgr

    d.redirectPolicyManager = redirectpolicy.NewRedirectPolicyManager(d.svc)

    d.k8sWatcher = watchers.NewK8sWatcher(
        d.endpointManager,
        d.nodeDiscovery.Manager,
        &d,
        d.policy,
        d.svc,
        d.datapath,
        d.redirectPolicyManager,
        d.bgpSpeaker,
        d.egressGatewayManager,
        option.Config,
    )
    nd.RegisterK8sNodeGetter(d.k8sWatcher)

    ipcache.IPIdentityCache.RegisterK8sSyncedChecker(&d)

    d.k8sWatcher.NodeChain.Register(d.endpointManager)
    d.k8sWatcher.NodeChain.Register(watchers.NewCiliumNodeUpdater(d.nodeDiscovery))

    d.redirectPolicyManager.RegisterSvcCache(&d.k8sWatcher.K8sSvcCache)
    d.redirectPolicyManager.RegisterGetStores(d.k8sWatcher)

    // Open or create BPF maps.
    d.initMaps()

    // Upsert restored CIDRs after the new ipcache has been opened above
    if len(restoredCIDRidentities) > 0 {
        ipcache.UpsertGeneratedIdentities(restoredCIDRidentities, nil)
    }

    d.svc.RestoreServices()

    d.k8sWatcher.RunK8sServiceHandler()
    treatRemoteNodeAsHost := option.Config.AlwaysAllowLocalhost() && !option.Config.EnableRemoteNodeIdentity
    policyApi.InitEntities(option.Config.ClusterName, treatRemoteNodeAsHost)

    // fetch old endpoints before k8s is configured.
    restoredEndpoints := d.fetchOldEndpoints(option.Config.StateDir)

    d.bootstrapFQDN(restoredEndpoints.possible, option.Config.ToFQDNsPreCache)

    if k8s.IsEnabled() {
        if err := d.k8sWatcher.WaitForCRDsToRegister(d.ctx); err != nil {
            return nil, restoredEndpoints, err
        }

        d.k8sWatcher.NodesInit(k8s.Client())

        if option.Config.IPAM == ipamOption.IPAMClusterPool {
            // Create the CiliumNode custom resource. This call will block until
            // the custom resource has been created
            d.nodeDiscovery.UpdateCiliumNodeResource()
        }

        k8s.WaitForNodeInformation(d.ctx, d.k8sWatcher)

        if option.Config.AllowLocalhost == option.AllowLocalhostAuto {
            option.Config.AllowLocalhost = option.AllowLocalhostAlways
            log.Info("k8s mode: Allowing localhost to reach local endpoints")
        }

    }

    if wgAgent := dp.WireguardAgent(); option.Config.EnableWireguard {
        if err := wgAgent.Init(mtuConfig); err != nil {
            log.WithError(err).Error("failed to initialize wireguard agent")
        }
    }

    bandwidth.ProbeBandwidthManager()

    d.deviceManager.Detect()
    finishKubeProxyReplacementInit(isKubeProxyReplacementStrict)

    if k8s.IsEnabled() {
        // Initialize d.k8sCachesSynced before any k8s watchers are alive, as they may
        // access it to check the status of k8s initialization
        cachesSynced := make(chan struct{})
        d.k8sCachesSynced = cachesSynced

        // Launch the K8s watchers in parallel as we continue to process other daemon options.
        d.k8sWatcher.InitK8sSubsystem(d.ctx, cachesSynced)
    }

    clearCiliumVeths()

    // Must init kvstore before starting node discovery
    if option.Config.KVStore == "" {
        log.Info("Skipping kvstore configuration")
    } else {
        d.initKVStore()
    }

    router4FromK8s, router6FromK8s := node.GetInternalIPv4Router(), node.GetIPv6Router()

    d.configureIPAM()

    d.startIPAM()

    if option.Config.EnableIPv4 {
        d.restoreCiliumHostIPs(false, router4FromK8s)
    }

    d.restoreOldEndpoints(restoredEndpoints, true)

    d.allocateIPs()

    // Must occur after d.allocateIPs(), see GH-14245 and its fix.
    d.nodeDiscovery.StartDiscovery()

    // Annotation of the k8s node must happen after discovery of the
    // PodCIDR range and allocation of the health IPs.
    if k8s.IsEnabled() && option.Config.AnnotateK8sNode {
        k8s.Client().AnnotateNode()
    }

    if option.Config.IPAM == ipamOption.IPAMCRD || option.Config.IPAM == ipamOption.IPAMENI || option.Config.IPAM == ipamOption.IPAMAzure || option.Config.IPAM == ipamOption.IPAMAlibabaCloud {
        if option.Config.EnableIPv4 {
            d.ipam.IPv4Allocator.RestoreFinished()
        }
    }

    if option.Config.DatapathMode != datapathOption.DatapathModeLBOnly {
        realIdentityAllocator := d.identityAllocator
        realIdentityAllocator.InitIdentityAllocator(k8s.CiliumClient(), nil)

        d.bootstrapClusterMesh(nodeMngr)
    }

    d.init()

    d.updateDNSDatapathRules(d.ctx)

    d.syncEndpointsAndHostIPs()

    controller.NewManager().UpdateController("sync-endpoints-and-host-ips",
        controller.ControllerParams{
            DoFunc: func(ctx ) error {
                return d.syncEndpointsAndHostIPs()
            },
            RunInterval: time.Minute,
            Context:     d.ctx,
        })

    loader.RestoreTemplates(option.Config.StateDir)

    // Start watcher for endpoint IP --> identity mappings in key-value store.
    // this needs to be done *after* init() for the daemon in that function,
    // we populate the IPCache with the host's IP(s).
    ipcache.InitIPIdentityWatcher()
    identitymanager.Subscribe(d.policy)

    return &d, restoredEndpoints, nil
}

    d := Daemon{ netConf: netConf, datapath: dp }
    d.svc = service.NewService(&d)
    d.identityAllocator = cache.NewCachingIdentityAllocator(&d)
    d.policy = policy.NewPolicyRepository()
    ...

// daemon/cmd/datapath.go

// initMaps opens all BPF maps (and creates them if they do not exist).
func (d *Daemon) initMaps() error {
    lxcmap.LXCMap.OpenOrCreate()
    ipcachemap.IPCache.OpenParallel()
    ...
    d.svc.InitMaps(Config.EnableIPv4, createSockRevNatMaps, Config.RestoreState)
    policymap.InitCallMap()

    for ep := range d.endpointManager.GetEndpoints()
        ep.InitMap()

    for ep := range d.endpointManager.GetEndpoints()
        for m := range ctmap.LocalMaps(ep, Config.EnableIPv4)
            m.Create()

    for m := range ctmap.GlobalMaps(Config.EnableIPv4)
        m.Create()

    ipv4Nat := nat.GlobalMaps(Config.EnableIPv4)
    ipv4Nat.Create()
    if Config.EnableNodePort
       neighborsmap.InitMaps(Config.EnableIPv4)

    // Set up the list of IPCache listeners in the daemon, to be used by syncEndpointsAndHostIPs()
    ipcache.IPIdentityCache.SetListeners()

    if !Config.RestoreState
        lxcmap.LXCMap.DeleteAll() // If we are not restoring state, all endpoints can be deleted.

    if Config.EnableSessionAffinity {
        lbmap.AffinityMatchMap.OpenOrCreate()
        lbmap.Affinity4Map.OpenOrCreate()
    }
}

    if Config.RestoreState
        d.svc.RestoreServices() // Read the service IDs of existing services from the BPF map and reserve them.

// pkg/service/service.go

// RestoreServices restores services from BPF maps.
//
// The method should be called once before establishing a connectivity to kube-apiserver.
func (s *Service) RestoreServices() error {
    s.restoreBackendsLocked() // Restore backend IDs
    s.restoreServicesLocked() // Restore service cache from BPF maps

    if option.Config.EnableSessionAffinity
        s.deleteOrphanAffinityMatchesLocked() // Remove no longer existing affinity matches

    s.deleteOrphanBackends() // Remove obsolete backends and release their IDs
}

    d.k8sWatcher.RunK8sServiceHandler()

// pkg/k8s/watchers/watcher.go

func (k *K8sWatcher) RunK8sServiceHandler() {
    go k.k8sServiceHandler()
}

func (k *K8sWatcher) k8sServiceHandler() {
    eventHandler := func(event k8s.ServiceEvent) {
        svc := event.Service

        switch event.Action {
        case k8s.UpdateService:
            k.addK8sSVCs(event.ID, event.OldService, svc, event.Endpoints)
            translator := k8s.NewK8sTranslator(event.ID, *event.Endpoints, false, svc.Labels, true)
            result := k.policyRepository.TranslateRules(translator)
            if result.NumToServicesRules > 0 // Only trigger policy updates if ToServices rules are in effect
                k.policyManager.TriggerPolicyUpdates(true, "Kubernetes service endpoint added")

        case k8s.DeleteService:
            ...
        }
    }

    for {
        event := <-k.K8sSvcCache.Events // Kubernetes service definition changed
        eventHandler(event)
    }
}

    k8s.RegisterCRDs()

    if Config.IPAM == ipamIPAMOperator
        d.nodeDiscovery.UpdateCiliumNodeResource()

    d.bootstrapIPAM()

// daemon/cmd/ipam.go

func (d *Daemon) bootstrapIPAM() {
    if Config.IPv4Range != AutoCIDR {
        allocCIDR := cidr.ParseCIDR(Config.IPv4Range)
        node.SetIPv4AllocRange(allocCIDR)
    }

    node.AutoComplete()

    // Set up ipam conf after init() because we might be running d.conf.KVStoreIPv4Registration
    d.ipam = ipam.NewIPAM(d.datapath.LocalNodeAddressing(), option.Config, d.nodeDiscovery, d.k8sWatcher)
}

    restoredEndpoints := d.restoreOldEndpoints(Config.StateDir, true)
    d.allocateIPs()

// daemon/cmd/state.go

// Perform the first step in restoring the endpoint structure,
// allocating their existing IP out of the CIDR block and then inserting the
// endpoints into the endpoints list. It needs to be followed by a call to
// regenerateRestoredEndpoints() once the endpoint builder is ready.
func (d *Daemon) restoreOldEndpoints(dir string, clean bool) (*endpointRestoreState) {
    state := &endpointRestoreState{
        restored: []*endpoint.Endpoint{},
        toClean:  []*endpoint.Endpoint{},
    }

    existingEndpoints = lxcmap.DumpToMap()             // get previous endpoint IDs from BPF map
    dirFiles := ioutil.ReadDir(dir)                    // state dir: `/var/run/cilium/`
    eptsID := endpoint.FilterEPDir(dirFiles)           // `/var/run/cilium/<ep_id>/lxc_config.h`

    possibleEPs := ReadEPsFromDirNames(dir, eptsID)    // parse endpoint ID from dir name
    for ep := range possibleEPs {
        ep.SetAllocator(d.identityAllocator)
        d.validateEndpoint(ep)  // further call allocateIPsLocked() to retain IP for this endpoint
        ep.SetDefaultConfiguration(true)

        state.restored.append(ep)                      // insert into restored list, will regen bpf for them
        delete(existingEndpoints, ep.IPv4.String())
    }

    for hostIP, info := range existingEndpoints        // for the remaining endpoints, delete them
        if ip := net.ParseIP(hostIP) && !info.IsHost() // from endpoint map
            lxcmap.DeleteEntry(ip)

    return state
}

    // Annotation must after discovery of the PodCIDR range and allocation of the health IPs.
    k8s.Client().AnnotateNode(nodeName, GetIPv4AllocRange(), GetInternalIPv4())

    d.bootstrapClusterMesh(nodeMngr)
    d.init()
    d.monitorAgent = monitoragent.NewAgent(context.TODO(), defaults.MonitorBufferPages)
    d.syncEndpointsAndHostIPs()

    controller.NewManager().UpdateController("sync-endpoints-and-host-ips", ControllerParams{
            DoFunc: func() { return d.syncEndpointsAndHostIPs() }, })

// daemon/cmd/daemon.go

func (d *Daemon) init() error {
    globalsDir := option.Config.GetGlobalsDir()
    os.MkdirAll(globalsDir, defaults.RuntimePathRights)
    os.Chdir(option.Config.StateDir)

    // Remove any old sockops and re-enable with _new_ programs if flag is set
    sockops.SockmapDisable()
    sockops.SkmsgDisable()

    d.createNodeConfigHeaderfile() // create /var/run/cilium/state/globals/node_config.h

    if Config.SockopsEnable {
        eppolicymap.CreateEPPolicyMap()
        sockops.SockmapEnable()
    }

    d.Datapath().Loader().Reinitialize(d.ctx, d, mtu, d.Datapath(), d.l7Proxy, d.ipam)
}

// daemon/cmd/datapath.go

// syncLXCMap adds local host enties to bpf lxcmap, as well as ipcache, if needed, and also
// notifies the daemon and network policy hosts cache if changes were made.
func (d *Daemon) syncEndpointsAndHostIPs() error {
    specialIdentities := []identity.IPIdentityPair{}

    addrs := d.datapath.LocalNodeAddressing().IPv4().LocalAddresses()
    for ip := range addrs
        specialIdentities.append(IPIdentityPair{IP: ip, ID: identity.ReservedIdentityHost})

    specialIdentities.append(identity.IPIdentityPair{
            IP:   net.IPv4zero,
            Mask: net.CIDRMask(0, net.IPv4len*8),
            ID:   identity.ReservedIdentityWorld,
        })

    existingEndpoints := lxcmap.DumpToMap()

    for ipIDPair := range specialIdentities {
        hostKey := node.GetIPsecKeyIdentity()
        isHost := ipIDPair.ID == identity.ReservedIdentityHost
        if isHost
            lxcmap.SyncHostEntry(ipIDPair.IP) // Added local ip to endpoint map

        delete(existingEndpoints, ipIDPair.IP.String())

        // Upsert will not propagate (reserved:foo->ID) mappings across the cluster, designed so.
        ipcache.IPIdentityCache.Upsert(ipIDPair.Prefix, hostKey,
            Identity{ID: ipIDPair.ID, Source: source.Local})
    }

    for hostIP, info := range existingEndpoints
        if ip := net.ParseIP(hostIP); info.IsHost()
            lxcmap.DeleteEntry(ip) // Removed outdated host ip from endpoint map
            ipcache.IPIdentityCache.Delete(hostIP, source.Local)
        }
}

    loader.RestoreTemplates(Config.StateDir) // restore previous BPF templates

// pkg/datapath/loader/cache.go

// RestoreTemplates populates the object cache from templates on the filesystem at the specified path.
func RestoreTemplates(stateDir string) error {
    path := filepath.Join(stateDir, defaults.TemplatesDir)
    RemoveAll(path)
}

    // Start watcher for endpoint IP --> identity mappings in key-value store.
    ipcache.InitIPIdentityWatcher()
    identitymanager.Subscribe(d.policy)

// pkg/ipcache/kvstore.go

func InitIPIdentityWatcher() {
    go func() {
        watcher = NewIPIdentityWatcher(kvstore.Client())
        watcher.Watch(context.TODO())
    }()
}

func (iw *IPIdentityWatcher) Watch(ctx ) {
    watcher := iw.backend.ListAndWatch(ctx, "endpointIPWatcher", IPIdentitiesPath) // cilium/state/ip/v1/

    for {
        select {
        case event, ok := <-watcher.Events:
            switch event.Typ {
            case EventTypeListDone:
                for listener := range IPIdentityCache.listeners
                    listener.OnIPIdentityCacheGC()

            case EventTypeCreate, EventTypeModify:
                json.Unmarshal(event.Value, &ipIDPair)
                IPIdentityCache.Upsert(ip, HostIP, ipIDPair.Key, k8sMeta, Identity{ipIDPair.ID, source.KVStore})

            case kvstore.EventTypeDelete: ...
            }
        }
    }
}

// pkg/maps/ctmap/gc/gc.go

// Enable enables the connection tracking garbage collection.
func Enable(ipv4, ipv6 bool, restoredEndpoints []*endpoint.Endpoint, mgr EndpointManager) {
    go func() {
        for {
            epsMap = make(map[string]*endpoint.Endpoint) // contains an IP -> EP mapping.

            gcStart = time.Now()
            eps := mgr.GetEndpoints()
            for e := range eps
                epsMap[e.GetIPv4Address()] = e

            if len(eps) > 0 || initialScan
                maxDeleteRatio = runGC(nil, ipv4, createGCFilter(initialScan, restoredEndpoints, emitEntryCB))

            for e := range eps
                runGC(e, ipv4, &ctmap.GCFilter{RemoveExpired: true, EmitCTEntryCB: emitEntryCB})
    }()
}

// daemon/cmd/daemon_main.go

func (d *Daemon) initKVStore() {
    controller.NewManager().UpdateController("kvstore-locks-gc", ControllerParams{
            // agent writes a file as lock before accessing a key in kvstore, run GC for those lock files
            DoFunc: func() { kvstore.RunLockGC() },
            RunInterval: defaults.KVStoreStaleLockTimeout,
        })

    kvstore.Setup(Config.KVStore, Config.KVStoreOpt) // connect to kvstore
}

// daemon/cmd/state.go

func (d *Daemon) initRestore(restoredEndpoints *endpointRestoreState) chan struct{} {
    // When we regenerate restored endpoints, it is guaranteed tha we have
    // received the full list of policies present at the time the daemon is bootstrapped.
    restoreComplete = d.regenerateRestoredEndpoints(restoredEndpoints)

    go func() {
        if d.clustermesh != nil               // wait for all cluster mesh to be synchronized
            d.clustermesh.ClustersSynced()    // with the datapath before proceeding.

        // Start controller which removes any leftover Kubernetes services that may have been deleted
        // while Cilium was not running. It will not run again unless updated elsewhere.
        // This means that if, for instance, a user manually adds a service via the CLI into the BPF maps,
        // that it will not be cleaned up by the daemon until it restarts.
        controller.NewManager().UpdateController("sync-lb-maps-with-k8s-services", ControllerParams{
                DoFunc: func() { return d.svc.SyncWithK8sFinished() },
            })
    }()
    return restoreComplete
}

//daemon/cmd/status.go

func (d *Daemon) startStatusCollector() {
    probes := []status.Probe{
        {
            Name: "check-locks",
            Probe: func() (interface{}, error) { ...  },
        }, {
            Name: "kvstore",
            Probe: func() { return kvstore.Client().Status() },
        }, {
            Name: "kubernetes",
            Probe: func() () { return d.getK8sStatus()
        }, {
            Name: "ipam", ...
        }, {
            Name: "node-monitor",
            Probe: func() () { return d.monitorAgent.State() },
        }, {
            Name: "cluster",
            Probe: func() {return &models.ClusterStatus{d.nodeDiscovery.LocalNode.Fullname() }
        }, {
            Name: "cilium-health", ...
        }, {
            Name: "l7-proxy",
            Probe: func() () { return d.l7Proxy.GetStatusModel() },
        }, {
            Name: "controllers",
            Probe: func() () { return controller.GetGlobalStatus(), nil },
        }, {
            Name: "clustermesh",
            Probe: func() () { return d.clustermesh.Status(), nil },
        },
    }

    d.statusCollector = status.NewCollector(probes, status.Config{})
}

// daemon/cmd/daemon.go

// sends an agent notification to the monitor
func (d *Daemon) SendNotification(typ, text string) {
    event := monitorAPI.AgentNotify{Type: typ, Text: text}
    return d.monitorAgent.SendEvent(MessageTypeAgent, event)
}