text:
LoadBalancer IPs don't seem to be being advertised (pod cidr works)

Is there an existing issue for this?

[X] I have searched the existing issues

What happened?
Hi All,
Not sure if this is me or my setup but Im pretty sure I've setup everything in a way where it should now be advertising external IP Loadbalancer routes from my worker for a plex pod, but when I check with FRR routing and opnsense (my upstream routers).  I can't see anything being advertised other than the podCidr range.
If I do a show ip bgp neighbor received routes it shows nothing from the LB pools and I've checked they are not being accidently filtered.
I've tried all wierd and wonderful ways of trying to select the ippool but whatever happens it is never advertised even though kubectl get all is showing the loadbalancerIP is there in the namespace attached to the service.
My config settings are as follows:
root@kube4master:~/k8s# kubectl get all -n plex -o wide
NAME                        READY   STATUS    RESTARTS   AGE   IP           NODE          NOMINATED NODE   READINESS GATES
pod/plex-798d4bfb9f-qdlj6   1/1     Running   0          23h   10.0.5.108   kube6worker   <none>           <none>

NAME              TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE   SELECTOR
service/plex-lb   LoadBalancer   10.43.109.182   10.100.198.10   32400:32624/TCP   22h   app=plex

NAME                   READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS   IMAGES                    SELECTOR
deployment.apps/plex   1/1     1            1           12d   plex4        linuxserver/plex:latest   app=plex

NAME                              DESIRED   CURRENT   READY   AGE   CONTAINERS   IMAGES                    SELECTOR
replicaset.apps/plex-575cf8f9fd   0         0         0       12d   plex4        linuxserver/plex:latest   app=plex,pod-template-hash=575cf8f9fd
replicaset.apps/plex-678f59468    0         0         0       12d   plex4        linuxserver/plex:latest   app=plex,pod-template-hash=678f59468
replicaset.apps/plex-77758bb5d5   0         0         0       12d   plex4        linuxserver/plex:latest   app=plex,pod-template-hash=77758bb5d5
replicaset.apps/plex-798d4bfb9f   1         1         1       10d   plex4        linuxserver/plex:latest   app=plex,pod-template-hash=798d4bfb9f
replicaset.apps/plex-c87dcc9d7    0         0         0       12d   plex4        linuxserver/plex:latest   app=plex,pod-template-hash=c87dcc9d7

---
apiVersion: "cilium.io/v2alpha1"
kind: CiliumBGPPeeringPolicy
metadata:
 name: 01-bgp-peering-policy-node-supa1
spec: # CiliumBGPPeeringPolicySpec
 nodeSelector:
   matchLabels:
     bgp-policy: node-supa1
 virtualRouters: # []CiliumBGPVirtualRouter
 - localASN: 100
   exportPodCIDR: false
   neighbors: # []CiliumBGPNeighbor
    - peerAddress: '10.12.89.1/32'
      peerASN: 65551
      #authSecretRef: secretname
      eBGPMultihopTTL: 3
      connectRetryTimeSeconds: 5
      holdTimeSeconds: 180
      keepAliveTimeSeconds: 60
      gracefulRestart:
        enabled: true
        restartTimeSeconds: 12
    - peerAddress: '10.12.95.1/24'
      peerASN: 65010
      #authSecretRef: secretname
      eBGPMultihopTTL: 3
      gracefulRestart:
        enabled: true
        restartTimeSeconds: 12
      families:
      - afi: ipv4
        safi: unicast
---
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
  name: "wizznet-pool"
spec:
  blocks:
  - cidr: "10.100.198.0/24"

another attempt
root@kube4master:~/k8s# cat bgp-config-pol-node-supa1.yaml
---
apiVersion: "cilium.io/v2alpha1"
kind: CiliumBGPPeeringPolicy
metadata:
 name: 01-bgp-peering-policy-node-supa1
spec: # CiliumBGPPeeringPolicySpec
 nodeSelector:
   matchLabels:
     bgp-policy: node-supa1
 virtualRouters: # []CiliumBGPVirtualRouter
 - localASN: 100
   exportPodCIDR: true
   podIPPoolSelector:
      matchLabels:
        io.cilium.podippool.name: wizznet-pool
#      matchExpressions:
#         - {key: somekey, operator: NotIn, values: ['never-used-value']}
   neighbors: # []CiliumBGPNeighbor
#    - peerAddress: '10.12.89.1/32'
#      peerASN: 65551
#      #authSecretRef: secretname
 #     eBGPMultihopTTL: 3
 #     connectRetryTimeSeconds: 5
 #     holdTimeSeconds: 180
 #     keepAliveTimeSeconds: 60
 #     gracefulRestart:
 #       enabled: true
 #       restartTimeSeconds: 12
    - peerAddress: '10.12.95.1/24'
      peerASN: 65010
      #authSecretRef: secretname
      eBGPMultihopTTL: 3
      gracefulRestart:
        enabled: true
        restartTimeSeconds: 12
#      families:
#      - afi: ipv4
#        safi: unicast
#      connectRetryTimeSeconds: 5
#      holdTimeSeconds: 180
#      keepAliveTimeSeconds: 60
#      gracefulRestart:
#        enabled: true
#        restartTimeSeconds: 12
#    - peerAddress: '10.12.95.11/32'
 #     peerASN: 65011
      #authSecretRef: secretname
#      eBGPMultihopTTL: 2
#      connectRetryTimeSeconds: 15
#      holdTimeSeconds: 5
#      keepAliveTimeSeconds: 5
#      gracefulRestart:
#        enabled: true
#        restartTimeSeconds: 12
#    - peerAddress: '10.12.95.12/32'
#      peerASN: 65012
#      #authSecretRef: secretname
#      eBGPMultihopTTL: 2
#      connectRetryTimeSeconds: 15
#      holdTimeSeconds: 5
#      keepAliveTimeSeconds: 5
#      gracefulRestart:
#        enabled: true
#        restartTimeSeconds: 12
---
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
  name: "wizznet-pool"
spec:
  blocks:
  - cidr: "10.100.198.0/24"

kubectl label nodes kube6worker bgp-policy=node-supa1 --overwrite
FRR upstream
*kube4master(10.12.95.151)                      4        100     11022     11056        0    0    0 09:10:25            2       11 N/A
*kube6worker(10.12.95.152)                      4        100     11029     11056        0    0    0 09:10:27            2       11 N/A
*kube1(10.12.95.155)                            4        100     11012     11056        0    0    0 09:10:27            0       11 N/A

supa1# show ip bgp neighbors 10.12.95.152 received-routes
BGP table version is 103, local router ID is 10.12.89.10, vrf id 0
Default local pref 100, local AS 65010
Status codes:  s suppressed, d damped, h history, * valid, > best, = multipath,
               i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes:  i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found

    Network          Next Hop            Metric LocPrf Weight Path
 *> 10.12.10.0/24    10.12.95.152                           0 100 65551 i
 *> 10.12.89.0/24    10.12.95.152                           0 100 65551 i

Total number of prefixes 2
supa1# show ip bgp neighbors 10.12.95.152 filtered-routes
BGP table version is 103, local router ID is 10.12.89.10, vrf id 0
Default local pref 100, local AS 65010
Status codes:  s suppressed, d damped, h history, * valid, > best, = multipath,
               i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes:  i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found

    Network          Next Hop            Metric LocPrf Weight Path

root@kube4master:~# cilium config view
agent-not-ready-taint-key                         node.cilium.io/agent-not-ready
arping-refresh-period                             30s
auto-direct-node-routes                           false
bgp-announce-lb-ip                                true
bpf-lb-acceleration                               disabled
bpf-lb-external-clusterip                         false
bpf-lb-map-max                                    65536
bpf-lb-sock                                       false
bpf-map-dynamic-size-ratio                        0.0025
bpf-policy-map-max                                16384
bpf-root                                          /sys/fs/bpf
cgroup-root                                       /run/cilium/cgroupv2
cilium-endpoint-gc-interval                       5m0s
cluster-id                                        0
cluster-name                                      default
cluster-pool-ipv4-cidr                            10.0.0.0/8
cluster-pool-ipv4-mask-size                       24
cni-exclusive                                     true
cni-log-file                                      /var/run/cilium/cilium-cni.log
custom-cni-conf                                   false
debug                                             true
debug-verbose
dnsproxy-enable-transparent-mode                  true
egress-gateway-reconciliation-trigger-interval    1s
enable-auto-protect-node-port-range               true
enable-bgp-control-plane                          true
enable-bpf-clock-probe                            false
enable-endpoint-health-checking                   true
enable-health-check-loadbalancer-ip               false
enable-health-check-nodeport                      true
enable-health-checking                            true
enable-hubble                                     true
enable-ipv4                                       true
enable-ipv4-big-tcp                               false
enable-ipv4-masquerade                            true
enable-ipv6                                       false
enable-ipv6-big-tcp                               false
enable-ipv6-masquerade                            true
enable-k8s-networkpolicy                          true
enable-k8s-terminating-endpoint                   true
enable-l2-neigh-discovery                         true
enable-l7-proxy                                   true
enable-local-redirect-policy                      false
enable-masquerade-to-route-source                 false
enable-metrics                                    true
enable-policy                                     default
enable-remote-node-identity                       true
enable-sctp                                       false
enable-svc-source-range-check                     true
enable-vtep                                       false
enable-well-known-identities                      false
enable-wireguard                                  true
enable-xt-socket-fallback                         true
external-envoy-proxy                              false
hubble-disable-tls                                false
hubble-export-file-max-backups                    5
hubble-export-file-max-size-mb                    10
hubble-listen-address                             :4244
hubble-socket-path                                /var/run/cilium/hubble.sock
hubble-tls-cert-file                              /var/lib/cilium/tls/hubble/server.crt
hubble-tls-client-ca-files                        /var/lib/cilium/tls/hubble/client-ca.crt
hubble-tls-key-file                               /var/lib/cilium/tls/hubble/server.key
identity-allocation-mode                          crd
identity-gc-interval                              15m0s
identity-heartbeat-timeout                        30m0s
install-no-conntrack-iptables-rules               false
ipam                                              cluster-pool
ipam-cilium-node-update-rate                      15s
k8s-client-burst                                  20
k8s-client-qps                                    10
kube-proxy-replacement                            strict
kube-proxy-replacement-healthz-bind-address
max-connected-clusters                            255
mesh-auth-enabled                                 true
mesh-auth-gc-interval                             5m0s
mesh-auth-queue-size                              1024
mesh-auth-rotated-identities-queue-size           1024
monitor-aggregation                               medium
monitor-aggregation-flags                         all
monitor-aggregation-interval                      5s
node-port-bind-protection                         true
nodes-gc-interval                                 5m0s
operator-api-serve-addr                           127.0.0.1:9234
operator-prometheus-serve-addr                    :9963
policy-cidr-match-mode
preallocate-bpf-maps                              false
procfs                                            /host/proc
proxy-connect-timeout                             2
proxy-max-connection-duration-seconds             0
proxy-max-requests-per-connection                 0
proxy-prometheus-port                             9964
remove-cilium-node-taints                         true
routing-mode                                      tunnel
service-no-backend-response                       reject
set-cilium-is-up-condition                        true
set-cilium-node-taints                            true
sidecar-istio-proxy-image                         cilium/istio_proxy
skip-cnp-status-startup-clean                     false
synchronize-k8s-nodes                             true
tofqdns-dns-reject-response-code                  refused
tofqdns-enable-dns-compression                    true
tofqdns-endpoint-max-ip-per-hostname              50
tofqdns-idle-connection-grace-period              0s
tofqdns-max-deferred-connection-deletes           10000
tofqdns-proxy-response-max-delay                  100ms
tunnel-protocol                                   vxlan
unmanaged-pod-watcher-interval                    15
vtep-cidr
vtep-endpoint
vtep-mac
vtep-mask
write-cni-conf-when-ready                         /host/etc/cni/net.d/05-cilium.conflist

I'm running out of things to check, I seemed to have no problems when using the older technique of using MetalLB.
Any help or pointers appreciated.
Thanks,
Jon.
Cilium Version
1.15.5
Kernel Version
Linux kube4master 6.8.4-3-pve #1 SMP PREEMPT_DYNAMIC PMX 6.8.4-3 (2024-05-02T11:55Z) x86_64 GNU/Linux
Kubernetes Version
k3s v1.29.5+k3s1
Regression
No response
Sysdump
No response
Relevant log output
No response
Anything else?
No response
Cilium Users Document

[ ] Are you a user of Cilium? Please add yourself to the Users doc

Code of Conduct

[X] I agree to follow this project's Code of Conduct

Have you tried serviceSelector?
https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v1/#advertising-service-virtual-ips

Thanks I'll try and see if it works.

@bodleytunes You may be running into the same issue that prompted https://github.com/cilium/cilium/pull/32694
Try changing:
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
  name: "wizznet-pool"
spec:
  blocks:
  - cidr: "10.100.198.0/24"

To:
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
  name: "wizznet-pool"
spec:
  cidrs:
  - cidr: "10.100.198.0/24"

If that solves it, you are hitting the same issue.  I merged a fix, but it literally just went in about a week ago so you're likely not running it :-).

I'll try both, had a few issues with my cluster but back up and running
again so ready to test.
On Sat, 15 Jun 2024 at 03:08, David Swafford @.***>
wrote:

@bodleytunes https://github.com/bodleytunes You may be running into the
same issue that prompted #32694
https://github.com/cilium/cilium/pull/32694
Try changing:
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
name: "wizznet-pool"
spec:
blocks:

cidr: "10.100.198.0/24"

To:
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
name: "wizznet-pool"
spec:
cidrs:

cidr: "10.100.198.0/24"

If that solves it, you are hitting the same issue. I merged a fix, but it
literally just went in about a week ago so you're likely not running it :-).
—
Reply to this email directly, view it on GitHub
https://github.com/cilium/cilium/issues/33076#issuecomment-2169040828,
or unsubscribe
https://github.com/notifications/unsubscribe-auth/AAEXFZZUTEYSF5JHATF63HDZHOOZBAVCNFSM6AAAAABJFC677OVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDCNRZGA2DAOBSHA
.
You are receiving this because you were mentioned.Message ID:
@.***>

I think I maybe stuck in the old mode of peering?
I initially used this config map
#apiVersion: v1
#kind: ConfigMap
#metadata:
#  name: bgp-config
#  namespace: kube-system
#data:
#  config.yaml: |
#    peers:
#      - peer-address: 10.12.89.10
#        peer-asn: 65010
#        my-asn: 100
#    address-pools:
#      - name: default
#        protocol: bgp
#        addresses:
#          - 10.100.198.0/24

but when I deleted it cilium completely stops working and asks for the
bgp-config to be applied again.  Its like I'm stuck between the old way
(metallb?, and the newer GoBGP)
I'll dig around a bit
On Sat, 15 Jun 2024 at 11:05, Jon Clayton @.***> wrote:

I'll try both, had a few issues with my cluster but back up and running
again so ready to test.
On Sat, 15 Jun 2024 at 03:08, David Swafford @.***>
wrote:

@bodleytunes https://github.com/bodleytunes You may be running into
the same issue that prompted #32694
https://github.com/cilium/cilium/pull/32694
Try changing:
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
name: "wizznet-pool"
spec:
blocks:

cidr: "10.100.198.0/24"

To:
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
name: "wizznet-pool"
spec:
cidrs:

cidr: "10.100.198.0/24"

If that solves it, you are hitting the same issue. I merged a fix, but it
literally just went in about a week ago so you're likely not running it :-).
—
Reply to this email directly, view it on GitHub
https://github.com/cilium/cilium/issues/33076#issuecomment-2169040828,
or unsubscribe
https://github.com/notifications/unsubscribe-auth/AAEXFZZUTEYSF5JHATF63HDZHOOZBAVCNFSM6AAAAABJFC677OVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDCNRZGA2DAOBSHA
.
You are receiving this because you were mentioned.Message ID:
@.***>

I seem to have got further with it now in that I'm seeing the load balancer ip appear and then  get advertised upstream, BUT its hunting between the load balancer ip and  constantly and so adding and removing the route.  Something very strange going on.
root@kube4master:~/k8s# kubectl get all -n plex
NAME                        READY   STATUS    RESTARTS        AGE
pod/plex-798d4bfb9f-qdlj6   1/1     Running   1 (3d16h ago)   5d16h

NAME              TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE
service/plex-lb   LoadBalancer   10.43.109.182   10.100.198.10   32400:32624/TCP   5d16h

NAME                   READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/plex   1/1     1            1           17d

NAME                              DESIRED   CURRENT   READY   AGE
replicaset.apps/plex-575cf8f9fd   0         0         0       17d
replicaset.apps/plex-678f59468    0         0         0       17d
replicaset.apps/plex-77758bb5d5   0         0         0       17d
replicaset.apps/plex-798d4bfb9f   1         1         1       14d
replicaset.apps/plex-c87dcc9d7    0         0         0       17d

root@kube4master:~/k8s# kubectl get all -n plex
NAME                        READY   STATUS    RESTARTS        AGE
pod/plex-798d4bfb9f-qdlj6   1/1     Running   1 (3d16h ago)   5d16h

NAME              TYPE           CLUSTER-IP      EXTERNAL-IP   PORT(S)           AGE
service/plex-lb   LoadBalancer   10.43.109.182   <pending>     32400:32624/TCP   5d16h

NAME                   READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/plex   1/1     1            1           17d

NAME                              DESIRED   CURRENT   READY   AGE
replicaset.apps/plex-575cf8f9fd   0         0         0       17d
replicaset.apps/plex-678f59468    0         0         0       17d
replicaset.apps/plex-77758bb5d5   0         0         0       17d
replicaset.apps/plex-798d4bfb9f   1         1         1       14d
replicaset.apps/plex-c87dcc9d7    0         0         0       17d

root@kube4master:~/k8s# kubectl -n kube-system logs -l k8s-app=cilium -c cilium-agent
time="2024-06-16T15:05:47Z" level=debug msg="Upserting service" backends="[10.0.5.23:32400]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceIP="{10.12.100.12 {TCP 32624} 0}" serviceName=plex-lb serviceNamespace=plex sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Acquired service ID" backends="[10.0.5.23:32400]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceID=11 serviceIP="{10.12.100.12 {TCP 32624} 0}" serviceName=plex-lb serviceNamespace=plex sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Deleting backends from session affinity match" backends="[]" serviceID=11 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="10.12.100.12:28799 (1)" svcVal="8 0 (2816) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="10.12.100.12:28799 (0)" svcVal="0 1 (2816) [0x42 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=11 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="received update" Key=10.12.95.1 Topic=Peer asn=100 attributes="[]" component=gobgp.BgpServerInstance nlri="[]" subsys=bgp-control-plane withdrawals="[10.100.198.10/32]"
time="2024-06-16T15:05:47Z" level=debug msg="create Destination" Nlri=10.100.198.10/32 Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="Removing withdrawals" Key=10.100.198.10/32 Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="Found withdrawals for path(s) that did not get installed" Key=10.100.198.10/32 Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="ip 10.100.198.10 op setBalancer error assigned IP not allowed by config msg IP allocated by controller not allowed by config" subsys=bgp-speaker
time="2024-06-16T15:05:47Z" level=debug msg="event endUpdate msg end of service update" subsys=bgp-speaker
time="2024-06-16T15:05:47Z" level=debug msg="Deleting backends from session affinity match" backends="[]" serviceID=14 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="10.12.100.10:28799 (1)" svcVal="13 0 (3584) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="10.12.100.10:28799 (0)" svcVal="0 1 (3584) [0x42 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=14 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="sent update" Key=10.12.95.1 State=BGP_FSM_ESTABLISHED Topic=Peer asn=100 attributes="[{Origin: i} {AsPath: 100} {Nexthop: 10.12.95.155}]" component=gobgp.BgpServerInstance nlri="[10.100.198.10/32]" subsys=bgp-control-plane withdrawals="[]"
time="2024-06-16T15:05:47Z" level=debug msg="received update" Key=10.12.95.1 Topic=Peer asn=100 attributes="[{Origin: i} {AsPath: 65010 100} {Nexthop: 10.12.95.155}]" component=gobgp.BgpServerInstance nlri="[10.100.198.10/32]" subsys=bgp-control-plane withdrawals="[]"
time="2024-06-16T15:05:47Z" level=debug msg="create Destination" Nlri=10.100.198.10/32 Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg=sent Key=10.12.95.1 State=BGP_FSM_ESTABLISHED Topic=Peer asn=100 component=gobgp.BgpServerInstance data="&{{[] 19 4} 0x661efa0}" subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="10.12.100.14:28799 (1)" svcVal="9 0 (2816) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="10.12.100.14:28799 (0)" svcVal="0 1 (2816) [0x42 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=11 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="received update" Key=10.12.89.1 Topic=Peer asn=100 attributes="[]" component=gobgp.BgpServerInstance nlri="[]" subsys=bgp-control-plane withdrawals="[10.100.198.10/32]"
time="2024-06-16T15:05:47Z" level=debug msg="Removing withdrawals" Key=10.100.198.10/32 Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=warning msg="No matching path for withdraw found, may be path was not installed into table" Key=10.100.198.10/32 Path="{ 10.100.198.10/32 | src: { 10.12.89.1 | as: 65551, id: 10.12.89.1 }, nh: <nil>, withdraw }" Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="received update" Key=10.12.95.1 Topic=Peer asn=100 attributes="[{Origin: i} {AsPath: 65010 100} {Nexthop: 10.12.95.155}]" component=gobgp.BgpServerInstance nlri="[10.100.198.10/32]" subsys=bgp-control-plane withdrawals="[]"
time="2024-06-16T15:05:47Z" level=debug msg="create Destination" Nlri=10.100.198.10/32 Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="received update" Key=10.12.89.1 Topic=Peer asn=100 attributes="[{Origin: i} {AsPath: 65551 65010 100} {Nexthop: 10.12.89.1}]" component=gobgp.BgpServerInstance nlri="[10.100.198.10/32]" subsys=bgp-control-plane withdrawals="[]"
time="2024-06-16T15:05:47Z" level=debug msg="create Destination" Nlri=10.100.198.10/32 Topic=Table asn=100 component=gobgp.BgpServerInstance subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="Deleting backends from session affinity match" backends="[]" serviceID=12 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="10.12.100.11:28799 (1)" svcVal="14 0 (3072) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="10.12.100.11:28799 (0)" svcVal="0 1 (3072) [0x42 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=12 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserting service" backends="[10.0.5.23:32400]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceIP="{0.0.0.0 {TCP 32624} 0}" serviceName=plex-lb serviceNamespace=plex sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Acquired service ID" backends="[10.0.5.23:32400]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceID=13 serviceIP="{0.0.0.0 {TCP 32624} 0}" serviceName=plex-lb serviceNamespace=plex sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Deleting backends from session affinity match" backends="[]" serviceID=13 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="0.0.0.0:28799 (1)" svcVal="14 0 (3328) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="0.0.0.0:28799 (0)" svcVal="0 1 (3328) [0x2 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=13 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Deleting backends from session affinity match" backends="[]" serviceID=4 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="10.12.100.16:20860 (1)" svcVal="4 0 (1024) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="10.12.100.16:20860 (0)" svcVal="0 1 (1024) [0x42 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=4 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserting service" backends="[10.0.0.221:80]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceIP="{0.0.0.0 {TCP 31825} 0}" serviceName=nginx-lb serviceNamespace=default sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Acquired service ID" backends="[10.0.0.221:80]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceID=5 serviceIP="{0.0.0.0 {TCP 31825} 0}" serviceName=nginx-lb serviceNamespace=default sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Deleting backends from session affinity match" backends="[]" serviceID=5 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="0.0.0.0:20860 (1)" svcVal="4 0 (1280) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="0.0.0.0:20860 (0)" svcVal="0 1 (1280) [0x2 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=5 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="No preflight reconciliation necessary for virtual router with local ASN 100" component=PreflightReconciler subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="Begin reconciling pod CIDR advertisements for virtual router with local ASN 100" component=exportPodCIDRReconciler subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="No reconciliation necessary" component=exportPodCIDRReconciler subsys=bgp-control-plane
time="2024-06-16T15:05:47Z" level=debug msg="Upserting service" backends="[10.0.0.221:80]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceIP="{0.0.0.0 {TCP 31825} 0}" serviceName=nginx-lb serviceNamespace=default sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Acquired service ID" backends="[10.0.0.221:80]" l7LBFrontendPorts="[]" l7LBProxyPort=0 loadBalancerSourceRanges="[]" serviceID=4 serviceIP="{0.0.0.0 {TCP 31825} 0}" serviceName=nginx-lb serviceNamespace=default sessionAffinity=false sessionAffinityTimeout=0 subsys=service svcExtTrafficPolicy=Cluster svcHealthCheckNodePort=0 svcIntTrafficPolicy=Cluster svcType=NodePort
time="2024-06-16T15:05:47Z" level=debug msg="Deleting backends from session affinity match" backends="[]" serviceID=4 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=1 subsys=map-lb svcKey="0.0.0.0:20860 (1)" svcVal="10 0 (1024) [0x0 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Upserted service entry" backendSlot=0 subsys=map-lb svcKey="0.0.0.0:20860 (0)" svcVal="0 1 (1024) [0x2 0x0]"
time="2024-06-16T15:05:47Z" level=debug msg="Adding backends to affinity match map" backends="[]" serviceID=4 subsys=service
time="2024-06-16T15:05:47Z" level=debug msg="Successfully completed reconciliation" component=Controller.Run subsys=bgp-control-plane

root@kube4master:~/k8s# kubectl get ciliumbgppeeringpolicies.cilium.io -o yaml
apiVersion: v1
items:
- apiVersion: cilium.io/v2alpha1
  kind: CiliumBGPPeeringPolicy
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"cilium.io/v2alpha1","kind":"CiliumBGPPeeringPolicy","metadata":{"annotations":{},"name":"01-bgp-peering-policy-node-supa1"},"spec":{"nodeSelector":{"matchLabels":{"bgp-policy":"node-supa1"}},"virtualRouters":[{"exportPodCIDR":true,"localASN":100,"neighbors":[{"connectRetryTimeSeconds":5,"eBGPMultihopTTL":3,"gracefulRestart":{"enabled":true,"restartTimeSeconds":12},"holdTimeSeconds":180,"keepAliveTimeSeconds":60,"peerASN":65551,"peerAddress":"10.12.89.1/32"},{"eBGPMultihopTTL":3,"gracefulRestart":{"enabled":true,"restartTimeSeconds":12},"peerASN":65010,"peerAddress":"10.12.95.1/24"}],"serviceSelector":{"matchLabels":{"color":"red"}}}]}}
    creationTimestamp: "2024-06-09T11:49:49Z"
    generation: 15
    name: 01-bgp-peering-policy-node-supa1
    resourceVersion: "5674876"
    uid: 32f224d2-dcf3-46bc-961b-34f043400ef9
  spec:
    nodeSelector:
      matchLabels:
        bgp-policy: node-supa1
    virtualRouters:
    - exportPodCIDR: true
      localASN: 100
      neighbors:
      - connectRetryTimeSeconds: 5
        eBGPMultihopTTL: 3
        gracefulRestart:
          enabled: true
          restartTimeSeconds: 12
        holdTimeSeconds: 180
        keepAliveTimeSeconds: 60
        peerASN: 65551
        peerAddress: 10.12.89.1/32
        peerPort: 179
      - connectRetryTimeSeconds: 120
        eBGPMultihopTTL: 3
        gracefulRestart:
          enabled: true
          restartTimeSeconds: 12
        holdTimeSeconds: 90
        keepAliveTimeSeconds: 30
        peerASN: 65010
        peerAddress: 10.12.95.1/24
        peerPort: 179
      serviceSelector:
        matchLabels:
          color: red
- apiVersion: cilium.io/v2alpha1
  kind: CiliumBGPPeeringPolicy
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"cilium.io/v2alpha1","kind":"CiliumBGPPeeringPolicy","metadata":{"annotations":{},"name":"01-bgp-peering-policy-node-supa2"},"spec":{"nodeSelector":{"matchLabels":{"bgp-policy":"node-supa2"}},"virtualRouters":[{"exportPodCIDR":true,"localASN":100,"neighbors":[{"eBGPMultihopTTL":3,"families":[{"afi":"ipv4","safi":"unicast"}],"gracefulRestart":{"enabled":true,"restartTimeSeconds":12},"peerASN":65011,"peerAddress":"10.12.95.1/24"}],"podIPPoolSelector":{"matchExpressions":[{"key":"somekey","operator":"NotIn","values":["never-used-value"]}]}}]}}
    creationTimestamp: "2024-06-09T11:49:51Z"
    generation: 4
    name: 01-bgp-peering-policy-node-supa2
    resourceVersion: "4457925"
    uid: 6cdc46ca-aae9-4aee-b895-c257fef37f6f
  spec:
    nodeSelector:
      matchLabels:
        bgp-policy: node-supa2
    virtualRouters:
    - exportPodCIDR: true
      localASN: 100
      neighbors:
      - connectRetryTimeSeconds: 120
        eBGPMultihopTTL: 3
        families:
        - afi: ipv4
          safi: unicast
        gracefulRestart:
          enabled: true
          restartTimeSeconds: 12
        holdTimeSeconds: 90
        keepAliveTimeSeconds: 30
        peerASN: 65011
        peerAddress: 10.12.95.1/24
        peerPort: 179
      podIPPoolSelector:
        matchExpressions:
        - key: somekey
          operator: NotIn
          values:
          - never-used-value
- apiVersion: cilium.io/v2alpha1
  kind: CiliumBGPPeeringPolicy
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"cilium.io/v2alpha1","kind":"CiliumBGPPeeringPolicy","metadata":{"annotations":{},"name":"01-bgp-peering-policy-node-supa3"},"spec":{"nodeSelector":{"matchLabels":{"bgp-policy":"node-supa3"}},"virtualRouters":[{"exportPodCIDR":true,"localASN":100,"neighbors":[{"eBGPMultihopTTL":3,"families":[{"afi":"ipv4","safi":"unicast"}],"gracefulRestart":{"enabled":true,"restartTimeSeconds":12},"peerASN":65012,"peerAddress":"10.12.95.1/24"}],"podIPPoolSelector":{"matchExpressions":[{"key":"somekey","operator":"NotIn","values":["never-used-value"]}]}}]}}
    creationTimestamp: "2024-06-09T11:49:53Z"
    generation: 4
    name: 01-bgp-peering-policy-node-supa3
    resourceVersion: "4458091"
    uid: 3c568b14-757f-48d8-bc82-89eba170e91b
  spec:
    nodeSelector:
      matchLabels:
        bgp-policy: node-supa3
    virtualRouters:
    - exportPodCIDR: true
      localASN: 100
      neighbors:
      - connectRetryTimeSeconds: 120
        eBGPMultihopTTL: 3
        families:
        - afi: ipv4
          safi: unicast
        gracefulRestart:
          enabled: true
          restartTimeSeconds: 12
        holdTimeSeconds: 90
        keepAliveTimeSeconds: 30
        peerASN: 65012
        peerAddress: 10.12.95.1/24
        peerPort: 179
      podIPPoolSelector:
        matchExpressions:
        - key: somekey
          operator: NotIn
          values:
          - never-used-value
kind: List
metadata:

root@kube4master:~/k8s# cat bgp-ip-pool.yaml
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
  name: "pool2"
spec:
  blocks:
  - cidr: "10.12.248.0/26"
  serviceSelector:
    matchLabels:
      color: red

---
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
  name: "wizznet-pool"
spec:
  blocks:
  - cidr: "10.100.198.0/24"
root@kube4master:~/k8s# kubectl get CiliumLoadBalancerIPPool
NAME           DISABLED   CONFLICTING   IPS AVAILABLE   AGE
pool2          false      False         61              6d20h
wizznet-pool   false      False         253             22d

time="2024-06-16T15:09:32Z" level=debug msg="ip 10.12.248.2 op setBalancer error assigned IP not allowed by config msg IP allocated by controller not allowed by config" subsys=bgp-speaker

when I google op setBalancer error assigned IP not allowed by config msg IP allocated by controller not allowed by config" subsys=bgp-speaker
I seem to get hits relating to metallb? which makes me think have I deployed both the old and new way of bgp speaking in Cilium somehow?  And are they fighting for control?

I think I installed it with the old bgp ?

shouldn't I have used  --set bgpControlPlane.enabled=true
Now wondering how to disable the old way?

Not sure if this fixed it but tried this and its looking better now!
helm upgrade cilium cilium/cilium --version 1.15.4   --namespace kube-system   --set global.mtu=1450   --set tunnel=wireguard   --set kubeProxyReplacement=strict   --set bgpControlPlane.enabled=true  --set bgp.enabled=false  --set bgp.announce.loadbalancerIP=true   --set k8sServiceHost=10.12.100.10   --set k8sServicePort=6443

 *= 10.12.248.1/32   10.12.95.16(kube6worker)
                                                           0 100 i
 *=                  10.12.95.151(kube4master)
                                                           0 100 i
 *>                  10.12.95.155(kube1)
                                                           0 100 i

I have the same problem that I can reproduce fairly reliably with 1.16.0.
Here's my config:
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPClusterConfig
metadata:
  name: cilium-bgp-test
  namespace: kube-system
spec:
  bgpInstances:
  - name: "cilium-test-64512"
    localASN: 64612
    peers:
    - name: "ubuntu"
      peerASN: 64612
      peerAddress: 192.168.128.128
      peerConfigRef:
        name: "cilium-peer-test"
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPPeerConfig
metadata:
  name: cilium-peer-test
  namespace: kube-system
spec:
  timers:
    holdTimeSeconds: 9
    keepAliveTimeSeconds: 3
  gracefulRestart:
    enabled: true
    restartTimeSeconds: 15
  families:
    - afi: ipv4
      safi: unicast
      advertisements:
        matchLabels:
          advertise: "bgp-test"
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPAdvertisement
metadata:
  name: bgp-advertisements-test
  namespace: kube-system
  labels:
    advertise: bgp-test
spec:
  advertisements:
    - advertisementType: "PodCIDR"
    - advertisementType: "Service"
      service:
        addresses:
          - ClusterIP
          - ExternalIP
          - LoadBalancerIP

And here's some IP pools and a service with using them
apiVersion: v1
kind: Service
metadata:
  annotations:
    lbipam.cilium.io/ips: "192.168.132.2"
  labels:
    ipam: "infra"
  name: argocd-server-test
  namespace: argocd
spec:
  externalTrafficPolicy: Cluster
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: 8080
    - name: https
      port: 443
      protocol: TCP
      targetPort: 8080
  selector:
    app.kubernetes.io/instance: argocd
    app.kubernetes.io/name: argocd-server
  type: LoadBalancer

192.168.128.128 is an ubuntu box with BIRD for testing. It has the following config:
#BGP Configuration
protocol bgp {
       import all;
       export where proto = "static_bgp";

       local as 64612;
   neighbor 192.168.130.141 as 64612;
}

protocol bgp {
       import all;
       export where proto = "static_bgp";

       local as 64612;
       neighbor 192.168.130.142 as 64612;
}

protocol bgp {
       import all;
       export where proto = "static_bgp";

       local as 64612;
       neighbor 192.168.130.151 as 64612;
}

protocol bgp {
       import all;
       export where proto = "static_bgp";

       local as 64612;
       neighbor 192.168.130.152 as 64612;
}

protocol bgp {
       import all;
       export where proto = "static_bgp";

       local as 64612;
       neighbor 192.168.130.153 as 64612;
}

where each neighbor is the static IP of my k8s nodes all of which are running Cilium DaemonSet pods.
Now I examine the routes received by BIRD on Ubuntu:
root@ubuntu-server-test2:~# birdc show route
BIRD 1.6.8 ready.
10.244.2.0/24      unreachable [bgp5 03:13:26 from 192.168.130.153] * (100/-) [i]
10.244.3.0/24      unreachable [bgp3 03:13:26 from 192.168.130.151] * (100/-) [i]
10.244.0.0/24      unreachable [bgp2 03:13:26 from 192.168.130.142] * (100/-) [i]
10.244.1.0/24      unreachable [bgp1 03:13:26 from 192.168.130.141] * (100/-) [i]
10.244.4.0/24      unreachable [bgp4 03:13:26 from 192.168.130.152] * (100/-) [i]
root@ubuntu-server-test2:~# birdc show proto
BIRD 1.6.8 ready.
name     proto    table    state  since       info
kernel1  Kernel   master   up     03:08:03
device1  Device   master   up     03:08:03
bgp1     BGP      master   up     03:12:47    Established
bgp2     BGP      master   up     03:12:48    Established
bgp3     BGP      master   up     03:12:50    Established
bgp4     BGP      master   up     03:12:49    Established
bgp5     BGP      master   up     03:12:48    Established

Next I will try this config (note I have commented out the PodCIDR advertisment type)
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPAdvertisement
metadata:
  name: bgp-advertisements-test
  namespace: kube-system
  labels:
    advertise: bgp-test
spec:
  advertisements:
    # - advertisementType: "PodCIDR"
    - advertisementType: "Service"
      service:
        addresses:
          - ClusterIP
          - ExternalIP
          - LoadBalancerIP

and all the POD CIDR routes disappear and no service routes are there in both cases
root@ubuntu-server-test2:~# birdc show proto
BIRD 1.6.8 ready.
name     proto    table    state  since       info
kernel1  Kernel   master   up     03:08:03
device1  Device   master   up     03:08:03
bgp1     BGP      master   up     03:12:47    Established
bgp2     BGP      master   up     03:12:48    Established
bgp3     BGP      master   up     03:12:50    Established
bgp4     BGP      master   up     03:12:49    Established
bgp5     BGP      master   up     03:12:48    Established
root@ubuntu-server-test2:~# birdc show route
BIRD 1.6.8 ready.

Here is the service status, it has been assigned the IP from the pool
apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"lbipam.cilium.io/ips":"192.168.132.2"},"labels":{"ipam":"infra"},"name":"argocd-server-test","namespace":"argocd"},"spec":{"externalTrafficPolicy":"Cluster","ports":[{"name":"http","port":80,"protocol":"TCP","targetPort":8080},{"name":"https","port":443,"protocol":"TCP","targetPort":8080}],"selector":{"app.kubernetes.io/instance":"argocd","app.kubernetes.io/name":"argocd-server"},"type":"LoadBalancer"}}
    lbipam.cilium.io/ips: 192.168.132.2
  creationTimestamp: "2024-07-31T01:13:46Z"
  labels:
    ipam: infra
  name: argocd-server-test
  namespace: argocd
  resourceVersion: "15966"
  uid: 897429b7-0230-4fed-b75c-8c1432cbbc99
spec:
  allocateLoadBalancerNodePorts: true
  clusterIP: 10.101.20.109
  clusterIPs:
  - 10.101.20.109
  externalTrafficPolicy: Cluster
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: http
    nodePort: 32417
    port: 80
    protocol: TCP
    targetPort: 8080
  - name: https
    nodePort: 31059
    port: 443
    protocol: TCP
    targetPort: 8080
  selector:
    app.kubernetes.io/instance: argocd
    app.kubernetes.io/name: argocd-server
  sessionAffinity: None
  type: LoadBalancer
status:
  conditions:
  - lastTransitionTime: "2024-07-31T01:13:46Z"
    message: ""
    reason: satisfied
    status: "True"
    type: cilium.io/IPAMRequestSatisfied
  loadBalancer:
    ingress:
    - ip: 192.168.132.2
      ipMode: VIP

I cannot do what @dswaffordcw suggests above as it gives me an invalid CRD error when I try to change spec.blocks to spec.cidrs.
Here is my Cilium config
agent-not-ready-taint-key                         node.cilium.io/agent-not-ready
arping-refresh-period                             30s
auto-direct-node-routes                           false
bgp-secrets-namespace                             kube-system
bpf-events-drop-enabled                           true
bpf-events-policy-verdict-enabled                 true
bpf-events-trace-enabled                          true
bpf-lb-acceleration                               disabled
bpf-lb-external-clusterip                         false
bpf-lb-map-max                                    65536
bpf-lb-sock                                       false
bpf-lb-sock-terminate-pod-connections             false
bpf-map-dynamic-size-ratio                        0.0025
bpf-policy-map-max                                16384
bpf-root                                          /sys/fs/bpf
cgroup-root                                       /sys/fs/cgroup
cilium-endpoint-gc-interval                       5m0s
cluster-id                                        0
cluster-name                                      default
clustermesh-enable-endpoint-sync                  false
clustermesh-enable-mcs-api                        false
cni-exclusive                                     true
cni-log-file                                      /var/run/cilium/cilium-cni.log
custom-cni-conf                                   false
datapath-mode                                     veth
debug                                             false
debug-verbose
direct-routing-skip-unreachable                   false
dnsproxy-enable-transparent-mode                  true
dnsproxy-socket-linger-timeout                    10
egress-gateway-reconciliation-trigger-interval    1s
enable-auto-protect-node-port-range               true
enable-bgp-control-plane                          true
enable-bpf-clock-probe                            false
enable-endpoint-health-checking                   true
enable-health-check-loadbalancer-ip               false
enable-health-check-nodeport                      true
enable-health-checking                            true
enable-hubble                                     true
enable-ipv4                                       true
enable-ipv4-big-tcp                               false
enable-ipv4-masquerade                            true
enable-ipv6                                       false
enable-ipv6-big-tcp                               false
enable-ipv6-masquerade                            true
enable-k8s-networkpolicy                          true
enable-k8s-terminating-endpoint                   true
enable-l2-announcements                           true
enable-l2-neigh-discovery                         true
enable-l7-proxy                                   true
enable-local-redirect-policy                      false
enable-masquerade-to-route-source                 false
enable-metrics                                    true
enable-node-selector-labels                       false
enable-policy                                     default
enable-runtime-device-detection                   true
enable-sctp                                       false
enable-svc-source-range-check                     true
enable-tcx                                        true
enable-vtep                                       false
enable-well-known-identities                      false
enable-xt-socket-fallback                         true
envoy-base-id                                     0
envoy-keep-cap-netbindservice                     false
external-envoy-proxy                              true
hubble-disable-tls                                false
hubble-export-file-max-backups                    5
hubble-export-file-max-size-mb                    10
hubble-listen-address                             :4244
hubble-socket-path                                /var/run/cilium/hubble.sock
hubble-tls-cert-file                              /var/lib/cilium/tls/hubble/server.crt
hubble-tls-client-ca-files                        /var/lib/cilium/tls/hubble/client-ca.crt
hubble-tls-key-file                               /var/lib/cilium/tls/hubble/server.key
identity-allocation-mode                          crd
identity-gc-interval                              15m0s
identity-heartbeat-timeout                        30m0s
install-no-conntrack-iptables-rules               false
ipam                                              kubernetes
ipam-cilium-node-update-rate                      15s
k8s-client-burst                                  20
k8s-client-qps                                    10
k8s-require-ipv4-pod-cidr                         false
k8s-require-ipv6-pod-cidr                         false
kube-proxy-replacement                            true
kube-proxy-replacement-healthz-bind-address
max-connected-clusters                            255
mesh-auth-enabled                                 true
mesh-auth-gc-interval                             5m0s
mesh-auth-queue-size                              1024
mesh-auth-rotated-identities-queue-size           1024
monitor-aggregation                               medium
monitor-aggregation-flags                         all
monitor-aggregation-interval                      5s
node-port-bind-protection                         true
nodeport-addresses
nodes-gc-interval                                 5m0s
operator-api-serve-addr                           127.0.0.1:9234
operator-prometheus-serve-addr                    :9963
policy-cidr-match-mode
preallocate-bpf-maps                              false
procfs                                            /host/proc
proxy-connect-timeout                             2
proxy-idle-timeout-seconds                        60
proxy-max-connection-duration-seconds             0
proxy-max-requests-per-connection                 0
proxy-xff-num-trusted-hops-egress                 0
proxy-xff-num-trusted-hops-ingress                0
remove-cilium-node-taints                         true
routing-mode                                      tunnel
service-no-backend-response                       reject
set-cilium-is-up-condition                        true
set-cilium-node-taints                            true
synchronize-k8s-nodes                             true
tofqdns-dns-reject-response-code                  refused
tofqdns-enable-dns-compression                    true
tofqdns-endpoint-max-ip-per-hostname              50
tofqdns-idle-connection-grace-period              0s
tofqdns-max-deferred-connection-deletes           10000
tofqdns-proxy-response-max-delay                  100ms
tunnel-protocol                                   vxlan
unmanaged-pod-watcher-interval                    15
vtep-cidr
vtep-endpoint
vtep-mac
vtep-mask
write-cni-conf-when-ready                         /host/etc/cni/net.d/05-cilium.conflist

I can say the LB advertisement worked for me with Cilium 1.14. I didn't check the routes like I did just now because it just worked.

@fernandesnikhil In your case it seems to be just incomplete configuration - you are missing a selector in the CiliumBGPAdvertisement for the Service advertisement type. Without selector, no services are advertised. PodCIDR advertisement is advertised even without the selector, as there is nothing to select in this case.
@RomLecat it would be super-helpful if you could share a sysdump of your system at the point it when is not advertising properly (cilium sysdump). If that is not possible, at least the following may help us: in a cilium pod on a node which is missing some advertisements, dump the following:

cilium-dbg bgp peers
cilium-dbg bgp routes advertised ipv4 unicast
cilium-dbg bgp routes advertised ipv6 unicast
cilium-dbg bgp route-policies
cilium service list.

Thanks!

@RomLecat Thanks! What is your Cilium version? Based on the naming in the cilium-dbg bgp route-policies output, this does not seem to be v1.16.0 (and the route policies seem to be the root cause here).

Ah, actually naming looks good, but we are indeed missing some route policies. Anyway, let's double-check the exact version.

It is 1.16.0, but I upgraded from 1.15.6, maybe that's why ? The running image is 1.16:
k -n kube-system get pod cilium-5hbtj -o yaml | grep image
    image: quay.io/cilium/cilium:v1.16.0@sha256:46ffa4ef3cf6d8885dcc4af5963b0683f7d59daa90d49ed9fb68d3b1627fe058

I anonymized the sysdump and re-compressed it to 7z cilium-sysdump.7z.zip
(otherwise it'd exceed the 25MB GitHub upload limit). It's 330MB uncompressed.
Make sure to remove the .zip at the end, it's only there since GitHub also does not allow 7z files.

@RomLecat thanks a lot, will take a closer look.

@RomLecat so the root-case why the svc VIPs are not advertised is that you are missing a route policy for your LB pools in the route-policies. The way it works ATM is that Service advertisement selector needs to match both Service and theCiliumLoadBalancerIPPool, e.g. for this to work:
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPAdvertisement
metadata:
  name: bgp-advertisements-internal
  namespace: kube-system
  labels:
    advertise: bgp-internal
spec:
  advertisements:
    - advertisementType: "PodCIDR"
    - advertisementType: "Service"
      service:
        addresses:
          - LoadBalancerIP
      selector:
        matchExpressions:
          - { key: "service-type", operator: In, values: [ "internal" ] }

You will need to label the CiliumLoadBalancerIPPool from which the service gets VIP with the service-type: internal label as well.
Please try that and let us know if it helped. As this may be a bit counter-intuitive, we will consider removing this requirement.

Either remove it, or document it with a big warning sign. It is actually very confusing.

@rastislavs Tagging the IP pool did it, thank you so much!
I think the documentation is a bit lacking in that regard (or perhaps I missed it).

Cool, thanks for confirming, will keep this issue open until this is either documented or fixed.

Actually, I think I spoke too soon.
It mostly resolved the services not advertised on the internal part, but after removing/readding services, the BGP announcements are still incorrect.
Three nodes are advertising the LoadBalancer IP on internal, where only 2 should, and the public part keeps advertising a service that is removed, and not advertising new services.

@RomLecat nginx-ingress-nginx-dmz-controller seems to have externalTrafficPolicy: Local, for this it would help to see cilium-dbg service list output as well as cilium-dbg bgp routes available ipv4 unicast.

@rastislavs I confirm it is on externalTrafficPolicy: local.
Here you go:
root@k8s-prd-worker-2:/home/cilium# cilium-dbg service list
ID    Frontend                                            Service Type   Backend
1     [ffff:fff:fff:701::2:dceb]:80                       ClusterIP      1 => [ffff:fff:fff:700:0:2:7:2c53]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:7:949c]:80 (active)
2     10.7.14.194:9000                                    ClusterIP      1 => 10.7.10.209:9000 (active)
3     [ffff:fff:fff:701::2:2f19]:9000                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:f872]:9000 (active)
4     [ffff:fff:fff:701::2:fbf2]:6767                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:966c]:6767 (active)
5     [ffff:fff:fff:701::2:fa59]:8083                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:92a7]:8083 (active)
6     [ffff:fff:fff:701::2:bf59]:9402                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:142d]:9402 (active)
7     [ffff:fff:fff:701::2:5bbd]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:7:cbd7]:10250 (active)
8     [ffff:fff:fff:701::2:ae09]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:a:b109]:9443 (active)
9     [ffff:fff:fff:701::2:1]:443                         ClusterIP      1 => [ffff:fff:fff:980::a:3]:6443 (active)
                                                                         2 => [ffff:fff:fff:980::a:1]:6443 (active)
                                                                         3 => [ffff:fff:fff:980::a:2]:6443 (active)
10    [ffff:fff:fff:701::2:efda]:7979                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:5988]:7979 (active)
11    [ffff:fff:fff:701::2:e748]:7979                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:b337]:7979 (active)
12    10.7.13.86:8191                                     ClusterIP      1 => 10.7.8.81:8191 (active)
13    [ffff:fff:fff:701::2:1670]:8191                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:2eac]:8191 (active)
14    [ffff:fff:fff:701::2:b422]:3003                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:4225]:3003 (active)
15    [ffff:fff:fff:701::2:203e]:3001                     ClusterIP
16    [ffff:fff:fff:701::2:c924]:6379                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:e913]:6379 (active)
17    [ffff:fff:fff:701::2:1589]:5432                     ClusterIP      1 => [ffff:fff:fff:700:0:2:8:1698]:5432 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:6e9]:5432 (active)
18    [ffff:fff:fff:701::2:851a]:5432                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:6e9]:5432 (active)
19    [ffff:fff:fff:701::2:f81e]:5432                     ClusterIP      1 => [ffff:fff:fff:700:0:2:8:1698]:5432 (active)
20    [ffff:fff:fff:701::2:8ad0]:8443                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:4382]:8443 (active)
21    [ffff:fff:fff:701::2:9773]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:7:4382]:9443 (active)
22    [ffff:fff:fff:701::2:a2ee]:8600                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:d0d0]:8600 (active)
23    10.7.15.186:3129                                    ClusterIP      1 => 10.7.10.195:3129 (active)
24    [ffff:fff:fff:701::2:153d]:3129                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
25    111.222.229.32:3129                                 LoadBalancer   1 => 10.7.10.195:3129 (active)
26    111.222.229.32:3129/i                               LoadBalancer   1 => 10.7.10.195:3129 (active)
27    [ffff:fff:fff:b002::]:3129                          LoadBalancer   1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
28    [ffff:fff:fff:b002::]:3129/i                        LoadBalancer   1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
29    0.0.0.0:31592                                       NodePort       1 => 10.7.10.195:3129 (active)
30    0.0.0.0:31592/i                                     NodePort       1 => 10.7.10.195:3129 (active)
31    [::]:31592                                          NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
32    [::]:31592/i                                        NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
33    10.4.2.82:31592                                     NodePort       1 => 10.7.10.195:3129 (active)
34    10.4.2.82:31592/i                                   NodePort       1 => 10.7.10.195:3129 (active)
35    10.9.8.22:31592                                     NodePort       1 => 10.7.10.195:3129 (active)
36    10.9.8.22:31592/i                                   NodePort       1 => 10.7.10.195:3129 (active)
37    [ffff:fff:fff:41::82]:31592                         NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
38    [ffff:fff:fff:41::82]:31592/i                       NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
39    [ffff:fff:fff:980::b:2]:31592                       NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
40    [ffff:fff:fff:980::b:2]:31592/i                     NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
41    [fdfa:a109:faec:1000:be24:11ff:feac:919d]:31592     NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
42    [fdfa:a109:faec:1000:be24:11ff:feac:919d]:31592/i   NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
43    [fdfa:a109:faec:1010::dc2]:31592                    NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
44    [fdfa:a109:faec:1010::dc2]:31592/i                  NodePort       1 => [ffff:fff:fff:700:0:2:a:6f09]:3129 (active)
45    [ffff:fff:fff:701::2:8a41]:5800                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:6f09]:5800 (active)
46    [ffff:fff:fff:701::2:5cfb]:8096                     ClusterIP      1 => [ffff:fff:fff:700:0:2:3:501c]:8096 (active)
47    [ffff:fff:fff:701::2:d94f]:443                      ClusterIP      1 => [ffff:fff:fff:980::b:2]:4244 (active)
48    [ffff:fff:fff:701::2:a]:53                          ClusterIP      1 => [ffff:fff:fff:700:0:2:4:95eb]:53 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:1:5286]:53 (active)
49    [ffff:fff:fff:701::2:a]:9153                        ClusterIP      1 => [ffff:fff:fff:700:0:2:4:95eb]:9153 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:1:5286]:9153 (active)
50    [ffff:fff:fff:701::2:ed5e]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:7:f06d]:9443 (active)
51    [ffff:fff:fff:701::2:cdb9]:8080                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:81d7]:8080 (active)
52    [ffff:fff:fff:701::2:bf42]:8080                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:7270]:8080 (active)
53    [ffff:fff:fff:701::2:a51d]:10050                    ClusterIP      1 => [ffff:fff:fff:980::a:2]:10050 (active)
                                                                         2 => [ffff:fff:fff:980::b:3]:10050 (active)
                                                                         3 => [ffff:fff:fff:980::b:1]:10050 (active)
                                                                         4 => [ffff:fff:fff:980::a:1]:10050 (active)
                                                                         5 => [ffff:fff:fff:980::c:2]:10050 (active)
                                                                         6 => [ffff:fff:fff:980::a:3]:10050 (active)
                                                                         7 => [ffff:fff:fff:980::c:1]:10050 (active)
                                                                         8 => [ffff:fff:fff:980::b:2]:10050 (active)
54    10.7.15.168:6980                                    ClusterIP      1 => 10.7.10.185:6980 (active)
55    [ffff:fff:fff:701::2:11c9]:6980                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:db07]:6980 (active)
100   [ffff:fff:fff:701::2:618c]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:8:8147]:8443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:8443 (active)
145   [ffff:fff:fff:701::2:1da3]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:8:e4b1]:8443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:8443 (active)
146   [ffff:fff:fff:701::2:c3a]:8080                      ClusterIP      1 => [ffff:fff:fff:700:0:2:4:a57]:8080 (active)
147   10.7.15.63:8945                                     ClusterIP      1 => 10.7.10.168:8945 (active)
148   10.7.15.110:32400                                   ClusterIP      1 => 10.7.11.234:32400 (active)
149   [ffff:fff:fff:701::2:cb7b]:32400                    ClusterIP      1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
150   10.7.12.108:32400                                   ClusterIP      1 => 10.7.11.234:32400 (active)
151   [ffff:fff:fff:701::2:d010]:32400                    ClusterIP      1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
152   111.222.229.23:32400                                LoadBalancer   1 => 10.7.11.234:32400 (active)
153   [ffff:fff:fff:811::b002:1]:32400                    LoadBalancer   1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
154   [fdfa:a109:faec:1010::dc2]:48374                    NodePort       1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
155   0.0.0.0:48374                                       NodePort       1 => 10.7.11.234:32400 (active)
156   [::]:48374                                          NodePort       1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
157   10.4.2.82:48374                                     NodePort       1 => 10.7.11.234:32400 (active)
158   10.9.8.22:48374                                     NodePort       1 => 10.7.11.234:32400 (active)
159   [ffff:fff:fff:41::82]:48374                         NodePort       1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
160   [ffff:fff:fff:980::b:2]:48374                       NodePort       1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
161   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:48374     NodePort       1 => [ffff:fff:fff:700:0:2:3:9da7]:32400 (active)
162   10.7.15.6:9696                                      ClusterIP      1 => 10.7.8.94:9696 (active)
163   [ffff:fff:fff:701::2:47e8]:9696                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:a00b]:9696 (active)
164   10.7.12.227:7878                                    ClusterIP      1 => 10.7.9.129:7878 (active)
165   [ffff:fff:fff:701::2:135a]:7878                     ClusterIP      1 => [ffff:fff:fff:700:0:2:8:e5a6]:7878 (active)
166   10.7.14.193:8080                                    ClusterIP      1 => 10.7.10.230:8080 (active)
167   [ffff:fff:fff:701::2:f3a8]:8080                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:cc9a]:8080 (active)
168   10.7.13.116:8080                                    ClusterIP      1 => 10.7.8.76:8080 (active)
169   [ffff:fff:fff:701::2:3a78]:8080                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:e32a]:8080 (active)
170   [ffff:fff:fff:701::2:5465]:7011                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:f936]:7011 (active)
171   10.7.15.57:8989                                     ClusterIP      1 => 10.7.10.236:8989 (active)
172   [ffff:fff:fff:701::2:5d93]:8989                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:ae7c]:8989 (active)
173   [ffff:fff:fff:701::2:a8a8]:5432                     ClusterIP      1 => [ffff:fff:fff:700:0:2:8:ac5e]:5432 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:f537]:5432 (active)
174   [ffff:fff:fff:701::2:9547]:5432                     ClusterIP      1 => [ffff:fff:fff:700:0:2:a:f537]:5432 (active)
175   [ffff:fff:fff:701::2:2442]:5432                     ClusterIP      1 => [ffff:fff:fff:700:0:2:8:ac5e]:5432 (active)
176   10.7.13.159:80                                      ClusterIP      1 => 10.7.8.62:80 (active)
177   [ffff:fff:fff:701::2:5283]:80                       ClusterIP      1 => [ffff:fff:fff:700:0:2:7:5683]:80 (active)
178   10.7.13.7:6666                                      ClusterIP      1 => 10.7.8.16:6666 (active)
179   [ffff:fff:fff:701::2:e7cd]:6666                     ClusterIP      1 => [ffff:fff:fff:700:0:2:7:2ba0]:6666 (active)
180   [ffff:fff:fff:701::2:2fa2]:8181                     ClusterIP
248   10.7.15.92:443                                      ClusterIP      1 => 10.7.10.231:443 (active)
                                                                         2 => 10.7.9.242:443 (active)
249   10.7.15.92:80                                       ClusterIP      1 => 10.7.10.231:80 (active)
                                                                         2 => 10.7.9.242:80 (active)
250   [ffff:fff:fff:701::2:627a]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:8:e4b1]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
251   [ffff:fff:fff:701::2:627a]:80                       ClusterIP      1 => [ffff:fff:fff:700:0:2:8:e4b1]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
252   10.9.8.22:30345                                     NodePort       1 => 10.7.10.231:443 (active)
253   10.9.8.22:30345/i                                   NodePort       1 => 10.7.10.231:443 (active)
                                                                         2 => 10.7.9.242:443 (active)
254   [ffff:fff:fff:41::82]:30345                         NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
255   [ffff:fff:fff:41::82]:30345/i                       NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
256   [ffff:fff:fff:980::b:2]:30345                       NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
257   [ffff:fff:fff:980::b:2]:30345/i                     NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
258   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:30345     NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
259   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:30345/i   NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
260   [fdfa:a109:faec:1010::dc2]:30345                    NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
261   [fdfa:a109:faec:1010::dc2]:30345/i                  NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
262   0.0.0.0:30345                                       NodePort       1 => 10.7.10.231:443 (active)
263   0.0.0.0:30345/i                                     NodePort       1 => 10.7.10.231:443 (active)
                                                                         2 => 10.7.9.242:443 (active)
264   [::]:30345                                          NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
265   [::]:30345/i                                        NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
266   10.4.2.82:30345                                     NodePort       1 => 10.7.10.231:443 (active)
267   10.4.2.82:30345/i                                   NodePort       1 => 10.7.10.231:443 (active)
                                                                         2 => 10.7.9.242:443 (active)
268   0.0.0.0:32125                                       NodePort       1 => 10.7.10.231:80 (active)
269   0.0.0.0:32125/i                                     NodePort       1 => 10.7.10.231:80 (active)
                                                                         2 => 10.7.9.242:80 (active)
270   [::]:32125                                          NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
271   [::]:32125/i                                        NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
272   10.4.2.82:32125                                     NodePort       1 => 10.7.10.231:80 (active)
273   10.4.2.82:32125/i                                   NodePort       1 => 10.7.10.231:80 (active)
                                                                         2 => 10.7.9.242:80 (active)
274   10.9.8.22:32125                                     NodePort       1 => 10.7.10.231:80 (active)
275   10.9.8.22:32125/i                                   NodePort       1 => 10.7.10.231:80 (active)
                                                                         2 => 10.7.9.242:80 (active)
276   [ffff:fff:fff:41::82]:32125                         NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
277   [ffff:fff:fff:41::82]:32125/i                       NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
278   [ffff:fff:fff:980::b:2]:32125                       NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
279   [ffff:fff:fff:980::b:2]:32125/i                     NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
280   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:32125     NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
281   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:32125/i   NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
282   [fdfa:a109:faec:1010::dc2]:32125                    NodePort       1 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
283   [fdfa:a109:faec:1010::dc2]:32125/i                  NodePort       1 => [ffff:fff:fff:700:0:2:8:e4b1]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
284   10.9.0.1:80                                         LoadBalancer   1 => 10.7.10.231:80 (active)
285   10.9.0.1:80/i                                       LoadBalancer   1 => 10.7.10.231:80 (active)
                                                                         2 => 10.7.9.242:80 (active)
286   10.9.0.1:443                                        LoadBalancer   1 => 10.7.10.231:443 (active)
287   10.9.0.1:443/i                                      LoadBalancer   1 => 10.7.10.231:443 (active)
                                                                         2 => 10.7.9.242:443 (active)
288   [ffff:fff:fff:981:b002::1]:80                       LoadBalancer   1 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
289   [ffff:fff:fff:981:b002::1]:80/i                     LoadBalancer   1 => [ffff:fff:fff:700:0:2:8:e4b1]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:80 (active)
290   [ffff:fff:fff:981:b002::1]:443                      LoadBalancer   1 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
291   [ffff:fff:fff:981:b002::1]:443/i                    LoadBalancer   1 => [ffff:fff:fff:700:0:2:8:e4b1]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:c2ed]:443 (active)
292   10.7.13.96:443                                      ClusterIP      1 => 10.7.9.141:443 (active)
                                                                         2 => 10.7.10.141:443 (active)
293   10.7.13.96:80                                       ClusterIP      1 => 10.7.9.141:80 (active)
                                                                         2 => 10.7.10.141:80 (active)
294   [ffff:fff:fff:701::2:c1b3]:80                       ClusterIP      1 => [ffff:fff:fff:700:0:2:8:8147]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
295   [ffff:fff:fff:701::2:c1b3]:443                      ClusterIP      1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:8:8147]:443 (active)
296   111.222.229.33:80                                   LoadBalancer   1 => 10.7.10.141:80 (active)
297   111.222.229.33:80/i                                 LoadBalancer   1 => 10.7.9.141:80 (active)
                                                                         2 => 10.7.10.141:80 (active)
298   111.222.229.33:443                                  LoadBalancer   1 => 10.7.10.141:443 (active)
299   111.222.229.33:443/i                                LoadBalancer   1 => 10.7.9.141:443 (active)
                                                                         2 => 10.7.10.141:443 (active)
300   [ffff:fff:fff:b002::1]:80                           LoadBalancer   1 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
301   [ffff:fff:fff:b002::1]:80/i                         LoadBalancer   1 => [ffff:fff:fff:700:0:2:8:8147]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
302   [ffff:fff:fff:b002::1]:443                          LoadBalancer   1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
303   [ffff:fff:fff:b002::1]:443/i                        LoadBalancer   1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:8:8147]:443 (active)
304   [fdfa:a109:faec:1010::dc2]:30328                    NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
305   [fdfa:a109:faec:1010::dc2]:30328/i                  NodePort       1 => [ffff:fff:fff:700:0:2:8:8147]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
306   0.0.0.0:30328                                       NodePort       1 => 10.7.10.141:80 (active)
307   0.0.0.0:30328/i                                     NodePort       1 => 10.7.9.141:80 (active)
                                                                         2 => 10.7.10.141:80 (active)
308   [::]:30328                                          NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
309   [::]:30328/i                                        NodePort       1 => [ffff:fff:fff:700:0:2:8:8147]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
310   10.4.2.82:30328                                     NodePort       1 => 10.7.10.141:80 (active)
311   10.4.2.82:30328/i                                   NodePort       1 => 10.7.9.141:80 (active)
                                                                         2 => 10.7.10.141:80 (active)
312   10.9.8.22:30328                                     NodePort       1 => 10.7.10.141:80 (active)
313   10.9.8.22:30328/i                                   NodePort       1 => 10.7.9.141:80 (active)
                                                                         2 => 10.7.10.141:80 (active)
314   [ffff:fff:fff:41::82]:30328                         NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
315   [ffff:fff:fff:41::82]:30328/i                       NodePort       1 => [ffff:fff:fff:700:0:2:8:8147]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
316   [ffff:fff:fff:980::b:2]:30328                       NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
317   [ffff:fff:fff:980::b:2]:30328/i                     NodePort       1 => [ffff:fff:fff:700:0:2:8:8147]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
318   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:30328     NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
319   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:30328/i   NodePort       1 => [ffff:fff:fff:700:0:2:8:8147]:80 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:a:e179]:80 (active)
320   [::]:31955                                          NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
321   [::]:31955/i                                        NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:8:8147]:443 (active)
322   10.4.2.82:31955                                     NodePort       1 => 10.7.10.141:443 (active)
323   10.4.2.82:31955/i                                   NodePort       1 => 10.7.9.141:443 (active)
                                                                         2 => 10.7.10.141:443 (active)
324   10.9.8.22:31955                                     NodePort       1 => 10.7.10.141:443 (active)
325   10.9.8.22:31955/i                                   NodePort       1 => 10.7.9.141:443 (active)
                                                                         2 => 10.7.10.141:443 (active)
326   [ffff:fff:fff:41::82]:31955                         NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
327   [ffff:fff:fff:41::82]:31955/i                       NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:8:8147]:443 (active)
328   [ffff:fff:fff:980::b:2]:31955                       NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
329   [ffff:fff:fff:980::b:2]:31955/i                     NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:8:8147]:443 (active)
330   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:31955     NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
331   [fdfa:a109:faec:1000:be24:11ff:feac:919d]:31955/i   NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:8:8147]:443 (active)
332   [fdfa:a109:faec:1010::dc2]:31955                    NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
333   [fdfa:a109:faec:1010::dc2]:31955/i                  NodePort       1 => [ffff:fff:fff:700:0:2:a:e179]:443 (active)
                                                                         2 => [ffff:fff:fff:700:0:2:8:8147]:443 (active)
334   0.0.0.0:31955                                       NodePort       1 => 10.7.10.141:443 (active)
335   0.0.0.0:31955/i                                     NodePort       1 => 10.7.9.141:443 (active)
                                                                         2 => 10.7.10.141:443 (active)

root@k8s-prd-worker-2:/home/cilium# cilium-dbg bgp routes available ipv4 unicast
VRouter   Prefix           NextHop   Age      Attrs
64512     10.7.10.128/25   0.0.0.0   55m25s   [{Origin: i} {Nexthop: 0.0.0.0}]
64512     10.9.0.1/32      0.0.0.0   28m32s   [{Origin: i} {Nexthop: 0.0.0.0}]
root@k8s-prd-worker-2:/home/cilium# cilium-dbg bgp routes available ipv6 unicast
VRouter   Prefix                         NextHop   Age      Attrs
64512     ffff:fff:fff:700:0:2:a:0/112   ::        56m28s   [{Origin: i} {MpReach(ipv6-unicast): {Nexthop: ::, NLRIs: [ffff:fff:fff:700:0:2:a:0/112]}}]
64512     ffff:fff:fff:981:b002::1/128   ::        29m35s   [{Origin: i} {MpReach(ipv6-unicast): {Nexthop: ::, NLRIs: [ffff:fff:fff:981:b002::1/128]}}]

Thanks @RomLecat , this indeed seems incorrect based on the information you gave here. IIUC, when cilium agent starts, it tends to advertise correctly, but after you add/modify/delete services, these changes are not reflected?
What I noticed in your sysdump is a lot of k8s resource watch warnings like these:
level=warning msg="github.com/cilium/cilium/pkg/k8s/resource/resource.go:821: watch of *v2.CiliumNode ended with: an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" subsys=klog
level=warning msg="github.com/cilium/cilium/pkg/k8s/resource/resource.go:821: watch of *v1.Service ended with: an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" subsys=klog
level=warning msg="github.com/cilium/cilium/pkg/k8s/resource/resource.go:821: watch of *v1.NetworkPolicy ended with: an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" subsys=klog
level=warning msg="github.com/cilium/cilium/pkg/k8s/resource/resource.go:821: watch of *v2alpha1.CiliumBGPNodeConfig ended with: an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" subsys=klog

These could be the root-cause of missing update events, but I am not sure what may be causing them. Can the k8s apiserver be overloaded? Could you maybe take a look at the k8s apiserver logs?

@rastislavs The errors are mostly happening when I restart the pods, but not once they're started.
This is likely because I restart all pods at once (eg. k -n kube-system delete pod -l k8s-app=cilium).
The API server logs does not show anything significant or abnormal.

Hi @RomLecat, would it be possible to get debug-level logs from Cilium when services are modified?

Hi @harsimran-pabla, sure, here you go:
I created a service with externalTrafficPolicy: cluster, then deleted it:
root@k8s-prd-worker-1:/home/cilium# cilium-dbg monitor -v | grep -vE 'cgroup|flow'
Listening for events on 8 CPUs with 64x4096 of shared memory
Press Ctrl-C to quit
time="2024-08-01T16:01:27Z" level=info msg="Initializing dissection cache..." subsys=monitor
>> Service upserted: {"id":416,"frontend-address":{"ip":"10.7.13.11","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":417,"frontend-address":{"ip":"ffff:fff:fff:701::2:8014","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":418,"frontend-address":{"ip":"10.9.0.2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":419,"frontend-address":{"ip":"ffff:fff:fff:981:b002::2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":420,"frontend-address":{"ip":"ffff:fff:fff:980::b:1","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":421,"frontend-address":{"ip":"fdfa:a109:faec:1000:be24:11ff:fe9c:eb32","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":422,"frontend-address":{"ip":"fdfa:a109:faec:1010::dc1","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":423,"frontend-address":{"ip":"0.0.0.0","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":424,"frontend-address":{"ip":"::","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":425,"frontend-address":{"ip":"10.4.2.81","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":426,"frontend-address":{"ip":"10.9.8.21","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":427,"frontend-address":{"ip":"ffff:fff:fff:41::81","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":416,"frontend-address":{"ip":"10.7.13.11","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":417,"frontend-address":{"ip":"ffff:fff:fff:701::2:8014","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":419,"frontend-address":{"ip":"ffff:fff:fff:981:b002::2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":418,"frontend-address":{"ip":"10.9.0.2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":420,"frontend-address":{"ip":"ffff:fff:fff:980::b:1","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":421,"frontend-address":{"ip":"fdfa:a109:faec:1000:be24:11ff:fe9c:eb32","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":422,"frontend-address":{"ip":"fdfa:a109:faec:1010::dc1","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":423,"frontend-address":{"ip":"0.0.0.0","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":424,"frontend-address":{"ip":"::","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":425,"frontend-address":{"ip":"10.4.2.81","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":426,"frontend-address":{"ip":"10.9.8.21","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":427,"frontend-address":{"ip":"ffff:fff:fff:41::81","port":30777},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":428,"frontend-address":{"ip":"10.7.13.136","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":429,"frontend-address":{"ip":"ffff:fff:fff:701::2:77c3","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":430,"frontend-address":{"ip":"fdfa:a109:faec:1000:be24:11ff:fe9c:eb32","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":431,"frontend-address":{"ip":"fdfa:a109:faec:1010::dc1","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":432,"frontend-address":{"ip":"0.0.0.0","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":433,"frontend-address":{"ip":"::","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":434,"frontend-address":{"ip":"10.4.2.81","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":435,"frontend-address":{"ip":"10.9.8.21","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":436,"frontend-address":{"ip":"ffff:fff:fff:41::81","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":437,"frontend-address":{"ip":"ffff:fff:fff:980::b:1","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":428,"frontend-address":{"ip":"10.7.13.136","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":429,"frontend-address":{"ip":"ffff:fff:fff:701::2:77c3","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":438,"frontend-address":{"ip":"193.228.229.34","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":439,"frontend-address":{"ip":"ffff:fff:fff:b002::2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":433,"frontend-address":{"ip":"::","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":434,"frontend-address":{"ip":"10.4.2.81","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":435,"frontend-address":{"ip":"10.9.8.21","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":436,"frontend-address":{"ip":"ffff:fff:fff:41::81","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":437,"frontend-address":{"ip":"ffff:fff:fff:980::b:1","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":430,"frontend-address":{"ip":"fdfa:a109:faec:1000:be24:11ff:fe9c:eb32","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":431,"frontend-address":{"ip":"fdfa:a109:faec:1010::dc1","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":432,"frontend-address":{"ip":"0.0.0.0","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":428,"frontend-address":{"ip":"10.7.13.136","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":429,"frontend-address":{"ip":"ffff:fff:fff:701::2:77c3","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":438,"frontend-address":{"ip":"193.228.229.34","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":439,"frontend-address":{"ip":"ffff:fff:fff:b002::2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":436,"frontend-address":{"ip":"ffff:fff:fff:41::81","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":437,"frontend-address":{"ip":"ffff:fff:fff:980::b:1","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":430,"frontend-address":{"ip":"fdfa:a109:faec:1000:be24:11ff:fe9c:eb32","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":431,"frontend-address":{"ip":"fdfa:a109:faec:1010::dc1","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":432,"frontend-address":{"ip":"0.0.0.0","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":433,"frontend-address":{"ip":"::","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":434,"frontend-address":{"ip":"10.4.2.81","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":435,"frontend-address":{"ip":"10.9.8.21","port":30967},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service deleted: {"id":416}
>> Service deleted: {"id":417}
>> Service deleted: {"id":425}
>> Service deleted: {"id":426}
>> Service deleted: {"id":427}
>> Service deleted: {"id":420}
>> Service deleted: {"id":421}
>> Service deleted: {"id":422}
>> Service deleted: {"id":423}
>> Service deleted: {"id":424}
>> Service deleted: {"id":418}
>> Service deleted: {"id":419}
>> Service deleted: {"id":428}
>> Service deleted: {"id":429}
>> Service deleted: {"id":432}
>> Service deleted: {"id":433}
>> Service deleted: {"id":434}
>> Service deleted: {"id":435}
>> Service deleted: {"id":436}
>> Service deleted: {"id":437}
>> Service deleted: {"id":430}
>> Service deleted: {"id":431}
>> Service deleted: {"id":438}
>> Service deleted: {"id":439}

It was advertised properly upon creation, but the advertisement remained upon removal.
The same debug from another node, when creating back the service:
root@k8s-prd-master-3:/home/cilium# cilium-dbg monitor -v | grep -vE 'cgroup|flow'
Listening for events on 4 CPUs with 64x4096 of shared memory
Press Ctrl-C to quit
time="2024-08-01T16:05:49Z" level=info msg="Initializing dissection cache..." subsys=monitor
>> Service upserted: {"id":320,"frontend-address":{"ip":"10.7.12.110","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":321,"frontend-address":{"ip":"2001:678:ea8:701::2:ff1b","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":322,"frontend-address":{"ip":"10.9.8.4","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":323,"frontend-address":{"ip":"2001:678:ea8:980::a:3","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":324,"frontend-address":{"ip":"0.0.0.0","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":325,"frontend-address":{"ip":"::","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":320,"frontend-address":{"ip":"10.7.12.110","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":321,"frontend-address":{"ip":"2001:678:ea8:701::2:ff1b","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":326,"frontend-address":{"ip":"10.9.0.2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":327,"frontend-address":{"ip":"2001:678:ea8:981:b002::2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":322,"frontend-address":{"ip":"10.9.8.4","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":323,"frontend-address":{"ip":"2001:678:ea8:980::a:3","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":324,"frontend-address":{"ip":"0.0.0.0","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":325,"frontend-address":{"ip":"::","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":320,"frontend-address":{"ip":"10.7.12.110","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":321,"frontend-address":{"ip":"2001:678:ea8:701::2:ff1b","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":326,"frontend-address":{"ip":"10.9.0.2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":327,"frontend-address":{"ip":"2001:678:ea8:981:b002::2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":323,"frontend-address":{"ip":"2001:678:ea8:980::a:3","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":324,"frontend-address":{"ip":"0.0.0.0","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":325,"frontend-address":{"ip":"::","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":322,"frontend-address":{"ip":"10.9.8.4","port":32014},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test","namespace":"default"}
>> Service upserted: {"id":328,"frontend-address":{"ip":"10.7.12.224","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":329,"frontend-address":{"ip":"2001:678:ea8:701::2:3b05","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":330,"frontend-address":{"ip":"0.0.0.0","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":331,"frontend-address":{"ip":"::","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":332,"frontend-address":{"ip":"10.9.8.4","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":333,"frontend-address":{"ip":"2001:678:ea8:980::a:3","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":328,"frontend-address":{"ip":"10.7.12.224","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":329,"frontend-address":{"ip":"2001:678:ea8:701::2:3b05","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":332,"frontend-address":{"ip":"10.9.8.4","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":333,"frontend-address":{"ip":"2001:678:ea8:980::a:3","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":330,"frontend-address":{"ip":"0.0.0.0","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":331,"frontend-address":{"ip":"::","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":328,"frontend-address":{"ip":"10.7.12.224","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":329,"frontend-address":{"ip":"2001:678:ea8:701::2:3b05","port":80},"backend-addresses":[],"type":"ClusterIP","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":334,"frontend-address":{"ip":"193.228.229.34","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":335,"frontend-address":{"ip":"2001:678:ea8:b002::2","port":80},"backend-addresses":[],"type":"LoadBalancer","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":332,"frontend-address":{"ip":"10.9.8.4","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":333,"frontend-address":{"ip":"2001:678:ea8:980::a:3","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":330,"frontend-address":{"ip":"0.0.0.0","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}
>> Service upserted: {"id":331,"frontend-address":{"ip":"::","port":30952},"backend-addresses":[],"type":"NodePort","traffic-policy":"Cluster","ext-traffic-policy":"Cluster","int-traffic-policy":"Cluster","name":"test-pub","namespace":"default"}

It never advertised the service.

Thanks @RomLecat . This is the cilium monitor output; I was asking for kubectl logs  output. I believe for enabling debug level logs, you'd need to set these helm values
debug:
  enabled: true

Oops, my bad. Here they are on one worker node.
This time the service never came up.
cilium-logs.txt

There are two: test which is internal (on which I'm not really focusing TBH) and test-pub, which is public (different BGP peer). I did not deleted them on the debug, only brought them up, and I did not check "test", only "test-pub" (which never got created).
Sorry, it's probably a bit confusing.

Ack, checking test-pub.

Hi @RomLecat, was this setup working with BGPv1 configuration?

I don't know, I was using NAT back on 1.15.

We have a theory about what could be an issue. We will try to reproduce it locally and provide a fix.

@harsimran-pabla Awesome, thanks for letting me know and for all your help, really appreciate it!

Isn’t this the same problem that I have #34146 ?
you have 2 BGP instances in your cluster config ?

@Jean-Daniel  Yes I have 2 instances. I think your issue is the same as this one.

I don't know if I should make a new issue but mine doesn't even have any peers despite configured.
image: quay.io/cilium/cilium:v1.16.0@sha256:46ffa4ef3cf6d8885dcc4af5963b0683f7d59daa90d49ed9fb68d3b1627fe058

./hack/k8s-cilium-exec.sh cilium-dbg bgp peers
==== detail from pod cilium-mhc9s , on node m1
Local AS   Peer AS   Peer Address   Session   Uptime   Family   Received   Advertised

==== detail from pod cilium-vl6w7 , on node w2
Local AS   Peer AS   Peer Address   Session   Uptime   Family   Received   Advertised

apiVersion: cilium.io/v2alpha1
kind: CiliumBGPClusterConfig
metadata:
  name: cilium-bgp
spec:
  nodeSelector:
    matchLabels:
      kubernetes.io/os: "linux"
  bgpInstances:
    - name: cluster
      localASN: 213021
      peers:
        - name: opnsense
          peerASN: 213021
          peerAddress: fd9d:7a72:44eb:d::1
          peerConfigRef:
            name: cilium-peer-v6
        - name: opnsense-v4
          peerASN: 213021
          peerAddress: 10.0.110.1
          peerConfigRef:
            name: cilium-peer-v4
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPPeerConfig
metadata:
  name: cilium-peer-v4
spec:
  families:
    - afi: ipv4
      safi: unicast
      advertisements:
        matchLabels:
          advertise: bgp
  gracefulRestart:
    enabled: true
    restartTimeSeconds: 15
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPPeerConfig
metadata:
  name: cilium-peer-v6
spec:
  families:
    - afi: ipv6
      safi: unicast
      advertisements:
        matchLabels:
          advertise: bgp
  gracefulRestart:
    enabled: true
    restartTimeSeconds: 15
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPAdvertisement
metadata:
  name: bgp-advertisements
  labels:
    advertise: bgp
spec:
  advertisements:
    - advertisementType: "Service"
      service:
        addresses:
          - ExternalIP
          - LoadBalancerIP
      selector:
        matchExpressions:
          - { key: somekey, operator: NotIn, values: ["never-used-value"] }
    - advertisementType: "PodCIDR"
      selector:
        matchExpressions:
          - { key: somekey, operator: NotIn, values: [ "never-used-value" ] }

@samip5 In your case it seems to be something else; please double-check that bgp control plane is enabled and you have restarted the cilium agents since enabling it. If the issue persists, please open a new issue with a sysdump.
This issue has been identified as a problem with 2 (and more) router instances per node, I was able to reproduce it for both BGPv1 and BGPv2 and will work on a fix.
In the meantime, the workaround is to use only single BGP instance per node. In BGPv2 it is possible to configure per-address-family and per-peer specific advertisements, so unless multiple local ASNs are required for some reason, this should allow for enough flexibility even for more complex setups.

@samip5 In your case it seems to be something else; please double-check that bgp control plane is enabled and you have restarted the cilium agents since enabling it. If the issue persists, please open a new issue with a sysdump.

Okay, seems like something was indeed wonky, but it resolved itself via node reboot.

This still appears to be an issue (on 1.17.0-pre.0 at least).
Using:
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
  name: pool
spec:
  blocks:
  - cidr: "10.235.0.0/18"
  - cidr: "2a10:4741:37:88:3::/80"
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPClusterConfig
metadata:
  name: bgp-spike
spec:
  bgpInstances:
  - name: spike
    localASN: 64513
    peers:
    - name: 305-1700-gw-v4
      peerASN: 64512
      peerAddress: 172.17.51.1
      peerConfigRef:
        name: cilium-peer
    - name: 305-1700-gw-v6
      peerASN: 64512
      peerAddress: 2a10:4741:37:51::1
      peerConfigRef:
        name: cilium-peer
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPPeerConfig
metadata:
  name: cilium-peer
spec:
  families:
  - afi: ipv4
    safi: unicast
    advertisements:
      matchLabels:
        advertise: "bgp"
  - afi: ipv6
    safi: unicast
    advertisements:
      matchLabels:
        advertise: "bgp"
  gracefulRestart:
    enabled: true
    restartTimeSeconds: 15
---
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPAdvertisement
metadata:
  name: bgp-advertisements
  labels:
    advertise: bgp
spec:
  advertisements:
  - advertisementType: "PodCIDR"
  - advertisementType: "Service"
    service:
      addresses:
      - ClusterIP
      - ExternalIP
      - LoadBalancerIP
    selector:
      matchExpressions:
      # All services
      - {key: somekey, operator: NotIn, values: ['never-used-value']}

I get all of my pod and clusterips advertized, but no LoadBalancerIPs.

@starcraft66 as you are using only single BGP instance, you may be hitting a different issue than what was fixed here. Could you please open a new issue and include sysdump in it? Thanks!

@rastislavs just opened https://github.com/cilium/cilium/issues/35121 to track this.

authors: Jean-Daniel RomLecat YutaroHayakawa bodleytunes dswaffordcw fernandesnikhil harsimran-pabla rastislavs samip5 starcraft66