
-------------
1) K3s logs
-------------

Dec 30 09:06:13 truenas k3s[6331]: W1230 09:06:13.570886    6331 genericapiserver.go:660] Skipping API flowcontrol.apiserver.k8s.io/v1alpha1 because it has no resources.
Dec 30 09:06:13 truenas k3s[6331]: W1230 09:06:13.581810    6331 genericapiserver.go:660] Skipping API apps/v1beta2 because it has no resources.
Dec 30 09:06:13 truenas k3s[6331]: W1230 09:06:13.581848    6331 genericapiserver.go:660] Skipping API apps/v1beta1 because it has no resources.
Dec 30 09:06:13 truenas k3s[6331]: W1230 09:06:13.586786    6331 genericapiserver.go:660] Skipping API admissionregistration.k8s.io/v1beta1 because it has no resources.
Dec 30 09:06:13 truenas k3s[6331]: W1230 09:06:13.586818    6331 genericapiserver.go:660] Skipping API admissionregistration.k8s.io/v1alpha1 because it has no resources.
Dec 30 09:06:13 truenas k3s[6331]: W1230 09:06:13.590363    6331 genericapiserver.go:660] Skipping API events.k8s.io/v1beta1 because it has no resources.
Dec 30 09:06:13 truenas k3s[6331]: W1230 09:06:13.611518    6331 genericapiserver.go:660] Skipping API apiregistration.k8s.io/v1beta1 because it has no resources.
Dec 30 09:06:14 truenas k3s[6331]: time="2023-12-30T09:06:14+01:00" level=info msg="Password verified locally for node ix-truenas"
Dec 30 09:06:14 truenas k3s[6331]: time="2023-12-30T09:06:14+01:00" level=info msg="certificate CN=ix-truenas signed by CN=k3s-server-ca@1703767655: notBefore=2023-12-28 12:47:35 +0000 UTC notAfter=2024-12-29 08:06:14 +0000 UTC"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.781326    6331 dynamic_cafile_content.go:157] "Starting controller" name="request-header::/mnt/SSD/ix-applications/k3s/server/tls/request-header-ca.crt"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.781331    6331 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/SSD/ix-applications/k3s/server/tls/client-ca.crt"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.781332    6331 dynamic_serving_content.go:132] "Starting controller" name="serving-cert::/mnt/SSD/ix-applications/k3s/server/tls/serving-kube-apiserver.crt::/mnt/SSD/ix-applications/k3s/server/tls/serving-kube-apiserver.key"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.781899    6331 secure_serving.go:210] Serving securely on 127.0.0.1:6444
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.782438    6331 dynamic_serving_content.go:132] "Starting controller" name="aggregator-proxy-cert::/mnt/SSD/ix-applications/k3s/server/tls/client-auth-proxy.crt::/mnt/SSD/ix-applications/k3s/server/tls/client-auth-proxy.key"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.782571    6331 aggregator.go:150] waiting for initial CRD sync...
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.783142    6331 crdregistration_controller.go:111] Starting crd-autoregister controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.783172    6331 shared_informer.go:270] Waiting for caches to sync for crd-autoregister
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.783178    6331 available_controller.go:423] Starting AvailableConditionController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.782443    6331 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.783691    6331 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.782499    6331 controller.go:83] Starting OpenAPI AggregationController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.786032    6331 apf_controller.go:361] Starting API Priority and Fairness config controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.786408    6331 gc_controller.go:78] Starting apiserver lease garbage collector
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.786669    6331 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.786704    6331 shared_informer.go:270] Waiting for caches to sync for cluster_authentication_trust_controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.786782    6331 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/SSD/ix-applications/k3s/server/tls/client-ca.crt"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.786933    6331 dynamic_cafile_content.go:157] "Starting controller" name="request-header::/mnt/SSD/ix-applications/k3s/server/tls/request-header-ca.crt"
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.787899    6331 controller.go:121] Starting legacy_token_tracking_controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.788142    6331 shared_informer.go:270] Waiting for caches to sync for configmaps
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.788434    6331 apiservice_controller.go:97] Starting APIServiceRegistrationController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.788603    6331 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.790514    6331 customresource_discovery_controller.go:288] Starting DiscoveryController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.790782    6331 controller.go:80] Starting OpenAPI V3 AggregationController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.792455    6331 controller.go:85] Starting OpenAPI controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.793316    6331 controller.go:85] Starting OpenAPI V3 controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.793869    6331 naming_controller.go:291] Starting NamingConditionController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.794458    6331 establishing_controller.go:76] Starting EstablishingController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.795112    6331 nonstructuralschema_controller.go:192] Starting NonStructuralSchemaConditionController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.795744    6331 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.796012    6331 crd_finalizer.go:266] Starting CRDFinalizer
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.869026    6331 controller.go:615] quota admission added evaluator for: leases.coordination.k8s.io
Dec 30 09:06:14 truenas k3s[6331]: time="2023-12-30T09:06:14+01:00" level=info msg="certificate CN=system:node:ix-truenas,O=system:nodes signed by CN=k3s-client-ca@1703767655: notBefore=2023-12-28 12:47:35 +0000 UTC notAfter=2024-12-29 08:06:14 +0000 UTC"
Dec 30 09:06:14 truenas k3s[6331]: E1230 09:06:14.882409    6331 controller.go:163] Error removing old endpoints from kubernetes service: no master IPs were listed in storage, refusing to erase all endpoints for the kubernetes service
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.883025    6331 shared_informer.go:277] Caches are synced for node_authorizer
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.883347    6331 shared_informer.go:277] Caches are synced for crd-autoregister
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.883525    6331 aggregator.go:152] initial CRD sync complete...
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.883895    6331 cache.go:39] Caches are synced for AvailableConditionController controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.884538    6331 autoregister_controller.go:141] Starting autoregister controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.884568    6331 cache.go:32] Waiting for caches to sync for autoregister controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.884590    6331 cache.go:39] Caches are synced for autoregister controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.887180    6331 shared_informer.go:277] Caches are synced for cluster_authentication_trust_controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.891987    6331 cache.go:39] Caches are synced for APIServiceRegistrationController controller
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.895860    6331 apf_controller.go:366] Running API Priority and Fairness config worker
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.895899    6331 apf_controller.go:369] Running API Priority and Fairness periodic rebalancing process
Dec 30 09:06:14 truenas k3s[6331]: I1230 09:06:14.904309    6331 shared_informer.go:277] Caches are synced for configmaps
Dec 30 09:06:15 truenas k3s[6331]: I1230 09:06:15.370300    6331 controller.go:132] OpenAPI AggregationController: action for item k8s_internal_local_delegation_chain_0000000000: Nothing (removed from the queue).
Dec 30 09:06:15 truenas k3s[6331]: I1230 09:06:15.789893    6331 storage_scheduling.go:111] all system priority classes are created successfully or already exist.
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Module overlay was already loaded"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Module br_netfilter was already loaded"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Set sysctl 'net/ipv4/conf/default/forwarding' to 1"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_established' to 86400"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_close_wait' to 3600"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Set sysctl 'net/ipv4/conf/all/forwarding' to 1"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Found nvidia container runtime at /usr/bin/nvidia-container-runtime"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Logging containerd to /mnt/SSD/ix-applications/k3s/agent/containerd/containerd.log"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Running containerd -c /mnt/SSD/ix-applications/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /mnt/SSD/ix-applications/k3s/agent/containerd"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Waiting for cloud-controller-manager privileges to become available"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Kube API server is now running"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="ETCD server is now running"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="k3s is up and running"
Dec 30 09:06:16 truenas k3s[6331]: W1230 09:06:16.834721    6331 feature_gate.go:241] Setting GA feature gate JobTrackingWithFinalizers=true. It will be removed in a future release.
Dec 30 09:06:16 truenas systemd[1]: Started k3s.service - Lightweight Kubernetes.
Dec 30 09:06:16 truenas k3s[6331]: I1230 09:06:16.854592    6331 controllermanager.go:152] Version: v1.26.6+k3s-e18037a7-dirty
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Creating service-controller event broadcaster"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Applying CRD helmcharts.helm.cattle.io"
Dec 30 09:06:16 truenas k3s[6331]: time="2023-12-30T09:06:16+01:00" level=info msg="Applying CRD helmchartconfigs.helm.cattle.io"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Applying CRD addons.k3s.cattle.io"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Creating k3s-supervisor event broadcaster"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing static file: /mnt/SSD/ix-applications/k3s/server/static/charts/traefik-10.19.300.tgz"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing static file: /mnt/SSD/ix-applications/k3s/server/static/charts/traefik-crd-10.19.300.tgz"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/nfs-csi.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/rolebindings.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/smb-csi.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/ccm.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/multus-daemonset.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/zfs-operator.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/coredns.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Writing manifest: /mnt/SSD/ix-applications/k3s/server/manifests/kuberouter.yaml"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Tunnel server egress proxy mode: agent"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting /v1, Kind=Node controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting /v1, Kind=Pod controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting k3s.cattle.io/v1, Kind=Addon controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Creating deploy event broadcaster"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting apps/v1, Kind=DaemonSet controller"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.205214    6331 event.go:294] "Event occurred" object="kube-system/ccm" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/ccm.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting discovery.k8s.io/v1, Kind=EndpointSlice controller"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.255434    6331 controller.go:615] quota admission added evaluator for: rolebindings.rbac.authorization.k8s.io
Dec 30 09:06:17 truenas k3s[6331]: W1230 09:06:17.255556    6331 controllermanager.go:288] "route" is disabled
Dec 30 09:06:17 truenas k3s[6331]: W1230 09:06:17.255581    6331 controllermanager.go:288] "cloud-node" is disabled
Dec 30 09:06:17 truenas k3s[6331]: W1230 09:06:17.255598    6331 controllermanager.go:288] "cloud-node-lifecycle" is disabled
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.256712    6331 controllermanager.go:311] Started "service"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.257985    6331 controller.go:227] Starting service controller
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.258025    6331 shared_informer.go:270] Waiting for caches to sync for service
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.258898    6331 event.go:294] "Event occurred" object="kube-system/ccm" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/ccm.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.264098    6331 controller.go:615] quota admission added evaluator for: addons.k3s.cattle.io
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.274268    6331 event.go:294] "Event occurred" object="kube-system/coredns" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/coredns.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.358743    6331 shared_informer.go:277] Caches are synced for service
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.367056    6331 controller.go:615] quota admission added evaluator for: deployments.apps
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.383183    6331 event.go:294] "Event occurred" object="kube-system/coredns" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/coredns.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.392658    6331 event.go:294] "Event occurred" object="kube-system/kuberouter" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/kuberouter.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting /v1, Kind=Secret controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting /v1, Kind=Node controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=warning msg="Unable to fetch coredns config map: configmaps \"coredns\" not found"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Creating helm-controller event broadcaster"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Cluster dns configmap already exists"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.432450    6331 event.go:294] "Event occurred" object="kube-system/kuberouter" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/kuberouter.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Updating TLS secret for kube-system/k3s-serving (count: 11): map[listener.cattle.io/cn-0.0.0.0:0.0.0.0 listener.cattle.io/cn-127.0.0.1:127.0.0.1 listener.cattle.io/cn-172.17.0.1:172.17.0.1 listener.cattle.io/cn-192.168.0.1:192.168.0.1 listener.cattle.io/cn-__1-f16284:::1 listener.cattle.io/cn-ix-truenas:ix-truenas listener.cattle.io/cn-kubernetes:kubernetes listener.cattle.io/cn-kubernetes.default:kubernetes.default listener.cattle.io/cn-kubernetes.default.svc:kubernetes.default.svc listener.cattle.io/cn-kubernetes.default.svc.cluster.local:kubernetes.default.svc.cluster.local listener.cattle.io/cn-localhost:localhost listener.cattle.io/fingerprint:SHA1=C9BD96783F7D4D07714D165CC30D3C25E5C8C7EF]"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Labels and annotations have been set successfully on node: ix-truenas"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.448461    6331 event.go:294] "Event occurred" object="kube-system/multus-daemonset" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/multus-daemonset.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.488276    6331 event.go:294] "Event occurred" object="kube-system/multus-daemonset" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/multus-daemonset.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.509119    6331 event.go:294] "Event occurred" object="kube-system/nfs-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/nfs-csi.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting /v1, Kind=ConfigMap controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChart controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChartConfig controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting batch/v1, Kind=Job controller"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Starting /v1, Kind=ServiceAccount controller"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.617644    6331 serving.go:355] Generated self-signed cert in-memory
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.719755    6331 event.go:294] "Event occurred" object="kube-system/nfs-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/nfs-csi.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.729706    6331 event.go:294] "Event occurred" object="kube-system/rolebindings" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/rolebindings.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="containerd is now running"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/mnt/SSD/ix-applications/k3s/agent/client-ca.crt --cluster-dns=172.17.0.10 --cluster-domain=cluster.local --config=/etc/rancher/k3s/kubelet_config.yaml --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ix-truenas --kubeconfig=/mnt/SSD/ix-applications/k3s/agent/kubelet.kubeconfig --node-ip=0.0.0.0 --node-labels= --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/mnt/SSD/ix-applications/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/etc/resolv.conf --serialize-image-pulls=false --tls-cert-file=/mnt/SSD/ix-applications/k3s/agent/serving-kubelet.crt --tls-private-key-file=/mnt/SSD/ix-applications/k3s/agent/serving-kubelet.key"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Connecting to proxy" url="wss://127.0.0.1:6443/v1-k3s/connect"
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Handling backend connection request [ix-truenas]"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.759170    6331 event.go:294] "Event occurred" object="kube-system/rolebindings" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/rolebindings.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: Flag --containerd has been deprecated, This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed.
Dec 30 09:06:17 truenas k3s[6331]: Flag --pod-infra-container-image has been deprecated, will be removed in 1.27. Image garbage collector will get sandbox image information from CRI.
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.760882    6331 server.go:197] "--pod-infra-container-image will not be pruned by the image garbage collector in kubelet and should also be set in the remote runtime"
Dec 30 09:06:17 truenas k3s[6331]: Flag --pod-infra-container-image has been deprecated, will be removed in 1.27. Image garbage collector will get sandbox image information from CRI.
Dec 30 09:06:17 truenas k3s[6331]: time="2023-12-30T09:06:17+01:00" level=info msg="Annotations and labels have already set on node: ix-truenas"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.770588    6331 event.go:294] "Event occurred" object="kube-system/smb-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/smb-csi.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.776221    6331 server.go:407] "Kubelet version" kubeletVersion="v1.26.6+k3s-e18037a7-dirty"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.776263    6331 server.go:409] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.779603    6331 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/SSD/ix-applications/k3s/agent/client-ca.crt"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.796456    6331 server.go:654] "--cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.800644    6331 container_manager_linux.go:267] "Container manager verified user specified cgroup-root exists" cgroupRoot=[]
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.800990    6331 container_manager_linux.go:272] "Creating Container Manager object based on Node Config" nodeConfig={RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: KubeletOOMScoreAdj:-999 ContainerRuntime: CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:systemd KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:true NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] CPUManagerPolicy:none CPUManagerPolicyOptions:map[] ExperimentalTopologyManagerScope:container CPUManagerReconcilePeriod:10s ExperimentalMemoryManagerPolicy:None ExperimentalMemoryManagerReservedMemory:[] ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none ExperimentalTopologyManagerPolicyOptions:map[]}
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.802322    6331 topology_manager.go:134] "Creating topology manager with policy per scope" topologyPolicyName="none" topologyScopeName="container"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.802753    6331 container_manager_linux.go:308] "Creating device plugin manager"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.805049    6331 state_mem.go:36] "Initialized new in-memory state store"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.812246    6331 kubelet.go:398] "Attempting to sync node with API server"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.812335    6331 kubelet.go:286] "Adding static pod path" path="/mnt/SSD/ix-applications/k3s/agent/pod-manifests"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.813779    6331 kubelet.go:297] "Adding apiserver pod source"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.814229    6331 apiserver.go:42] "Waiting for node sync before watching apiserver pods"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.814857    6331 apiserver.go:52] "Watching apiserver"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.820775    6331 kuberuntime_manager.go:244] "Container runtime initialized" containerRuntime="containerd" version="truenas/v1.7.1-k3s1" apiVersion="v1"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.829143    6331 nodeshutdown_manager_linux.go:137] "Creating node shutdown manager" shutdownGracePeriodRequested="15s" shutdownGracePeriodCriticalPods="10s" shutdownGracePeriodByPodPriority=[{Priority:0 ShutdownGracePeriodSeconds:5} {Priority:2000000000 ShutdownGracePeriodSeconds:10}]
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.830194    6331 server.go:1181] "Started kubelet"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.831196    6331 server.go:161] "Starting to listen" address="0.0.0.0" port=10250
Dec 30 09:06:17 truenas k3s[6331]: E1230 09:06:17.833549    6331 cri_stats_provider.go:455] "Failed to get the info of the filesystem with mountpoint" err="unable to find data in memory cache" mountpoint="/mnt/SSD/ix-applications/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs"
Dec 30 09:06:17 truenas k3s[6331]: E1230 09:06:17.833824    6331 kubelet.go:1386] "Image garbage collection failed once. Stats initialization may not have completed yet" err="invalid capacity 0 on image filesystem"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.834176    6331 server.go:451] "Adding debug handlers to kubelet server"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.834275    6331 fs_resource_analyzer.go:67] "Starting FS ResourceAnalyzer"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.836230    6331 event.go:294] "Event occurred" object="kube-system/smb-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/smb-csi.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.837648    6331 volume_manager.go:293] "Starting Kubelet Volume Manager"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.837878    6331 desired_state_of_world_populator.go:151] "Desired state populator starts to run"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.878955    6331 event.go:294] "Event occurred" object="kube-system/zfs-operator" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/zfs-operator.yaml\""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.899105    6331 cpu_manager.go:214] "Starting CPU manager" policy="none"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.899141    6331 cpu_manager.go:215] "Reconciling" reconcilePeriod="10s"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.899197    6331 state_mem.go:36] "Initialized new in-memory state store"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.900555    6331 state_mem.go:88] "Updated default CPUSet" cpuSet=""
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.900589    6331 state_mem.go:96] "Updated CPUSet assignments" assignments=map[]
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.900756    6331 policy_none.go:49] "None policy: Start"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.903360    6331 memory_manager.go:169] "Starting memorymanager" policy="None"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.903479    6331 state_mem.go:35] "Initializing new in-memory state store"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.904233    6331 state_mem.go:75] "Updated machine memory state"
Dec 30 09:06:17 truenas k3s[6331]: I1230 09:06:17.996836    6331 plugin_manager.go:118] "Starting Kubelet Plugin Manager"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.002959    6331 kuberuntime_manager.go:1114] "Updating runtime config through cri with podcidr" CIDR="172.16.0.0/16"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.004800    6331 kubelet_network.go:61] "Updating Pod CIDR" originalPodCIDR="" newPodCIDR="172.16.0.0/16"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.081103    6331 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv4
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.114745    6331 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv6
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.116335    6331 status_manager.go:176] "Starting to sync pod status with apiserver"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.116461    6331 kubelet.go:2113] "Starting kubelet main sync loop"
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.116611    6331 kubelet.go:2137] "Skipping pod synchronization" err="PLEG is not healthy: pleg has yet to be successful"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.164783    6331 kubelet_node_status.go:70] "Attempting to register node" node="ix-truenas"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.180064    6331 kubelet_node_status.go:108] "Node was previously registered" node="ix-truenas"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.180226    6331 kubelet_node_status.go:73] "Successfully registered node" node="ix-truenas"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.218759    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.220925    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.221089    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.221505    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.221564    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.221624    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.221741    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.221868    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.221983    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222146    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222269    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222330    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222450    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222509    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222746    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222873    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.222997    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223113    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223244    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223347    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223487    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223609    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223669    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223795    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223891    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223936    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="9b02bc2ee44e19d18cd68470131ff6780f9f61623dfa37a356ce21101301b63c"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.223983    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="05489193a9d4d45961fcfbc326349ad56cbf50190967a121441446cbf4f5fd3e"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224040    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="42ecc814a22efc4b61eb7fd0b2efc6d21f6b84cf801b94eca8e823c91c460a7d"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224080    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="ec001912ab15dacaa89e33bc6d3b4be11e994091bcde2620224ea1ae209a5eb8"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224119    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="1efd8d34cbf6ca321d9cb14b9c2093e77022d8a20794edb8438c29eeded8c443"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224161    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="e119d76c3885a0e6459ff8a57c49e2842e8ee498279aff5f56a98c83f926b605"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224201    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="17ead4815e76dccb43c97598d0187e696cca7b8491fd20685fdb072e2b399701"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224239    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="f75a75f3a13e29fea2c045233ab220ed231c784c2bc77612d5a0c8a75921af4b"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224274    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="f507e3764c5c3e39c9826443af45da962894801a63f11f5bda41050ae9c49479"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224315    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="01491487dde7297ef7e3e571ccc0e24064e9a65feb9160385ee8632794264b15"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224365    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224409    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="49e1fdcbf51d1a289398a2bad756bd9c709812c2b631256909ba41dded4888e6"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224449    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="911d3b12273935b6282be8d03209e8ace878f104217c1ef3e2ab7ee088e3ee48"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224486    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224523    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224562    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224603    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="a4ba5c1992af155a76ef5e99b4f43a35723b18f3d59d02c54cbd36f4db815b08"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.224653    6331 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="1c1f01c19905b4d340c50df01cf1591319d3fdd8ed2e758e3413cb42659054cf"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.246321    6331 desired_state_of_world_populator.go:159] "Finished populating initial desired state of world"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.255759    6331 server.go:152] "Starting Kubernetes Scheduler" version="v1.26.6+k3s-e18037a7-dirty"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.255793    6331 server.go:154] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261547    6331 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261547    6331 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261601    6331 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261582    6331 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261673    6331 secure_serving.go:210] Serving securely on 127.0.0.1:10259
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261772    6331 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261868    6331 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.261891    6331 shared_informer.go:270] Waiting for caches to sync for RequestHeaderAuthRequestController
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.342593    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-registration-dir\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343478    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-1\") pod \"duplicati-ix-chart-6c986975cb-tbnph\" (UID: \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343586    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-config\" (UniqueName: \"kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config\") pod \"nvidia-device-plugin-daemonset-bl5th\" (UID: \"aa9b377b-bd02-4e51-83ed-376a36c0d803\") " pod="kube-system/nvidia-device-plugin-daemonset-bl5th"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343653    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-p7cp4\" (UniqueName: \"kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4\") pod \"duplicati-ix-chart-6c986975cb-787sr\" (UID: \"ea325533-21c4-494e-b08f-ee25cb2a136f\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-787sr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343721    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-plugin\" (UniqueName: \"kubernetes.io/host-path/aa9b377b-bd02-4e51-83ed-376a36c0d803-device-plugin\") pod \"nvidia-device-plugin-daemonset-bl5th\" (UID: \"aa9b377b-bd02-4e51-83ed-376a36c0d803\") " pod="kube-system/nvidia-device-plugin-daemonset-bl5th"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343779    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-2ccgt\" (UniqueName: \"kubernetes.io/projected/e545eb4a-9b4c-426e-b86e-e2629dd4ea7f-kube-api-access-2ccgt\") pod \"snapshot-controller-546868dfb4-l2fdd\" (UID: \"e545eb4a-9b4c-426e-b86e-e2629dd4ea7f\") " pod="kube-system/snapshot-controller-546868dfb4-l2fdd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343832    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"mountpoint-dir\" (UniqueName: \"kubernetes.io/host-path/4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e-mountpoint-dir\") pod \"csi-smb-node-j9tdh\" (UID: \"4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e\") " pod="kube-system/csi-smb-node-j9tdh"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343876    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-577zh\" (UniqueName: \"kubernetes.io/projected/7c61dd38-d4e4-4a38-a0eb-48077635fa65-kube-api-access-577zh\") pod \"openebs-zfs-controller-0\" (UID: \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\") " pod="kube-system/openebs-zfs-controller-0"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343915    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-1\") pod \"duplicati-ix-chart-6c986975cb-8t9rq\" (UID: \"46548d19-6f35-4e2a-a247-dbad3afad7d4\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.343983    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/7c61dd38-d4e4-4a38-a0eb-48077635fa65-socket-dir\") pod \"openebs-zfs-controller-0\" (UID: \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\") " pod="kube-system/openebs-zfs-controller-0"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344038    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"logs\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-logs\") pod \"unifi-controller-85b5667bb-qfv5n\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-qfv5n"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344093    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-0\") pod \"duplicati-ix-chart-6c986975cb-st8nr\" (UID: \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344152    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-mb77q\" (UniqueName: \"kubernetes.io/projected/cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c-kube-api-access-mb77q\") pod \"snapshot-controller-546868dfb4-f5hbl\" (UID: \"cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c\") " pod="kube-system/snapshot-controller-546868dfb4-f5hbl"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344212    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-kwpvz\" (UniqueName: \"kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz\") pod \"duplicati-ix-chart-6c986975cb-8t9rq\" (UID: \"46548d19-6f35-4e2a-a247-dbad3afad7d4\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344267    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-1\") pod \"duplicati-ix-chart-6c986975cb-s9wr5\" (UID: \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344316    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344371    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-7xcxs\" (UniqueName: \"kubernetes.io/projected/aa9b377b-bd02-4e51-83ed-376a36c0d803-kube-api-access-7xcxs\") pod \"nvidia-device-plugin-daemonset-bl5th\" (UID: \"aa9b377b-bd02-4e51-83ed-376a36c0d803\") " pod="kube-system/nvidia-device-plugin-daemonset-bl5th"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344444    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cert\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-cert\") pod \"unifi-controller-85b5667bb-qfv5n\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-qfv5n"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344500    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/4737d07c-4267-43f0-8177-6d8ca1392ee6-pods-mount-dir\") pod \"csi-nfs-controller-7b74694749-7w5ts\" (UID: \"4737d07c-4267-43f0-8177-6d8ca1392ee6\") " pod="kube-system/csi-nfs-controller-7b74694749-7w5ts"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344555    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/679c0911-9c04-4dcc-94a9-0879fbfe8c3e-pods-mount-dir\") pod \"csi-nfs-node-q2tpx\" (UID: \"679c0911-9c04-4dcc-94a9-0879fbfe8c3e\") " pod="kube-system/csi-nfs-node-q2tpx"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344611    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/host-path/679c0911-9c04-4dcc-94a9-0879fbfe8c3e-socket-dir\") pod \"csi-nfs-node-q2tpx\" (UID: \"679c0911-9c04-4dcc-94a9-0879fbfe8c3e\") " pod="kube-system/csi-nfs-node-q2tpx"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344672    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-volume-duplicati-duplicati-config\") pod \"duplicati-ix-chart-6c986975cb-tbnph\" (UID: \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344721    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/31dfa589-323f-4aec-9109-a2f7065d90c7-socket-dir\") pod \"csi-smb-controller-7fbbb8fb6f-hgmnb\" (UID: \"31dfa589-323f-4aec-9109-a2f7065d90c7\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-hgmnb"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344800    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-0\") pod \"duplicati-ix-chart-6c986975cb-k7ndd\" (UID: \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344880    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-9dr4d\" (UniqueName: \"kubernetes.io/projected/4737d07c-4267-43f0-8177-6d8ca1392ee6-kube-api-access-9dr4d\") pod \"csi-nfs-controller-7b74694749-7w5ts\" (UID: \"4737d07c-4267-43f0-8177-6d8ca1392ee6\") " pod="kube-system/csi-nfs-controller-7b74694749-7w5ts"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344920    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-plugin-dir\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.344975    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-tmp\") pod \"unifi-controller-85b5667bb-qfv5n\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-qfv5n"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345035    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/4737d07c-4267-43f0-8177-6d8ca1392ee6-socket-dir\") pod \"csi-nfs-controller-7b74694749-7w5ts\" (UID: \"4737d07c-4267-43f0-8177-6d8ca1392ee6\") " pod="kube-system/csi-nfs-controller-7b74694749-7w5ts"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345102    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-host-root\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345161    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-volume-duplicati-duplicati-config\") pod \"duplicati-ix-chart-6c986975cb-k7ndd\" (UID: \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345233    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-volume-duplicati-duplicati-config\") pod \"duplicati-ix-chart-6c986975cb-s9wr5\" (UID: \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345291    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-xbmql\" (UniqueName: \"kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql\") pod \"duplicati-ix-chart-6c986975cb-s9wr5\" (UID: \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345357    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-0\") pod \"duplicati-ix-chart-6c986975cb-tbnph\" (UID: \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345427    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-volume-duplicati-duplicati-config\") pod \"duplicati-ix-chart-6c986975cb-787sr\" (UID: \"ea325533-21c4-494e-b08f-ee25cb2a136f\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-787sr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345485    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-rtqfc\" (UniqueName: \"kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc\") pod \"duplicati-ix-chart-6c986975cb-k7ndd\" (UID: \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345535    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/679c0911-9c04-4dcc-94a9-0879fbfe8c3e-registration-dir\") pod \"csi-nfs-node-q2tpx\" (UID: \"679c0911-9c04-4dcc-94a9-0879fbfe8c3e\") " pod="kube-system/csi-nfs-node-q2tpx"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345609    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-encr-keys\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345667    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-0\") pod \"duplicati-ix-chart-6c986975cb-787sr\" (UID: \"ea325533-21c4-494e-b08f-ee25cb2a136f\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-787sr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345725    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-45b7k\" (UniqueName: \"kubernetes.io/projected/25715d19-0754-4b80-9a23-07c7d6de4edf-kube-api-access-45b7k\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345797    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-1\") pod \"duplicati-ix-chart-6c986975cb-k7ndd\" (UID: \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345877    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-volume-duplicati-duplicati-config\") pod \"duplicati-ix-chart-6c986975cb-8t9rq\" (UID: \"46548d19-6f35-4e2a-a247-dbad3afad7d4\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345920    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-device-dir\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.345988    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-volume-duplicati-duplicati-config\") pod \"duplicati-ix-chart-6c986975cb-st8nr\" (UID: \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346033    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-z8lf2\" (UniqueName: \"kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2\") pod \"duplicati-ix-chart-6c986975cb-st8nr\" (UID: \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346067    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/host-path/4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e-socket-dir\") pod \"csi-smb-node-j9tdh\" (UID: \"4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e\") " pod="kube-system/csi-smb-node-j9tdh"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346124    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e-registration-dir\") pod \"csi-smb-node-j9tdh\" (UID: \"4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e\") " pod="kube-system/csi-smb-node-j9tdh"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346184    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-0\") pod \"duplicati-ix-chart-6c986975cb-s9wr5\" (UID: \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346231    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-0\") pod \"duplicati-ix-chart-6c986975cb-8t9rq\" (UID: \"46548d19-6f35-4e2a-a247-dbad3afad7d4\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346287    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-7qmp9\" (UniqueName: \"kubernetes.io/projected/679c0911-9c04-4dcc-94a9-0879fbfe8c3e-kube-api-access-7qmp9\") pod \"csi-nfs-node-q2tpx\" (UID: \"679c0911-9c04-4dcc-94a9-0879fbfe8c3e\") " pod="kube-system/csi-nfs-node-q2tpx"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346323    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"data\" (UniqueName: \"kubernetes.io/host-path/87257916-cd95-49d3-bb9a-2ca265e41e37-data\") pod \"unifi-controller-85b5667bb-qfv5n\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-qfv5n"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346377    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-ldqkp\" (UniqueName: \"kubernetes.io/projected/31dfa589-323f-4aec-9109-a2f7065d90c7-kube-api-access-ldqkp\") pod \"csi-smb-controller-7fbbb8fb6f-hgmnb\" (UID: \"31dfa589-323f-4aec-9109-a2f7065d90c7\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-hgmnb"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346448    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-1\") pod \"duplicati-ix-chart-6c986975cb-st8nr\" (UID: \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346523    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-1\") pod \"duplicati-ix-chart-6c986975cb-787sr\" (UID: \"ea325533-21c4-494e-b08f-ee25cb2a136f\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-787sr"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346593    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-m49k6\" (UniqueName: \"kubernetes.io/projected/4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e-kube-api-access-m49k6\") pod \"csi-smb-node-j9tdh\" (UID: \"4c9187a2-5ad2-4c5a-81b2-d433eaf86d8e\") " pod="kube-system/csi-smb-node-j9tdh"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346650    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-pods-mount-dir\") pod \"openebs-zfs-node-trztd\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") " pod="kube-system/openebs-zfs-node-trztd"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346710    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-hrpkr\" (UniqueName: \"kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr\") pod \"duplicati-ix-chart-6c986975cb-tbnph\" (UID: \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\") " pod="ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.346753    6331 reconciler.go:41] "Reconciler: start to sync state"
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.362303    6331 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.362303    6331 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.362364    6331 shared_informer.go:277] Caches are synced for RequestHeaderAuthRequestController
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.456288    6331 configmap.go:199] Couldn't get configMap kube-system/nvidia-device-plugin-config: object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.456381    6331 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.456887    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config podName:aa9b377b-bd02-4e51-83ed-376a36c0d803 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:18.956397189 +0100 CET m=+8.072515633 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "plugin-config" (UniqueName: "kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config") pod "nvidia-device-plugin-daemonset-bl5th" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803") : object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.456937    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs podName:25715d19-0754-4b80-9a23-07c7d6de4edf nodeName:}" failed. No retries permitted until 2023-12-30 09:06:18.95691078 +0100 CET m=+8.073029215 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs") pod "openebs-zfs-node-trztd" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf") : object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:18 truenas k3s[6331]: time="2023-12-30T09:06:18+01:00" level=info msg="Stopped tunnel to 127.0.0.1:6443"
Dec 30 09:06:18 truenas k3s[6331]: time="2023-12-30T09:06:18+01:00" level=info msg="Connecting to proxy" url="wss://192.168.0.1:6443/v1-k3s/connect"
Dec 30 09:06:18 truenas k3s[6331]: time="2023-12-30T09:06:18+01:00" level=info msg="Proxy done" err="context canceled" url="wss://127.0.0.1:6443/v1-k3s/connect"
Dec 30 09:06:18 truenas k3s[6331]: time="2023-12-30T09:06:18+01:00" level=info msg="error in remotedialer server [400]: websocket: close 1006 (abnormal closure): unexpected EOF"
Dec 30 09:06:18 truenas k3s[6331]: time="2023-12-30T09:06:18+01:00" level=info msg="Handling backend connection request [ix-truenas]"
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.958959    6331 configmap.go:199] Couldn't get configMap kube-system/nvidia-device-plugin-config: object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.959031    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config podName:aa9b377b-bd02-4e51-83ed-376a36c0d803 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:19.959012624 +0100 CET m=+9.075131040 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "plugin-config" (UniqueName: "kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config") pod "nvidia-device-plugin-daemonset-bl5th" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803") : object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.959041    6331 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:18 truenas k3s[6331]: E1230 09:06:18.959112    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs podName:25715d19-0754-4b80-9a23-07c7d6de4edf nodeName:}" failed. No retries permitted until 2023-12-30 09:06:19.95909008 +0100 CET m=+9.075208498 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs") pod "openebs-zfs-node-trztd" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf") : object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:18 truenas k3s[6331]: I1230 09:06:18.999074    6331 serving.go:355] Generated self-signed cert in-memory
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180173    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a\": cni plugin not initialized" podSandboxID="e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180189    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16\": cni plugin not initialized" podSandboxID="d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180217    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455\": cni plugin not initialized" podSandboxID="21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180246    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16}
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180180    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3\": cni plugin not initialized" podSandboxID="2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180271    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455}
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180293    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3}
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180231    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a}
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180180    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61\": cni plugin not initialized" podSandboxID="565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180517    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61}
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180834    6331 kubelet.go:1899] failed to "KillPodSandbox" for "e545eb4a-9b4c-426e-b86e-e2629dd4ea7f" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61\": cni plugin not initialized"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180844    6331 kubelet.go:1899] failed to "KillPodSandbox" for "7c61dd38-d4e4-4a38-a0eb-48077635fa65" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3\": cni plugin not initialized"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180853    6331 kubelet.go:1899] failed to "KillPodSandbox" for "aa9b377b-bd02-4e51-83ed-376a36c0d803" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a\": cni plugin not initialized"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180858    6331 kubelet.go:1899] failed to "KillPodSandbox" for "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455\": cni plugin not initialized"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180909    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"e545eb4a-9b4c-426e-b86e-e2629dd4ea7f\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61\\\": cni plugin not initialized\"" pod="kube-system/snapshot-controller-546868dfb4-l2fdd" podUID=e545eb4a-9b4c-426e-b86e-e2629dd4ea7f
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.180970    6331 kubelet.go:1899] failed to "KillPodSandbox" for "cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16\": cni plugin not initialized"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.182232    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3\\\": cni plugin not initialized\"" pod="kube-system/openebs-zfs-controller-0" podUID=7c61dd38-d4e4-4a38-a0eb-48077635fa65
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.183363    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"aa9b377b-bd02-4e51-83ed-376a36c0d803\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a\\\": cni plugin not initialized\"" pod="kube-system/nvidia-device-plugin-daemonset-bl5th" podUID=aa9b377b-bd02-4e51-83ed-376a36c0d803
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.184485    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455\\\": cni plugin not initialized\"" pod="ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5" podUID=5f64faad-9fcf-478b-a4a5-e1e8e9e5c409
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.185592    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16\\\": cni plugin not initialized\"" pod="kube-system/snapshot-controller-546868dfb4-f5hbl" podUID=cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.223390    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.223442    6331 projected.go:198] Error preparing data for projected volume kube-api-access-rtqfc for pod ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.223534    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc podName:7ee71e90-c82b-41b2-b1eb-7b58096f348a nodeName:}" failed. No retries permitted until 2023-12-30 09:06:19.723505147 +0100 CET m=+8.839623575 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-rtqfc" (UniqueName: "kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc") pod "duplicati-ix-chart-6c986975cb-k7ndd" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.261763    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-9dr4d\" (UniqueName: \"kubernetes.io/projected/4737d07c-4267-43f0-8177-6d8ca1392ee6-kube-api-access-9dr4d\") pod \"4737d07c-4267-43f0-8177-6d8ca1392ee6\" (UID: \"4737d07c-4267-43f0-8177-6d8ca1392ee6\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.261825    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-host-root\") pod \"25715d19-0754-4b80-9a23-07c7d6de4edf\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.261871    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-device-dir\") pod \"25715d19-0754-4b80-9a23-07c7d6de4edf\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.261919    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-tmp\") pod \"87257916-cd95-49d3-bb9a-2ca265e41e37\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.261973    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"logs\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-logs\") pod \"87257916-cd95-49d3-bb9a-2ca265e41e37\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262023    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/4737d07c-4267-43f0-8177-6d8ca1392ee6-pods-mount-dir\") pod \"4737d07c-4267-43f0-8177-6d8ca1392ee6\" (UID: \"4737d07c-4267-43f0-8177-6d8ca1392ee6\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262074    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/31dfa589-323f-4aec-9109-a2f7065d90c7-socket-dir\") pod \"31dfa589-323f-4aec-9109-a2f7065d90c7\" (UID: \"31dfa589-323f-4aec-9109-a2f7065d90c7\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262125    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-plugin-dir\") pod \"25715d19-0754-4b80-9a23-07c7d6de4edf\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262176    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-encr-keys\") pod \"25715d19-0754-4b80-9a23-07c7d6de4edf\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262233    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-registration-dir\") pod \"25715d19-0754-4b80-9a23-07c7d6de4edf\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262294    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-pods-mount-dir\") pod \"25715d19-0754-4b80-9a23-07c7d6de4edf\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262351    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"cert\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-cert\") pod \"87257916-cd95-49d3-bb9a-2ca265e41e37\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262402    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"data\" (UniqueName: \"kubernetes.io/host-path/87257916-cd95-49d3-bb9a-2ca265e41e37-data\") pod \"87257916-cd95-49d3-bb9a-2ca265e41e37\" (UID: \"87257916-cd95-49d3-bb9a-2ca265e41e37\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.262467    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/4737d07c-4267-43f0-8177-6d8ca1392ee6-socket-dir\") pod \"4737d07c-4267-43f0-8177-6d8ca1392ee6\" (UID: \"4737d07c-4267-43f0-8177-6d8ca1392ee6\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263199    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-device-dir" (OuterVolumeSpecName: "device-dir") pod "25715d19-0754-4b80-9a23-07c7d6de4edf" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf"). InnerVolumeSpecName "device-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263287    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-host-root" (OuterVolumeSpecName: "host-root") pod "25715d19-0754-4b80-9a23-07c7d6de4edf" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf"). InnerVolumeSpecName "host-root". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263344    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-encr-keys" (OuterVolumeSpecName: "encr-keys") pod "25715d19-0754-4b80-9a23-07c7d6de4edf" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf"). InnerVolumeSpecName "encr-keys". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: W1230 09:06:19.263357    6331 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/4737d07c-4267-43f0-8177-6d8ca1392ee6/volumes/kubernetes.io~empty-dir/socket-dir: clearQuota called, but quotas disabled
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263389    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-registration-dir" (OuterVolumeSpecName: "registration-dir") pod "25715d19-0754-4b80-9a23-07c7d6de4edf" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf"). InnerVolumeSpecName "registration-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263442    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-pods-mount-dir" (OuterVolumeSpecName: "pods-mount-dir") pod "25715d19-0754-4b80-9a23-07c7d6de4edf" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf"). InnerVolumeSpecName "pods-mount-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263487    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/87257916-cd95-49d3-bb9a-2ca265e41e37-data" (OuterVolumeSpecName: "data") pod "87257916-cd95-49d3-bb9a-2ca265e41e37" (UID: "87257916-cd95-49d3-bb9a-2ca265e41e37"). InnerVolumeSpecName "data". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263532    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/4737d07c-4267-43f0-8177-6d8ca1392ee6-socket-dir" (OuterVolumeSpecName: "socket-dir") pod "4737d07c-4267-43f0-8177-6d8ca1392ee6" (UID: "4737d07c-4267-43f0-8177-6d8ca1392ee6"). InnerVolumeSpecName "socket-dir". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263536    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/4737d07c-4267-43f0-8177-6d8ca1392ee6-pods-mount-dir" (OuterVolumeSpecName: "pods-mount-dir") pod "4737d07c-4267-43f0-8177-6d8ca1392ee6" (UID: "4737d07c-4267-43f0-8177-6d8ca1392ee6"). InnerVolumeSpecName "pods-mount-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: W1230 09:06:19.263643    6331 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/87257916-cd95-49d3-bb9a-2ca265e41e37/volumes/kubernetes.io~empty-dir/tmp: clearQuota called, but quotas disabled
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263831    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-tmp" (OuterVolumeSpecName: "tmp") pod "87257916-cd95-49d3-bb9a-2ca265e41e37" (UID: "87257916-cd95-49d3-bb9a-2ca265e41e37"). InnerVolumeSpecName "tmp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.263886    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-plugin-dir" (OuterVolumeSpecName: "plugin-dir") pod "25715d19-0754-4b80-9a23-07c7d6de4edf" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf"). InnerVolumeSpecName "plugin-dir". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: W1230 09:06:19.264058    6331 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/87257916-cd95-49d3-bb9a-2ca265e41e37/volumes/kubernetes.io~empty-dir/cert: clearQuota called, but quotas disabled
Dec 30 09:06:19 truenas k3s[6331]: W1230 09:06:19.264127    6331 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/31dfa589-323f-4aec-9109-a2f7065d90c7/volumes/kubernetes.io~empty-dir/socket-dir: clearQuota called, but quotas disabled
Dec 30 09:06:19 truenas k3s[6331]: W1230 09:06:19.264182    6331 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/87257916-cd95-49d3-bb9a-2ca265e41e37/volumes/kubernetes.io~empty-dir/logs: clearQuota called, but quotas disabled
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.264197    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-cert" (OuterVolumeSpecName: "cert") pod "87257916-cd95-49d3-bb9a-2ca265e41e37" (UID: "87257916-cd95-49d3-bb9a-2ca265e41e37"). InnerVolumeSpecName "cert". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.264272    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/31dfa589-323f-4aec-9109-a2f7065d90c7-socket-dir" (OuterVolumeSpecName: "socket-dir") pod "31dfa589-323f-4aec-9109-a2f7065d90c7" (UID: "31dfa589-323f-4aec-9109-a2f7065d90c7"). InnerVolumeSpecName "socket-dir". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.264342    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-logs" (OuterVolumeSpecName: "logs") pod "87257916-cd95-49d3-bb9a-2ca265e41e37" (UID: "87257916-cd95-49d3-bb9a-2ca265e41e37"). InnerVolumeSpecName "logs". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.265744    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/4737d07c-4267-43f0-8177-6d8ca1392ee6-kube-api-access-9dr4d" (OuterVolumeSpecName: "kube-api-access-9dr4d") pod "4737d07c-4267-43f0-8177-6d8ca1392ee6" (UID: "4737d07c-4267-43f0-8177-6d8ca1392ee6"). InnerVolumeSpecName "kube-api-access-9dr4d". PluginName "kubernetes.io/projected", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363763    6331 reconciler_common.go:295] "Volume detached for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-registration-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363815    6331 reconciler_common.go:295] "Volume detached for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-pods-mount-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363835    6331 reconciler_common.go:295] "Volume detached for volume \"cert\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-cert\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363854    6331 reconciler_common.go:295] "Volume detached for volume \"data\" (UniqueName: \"kubernetes.io/host-path/87257916-cd95-49d3-bb9a-2ca265e41e37-data\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363874    6331 reconciler_common.go:295] "Volume detached for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/4737d07c-4267-43f0-8177-6d8ca1392ee6-socket-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363897    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-9dr4d\" (UniqueName: \"kubernetes.io/projected/4737d07c-4267-43f0-8177-6d8ca1392ee6-kube-api-access-9dr4d\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363917    6331 reconciler_common.go:295] "Volume detached for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-host-root\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363936    6331 reconciler_common.go:295] "Volume detached for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-device-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363955    6331 reconciler_common.go:295] "Volume detached for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-tmp\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363974    6331 reconciler_common.go:295] "Volume detached for volume \"logs\" (UniqueName: \"kubernetes.io/empty-dir/87257916-cd95-49d3-bb9a-2ca265e41e37-logs\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.363994    6331 reconciler_common.go:295] "Volume detached for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/4737d07c-4267-43f0-8177-6d8ca1392ee6-pods-mount-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.364013    6331 reconciler_common.go:295] "Volume detached for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/31dfa589-323f-4aec-9109-a2f7065d90c7-socket-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.364032    6331 reconciler_common.go:295] "Volume detached for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-plugin-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.364051    6331 reconciler_common.go:295] "Volume detached for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/25715d19-0754-4b80-9a23-07c7d6de4edf-encr-keys\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.464213    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-45b7k\" (UniqueName: \"kubernetes.io/projected/25715d19-0754-4b80-9a23-07c7d6de4edf-kube-api-access-45b7k\") pod \"25715d19-0754-4b80-9a23-07c7d6de4edf\" (UID: \"25715d19-0754-4b80-9a23-07c7d6de4edf\") "
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.466452    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/25715d19-0754-4b80-9a23-07c7d6de4edf-kube-api-access-45b7k" (OuterVolumeSpecName: "kube-api-access-45b7k") pod "25715d19-0754-4b80-9a23-07c7d6de4edf" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf"). InnerVolumeSpecName "kube-api-access-45b7k". PluginName "kubernetes.io/projected", VolumeGidValue ""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.508158    6331 event.go:294] "Event occurred" object="ix-truenas" fieldPath="" kind="Node" apiVersion="" type="Normal" reason="NodePasswordValidationComplete" message="Deferred node password secret validation complete"
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.565797    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-45b7k\" (UniqueName: \"kubernetes.io/projected/25715d19-0754-4b80-9a23-07c7d6de4edf-kube-api-access-45b7k\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.613114    6331 request.go:690] Waited for 1.157382419s due to client-side throttling, not priority and fairness, request: POST:https://127.0.0.1:6443/api/v1/namespaces/ix-duplicati/serviceaccounts/default/token
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.624479    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.624504    6331 projected.go:198] Error preparing data for projected volume kube-api-access-z8lf2 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.624563    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2 podName:a8fadcd0-3556-42ea-9ed3-b43c57dbed99 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:20.12454434 +0100 CET m=+9.240662757 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-z8lf2" (UniqueName: "kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2") pod "duplicati-ix-chart-6c986975cb-st8nr" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.740234    6331 event.go:294] "Event occurred" object="kube-system/zfs-operator" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/SSD/ix-applications/k3s/server/manifests/zfs-operator.yaml\""
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.767135    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.767166    6331 projected.go:198] Error preparing data for projected volume kube-api-access-rtqfc for pod ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.767243    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc podName:7ee71e90-c82b-41b2-b1eb-7b58096f348a nodeName:}" failed. No retries permitted until 2023-12-30 09:06:20.767216865 +0100 CET m=+9.883335291 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-rtqfc" (UniqueName: "kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc") pod "duplicati-ix-chart-6c986975cb-k7ndd" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.819623    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.819651    6331 projected.go:198] Error preparing data for projected volume kube-api-access-xbmql for pod ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.819724    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql podName:5f64faad-9fcf-478b-a4a5-e1e8e9e5c409 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:20.319693108 +0100 CET m=+9.435811524 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-xbmql" (UniqueName: "kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql") pod "duplicati-ix-chart-6c986975cb-s9wr5" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.927392    6331 controllermanager.go:182] Version: v1.26.6+k3s-e18037a7-dirty
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.927433    6331 controllermanager.go:184] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932646    6331 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932670    6331 shared_informer.go:270] Waiting for caches to sync for RequestHeaderAuthRequestController
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932686    6331 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932707    6331 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932707    6331 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932727    6331 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932853    6331 secure_serving.go:210] Serving securely on 127.0.0.1:10257
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.932906    6331 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.968397    6331 configmap.go:199] Couldn't get configMap kube-system/nvidia-device-plugin-config: object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.968463    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config podName:aa9b377b-bd02-4e51-83ed-376a36c0d803 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:21.96844488 +0100 CET m=+11.084563293 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "plugin-config" (UniqueName: "kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config") pod "nvidia-device-plugin-daemonset-bl5th" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803") : object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.968397    6331 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:19 truenas k3s[6331]: E1230 09:06:19.968515    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs podName:25715d19-0754-4b80-9a23-07c7d6de4edf nodeName:}" failed. No retries permitted until 2023-12-30 09:06:21.968502983 +0100 CET m=+11.084621400 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs") pod "openebs-zfs-node-trztd" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf") : object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:19 truenas k3s[6331]: I1230 09:06:19.994335    6331 shared_informer.go:270] Waiting for caches to sync for tokens
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.001767    6331 node_lifecycle_controller.go:492] Controller will reconcile labels.
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.002276    6331 node_lifecycle_controller.go:527] Sending events to api server.
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.002327    6331 node_lifecycle_controller.go:538] Starting node controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.002342    6331 shared_informer.go:270] Waiting for caches to sync for taint
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.002493    6331 controllermanager.go:622] Started "nodelifecycle"
Dec 30 09:06:20 truenas k3s[6331]: W1230 09:06:20.002930    6331 core.go:221] configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.
Dec 30 09:06:20 truenas k3s[6331]: W1230 09:06:20.002978    6331 controllermanager.go:600] Skipping "route"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.008297    6331 controllermanager.go:622] Started "attachdetach"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.009531    6331 attach_detach_controller.go:328] Starting attach detach controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.009561    6331 shared_informer.go:270] Waiting for caches to sync for attach detach
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.012408    6331 controllermanager.go:622] Started "clusterrole-aggregation"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.013893    6331 clusterroleaggregation_controller.go:188] Starting ClusterRoleAggregator
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.013927    6331 shared_informer.go:270] Waiting for caches to sync for ClusterRoleAggregator
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.017674    6331 controllermanager.go:622] Started "persistentvolume-expander"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.017905    6331 expand_controller.go:340] Starting expand controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.017932    6331 shared_informer.go:270] Waiting for caches to sync for expand
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.022022    6331 controllermanager.go:622] Started "root-ca-cert-publisher"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.022696    6331 publisher.go:101] Starting root CA certificate configmap publisher
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.022718    6331 shared_informer.go:270] Waiting for caches to sync for crt configmap
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.026635    6331 controllermanager.go:622] Started "endpoint"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.026943    6331 endpoints_controller.go:178] Starting endpoint controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.026975    6331 shared_informer.go:270] Waiting for caches to sync for endpoint
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.031353    6331 controllermanager.go:622] Started "replicationcontroller"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.032195    6331 replica_set.go:201] Starting replicationcontroller controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.032223    6331 shared_informer.go:270] Waiting for caches to sync for ReplicationController
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.032742    6331 shared_informer.go:277] Caches are synced for RequestHeaderAuthRequestController
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.032770    6331 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.032782    6331 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.064444    6331 controllermanager.go:622] Started "namespace"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.065035    6331 namespace_controller.go:195] Starting namespace controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.065058    6331 shared_informer.go:270] Waiting for caches to sync for namespace
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.069448    6331 controllermanager.go:622] Started "csrapproving"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.069555    6331 certificate_controller.go:112] Starting certificate controller "csrapproving"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.069582    6331 shared_informer.go:270] Waiting for caches to sync for certificate-csrapproving
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.073289    6331 core.go:207] failed to start cloud node lifecycle controller: no cloud provider provided
Dec 30 09:06:20 truenas k3s[6331]: W1230 09:06:20.073327    6331 controllermanager.go:600] Skipping "cloud-node-lifecycle"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.077924    6331 certificate_controller.go:112] Starting certificate controller "csrsigning-kubelet-serving"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.077955    6331 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-kubelet-serving
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.077964    6331 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/SSD/ix-applications/k3s/server/tls/server-ca.nochain.crt::/mnt/SSD/ix-applications/k3s/server/tls/server-ca.key"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.078422    6331 certificate_controller.go:112] Starting certificate controller "csrsigning-kubelet-client"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.078465    6331 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-kubelet-client
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.078470    6331 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/SSD/ix-applications/k3s/server/tls/client-ca.nochain.crt::/mnt/SSD/ix-applications/k3s/server/tls/client-ca.key"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.079025    6331 certificate_controller.go:112] Starting certificate controller "csrsigning-kube-apiserver-client"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.079048    6331 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-kube-apiserver-client
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.079057    6331 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/SSD/ix-applications/k3s/server/tls/client-ca.nochain.crt::/mnt/SSD/ix-applications/k3s/server/tls/client-ca.key"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.079528    6331 controllermanager.go:622] Started "csrsigning"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.079720    6331 certificate_controller.go:112] Starting certificate controller "csrsigning-legacy-unknown"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.079758    6331 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/SSD/ix-applications/k3s/server/tls/server-ca.nochain.crt::/mnt/SSD/ix-applications/k3s/server/tls/server-ca.key"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.079769    6331 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-legacy-unknown
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.083551    6331 controllermanager.go:622] Started "ttl-after-finished"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.083743    6331 ttlafterfinished_controller.go:104] Starting TTL after finished controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.083779    6331 shared_informer.go:270] Waiting for caches to sync for TTL after finished
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.094480    6331 shared_informer.go:277] Caches are synced for tokens
Dec 30 09:06:20 truenas k3s[6331]: W1230 09:06:20.114395    6331 shared_informer.go:548] resyncPeriod 14h56m58.735341042s is smaller than resyncCheckPeriod 16h42m32.146361263s and the informer has already started. Changing it to 16h42m32.146361263s
Dec 30 09:06:20 truenas k3s[6331]: W1230 09:06:20.114558    6331 shared_informer.go:548] resyncPeriod 12h42m43.992986892s is smaller than resyncCheckPeriod 16h42m32.146361263s and the informer has already started. Changing it to 16h42m32.146361263s
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.115424    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serviceaccounts
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.115936    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for poddisruptionbudgets.policy
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116021    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for rolebindings.rbac.authorization.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116530    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsrestores.zfs.openebs.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116594    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for deployments.apps
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116639    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmchartconfigs.helm.cattle.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116683    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsnodes.zfs.openebs.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116754    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for replicasets.apps
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116796    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmcharts.helm.cattle.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116836    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for volumesnapshots.snapshot.storage.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116881    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsbackups.zfs.openebs.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116919    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for podtemplates
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116949    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpoints
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.116983    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for statefulsets.apps
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117014    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for daemonsets.apps
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117056    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for networkpolicies.networking.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117111    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for csistoragecapacities.storage.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117199    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for controllerrevisions.apps
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117241    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for roles.rbac.authorization.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117656    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpointslices.discovery.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117706    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsvolumes.zfs.openebs.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117746    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfssnapshots.zfs.openebs.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117814    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for jobs.batch
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117864    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for leases.coordination.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.117910    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for limitranges
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118018    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for horizontalpodautoscalers.autoscaling
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118056    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for cronjobs.batch
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118100    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for addons.k3s.cattle.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118139    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for network-attachment-definitions.k8s.cni.cncf.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118433    6331 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingresses.networking.k8s.io
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118466    6331 controllermanager.go:622] Started "resourcequota"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118601    6331 resource_quota_controller.go:277] Starting resource quota controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.118636    6331 shared_informer.go:270] Waiting for caches to sync for resource quota
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.119305    6331 resource_quota_monitor.go:295] QuotaMonitor running
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.124079    6331 controllermanager.go:622] Started "daemonset"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.124951    6331 daemon_controller.go:267] Starting daemon sets controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.124991    6331 shared_informer.go:270] Waiting for caches to sync for daemon sets
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.129930    6331 controllermanager.go:622] Started "job"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.130152    6331 job_controller.go:191] Starting job controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.130180    6331 shared_informer.go:270] Waiting for caches to sync for job
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.137482    6331 controllermanager.go:622] Started "disruption"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.137666    6331 disruption.go:424] Sending events to api server.
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.137729    6331 disruption.go:435] Starting disruption controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.137744    6331 shared_informer.go:270] Waiting for caches to sync for disruption
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.141600    6331 controllermanager.go:622] Started "statefulset"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.141789    6331 stateful_set.go:152] Starting stateful set controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.141816    6331 shared_informer.go:270] Waiting for caches to sync for stateful set
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.145547    6331 controllermanager.go:622] Started "tokencleaner"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.145758    6331 tokencleaner.go:111] Starting token cleaner controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.145785    6331 shared_informer.go:270] Waiting for caches to sync for token_cleaner
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.145806    6331 shared_informer.go:277] Caches are synced for token_cleaner
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.150277    6331 controllermanager.go:622] Started "persistentvolume-binder"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.150591    6331 pv_controller_base.go:318] Starting persistent volume controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.150624    6331 shared_informer.go:270] Waiting for caches to sync for persistent volume
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.153945    6331 controllermanager.go:622] Started "pvc-protection"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.154114    6331 pvc_protection_controller.go:99] "Starting PVC protection controller"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.154133    6331 shared_informer.go:270] Waiting for caches to sync for PVC protection
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.158039    6331 controllermanager.go:622] Started "serviceaccount"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.159152    6331 serviceaccounts_controller.go:111] Starting service account controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.159183    6331 shared_informer.go:270] Waiting for caches to sync for service account
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169641    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-volume-duplicati-duplicati-config\") pod \"ea325533-21c4-494e-b08f-ee25cb2a136f\" (UID: \"ea325533-21c4-494e-b08f-ee25cb2a136f\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169709    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-volume-duplicati-duplicati-config\") pod \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\" (UID: \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169757    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-0\") pod \"46548d19-6f35-4e2a-a247-dbad3afad7d4\" (UID: \"46548d19-6f35-4e2a-a247-dbad3afad7d4\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169781    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-volume-duplicati-duplicati-config" (OuterVolumeSpecName: "ix-host-volume-duplicati-duplicati-config") pod "ea325533-21c4-494e-b08f-ee25cb2a136f" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f"). InnerVolumeSpecName "ix-host-volume-duplicati-duplicati-config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169831    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-0\") pod \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\" (UID: \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169883    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-0" (OuterVolumeSpecName: "ix-host-path-duplicati-0") pod "46548d19-6f35-4e2a-a247-dbad3afad7d4" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4"). InnerVolumeSpecName "ix-host-path-duplicati-0". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169910    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-0\") pod \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\" (UID: \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169968    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-0" (OuterVolumeSpecName: "ix-host-path-duplicati-0") pod "7ee71e90-c82b-41b2-b1eb-7b58096f348a" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a"). InnerVolumeSpecName "ix-host-path-duplicati-0". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170012    6331 garbagecollector.go:154] Starting garbage collector controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170042    6331 shared_informer.go:270] Waiting for caches to sync for garbage collector
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170048    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-0" (OuterVolumeSpecName: "ix-host-path-duplicati-0") pod "ea325533-21c4-494e-b08f-ee25cb2a136f" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f"). InnerVolumeSpecName "ix-host-path-duplicati-0". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170063    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-0" (OuterVolumeSpecName: "ix-host-path-duplicati-0") pod "a18a85bb-a765-4ff3-90cc-6436a0c95e08" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08"). InnerVolumeSpecName "ix-host-path-duplicati-0". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.169891    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-volume-duplicati-duplicati-config" (OuterVolumeSpecName: "ix-host-volume-duplicati-duplicati-config") pod "7ee71e90-c82b-41b2-b1eb-7b58096f348a" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a"). InnerVolumeSpecName "ix-host-volume-duplicati-duplicati-config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170018    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-0\") pod \"ea325533-21c4-494e-b08f-ee25cb2a136f\" (UID: \"ea325533-21c4-494e-b08f-ee25cb2a136f\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170165    6331 controllermanager.go:622] Started "garbagecollector"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170199    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-1\") pod \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\" (UID: \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170256    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-volume-duplicati-duplicati-config\") pod \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\" (UID: \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170289    6331 graph_builder.go:291] GraphBuilder running
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170318    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-1" (OuterVolumeSpecName: "ix-host-path-duplicati-1") pod "a8fadcd0-3556-42ea-9ed3-b43c57dbed99" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99"). InnerVolumeSpecName "ix-host-path-duplicati-1". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170301    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-1\") pod \"46548d19-6f35-4e2a-a247-dbad3afad7d4\" (UID: \"46548d19-6f35-4e2a-a247-dbad3afad7d4\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170344    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-1" (OuterVolumeSpecName: "ix-host-path-duplicati-1") pod "46548d19-6f35-4e2a-a247-dbad3afad7d4" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4"). InnerVolumeSpecName "ix-host-path-duplicati-1". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170368    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-volume-duplicati-duplicati-config" (OuterVolumeSpecName: "ix-host-volume-duplicati-duplicati-config") pod "a18a85bb-a765-4ff3-90cc-6436a0c95e08" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08"). InnerVolumeSpecName "ix-host-volume-duplicati-duplicati-config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170445    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-1\") pod \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\" (UID: \"a18a85bb-a765-4ff3-90cc-6436a0c95e08\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170502    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-1" (OuterVolumeSpecName: "ix-host-path-duplicati-1") pod "a18a85bb-a765-4ff3-90cc-6436a0c95e08" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08"). InnerVolumeSpecName "ix-host-path-duplicati-1". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170544    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-volume-duplicati-duplicati-config\") pod \"46548d19-6f35-4e2a-a247-dbad3afad7d4\" (UID: \"46548d19-6f35-4e2a-a247-dbad3afad7d4\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170604    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-volume-duplicati-duplicati-config" (OuterVolumeSpecName: "ix-host-volume-duplicati-duplicati-config") pod "46548d19-6f35-4e2a-a247-dbad3afad7d4" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4"). InnerVolumeSpecName "ix-host-volume-duplicati-duplicati-config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170638    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-1\") pod \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\" (UID: \"7ee71e90-c82b-41b2-b1eb-7b58096f348a\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170678    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-1" (OuterVolumeSpecName: "ix-host-path-duplicati-1") pod "7ee71e90-c82b-41b2-b1eb-7b58096f348a" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a"). InnerVolumeSpecName "ix-host-path-duplicati-1". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170732    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-1\") pod \"ea325533-21c4-494e-b08f-ee25cb2a136f\" (UID: \"ea325533-21c4-494e-b08f-ee25cb2a136f\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170807    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-1" (OuterVolumeSpecName: "ix-host-path-duplicati-1") pod "ea325533-21c4-494e-b08f-ee25cb2a136f" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f"). InnerVolumeSpecName "ix-host-path-duplicati-1". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170830    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-volume-duplicati-duplicati-config\") pod \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\" (UID: \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170889    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-volume-duplicati-duplicati-config" (OuterVolumeSpecName: "ix-host-volume-duplicati-duplicati-config") pod "a8fadcd0-3556-42ea-9ed3-b43c57dbed99" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99"). InnerVolumeSpecName "ix-host-volume-duplicati-duplicati-config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170938    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-0\") pod \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\" (UID: \"a8fadcd0-3556-42ea-9ed3-b43c57dbed99\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.170991    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-0" (OuterVolumeSpecName: "ix-host-path-duplicati-0") pod "a8fadcd0-3556-42ea-9ed3-b43c57dbed99" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99"). InnerVolumeSpecName "ix-host-path-duplicati-0". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.171357    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.171383    6331 projected.go:198] Error preparing data for projected volume kube-api-access-z8lf2 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171427    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-0\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.171456    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2 podName:a8fadcd0-3556-42ea-9ed3-b43c57dbed99 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:21.171431114 +0100 CET m=+10.287549544 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-z8lf2" (UniqueName: "kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2") pod "duplicati-ix-chart-6c986975cb-st8nr" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171499    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-volume-duplicati-duplicati-config\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171532    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-volume-duplicati-duplicati-config\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171559    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-0\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171587    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-0\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171614    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-0\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171641    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-0\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171668    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-path-duplicati-1\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171718    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-volume-duplicati-duplicati-config\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171751    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-path-duplicati-1\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171781    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/46548d19-6f35-4e2a-a247-dbad3afad7d4-ix-host-volume-duplicati-duplicati-config\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171807    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/a18a85bb-a765-4ff3-90cc-6436a0c95e08-ix-host-path-duplicati-1\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171833    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/7ee71e90-c82b-41b2-b1eb-7b58096f348a-ix-host-path-duplicati-1\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171859    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/ea325533-21c4-494e-b08f-ee25cb2a136f-ix-host-path-duplicati-1\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.171891    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-ix-host-volume-duplicati-duplicati-config\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.174361    6331 controllermanager.go:622] Started "deployment"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.174699    6331 deployment_controller.go:154] "Starting controller" controller="deployment"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.174721    6331 shared_informer.go:270] Waiting for caches to sync for deployment
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.178254    6331 controllermanager.go:622] Started "replicaset"
Dec 30 09:06:20 truenas k3s[6331]: W1230 09:06:20.178292    6331 controllermanager.go:587] "bootstrapsigner" is disabled
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.178352    6331 replica_set.go:201] Starting replicaset controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.178372    6331 shared_informer.go:270] Waiting for caches to sync for ReplicaSet
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181311    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16\": cni plugin not initialized" podSandboxID="d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181397    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16}
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181473    6331 kubelet.go:1899] failed to "KillPodSandbox" for "cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16\": cni plugin not initialized"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181498    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61\": cni plugin not initialized" podSandboxID="565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181532    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"d023836f022081fbbe6a170b3fa37af511c1f0f2911b2b47cf5a37d8657ced16\\\": cni plugin not initialized\"" pod="kube-system/snapshot-controller-546868dfb4-f5hbl" podUID=cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181550    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61}
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181619    6331 kubelet.go:1899] failed to "KillPodSandbox" for "e545eb4a-9b4c-426e-b86e-e2629dd4ea7f" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61\": cni plugin not initialized"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181687    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a\": cni plugin not initialized" podSandboxID="e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181736    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a}
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181797    6331 kubelet.go:1899] failed to "KillPodSandbox" for "aa9b377b-bd02-4e51-83ed-376a36c0d803" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a\": cni plugin not initialized"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181901    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3\": cni plugin not initialized" podSandboxID="2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181921    6331 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455\": cni plugin not initialized" podSandboxID="21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181935    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3}
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181968    6331 kuberuntime_manager.go:965] "Failed to stop sandbox" podSandboxID={Type:containerd ID:21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455}
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.181982    6331 kubelet.go:1899] failed to "KillPodSandbox" for "7c61dd38-d4e4-4a38-a0eb-48077635fa65" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3\": cni plugin not initialized"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.182032    6331 kubelet.go:1899] failed to "KillPodSandbox" for "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409" with KillPodSandboxError: "rpc error: code = Unknown desc = failed to destroy network for sandbox \"21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455\": cni plugin not initialized"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.182800    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"e545eb4a-9b4c-426e-b86e-e2629dd4ea7f\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"565cbc9459c5dd6c13d0464b56c66d69bfafc55d7e5e5f16c1e8821302f7fe61\\\": cni plugin not initialized\"" pod="kube-system/snapshot-controller-546868dfb4-l2fdd" podUID=e545eb4a-9b4c-426e-b86e-e2629dd4ea7f
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.183967    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"aa9b377b-bd02-4e51-83ed-376a36c0d803\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"e9154c8e8a60225a65351d1c8dddf07903600b2f5a668acc0e07a23d41f4606a\\\": cni plugin not initialized\"" pod="kube-system/nvidia-device-plugin-daemonset-bl5th" podUID=aa9b377b-bd02-4e51-83ed-376a36c0d803
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.185965    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"2ecfae4c13b5a1c3a20e661f2a19800a98d38351dfb147c5c35a0ae5afb8e5e3\\\": cni plugin not initialized\"" pod="kube-system/openebs-zfs-controller-0" podUID=7c61dd38-d4e4-4a38-a0eb-48077635fa65
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.188003    6331 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"KillPodSandbox\" for \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\" with KillPodSandboxError: \"rpc error: code = Unknown desc = failed to destroy network for sandbox \\\"21e8beca8e947da46ab065bc41f883fb0fc8487db47d252973c97456c10cb455\\\": cni plugin not initialized\"" pod="ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5" podUID=5f64faad-9fcf-478b-a4a5-e1e8e9e5c409
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.250134    6331 controllermanager.go:622] Started "cronjob"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.250331    6331 cronjob_controllerv2.go:137] "Starting cronjob controller v2"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.250814    6331 shared_informer.go:270] Waiting for caches to sync for cronjob
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.299312    6331 controllermanager.go:622] Started "csrcleaner"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.299436    6331 cleaner.go:82] Starting CSR cleaner controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.349843    6331 controllermanager.go:622] Started "ttl"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.349988    6331 ttl_controller.go:120] Starting TTL controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.350008    6331 shared_informer.go:270] Waiting for caches to sync for TTL
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.373597    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.373628    6331 projected.go:198] Error preparing data for projected volume kube-api-access-xbmql for pod ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.373692    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql podName:5f64faad-9fcf-478b-a4a5-e1e8e9e5c409 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:21.373671414 +0100 CET m=+10.489789831 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-xbmql" (UniqueName: "kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql") pod "duplicati-ix-chart-6c986975cb-s9wr5" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.399047    6331 controllermanager.go:622] Started "endpointslice"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.399283    6331 endpointslice_controller.go:257] Starting endpoint slice controller
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.399302    6331 shared_informer.go:270] Waiting for caches to sync for endpoint_slice
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.426696    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.426734    6331 projected.go:198] Error preparing data for projected volume kube-api-access-p7cp4 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-787sr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.426821    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4 podName:ea325533-21c4-494e-b08f-ee25cb2a136f nodeName:}" failed. No retries permitted until 2023-12-30 09:06:20.92679267 +0100 CET m=+10.042911099 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-p7cp4" (UniqueName: "kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4") pod "duplicati-ix-chart-6c986975cb-787sr" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.454671    6331 kubelet_node_status.go:493] "Fast updating node status as it just became ready"
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.778329    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.778364    6331 projected.go:198] Error preparing data for projected volume kube-api-access-rtqfc for pod ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.778445    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc podName:7ee71e90-c82b-41b2-b1eb-7b58096f348a nodeName:}" failed. No retries permitted until 2023-12-30 09:06:22.778416279 +0100 CET m=+11.894534708 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-rtqfc" (UniqueName: "kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc") pod "duplicati-ix-chart-6c986975cb-k7ndd" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: time="2023-12-30T09:06:20+01:00" level=info msg="Tunnel authorizer set Kubelet Port 10250"
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.878359    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-ldqkp\" (UniqueName: \"kubernetes.io/projected/31dfa589-323f-4aec-9109-a2f7065d90c7-kube-api-access-ldqkp\") pod \"31dfa589-323f-4aec-9109-a2f7065d90c7\" (UID: \"31dfa589-323f-4aec-9109-a2f7065d90c7\") "
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.880578    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/31dfa589-323f-4aec-9109-a2f7065d90c7-kube-api-access-ldqkp" (OuterVolumeSpecName: "kube-api-access-ldqkp") pod "31dfa589-323f-4aec-9109-a2f7065d90c7" (UID: "31dfa589-323f-4aec-9109-a2f7065d90c7"). InnerVolumeSpecName "kube-api-access-ldqkp". PluginName "kubernetes.io/projected", VolumeGidValue ""
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.915128    6331 controller.go:615] quota admission added evaluator for: daemonsets.apps
Dec 30 09:06:20 truenas k3s[6331]: I1230 09:06:20.980006    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-ldqkp\" (UniqueName: \"kubernetes.io/projected/31dfa589-323f-4aec-9109-a2f7065d90c7-kube-api-access-ldqkp\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.980086    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.980121    6331 projected.go:198] Error preparing data for projected volume kube-api-access-p7cp4 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-787sr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:20 truenas k3s[6331]: E1230 09:06:20.980214    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4 podName:ea325533-21c4-494e-b08f-ee25cb2a136f nodeName:}" failed. No retries permitted until 2023-12-30 09:06:21.980182257 +0100 CET m=+11.096300685 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-p7cp4" (UniqueName: "kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4") pod "duplicati-ix-chart-6c986975cb-787sr" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.181102    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.181138    6331 projected.go:198] Error preparing data for projected volume kube-api-access-z8lf2 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.181231    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2 podName:a8fadcd0-3556-42ea-9ed3-b43c57dbed99 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:23.181201109 +0100 CET m=+12.297319548 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-z8lf2" (UniqueName: "kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2") pod "duplicati-ix-chart-6c986975cb-st8nr" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.382303    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.382339    6331 projected.go:198] Error preparing data for projected volume kube-api-access-xbmql for pod ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.382432    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql podName:5f64faad-9fcf-478b-a4a5-e1e8e9e5c409 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:23.382400891 +0100 CET m=+12.498519321 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-xbmql" (UniqueName: "kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql") pod "duplicati-ix-chart-6c986975cb-s9wr5" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.427413    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.427450    6331 projected.go:198] Error preparing data for projected volume kube-api-access-hrpkr for pod ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.427538    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr podName:a18a85bb-a765-4ff3-90cc-6436a0c95e08 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:21.927508411 +0100 CET m=+11.043626840 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-hrpkr" (UniqueName: "kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr") pod "duplicati-ix-chart-6c986975cb-tbnph" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.624743    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.624782    6331 projected.go:198] Error preparing data for projected volume kube-api-access-kwpvz for pod ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.624872    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz podName:46548d19-6f35-4e2a-a247-dbad3afad7d4 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:22.124842666 +0100 CET m=+11.240961098 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-kwpvz" (UniqueName: "kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz") pod "duplicati-ix-chart-6c986975cb-8t9rq" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987322    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987377    6331 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987469    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs podName:25715d19-0754-4b80-9a23-07c7d6de4edf nodeName:}" failed. No retries permitted until 2023-12-30 09:06:25.987437362 +0100 CET m=+15.103555807 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs") pod "openebs-zfs-node-trztd" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf") : object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987470    6331 configmap.go:199] Couldn't get configMap kube-system/nvidia-device-plugin-config: object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987540    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config podName:aa9b377b-bd02-4e51-83ed-376a36c0d803 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:25.987519495 +0100 CET m=+15.103637910 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "plugin-config" (UniqueName: "kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config") pod "nvidia-device-plugin-daemonset-bl5th" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803") : object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987379    6331 projected.go:198] Error preparing data for projected volume kube-api-access-hrpkr for pod ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987576    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987598    6331 projected.go:198] Error preparing data for projected volume kube-api-access-p7cp4 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-787sr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987612    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr podName:a18a85bb-a765-4ff3-90cc-6436a0c95e08 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:22.987589895 +0100 CET m=+12.103708329 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-hrpkr" (UniqueName: "kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr") pod "duplicati-ix-chart-6c986975cb-tbnph" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:21 truenas k3s[6331]: E1230 09:06:21.987666    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4 podName:ea325533-21c4-494e-b08f-ee25cb2a136f nodeName:}" failed. No retries permitted until 2023-12-30 09:06:23.987640832 +0100 CET m=+13.103759256 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-p7cp4" (UniqueName: "kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4") pod "duplicati-ix-chart-6c986975cb-787sr" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.188036    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.188073    6331 projected.go:198] Error preparing data for projected volume kube-api-access-kwpvz for pod ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.188163    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz podName:46548d19-6f35-4e2a-a247-dbad3afad7d4 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:23.188128013 +0100 CET m=+12.304246435 (durationBeforeRetry 1s). Error: MountVolume.SetUp failed for volume "kube-api-access-kwpvz" (UniqueName: "kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz") pod "duplicati-ix-chart-6c986975cb-8t9rq" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.792124    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.792164    6331 projected.go:198] Error preparing data for projected volume kube-api-access-rtqfc for pod ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.792251    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc podName:7ee71e90-c82b-41b2-b1eb-7b58096f348a nodeName:}" failed. No retries permitted until 2023-12-30 09:06:26.792221823 +0100 CET m=+15.908340259 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-rtqfc" (UniqueName: "kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc") pod "duplicati-ix-chart-6c986975cb-k7ndd" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.994436    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.994476    6331 projected.go:198] Error preparing data for projected volume kube-api-access-hrpkr for pod ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:22 truenas k3s[6331]: E1230 09:06:22.994549    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr podName:a18a85bb-a765-4ff3-90cc-6436a0c95e08 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:24.994528768 +0100 CET m=+14.110647183 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-hrpkr" (UniqueName: "kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr") pod "duplicati-ix-chart-6c986975cb-tbnph" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.195367    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.195400    6331 projected.go:198] Error preparing data for projected volume kube-api-access-kwpvz for pod ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.195456    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz podName:46548d19-6f35-4e2a-a247-dbad3afad7d4 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:25.195436157 +0100 CET m=+14.311554574 (durationBeforeRetry 2s). Error: MountVolume.SetUp failed for volume "kube-api-access-kwpvz" (UniqueName: "kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz") pod "duplicati-ix-chart-6c986975cb-8t9rq" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.195370    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.195484    6331 projected.go:198] Error preparing data for projected volume kube-api-access-z8lf2 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.195553    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2 podName:a8fadcd0-3556-42ea-9ed3-b43c57dbed99 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:27.195530493 +0100 CET m=+16.311648911 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-z8lf2" (UniqueName: "kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2") pod "duplicati-ix-chart-6c986975cb-st8nr" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.396981    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.397023    6331 projected.go:198] Error preparing data for projected volume kube-api-access-xbmql for pod ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:23 truenas k3s[6331]: E1230 09:06:23.397114    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql podName:5f64faad-9fcf-478b-a4a5-e1e8e9e5c409 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:27.397084263 +0100 CET m=+16.513202695 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-xbmql" (UniqueName: "kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql") pod "duplicati-ix-chart-6c986975cb-s9wr5" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:24 truenas k3s[6331]: E1230 09:06:24.001347    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:24 truenas k3s[6331]: E1230 09:06:24.001393    6331 projected.go:198] Error preparing data for projected volume kube-api-access-p7cp4 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-787sr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:24 truenas k3s[6331]: E1230 09:06:24.001484    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4 podName:ea325533-21c4-494e-b08f-ee25cb2a136f nodeName:}" failed. No retries permitted until 2023-12-30 09:06:28.001456234 +0100 CET m=+17.117574686 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-p7cp4" (UniqueName: "kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4") pod "duplicati-ix-chart-6c986975cb-787sr" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:25 truenas k3s[6331]: I1230 09:06:25.007461    6331 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: smb.csi.k8s.io endpoint: /var/lib/kubelet/plugins/smb.csi.k8s.io/csi.sock versions: 1.0.0
Dec 30 09:06:25 truenas k3s[6331]: I1230 09:06:25.007530    6331 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: smb.csi.k8s.io at endpoint: /var/lib/kubelet/plugins/smb.csi.k8s.io/csi.sock
Dec 30 09:06:25 truenas k3s[6331]: E1230 09:06:25.009393    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:25 truenas k3s[6331]: E1230 09:06:25.009431    6331 projected.go:198] Error preparing data for projected volume kube-api-access-hrpkr for pod ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:25 truenas k3s[6331]: E1230 09:06:25.009529    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr podName:a18a85bb-a765-4ff3-90cc-6436a0c95e08 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:29.009498533 +0100 CET m=+18.125616958 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-hrpkr" (UniqueName: "kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr") pod "duplicati-ix-chart-6c986975cb-tbnph" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:25 truenas k3s[6331]: E1230 09:06:25.211331    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:25 truenas k3s[6331]: E1230 09:06:25.211366    6331 projected.go:198] Error preparing data for projected volume kube-api-access-kwpvz for pod ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:25 truenas k3s[6331]: E1230 09:06:25.211451    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz podName:46548d19-6f35-4e2a-a247-dbad3afad7d4 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:29.211413936 +0100 CET m=+18.327532363 (durationBeforeRetry 4s). Error: MountVolume.SetUp failed for volume "kube-api-access-kwpvz" (UniqueName: "kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz") pod "duplicati-ix-chart-6c986975cb-8t9rq" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:26 truenas k3s[6331]: I1230 09:06:26.003945    6331 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: nfs.csi.k8s.io endpoint: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock versions: 1.0.0
Dec 30 09:06:26 truenas k3s[6331]: I1230 09:06:26.004000    6331 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: nfs.csi.k8s.io at endpoint: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
Dec 30 09:06:26 truenas k3s[6331]: E1230 09:06:26.017131    6331 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:26 truenas k3s[6331]: E1230 09:06:26.017131    6331 configmap.go:199] Couldn't get configMap kube-system/nvidia-device-plugin-config: object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:26 truenas k3s[6331]: E1230 09:06:26.017206    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs podName:25715d19-0754-4b80-9a23-07c7d6de4edf nodeName:}" failed. No retries permitted until 2023-12-30 09:06:34.017186199 +0100 CET m=+23.133304618 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs") pod "openebs-zfs-node-trztd" (UID: "25715d19-0754-4b80-9a23-07c7d6de4edf") : object "kube-system"/"openebs-zfspv-bin" not registered
Dec 30 09:06:26 truenas k3s[6331]: E1230 09:06:26.017297    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config podName:aa9b377b-bd02-4e51-83ed-376a36c0d803 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:34.017273639 +0100 CET m=+23.133392071 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "plugin-config" (UniqueName: "kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config") pod "nvidia-device-plugin-daemonset-bl5th" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803") : object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:26 truenas k3s[6331]: E1230 09:06:26.821237    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:26 truenas k3s[6331]: E1230 09:06:26.821274    6331 projected.go:198] Error preparing data for projected volume kube-api-access-rtqfc for pod ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:26 truenas k3s[6331]: E1230 09:06:26.821348    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc podName:7ee71e90-c82b-41b2-b1eb-7b58096f348a nodeName:}" failed. No retries permitted until 2023-12-30 09:06:34.821324002 +0100 CET m=+23.937442429 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-rtqfc" (UniqueName: "kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc") pod "duplicati-ix-chart-6c986975cb-k7ndd" (UID: "7ee71e90-c82b-41b2-b1eb-7b58096f348a") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:27 truenas k3s[6331]: E1230 09:06:27.224689    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:27 truenas k3s[6331]: E1230 09:06:27.224727    6331 projected.go:198] Error preparing data for projected volume kube-api-access-z8lf2 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:27 truenas k3s[6331]: E1230 09:06:27.224805    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2 podName:a8fadcd0-3556-42ea-9ed3-b43c57dbed99 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:35.224779298 +0100 CET m=+24.340897725 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-z8lf2" (UniqueName: "kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2") pod "duplicati-ix-chart-6c986975cb-st8nr" (UID: "a8fadcd0-3556-42ea-9ed3-b43c57dbed99") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:27 truenas k3s[6331]: E1230 09:06:27.425810    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:27 truenas k3s[6331]: E1230 09:06:27.425847    6331 projected.go:198] Error preparing data for projected volume kube-api-access-xbmql for pod ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:27 truenas k3s[6331]: E1230 09:06:27.425922    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql podName:5f64faad-9fcf-478b-a4a5-e1e8e9e5c409 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:35.425897411 +0100 CET m=+24.542015838 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-xbmql" (UniqueName: "kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql") pod "duplicati-ix-chart-6c986975cb-s9wr5" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:28 truenas k3s[6331]: E1230 09:06:28.029539    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:28 truenas k3s[6331]: E1230 09:06:28.029578    6331 projected.go:198] Error preparing data for projected volume kube-api-access-p7cp4 for pod ix-duplicati/duplicati-ix-chart-6c986975cb-787sr: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:28 truenas k3s[6331]: E1230 09:06:28.029657    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4 podName:ea325533-21c4-494e-b08f-ee25cb2a136f nodeName:}" failed. No retries permitted until 2023-12-30 09:06:36.029630066 +0100 CET m=+25.145748496 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-p7cp4" (UniqueName: "kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4") pod "duplicati-ix-chart-6c986975cb-787sr" (UID: "ea325533-21c4-494e-b08f-ee25cb2a136f") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:29 truenas k3s[6331]: E1230 09:06:29.037129    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:29 truenas k3s[6331]: E1230 09:06:29.037168    6331 projected.go:198] Error preparing data for projected volume kube-api-access-hrpkr for pod ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:29 truenas k3s[6331]: E1230 09:06:29.037249    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr podName:a18a85bb-a765-4ff3-90cc-6436a0c95e08 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:37.037223503 +0100 CET m=+26.153341933 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-hrpkr" (UniqueName: "kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr") pod "duplicati-ix-chart-6c986975cb-tbnph" (UID: "a18a85bb-a765-4ff3-90cc-6436a0c95e08") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:29 truenas k3s[6331]: E1230 09:06:29.238768    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:29 truenas k3s[6331]: E1230 09:06:29.238810    6331 projected.go:198] Error preparing data for projected volume kube-api-access-kwpvz for pod ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:29 truenas k3s[6331]: E1230 09:06:29.238895    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz podName:46548d19-6f35-4e2a-a247-dbad3afad7d4 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:37.238870606 +0100 CET m=+26.354989043 (durationBeforeRetry 8s). Error: MountVolume.SetUp failed for volume "kube-api-access-kwpvz" (UniqueName: "kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz") pod "duplicati-ix-chart-6c986975cb-8t9rq" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.468542    6331 range_allocator.go:109] No Secondary Service CIDR provided. Skipping filtering out secondary service addresses.
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.468673    6331 controllermanager.go:622] Started "nodeipam"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.468905    6331 node_ipam_controller.go:155] Starting ipam controller
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.468940    6331 shared_informer.go:270] Waiting for caches to sync for node
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.472917    6331 controllermanager.go:622] Started "endpointslicemirroring"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.473496    6331 endpointslicemirroring_controller.go:211] Starting EndpointSliceMirroring controller
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.473518    6331 shared_informer.go:270] Waiting for caches to sync for endpoint_slice_mirroring
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.476744    6331 controllermanager.go:622] Started "podgc"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.477311    6331 gc_controller.go:102] Starting GC controller
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.477335    6331 shared_informer.go:270] Waiting for caches to sync for GC
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.480916    6331 core.go:92] Failed to start service controller: WARNING: no cloud provider provided, services of type LoadBalancer will fail
Dec 30 09:06:30 truenas k3s[6331]: W1230 09:06:30.480963    6331 controllermanager.go:600] Skipping "service"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.496343    6331 controllermanager.go:622] Started "horizontalpodautoscaling"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.497123    6331 horizontal.go:181] Starting HPA controller
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.497157    6331 shared_informer.go:270] Waiting for caches to sync for HPA
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.500040    6331 controllermanager.go:622] Started "pv-protection"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.500210    6331 pv_protection_controller.go:75] Starting PV protection controller
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.500227    6331 shared_informer.go:270] Waiting for caches to sync for PV protection
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.502882    6331 controllermanager.go:622] Started "ephemeral-volume"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.503272    6331 controller.go:169] Starting ephemeral volume controller
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.503286    6331 shared_informer.go:270] Waiting for caches to sync for ephemeral
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.508706    6331 shared_informer.go:270] Waiting for caches to sync for resource quota
Dec 30 09:06:30 truenas k3s[6331]: W1230 09:06:30.523073    6331 actual_state_of_world.go:541] Failed to update statusUpdateNeeded field in actual state of world: Failed to set statusUpdateNeeded to needed true, because nodeName="ix-truenas" does not exist
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.531152    6331 shared_informer.go:277] Caches are synced for job
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.532361    6331 shared_informer.go:277] Caches are synced for ReplicationController
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.541876    6331 shared_informer.go:277] Caches are synced for stateful set
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.547459    6331 shared_informer.go:270] Waiting for caches to sync for garbage collector
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.550100    6331 shared_informer.go:277] Caches are synced for TTL
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.551094    6331 shared_informer.go:277] Caches are synced for persistent volume
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.551125    6331 shared_informer.go:277] Caches are synced for cronjob
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.554730    6331 shared_informer.go:277] Caches are synced for PVC protection
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.559319    6331 shared_informer.go:277] Caches are synced for service account
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.565237    6331 shared_informer.go:277] Caches are synced for namespace
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.569031    6331 shared_informer.go:277] Caches are synced for node
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.569075    6331 range_allocator.go:167] Sending events to api server.
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.569111    6331 range_allocator.go:171] Starting range CIDR allocator
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.569124    6331 shared_informer.go:270] Waiting for caches to sync for cidrallocator
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.569143    6331 shared_informer.go:277] Caches are synced for cidrallocator
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.569623    6331 shared_informer.go:277] Caches are synced for certificate-csrapproving
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.574526    6331 shared_informer.go:277] Caches are synced for endpoint_slice_mirroring
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.577764    6331 shared_informer.go:277] Caches are synced for GC
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578010    6331 gc_controller.go:207] "Garbage collecting pods" numPods=12
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578067    6331 shared_informer.go:277] Caches are synced for certificate-csrsigning-kubelet-serving
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578129    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-duplicati/duplicati-ix-chart-6c986975cb-k7ndd"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578156    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-duplicati/duplicati-ix-chart-6c986975cb-tbnph"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578170    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/coredns-59b4f5bbd5-t9zfd"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578183    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-sonarr/sonarr-8f694b79d-5vsdq"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578215    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-jellyfin/jellyfin-64c4d7b66b-rdpxz"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578167    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-radarr/radarr-78fb8b865b-lbxwm"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578217    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-wg-easy/wg-easy-568f69bb5d-vrsw9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578189    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-gitea/gitea-6f7479bdf4-b4l9w"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578192    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-sabnzbd/sabnzbd-9f94c7bfd-tcprh"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578243    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-duplicati/duplicati-ix-chart-6c986975cb-st8nr"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578282    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-gitea/gitea-postgres-86c8c597f5-6wrsl"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578139    6331 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-duplicati/duplicati-ix-chart-6c986975cb-787sr"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.578536    6331 shared_informer.go:277] Caches are synced for certificate-csrsigning-kubelet-client
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.579680    6331 shared_informer.go:277] Caches are synced for certificate-csrsigning-kube-apiserver-client
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.579834    6331 shared_informer.go:277] Caches are synced for certificate-csrsigning-legacy-unknown
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.584094    6331 shared_informer.go:277] Caches are synced for TTL after finished
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.607379    6331 shared_informer.go:277] Caches are synced for ephemeral
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.607467    6331 shared_informer.go:277] Caches are synced for taint
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.607830    6331 node_lifecycle_controller.go:1438] Initializing eviction metric for zone:
Dec 30 09:06:30 truenas k3s[6331]: W1230 09:06:30.608458    6331 node_lifecycle_controller.go:1053] Missing timestamp for Node ix-truenas. Assuming now as a timestamp.
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.608544    6331 node_lifecycle_controller.go:1254] Controller detected that zone  is now in state Normal.
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.609005    6331 taint_manager.go:206] "Starting NoExecuteTaintManager"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.609042    6331 taint_manager.go:211] "Sending events to api server"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.610215    6331 event.go:294] "Event occurred" object="ix-truenas" fieldPath="" kind="Node" apiVersion="v1" type="Normal" reason="RegisteredNode" message="Node ix-truenas event: Registered Node ix-truenas in Controller"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.613330    6331 shared_informer.go:277] Caches are synced for HPA
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.613478    6331 shared_informer.go:277] Caches are synced for endpoint_slice
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.614696    6331 shared_informer.go:277] Caches are synced for ClusterRoleAggregator
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.615561    6331 shared_informer.go:277] Caches are synced for PV protection
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.618121    6331 shared_informer.go:277] Caches are synced for expand
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.622766    6331 shared_informer.go:277] Caches are synced for crt configmap
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.625844    6331 shared_informer.go:277] Caches are synced for daemon sets
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.638059    6331 shared_informer.go:277] Caches are synced for endpoint
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.638194    6331 shared_informer.go:277] Caches are synced for disruption
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.664621    6331 controller.go:615] quota admission added evaluator for: endpointslices.discovery.k8s.io
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.675485    6331 shared_informer.go:277] Caches are synced for deployment
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.679389    6331 shared_informer.go:277] Caches are synced for ReplicaSet
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.691393    6331 controller.go:615] quota admission added evaluator for: endpoints
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.719825    6331 event.go:294] "Event occurred" object="kube-system/openebs-zfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Warning" reason="FailedDaemonPod" message="Found failed daemon pod kube-system/openebs-zfs-node-trztd on node ix-truenas, will try to kill it"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.736336    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.736921    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="liveness-probe"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737351    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="c6036fbb-5430-4134-968c-f1540cfde0d9" containerName="gitea"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737391    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="3bbaa9f0-a0cb-46e8-95db-f4a414b0b24e" containerName="gitea"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737429    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="csi-provisioner"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737459    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="25715d19-0754-4b80-9a23-07c7d6de4edf" containerName="openebs-zfs-plugin"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737493    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="fe85ed7d-d0dd-4ea8-9d70-23365d4cf3a4" containerName="radarr"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737518    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="94a034f8-6855-4601-8c87-70d8795eda28" containerName="sabnzbd-init-01-permissions"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737544    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="d9b261b0-692c-49d3-ae73-9c02244c171b" containerName="wg-easy"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737575    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="31dfa589-323f-4aec-9109-a2f7065d90c7" containerName="smb"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737599    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="94a034f8-6855-4601-8c87-70d8795eda28" containerName="sabnzbd"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737623    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="nfs"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737646    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="8f7c4b03-d460-4a04-81d5-baa12c90108b" containerName="coredns"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737837    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="csi-snapshotter"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737867    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="87257916-cd95-49d3-bb9a-2ca265e41e37" containerName="unifi-controller-init-01-permissions"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737894    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="3bbaa9f0-a0cb-46e8-95db-f4a414b0b24e" containerName="gitea-init-postgres-wait"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737928    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="31dfa589-323f-4aec-9109-a2f7065d90c7" containerName="csi-provisioner"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737953    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="31dfa589-323f-4aec-9109-a2f7065d90c7" containerName="liveness-probe"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.737988    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="25715d19-0754-4b80-9a23-07c7d6de4edf" containerName="csi-node-driver-registrar"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.738012    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="51073c96-555b-4ab7-b75d-4ffe71922fb0" containerName="jellyfin"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.738034    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="272ccf49-d49b-4e04-8dd8-37710fdd1fb1" containerName="sonarr"
Dec 30 09:06:30 truenas k3s[6331]: E1230 09:06:30.738057    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="87257916-cd95-49d3-bb9a-2ca265e41e37" containerName="unifi-controller"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.738866    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="31dfa589-323f-4aec-9109-a2f7065d90c7" containerName="csi-provisioner"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.738900    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="31dfa589-323f-4aec-9109-a2f7065d90c7" containerName="smb"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.738923    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="25715d19-0754-4b80-9a23-07c7d6de4edf" containerName="csi-node-driver-registrar"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.738961    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="csi-provisioner"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.738984    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="51073c96-555b-4ab7-b75d-4ffe71922fb0" containerName="jellyfin"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739008    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="csi-snapshotter"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739041    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="31dfa589-323f-4aec-9109-a2f7065d90c7" containerName="liveness-probe"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739305    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="fe85ed7d-d0dd-4ea8-9d70-23365d4cf3a4" containerName="radarr"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739333    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="8f7c4b03-d460-4a04-81d5-baa12c90108b" containerName="coredns"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739367    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="87257916-cd95-49d3-bb9a-2ca265e41e37" containerName="unifi-controller-init-01-permissions"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739390    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="87257916-cd95-49d3-bb9a-2ca265e41e37" containerName="unifi-controller"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739415    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="94a034f8-6855-4601-8c87-70d8795eda28" containerName="sabnzbd"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739446    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="94a034f8-6855-4601-8c87-70d8795eda28" containerName="sabnzbd-init-01-permissions"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739468    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="nfs"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739490    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="d9b261b0-692c-49d3-ae73-9c02244c171b" containerName="wg-easy"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739511    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="272ccf49-d49b-4e04-8dd8-37710fdd1fb1" containerName="sonarr"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739542    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="25715d19-0754-4b80-9a23-07c7d6de4edf" containerName="openebs-zfs-plugin"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739564    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="3bbaa9f0-a0cb-46e8-95db-f4a414b0b24e" containerName="gitea"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739585    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="4737d07c-4267-43f0-8177-6d8ca1392ee6" containerName="liveness-probe"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739616    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="c6036fbb-5430-4134-968c-f1540cfde0d9" containerName="gitea"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.739638    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="3bbaa9f0-a0cb-46e8-95db-f4a414b0b24e" containerName="gitea-init-postgres-wait"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.740193    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.748528    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-p7cp4\" (UniqueName: \"kubernetes.io/projected/ea325533-21c4-494e-b08f-ee25cb2a136f-kube-api-access-p7cp4\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.748913    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-hrpkr\" (UniqueName: \"kubernetes.io/projected/a18a85bb-a765-4ff3-90cc-6436a0c95e08-kube-api-access-hrpkr\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.749133    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-rtqfc\" (UniqueName: \"kubernetes.io/projected/7ee71e90-c82b-41b2-b1eb-7b58096f348a-kube-api-access-rtqfc\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.757108    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.757723    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.760925    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.765388    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.771881    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.810133    6331 shared_informer.go:277] Caches are synced for attach detach
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.826164    6331 event.go:294] "Event occurred" object="kube-system/csi-nfs-controller-7b74694749" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: csi-nfs-controller-7b74694749-kv5tk"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.827628    6331 event.go:294] "Event occurred" object="kube-system/openebs-zfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulDelete" message="Deleted pod: openebs-zfs-node-trztd"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.834228    6331 event.go:294] "Event occurred" object="kube-system/csi-smb-controller-7fbbb8fb6f" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: csi-smb-controller-7fbbb8fb6f-67q8b"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.852728    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"postgresdata\" (UniqueName: \"kubernetes.io/host-path/552167b9-efa1-424b-8a26-27c144a1cf8d-postgresdata\") pod \"gitea-postgres-86c8c597f5-4x25r\" (UID: \"552167b9-efa1-424b-8a26-27c144a1cf8d\") " pod="ix-gitea/gitea-postgres-86c8c597f5-4x25r"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853305    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/58d652d4-5156-412d-a682-bdab69636ece-tmp\") pod \"sonarr-8f694b79d-mlrc8\" (UID: \"58d652d4-5156-412d-a682-bdab69636ece\") " pod="ix-sonarr/sonarr-8f694b79d-mlrc8"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853365    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/58d652d4-5156-412d-a682-bdab69636ece-config\") pod \"sonarr-8f694b79d-mlrc8\" (UID: \"58d652d4-5156-412d-a682-bdab69636ece\") " pod="ix-sonarr/sonarr-8f694b79d-mlrc8"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853426    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/6172a6be-5f0c-4902-9c07-b2dfc03aaacd-tmp\") pod \"radarr-78fb8b865b-zj49j\" (UID: \"6172a6be-5f0c-4902-9c07-b2dfc03aaacd\") " pod="ix-radarr/radarr-78fb8b865b-zj49j"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853497    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"custom-config-volume\" (UniqueName: \"kubernetes.io/configmap/fb38264f-c9c8-4717-a9cd-28bd2c4d829c-custom-config-volume\") pod \"coredns-59b4f5bbd5-z82vr\" (UID: \"fb38264f-c9c8-4717-a9cd-28bd2c4d829c\") " pod="kube-system/coredns-59b4f5bbd5-z82vr"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853563    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"gitea-temp\" (UniqueName: \"kubernetes.io/empty-dir/8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8-gitea-temp\") pod \"gitea-6f7479bdf4-vkdwt\" (UID: \"8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8\") " pod="ix-gitea/gitea-6f7479bdf4-vkdwt"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853646    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cert\" (UniqueName: \"kubernetes.io/secret/8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8-cert\") pod \"gitea-6f7479bdf4-vkdwt\" (UID: \"8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8\") " pod="ix-gitea/gitea-6f7479bdf4-vkdwt"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853710    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"sabnzbd-0\" (UniqueName: \"kubernetes.io/host-path/65991137-892f-4484-aaea-b8f6483de655-sabnzbd-0\") pod \"sabnzbd-9f94c7bfd-6929h\" (UID: \"65991137-892f-4484-aaea-b8f6483de655\") " pod="ix-sabnzbd/sabnzbd-9f94c7bfd-6929h"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853774    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/a0437ed4-83eb-49d4-9b44-e1bfe406f19d-config\") pod \"wg-easy-568f69bb5d-pnbxp\" (UID: \"a0437ed4-83eb-49d4-9b44-e1bfe406f19d\") " pod="ix-wg-easy/wg-easy-568f69bb5d-pnbxp"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853829    6331 event.go:294] "Event occurred" object="ix-unifi-controller/unifi-controller-85b5667bb" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: unifi-controller-85b5667bb-t5w66"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853831    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"radarr-0\" (UniqueName: \"kubernetes.io/host-path/6172a6be-5f0c-4902-9c07-b2dfc03aaacd-radarr-0\") pod \"radarr-78fb8b865b-zj49j\" (UID: \"6172a6be-5f0c-4902-9c07-b2dfc03aaacd\") " pod="ix-radarr/radarr-78fb8b865b-zj49j"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853926    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"data\" (UniqueName: \"kubernetes.io/host-path/8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8-data\") pod \"gitea-6f7479bdf4-vkdwt\" (UID: \"8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8\") " pod="ix-gitea/gitea-6f7479bdf4-vkdwt"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.853989    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/a0437ed4-83eb-49d4-9b44-e1bfe406f19d-tmp\") pod \"wg-easy-568f69bb5d-pnbxp\" (UID: \"a0437ed4-83eb-49d4-9b44-e1bfe406f19d\") " pod="ix-wg-easy/wg-easy-568f69bb5d-pnbxp"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854050    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/fb38264f-c9c8-4717-a9cd-28bd2c4d829c-config-volume\") pod \"coredns-59b4f5bbd5-z82vr\" (UID: \"fb38264f-c9c8-4717-a9cd-28bd2c4d829c\") " pod="kube-system/coredns-59b4f5bbd5-z82vr"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854133    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/65991137-892f-4484-aaea-b8f6483de655-config\") pod \"sabnzbd-9f94c7bfd-6929h\" (UID: \"65991137-892f-4484-aaea-b8f6483de655\") " pod="ix-sabnzbd/sabnzbd-9f94c7bfd-6929h"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854195    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/65991137-892f-4484-aaea-b8f6483de655-tmp\") pod \"sabnzbd-9f94c7bfd-6929h\" (UID: \"65991137-892f-4484-aaea-b8f6483de655\") " pod="ix-sabnzbd/sabnzbd-9f94c7bfd-6929h"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854257    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"sonarr-0\" (UniqueName: \"kubernetes.io/host-path/58d652d4-5156-412d-a682-bdab69636ece-sonarr-0\") pod \"sonarr-8f694b79d-mlrc8\" (UID: \"58d652d4-5156-412d-a682-bdab69636ece\") " pod="ix-sonarr/sonarr-8f694b79d-mlrc8"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854313    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8-config\") pod \"gitea-6f7479bdf4-vkdwt\" (UID: \"8e8380f5-5c36-46b7-b70f-62bdcfd3dfb8\") " pod="ix-gitea/gitea-6f7479bdf4-vkdwt"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854381    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-58rgs\" (UniqueName: \"kubernetes.io/projected/fb38264f-c9c8-4717-a9cd-28bd2c4d829c-kube-api-access-58rgs\") pod \"coredns-59b4f5bbd5-z82vr\" (UID: \"fb38264f-c9c8-4717-a9cd-28bd2c4d829c\") " pod="kube-system/coredns-59b4f5bbd5-z82vr"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854440    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/6172a6be-5f0c-4902-9c07-b2dfc03aaacd-config\") pod \"radarr-78fb8b865b-zj49j\" (UID: \"6172a6be-5f0c-4902-9c07-b2dfc03aaacd\") " pod="ix-radarr/radarr-78fb8b865b-zj49j"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854490    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-z8lf2\" (UniqueName: \"kubernetes.io/projected/a8fadcd0-3556-42ea-9ed3-b43c57dbed99-kube-api-access-z8lf2\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.854525    6331 reconciler_common.go:295] "Volume detached for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/25715d19-0754-4b80-9a23-07c7d6de4edf-chroot-zfs\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.861268    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.879019    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.894418    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.913712    6331 event.go:294] "Event occurred" object="kube-system/openebs-zfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.932521    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.954735    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/5908f6cd-e1b2-43d8-b50c-9a079d1b3813-socket-dir\") pod \"csi-nfs-controller-7b74694749-kv5tk\" (UID: \"5908f6cd-e1b2-43d8-b50c-9a079d1b3813\") " pod="kube-system/csi-nfs-controller-7b74694749-kv5tk"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.954851    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/5908f6cd-e1b2-43d8-b50c-9a079d1b3813-pods-mount-dir\") pod \"csi-nfs-controller-7b74694749-kv5tk\" (UID: \"5908f6cd-e1b2-43d8-b50c-9a079d1b3813\") " pod="kube-system/csi-nfs-controller-7b74694749-kv5tk"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.954915    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/45b7ac13-52e0-4f98-96f7-c9bf830cc478-encr-keys\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955029    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/45b7ac13-52e0-4f98-96f7-c9bf830cc478-pods-mount-dir\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955138    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"data\" (UniqueName: \"kubernetes.io/host-path/6cbdde0e-7e6e-435b-a60e-decd7bd7025d-data\") pod \"unifi-controller-85b5667bb-t5w66\" (UID: \"6cbdde0e-7e6e-435b-a60e-decd7bd7025d\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-t5w66"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955401    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/45b7ac13-52e0-4f98-96f7-c9bf830cc478-device-dir\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955474    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-sv2pl\" (UniqueName: \"kubernetes.io/projected/45b7ac13-52e0-4f98-96f7-c9bf830cc478-kube-api-access-sv2pl\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955627    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/45b7ac13-52e0-4f98-96f7-c9bf830cc478-host-root\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955802    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-9tv6w\" (UniqueName: \"kubernetes.io/projected/08d3b23e-b3fd-4563-96ae-c2167ab046ff-kube-api-access-9tv6w\") pod \"csi-smb-controller-7fbbb8fb6f-67q8b\" (UID: \"08d3b23e-b3fd-4563-96ae-c2167ab046ff\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-67q8b"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955909    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/08d3b23e-b3fd-4563-96ae-c2167ab046ff-socket-dir\") pod \"csi-smb-controller-7fbbb8fb6f-67q8b\" (UID: \"08d3b23e-b3fd-4563-96ae-c2167ab046ff\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-67q8b"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.955967    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/6cbdde0e-7e6e-435b-a60e-decd7bd7025d-tmp\") pod \"unifi-controller-85b5667bb-t5w66\" (UID: \"6cbdde0e-7e6e-435b-a60e-decd7bd7025d\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-t5w66"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.956052    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/45b7ac13-52e0-4f98-96f7-c9bf830cc478-plugin-dir\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.956137    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/45b7ac13-52e0-4f98-96f7-c9bf830cc478-registration-dir\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.956242    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-vlb8l\" (UniqueName: \"kubernetes.io/projected/5908f6cd-e1b2-43d8-b50c-9a079d1b3813-kube-api-access-vlb8l\") pod \"csi-nfs-controller-7b74694749-kv5tk\" (UID: \"5908f6cd-e1b2-43d8-b50c-9a079d1b3813\") " pod="kube-system/csi-nfs-controller-7b74694749-kv5tk"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.956323    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cert\" (UniqueName: \"kubernetes.io/empty-dir/6cbdde0e-7e6e-435b-a60e-decd7bd7025d-cert\") pod \"unifi-controller-85b5667bb-t5w66\" (UID: \"6cbdde0e-7e6e-435b-a60e-decd7bd7025d\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-t5w66"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.956714    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"logs\" (UniqueName: \"kubernetes.io/empty-dir/6cbdde0e-7e6e-435b-a60e-decd7bd7025d-logs\") pod \"unifi-controller-85b5667bb-t5w66\" (UID: \"6cbdde0e-7e6e-435b-a60e-decd7bd7025d\") " pod="ix-unifi-controller/unifi-controller-85b5667bb-t5w66"
Dec 30 09:06:30 truenas k3s[6331]: I1230 09:06:30.956799    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/45b7ac13-52e0-4f98-96f7-c9bf830cc478-chroot-zfs\") pod \"openebs-zfs-node-z2fg9\" (UID: \"45b7ac13-52e0-4f98-96f7-c9bf830cc478\") " pod="kube-system/openebs-zfs-node-z2fg9"
Dec 30 09:06:31 truenas k3s[6331]: I1230 09:06:31.009026    6331 shared_informer.go:277] Caches are synced for resource quota
Dec 30 09:06:31 truenas k3s[6331]: I1230 09:06:31.019505    6331 shared_informer.go:277] Caches are synced for resource quota
Dec 30 09:06:31 truenas k3s[6331]: I1230 09:06:31.148147    6331 shared_informer.go:277] Caches are synced for garbage collector
Dec 30 09:06:31 truenas k3s[6331]: I1230 09:06:31.170654    6331 shared_informer.go:277] Caches are synced for garbage collector
Dec 30 09:06:31 truenas k3s[6331]: I1230 09:06:31.170686    6331 garbagecollector.go:163] Garbage collector: all resource monitors have synced. Proceeding to collect garbage
Dec 30 09:06:31 truenas k3s[6331]: I1230 09:06:31.812040    6331 request.go:690] Waited for 1.039173967s due to client-side throttling, not priority and fairness, request: GET:https://127.0.0.1:6443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dcoredns-custom&limit=500&resourceVersion=0
Dec 30 09:06:31 truenas k3s[6331]: E1230 09:06:31.955754    6331 configmap.go:199] Couldn't get configMap kube-system/coredns: failed to sync configmap cache: timed out waiting for the condition
Dec 30 09:06:31 truenas k3s[6331]: E1230 09:06:31.955864    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/fb38264f-c9c8-4717-a9cd-28bd2c4d829c-config-volume podName:fb38264f-c9c8-4717-a9cd-28bd2c4d829c nodeName:}" failed. No retries permitted until 2023-12-30 09:06:32.455834948 +0100 CET m=+21.571953370 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "config-volume" (UniqueName: "kubernetes.io/configmap/fb38264f-c9c8-4717-a9cd-28bd2c4d829c-config-volume") pod "coredns-59b4f5bbd5-z82vr" (UID: "fb38264f-c9c8-4717-a9cd-28bd2c4d829c") : failed to sync configmap cache: timed out waiting for the condition
Dec 30 09:06:32 truenas k3s[6331]: E1230 09:06:32.057635    6331 configmap.go:199] Couldn't get configMap kube-system/openebs-zfspv-bin: failed to sync configmap cache: timed out waiting for the condition
Dec 30 09:06:32 truenas k3s[6331]: E1230 09:06:32.057707    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/45b7ac13-52e0-4f98-96f7-c9bf830cc478-chroot-zfs podName:45b7ac13-52e0-4f98-96f7-c9bf830cc478 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:32.557684644 +0100 CET m=+21.673803059 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "chroot-zfs" (UniqueName: "kubernetes.io/configmap/45b7ac13-52e0-4f98-96f7-c9bf830cc478-chroot-zfs") pod "openebs-zfs-node-z2fg9" (UID: "45b7ac13-52e0-4f98-96f7-c9bf830cc478") : failed to sync configmap cache: timed out waiting for the condition
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.153111    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=25715d19-0754-4b80-9a23-07c7d6de4edf path="/var/lib/kubelet/pods/25715d19-0754-4b80-9a23-07c7d6de4edf/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.157923    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=272ccf49-d49b-4e04-8dd8-37710fdd1fb1 path="/var/lib/kubelet/pods/272ccf49-d49b-4e04-8dd8-37710fdd1fb1/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.162142    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=3bbaa9f0-a0cb-46e8-95db-f4a414b0b24e path="/var/lib/kubelet/pods/3bbaa9f0-a0cb-46e8-95db-f4a414b0b24e/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.166251    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=51073c96-555b-4ab7-b75d-4ffe71922fb0 path="/var/lib/kubelet/pods/51073c96-555b-4ab7-b75d-4ffe71922fb0/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.169787    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=7ee71e90-c82b-41b2-b1eb-7b58096f348a path="/var/lib/kubelet/pods/7ee71e90-c82b-41b2-b1eb-7b58096f348a/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.172634    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=8f7c4b03-d460-4a04-81d5-baa12c90108b path="/var/lib/kubelet/pods/8f7c4b03-d460-4a04-81d5-baa12c90108b/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.188666    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=94a034f8-6855-4601-8c87-70d8795eda28 path="/var/lib/kubelet/pods/94a034f8-6855-4601-8c87-70d8795eda28/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.192629    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=a18a85bb-a765-4ff3-90cc-6436a0c95e08 path="/var/lib/kubelet/pods/a18a85bb-a765-4ff3-90cc-6436a0c95e08/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.194313    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=c6036fbb-5430-4134-968c-f1540cfde0d9 path="/var/lib/kubelet/pods/c6036fbb-5430-4134-968c-f1540cfde0d9/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.197119    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=d9b261b0-692c-49d3-ae73-9c02244c171b path="/var/lib/kubelet/pods/d9b261b0-692c-49d3-ae73-9c02244c171b/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.200218    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=ea325533-21c4-494e-b08f-ee25cb2a136f path="/var/lib/kubelet/pods/ea325533-21c4-494e-b08f-ee25cb2a136f/volumes"
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.203540    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=fe85ed7d-d0dd-4ea8-9d70-23365d4cf3a4 path="/var/lib/kubelet/pods/fe85ed7d-d0dd-4ea8-9d70-23365d4cf3a4/volumes"
Dec 30 09:06:32 truenas k3s[6331]: {"level":"warn","ts":"2023-12-30T09:06:32.346+0100","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002298e00/kine.sock","attempt":0,"error":"rpc error: code = Canceled desc = context canceled"}
Dec 30 09:06:32 truenas k3s[6331]: time="2023-12-30T09:06:32+01:00" level=error msg="error in txn: context canceled"
Dec 30 09:06:32 truenas k3s[6331]: E1230 09:06:32.347588    6331 finisher.go:175] FinishRequest: post-timeout activity - time-elapsed: 19.815µs, panicked: false, err: context canceled, panic-reason: <nil>
Dec 30 09:06:32 truenas k3s[6331]: E1230 09:06:32.347990    6331 writers.go:122] apiserver was unable to write a JSON response: http: Handler timeout
Dec 30 09:06:32 truenas k3s[6331]: E1230 09:06:32.350097    6331 status.go:71] apiserver received an error that is not an metav1.Status: &errors.errorString{s:"http: Handler timeout"}: http: Handler timeout
Dec 30 09:06:32 truenas k3s[6331]: E1230 09:06:32.351002    6331 writers.go:135] apiserver was unable to write a fallback JSON response: http: Handler timeout
Dec 30 09:06:32 truenas k3s[6331]: E1230 09:06:32.352286    6331 timeout.go:142] post-timeout activity - time-elapsed: 8.198529ms, POST "/api/v1/namespaces/ix-gitea/events" result: <nil>
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.470163    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-577zh\" (UniqueName: \"kubernetes.io/projected/7c61dd38-d4e4-4a38-a0eb-48077635fa65-kube-api-access-577zh\") pod \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\" (UID: \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\") "
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.471160    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-2ccgt\" (UniqueName: \"kubernetes.io/projected/e545eb4a-9b4c-426e-b86e-e2629dd4ea7f-kube-api-access-2ccgt\") pod \"e545eb4a-9b4c-426e-b86e-e2629dd4ea7f\" (UID: \"e545eb4a-9b4c-426e-b86e-e2629dd4ea7f\") "
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.471740    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/7c61dd38-d4e4-4a38-a0eb-48077635fa65-socket-dir\") pod \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\" (UID: \"7c61dd38-d4e4-4a38-a0eb-48077635fa65\") "
Dec 30 09:06:32 truenas k3s[6331]: W1230 09:06:32.474203    6331 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/7c61dd38-d4e4-4a38-a0eb-48077635fa65/volumes/kubernetes.io~empty-dir/socket-dir: clearQuota called, but quotas disabled
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.474694    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/7c61dd38-d4e4-4a38-a0eb-48077635fa65-socket-dir" (OuterVolumeSpecName: "socket-dir") pod "7c61dd38-d4e4-4a38-a0eb-48077635fa65" (UID: "7c61dd38-d4e4-4a38-a0eb-48077635fa65"). InnerVolumeSpecName "socket-dir". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.479562    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/e545eb4a-9b4c-426e-b86e-e2629dd4ea7f-kube-api-access-2ccgt" (OuterVolumeSpecName: "kube-api-access-2ccgt") pod "e545eb4a-9b4c-426e-b86e-e2629dd4ea7f" (UID: "e545eb4a-9b4c-426e-b86e-e2629dd4ea7f"). InnerVolumeSpecName "kube-api-access-2ccgt". PluginName "kubernetes.io/projected", VolumeGidValue ""
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.482569    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/7c61dd38-d4e4-4a38-a0eb-48077635fa65-kube-api-access-577zh" (OuterVolumeSpecName: "kube-api-access-577zh") pod "7c61dd38-d4e4-4a38-a0eb-48077635fa65" (UID: "7c61dd38-d4e4-4a38-a0eb-48077635fa65"). InnerVolumeSpecName "kube-api-access-577zh". PluginName "kubernetes.io/projected", VolumeGidValue ""
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.573897    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-577zh\" (UniqueName: \"kubernetes.io/projected/7c61dd38-d4e4-4a38-a0eb-48077635fa65-kube-api-access-577zh\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.574560    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-2ccgt\" (UniqueName: \"kubernetes.io/projected/e545eb4a-9b4c-426e-b86e-e2629dd4ea7f-kube-api-access-2ccgt\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:32 truenas k3s[6331]: I1230 09:06:32.574613    6331 reconciler_common.go:295] "Volume detached for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/7c61dd38-d4e4-4a38-a0eb-48077635fa65-socket-dir\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.283462    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-mb77q\" (UniqueName: \"kubernetes.io/projected/cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c-kube-api-access-mb77q\") pod \"cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c\" (UID: \"cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c\") "
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.286405    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c-kube-api-access-mb77q" (OuterVolumeSpecName: "kube-api-access-mb77q") pod "cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c" (UID: "cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c"). InnerVolumeSpecName "kube-api-access-mb77q". PluginName "kubernetes.io/projected", VolumeGidValue ""
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.384495    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-mb77q\" (UniqueName: \"kubernetes.io/projected/cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c-kube-api-access-mb77q\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.647442    6331 event.go:294] "Event occurred" object="kube-system/openebs-zfs-controller" fieldPath="" kind="StatefulSet" apiVersion="apps/v1" type="Warning" reason="RecreatingFailedPod" message="StatefulSet kube-system/openebs-zfs-controller is recreating failed Pod openebs-zfs-controller-0"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.670889    6331 event.go:294] "Event occurred" object="kube-system/openebs-zfs-controller" fieldPath="" kind="StatefulSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulDelete" message="delete Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.687786    6331 event.go:294] "Event occurred" object="kube-system/openebs-zfs-controller" fieldPath="" kind="StatefulSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="create Pod openebs-zfs-controller-0 in StatefulSet openebs-zfs-controller successful"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763305    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.763439    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="e545eb4a-9b4c-426e-b86e-e2629dd4ea7f" containerName="snapshot-controller"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.763475    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c" containerName="snapshot-controller"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.763498    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="csi-snapshotter"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.763525    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="csi-resizer"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.763549    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="openebs-zfs-plugin"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.763582    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="csi-provisioner"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.763605    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="snapshot-controller"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763687    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="cfbcd6c8-bdad-43e9-95f0-c6ae5da8859c" containerName="snapshot-controller"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763793    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="snapshot-controller"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763826    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="csi-snapshotter"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763851    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="csi-resizer"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763874    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="e545eb4a-9b4c-426e-b86e-e2629dd4ea7f" containerName="snapshot-controller"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763893    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="openebs-zfs-plugin"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.763918    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="7c61dd38-d4e4-4a38-a0eb-48077635fa65" containerName="csi-provisioner"
Dec 30 09:06:33 truenas k3s[6331]: E1230 09:06:33.781400    6331 stateful_set.go:422] error syncing StatefulSet kube-system/openebs-zfs-controller, requeuing: pods "openebs-zfs-controller-0" already exists, the server was not able to generate a unique name for the object
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.792537    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/9d9f35ec-9de0-4343-be30-43e793ef7b3f-socket-dir\") pod \"openebs-zfs-controller-0\" (UID: \"9d9f35ec-9de0-4343-be30-43e793ef7b3f\") " pod="kube-system/openebs-zfs-controller-0"
Dec 30 09:06:33 truenas k3s[6331]: I1230 09:06:33.792630    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-wmpqf\" (UniqueName: \"kubernetes.io/projected/9d9f35ec-9de0-4343-be30-43e793ef7b3f-kube-api-access-wmpqf\") pod \"openebs-zfs-controller-0\" (UID: \"9d9f35ec-9de0-4343-be30-43e793ef7b3f\") " pod="kube-system/openebs-zfs-controller-0"
Dec 30 09:06:34 truenas k3s[6331]: E1230 09:06:34.019095    6331 configmap.go:199] Couldn't get configMap kube-system/nvidia-device-plugin-config: object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:34 truenas k3s[6331]: E1230 09:06:34.019202    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config podName:aa9b377b-bd02-4e51-83ed-376a36c0d803 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:50.019173777 +0100 CET m=+39.135292219 (durationBeforeRetry 16s). Error: MountVolume.SetUp failed for volume "plugin-config" (UniqueName: "kubernetes.io/configmap/aa9b377b-bd02-4e51-83ed-376a36c0d803-plugin-config") pod "nvidia-device-plugin-daemonset-bl5th" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803") : object "kube-system"/"nvidia-device-plugin-config" not registered
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.123781    6331 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=7c61dd38-d4e4-4a38-a0eb-48077635fa65 path="/var/lib/kubelet/pods/7c61dd38-d4e4-4a38-a0eb-48077635fa65/volumes"
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.320283    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-volume-duplicati-duplicati-config\") pod \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\" (UID: \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\") "
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.320372    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-1\") pod \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\" (UID: \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\") "
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.320451    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-7xcxs\" (UniqueName: \"kubernetes.io/projected/aa9b377b-bd02-4e51-83ed-376a36c0d803-kube-api-access-7xcxs\") pod \"aa9b377b-bd02-4e51-83ed-376a36c0d803\" (UID: \"aa9b377b-bd02-4e51-83ed-376a36c0d803\") "
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.320506    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-0\") pod \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\" (UID: \"5f64faad-9fcf-478b-a4a5-e1e8e9e5c409\") "
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.320564    6331 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"device-plugin\" (UniqueName: \"kubernetes.io/host-path/aa9b377b-bd02-4e51-83ed-376a36c0d803-device-plugin\") pod \"aa9b377b-bd02-4e51-83ed-376a36c0d803\" (UID: \"aa9b377b-bd02-4e51-83ed-376a36c0d803\") "
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.321344    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-volume-duplicati-duplicati-config" (OuterVolumeSpecName: "ix-host-volume-duplicati-duplicati-config") pod "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409"). InnerVolumeSpecName "ix-host-volume-duplicati-duplicati-config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.321402    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-1" (OuterVolumeSpecName: "ix-host-path-duplicati-1") pod "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409"). InnerVolumeSpecName "ix-host-path-duplicati-1". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.322445    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-0" (OuterVolumeSpecName: "ix-host-path-duplicati-0") pod "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409"). InnerVolumeSpecName "ix-host-path-duplicati-0". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.322521    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/aa9b377b-bd02-4e51-83ed-376a36c0d803-device-plugin" (OuterVolumeSpecName: "device-plugin") pod "aa9b377b-bd02-4e51-83ed-376a36c0d803" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803"). InnerVolumeSpecName "device-plugin". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.332462    6331 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/aa9b377b-bd02-4e51-83ed-376a36c0d803-kube-api-access-7xcxs" (OuterVolumeSpecName: "kube-api-access-7xcxs") pod "aa9b377b-bd02-4e51-83ed-376a36c0d803" (UID: "aa9b377b-bd02-4e51-83ed-376a36c0d803"). InnerVolumeSpecName "kube-api-access-7xcxs". PluginName "kubernetes.io/projected", VolumeGidValue ""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.421242    6331 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-7xcxs\" (UniqueName: \"kubernetes.io/projected/aa9b377b-bd02-4e51-83ed-376a36c0d803-kube-api-access-7xcxs\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.421294    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-0\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-0\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.421326    6331 reconciler_common.go:295] "Volume detached for volume \"device-plugin\" (UniqueName: \"kubernetes.io/host-path/aa9b377b-bd02-4e51-83ed-376a36c0d803-device-plugin\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.421362    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-volume-duplicati-duplicati-config\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-volume-duplicati-duplicati-config\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.421391    6331 reconciler_common.go:295] "Volume detached for volume \"ix-host-path-duplicati-1\" (UniqueName: \"kubernetes.io/host-path/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-ix-host-path-duplicati-1\") on node \"ix-truenas\" DevicePath \"\""
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.670127    6331 event.go:294] "Event occurred" object="ix-duplicati/duplicati-ix-chart-6c986975cb" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: duplicati-ix-chart-6c986975cb-x8ldc"
Dec 30 09:06:34 truenas k3s[6331]: I1230 09:06:34.688158    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:35 truenas k3s[6331]: I1230 09:06:35.044676    6331 event.go:294] "Event occurred" object="kube-system/snapshot-controller-546868dfb4" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: snapshot-controller-546868dfb4-278c9"
Dec 30 09:06:35 truenas k3s[6331]: I1230 09:06:35.087305    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:35 truenas k3s[6331]: E1230 09:06:35.087434    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="5f64faad-9fcf-478b-a4a5-e1e8e9e5c409" containerName="ix-chart"
Dec 30 09:06:35 truenas k3s[6331]: E1230 09:06:35.087463    6331 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="aa9b377b-bd02-4e51-83ed-376a36c0d803" containerName="nvidia-device-plugin-ctr"
Dec 30 09:06:35 truenas k3s[6331]: I1230 09:06:35.087537    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="5f64faad-9fcf-478b-a4a5-e1e8e9e5c409" containerName="ix-chart"
Dec 30 09:06:35 truenas k3s[6331]: I1230 09:06:35.087561    6331 memory_manager.go:346] "RemoveStaleState removing state" podUID="aa9b377b-bd02-4e51-83ed-376a36c0d803" containerName="nvidia-device-plugin-ctr"
Dec 30 09:06:35 truenas k3s[6331]: I1230 09:06:35.127511    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-9t9b9\" (UniqueName: \"kubernetes.io/projected/5409416b-252e-45e6-b5be-240ef907be98-kube-api-access-9t9b9\") pod \"snapshot-controller-546868dfb4-278c9\" (UID: \"5409416b-252e-45e6-b5be-240ef907be98\") " pod="kube-system/snapshot-controller-546868dfb4-278c9"
Dec 30 09:06:35 truenas k3s[6331]: E1230 09:06:35.430840    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:35 truenas k3s[6331]: E1230 09:06:35.438487    6331 projected.go:198] Error preparing data for projected volume kube-api-access-xbmql for pod ix-duplicati/duplicati-ix-chart-6c986975cb-s9wr5: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:35 truenas k3s[6331]: E1230 09:06:35.440616    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql podName:5f64faad-9fcf-478b-a4a5-e1e8e9e5c409 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:51.440578261 +0100 CET m=+40.556696694 (durationBeforeRetry 16s). Error: MountVolume.SetUp failed for volume "kube-api-access-xbmql" (UniqueName: "kubernetes.io/projected/5f64faad-9fcf-478b-a4a5-e1e8e9e5c409-kube-api-access-xbmql") pod "duplicati-ix-chart-6c986975cb-s9wr5" (UID: "5f64faad-9fcf-478b-a4a5-e1e8e9e5c409") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:36 truenas k3s[6331]: I1230 09:06:36.042673    6331 event.go:294] "Event occurred" object="kube-system/snapshot-controller-546868dfb4" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: snapshot-controller-546868dfb4-srv7x"
Dec 30 09:06:36 truenas k3s[6331]: I1230 09:06:36.054349    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:36 truenas k3s[6331]: I1230 09:06:36.144433    6331 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-ql5sn\" (UniqueName: \"kubernetes.io/projected/4830479b-7dff-452f-89aa-052a5b579985-kube-api-access-ql5sn\") pod \"snapshot-controller-546868dfb4-srv7x\" (UID: \"4830479b-7dff-452f-89aa-052a5b579985\") " pod="kube-system/snapshot-controller-546868dfb4-srv7x"
Dec 30 09:06:37 truenas k3s[6331]: E1230 09:06:37.284842    6331 projected.go:292] Couldn't get configMap ix-duplicati/kube-root-ca.crt: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:37 truenas k3s[6331]: E1230 09:06:37.290250    6331 projected.go:198] Error preparing data for projected volume kube-api-access-kwpvz for pod ix-duplicati/duplicati-ix-chart-6c986975cb-8t9rq: object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:37 truenas k3s[6331]: E1230 09:06:37.290617    6331 nestedpendingoperations.go:348] Operation for "{volumeName:kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz podName:46548d19-6f35-4e2a-a247-dbad3afad7d4 nodeName:}" failed. No retries permitted until 2023-12-30 09:06:53.290573129 +0100 CET m=+42.406691561 (durationBeforeRetry 16s). Error: MountVolume.SetUp failed for volume "kube-api-access-kwpvz" (UniqueName: "kubernetes.io/projected/46548d19-6f35-4e2a-a247-dbad3afad7d4-kube-api-access-kwpvz") pod "duplicati-ix-chart-6c986975cb-8t9rq" (UID: "46548d19-6f35-4e2a-a247-dbad3afad7d4") : object "ix-duplicati"/"kube-root-ca.crt" not registered
Dec 30 09:06:40 truenas k3s[6331]: I1230 09:06:40.030326    6331 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-gitea/gitea-postgres-86c8c597f5-4x25r" podStartSLOduration=1228.029836853 pod.CreationTimestamp="2023-12-30 08:46:12 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2023-12-30 09:06:40.028674361 +0100 CET m=+29.144792818" watchObservedRunningTime="2023-12-30 09:06:40.029836853 +0100 CET m=+29.145955280"
Dec 30 09:06:40 truenas k3s[6331]: I1230 09:06:40.556831    6331 controller.go:615] quota admission added evaluator for: zfsnodes.zfs.openebs.io
Dec 30 09:06:40 truenas k3s[6331]: I1230 09:06:40.827583    6331 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-wg-easy/wg-easy-568f69bb5d-pnbxp" podStartSLOduration=1226.827517917 pod.CreationTimestamp="2023-12-30 08:46:14 +0100 CET" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2023-12-30 09:06:40.443519567 +0100 CET m=+29.559638022" watchObservedRunningTime="2023-12-30 09:06:40.827517917 +0100 CET m=+29.943636357"
Dec 30 09:06:40 truenas k3s[6331]: I1230 09:06:40.838329    6331 event.go:294] "Event occurred" object="ix-duplicati/duplicati-ix-chart-6c986975cb" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: duplicati-ix-chart-6c986975cb-sv5j5"
Dec 30 09:06:40 truenas k3s[6331]: I1230 09:06:40.864424    6331 topology_manager.go:210] "Topology Admit Handler"
Dec 30 09:06:42 truenas k3s[6331]: I1230 09:06:42.010849    6331 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: zfs.csi.openebs.io endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock versions: 1.0.0
Dec 30 09:06:42 truenas k3s[6331]: I1230 09:06:42.010893    6331 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: zfs.csi.openebs.io at endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
