
-------------
1) K3s logs
-------------

Feb 23 19:30:39 truenas k3s[6488]: time="2024-02-23T19:30:39-08:00" level=info msg="Configuring database table schema and indexes, this may take a moment..."
Feb 23 19:30:39 truenas k3s[6488]: time="2024-02-23T19:30:39-08:00" level=info msg="Database tables and indexes are up to date"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Kine available at unix://kine.sock"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Reconciling bootstrap data between datastore and disk"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Running kube-apiserver --advertise-address=192.168.0.132 --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/log/k3s_server_audit.log --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/mnt/Test/ix-applications/k3s/server/tls/temporary-certs --client-ca-file=/mnt/Test/ix-applications/k3s/server/tls/client-ca.crt --egress-selector-config-file=/mnt/Test/ix-applications/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --etcd-servers=unix://kine.sock --feature-gates=MixedProtocolLBService=true --kubelet-certificate-authority=/mnt/Test/ix-applications/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/mnt/Test/ix-applications/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/mnt/Test/ix-applications/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/mnt/Test/ix-applications/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/mnt/Test/ix-applications/k3s/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/mnt/Test/ix-applications/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/mnt/Test/ix-applications/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/mnt/Test/ix-applications/k3s/server/tls/service.current.key --service-cluster-ip-range=172.17.0.0/16 --service-node-port-range=9000-65535 --storage-backend=etcd3 --tls-cert-file=/mnt/Test/ix-applications/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/mnt/Test/ix-applications/k3s/server/tls/serving-kube-apiserver.key"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Running kube-scheduler --authentication-kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/scheduler.kubeconfig --leader-elect=false --profiling=false --secure-port=10259"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Waiting for API server to become available"
Feb 23 19:30:40 truenas k3s[6488]: W0223 19:30:40.637081    6488 feature_gate.go:241] Setting GA feature gate MixedProtocolLBService=true. It will be removed in a future release.
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cluster-cidr=172.16.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/mnt/Test/ix-applications/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/mnt/Test/ix-applications/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/mnt/Test/ix-applications/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/mnt/Test/ix-applications/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/mnt/Test/ix-applications/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/mnt/Test/ix-applications/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/mnt/Test/ix-applications/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/mnt/Test/ix-applications/k3s/server/tls/server-ca.key --controllers=*,tokencleaner --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/controller.kubeconfig --leader-elect=false --node-cidr-mask-size=16 --profiling=false --root-ca-file=/mnt/Test/ix-applications/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/mnt/Test/ix-applications/k3s/server/tls/service.current.key --service-cluster-ip-range=172.17.0.0/16 --terminated-pod-gc-threshold=5 --use-service-account-credentials=true"
Feb 23 19:30:40 truenas k3s[6488]: I0223 19:30:40.640975    6488 server.go:571] external host was not specified, using 192.168.0.132
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Running cloud-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/cloud-controller.kubeconfig --authorization-kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/cloud-controller.kubeconfig --bind-address=127.0.0.1 --cloud-config=/mnt/Test/ix-applications/k3s/server/etc/cloud-config.yaml --cloud-provider=k3s --cluster-cidr=172.16.0.0/16 --configure-cloud-routes=false --controllers=*,-route,-cloud-node,-cloud-node-lifecycle --kubeconfig=/mnt/Test/ix-applications/k3s/server/cred/cloud-controller.kubeconfig --leader-elect=false --leader-elect-resource-name=k3s-cloud-controller-manager --node-status-update-frequency=1m0s --profiling=false --secure-port=0"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Server node token is available at /mnt/Test/ix-applications/k3s/server/token"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="To join server node to cluster: k3s server -s https://192.168.0.132:6443 -t ${SERVER_NODE_TOKEN}"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Agent node token is available at /mnt/Test/ix-applications/k3s/server/agent-token"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="To join agent node to cluster: k3s agent -s https://192.168.0.132:6443 -t ${AGENT_NODE_TOKEN}"
Feb 23 19:30:40 truenas k3s[6488]: I0223 19:30:40.645799    6488 server.go:172] Version: v1.26.6+k3s-e18037a7-dirty
Feb 23 19:30:40 truenas k3s[6488]: I0223 19:30:40.645983    6488 server.go:174] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Wrote kubeconfig /etc/rancher/k3s/k3s.yaml"
Feb 23 19:30:40 truenas k3s[6488]: time="2024-02-23T19:30:40-08:00" level=info msg="Run: k3s kubectl"
Feb 23 19:30:40 truenas k3s[6488]: I0223 19:30:40.866773    6488 shared_informer.go:270] Waiting for caches to sync for node_authorizer
Feb 23 19:30:40 truenas k3s[6488]: I0223 19:30:40.873930    6488 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
Feb 23 19:30:40 truenas k3s[6488]: I0223 19:30:40.873962    6488 plugins.go:161] Loaded 12 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook,ResourceQuota.
Feb 23 19:30:40 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:30:40.894-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:30:40 truenas k3s[6488]: W0223 19:30:40.924469    6488 genericapiserver.go:660] Skipping API apiextensions.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:40 truenas k3s[6488]: I0223 19:30:40.926631    6488 instance.go:277] Using reconciler: lease
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.083016    6488 instance.go:621] API group "internal.apiserver.k8s.io" is not enabled, skipping.
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.197873    6488 instance.go:621] API group "resource.k8s.io" is not enabled, skipping.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.265206    6488 genericapiserver.go:660] Skipping API authentication.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.265235    6488 genericapiserver.go:660] Skipping API authentication.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.267422    6488 genericapiserver.go:660] Skipping API authorization.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.272919    6488 genericapiserver.go:660] Skipping API autoscaling/v2beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.272945    6488 genericapiserver.go:660] Skipping API autoscaling/v2beta2 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.275969    6488 genericapiserver.go:660] Skipping API batch/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.277505    6488 genericapiserver.go:660] Skipping API certificates.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.278593    6488 genericapiserver.go:660] Skipping API coordination.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.278623    6488 genericapiserver.go:660] Skipping API discovery.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.281712    6488 genericapiserver.go:660] Skipping API networking.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.281722    6488 genericapiserver.go:660] Skipping API networking.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.282782    6488 genericapiserver.go:660] Skipping API node.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.282790    6488 genericapiserver.go:660] Skipping API node.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.282817    6488 genericapiserver.go:660] Skipping API policy/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.286157    6488 genericapiserver.go:660] Skipping API rbac.authorization.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.286169    6488 genericapiserver.go:660] Skipping API rbac.authorization.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.287181    6488 genericapiserver.go:660] Skipping API scheduling.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.287190    6488 genericapiserver.go:660] Skipping API scheduling.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.290701    6488 genericapiserver.go:660] Skipping API storage.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.294090    6488 genericapiserver.go:660] Skipping API flowcontrol.apiserver.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.294101    6488 genericapiserver.go:660] Skipping API flowcontrol.apiserver.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.297238    6488 genericapiserver.go:660] Skipping API apps/v1beta2 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.297248    6488 genericapiserver.go:660] Skipping API apps/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.298631    6488 genericapiserver.go:660] Skipping API admissionregistration.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.298641    6488 genericapiserver.go:660] Skipping API admissionregistration.k8s.io/v1alpha1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.299720    6488 genericapiserver.go:660] Skipping API events.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: W0223 19:30:41.328426    6488 genericapiserver.go:660] Skipping API apiregistration.k8s.io/v1beta1 because it has no resources.
Feb 23 19:30:41 truenas k3s[6488]: time="2024-02-23T19:30:41-08:00" level=info msg="Password verified locally for node ix-truenas"
Feb 23 19:30:41 truenas k3s[6488]: time="2024-02-23T19:30:41-08:00" level=info msg="certificate CN=ix-truenas signed by CN=k3s-server-ca@1708472647: notBefore=2024-02-20 23:44:07 +0000 UTC notAfter=2025-02-23 03:30:41 +0000 UTC"
Feb 23 19:30:41 truenas k3s[6488]: time="2024-02-23T19:30:41-08:00" level=info msg="certificate CN=system:node:ix-truenas,O=system:nodes signed by CN=k3s-client-ca@1708472647: notBefore=2024-02-20 23:44:07 +0000 UTC notAfter=2025-02-23 03:30:41 +0000 UTC"
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.982440    6488 secure_serving.go:210] Serving securely on 127.0.0.1:6444
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.982711    6488 controller.go:83] Starting OpenAPI AggregationController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.982727    6488 available_controller.go:423] Starting AvailableConditionController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.982947    6488 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983009    6488 apf_controller.go:361] Starting API Priority and Fairness config controller
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983268    6488 apiservice_controller.go:97] Starting APIServiceRegistrationController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983279    6488 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983283    6488 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983279    6488 dynamic_serving_content.go:132] "Starting controller" name="serving-cert::/mnt/Test/ix-applications/k3s/server/tls/serving-kube-apiserver.crt::/mnt/Test/ix-applications/k3s/server/tls/serving-kube-apiserver.key"
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983346    6488 controller.go:80] Starting OpenAPI V3 AggregationController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983360    6488 customresource_discovery_controller.go:288] Starting DiscoveryController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983285    6488 dynamic_cafile_content.go:157] "Starting controller" name="request-header::/mnt/Test/ix-applications/k3s/server/tls/request-header-ca.crt"
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983286    6488 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/Test/ix-applications/k3s/server/tls/client-ca.crt"
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983756    6488 establishing_controller.go:76] Starting EstablishingController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983766    6488 naming_controller.go:291] Starting NamingConditionController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983828    6488 controller.go:85] Starting OpenAPI controller
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.983968    6488 nonstructuralschema_controller.go:192] Starting NonStructuralSchemaConditionController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984120    6488 apiapproval_controller.go:186] Starting KubernetesAPIApprovalPolicyConformantConditionController
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984151    6488 crd_finalizer.go:266] Starting CRDFinalizer
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984173    6488 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/Test/ix-applications/k3s/server/tls/client-ca.crt"
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984254    6488 dynamic_cafile_content.go:157] "Starting controller" name="request-header::/mnt/Test/ix-applications/k3s/server/tls/request-header-ca.crt"
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984304    6488 controller.go:85] Starting OpenAPI V3 controller
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984456    6488 aggregator.go:150] waiting for initial CRD sync...
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984567    6488 crdregistration_controller.go:111] Starting crd-autoregister controller
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984576    6488 shared_informer.go:270] Waiting for caches to sync for crd-autoregister
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984952    6488 controller.go:121] Starting legacy_token_tracking_controller
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984973    6488 shared_informer.go:270] Waiting for caches to sync for configmaps
Feb 23 19:30:41 truenas k3s[6488]: I0223 19:30:41.984954    6488 gc_controller.go:78] Starting apiserver lease garbage collector
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.016074    6488 cluster_authentication_trust_controller.go:440] Starting cluster_authentication_trust_controller controller
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.016081    6488 dynamic_serving_content.go:132] "Starting controller" name="aggregator-proxy-cert::/mnt/Test/ix-applications/k3s/server/tls/client-auth-proxy.crt::/mnt/Test/ix-applications/k3s/server/tls/client-auth-proxy.key"
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.016095    6488 shared_informer.go:270] Waiting for caches to sync for cluster_authentication_trust_controller
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.067074    6488 shared_informer.go:277] Caches are synced for node_authorizer
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.083119    6488 cache.go:39] Caches are synced for AvailableConditionController controller
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.083202    6488 apf_controller.go:366] Running API Priority and Fairness config worker
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.083218    6488 apf_controller.go:369] Running API Priority and Fairness periodic rebalancing process
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.083374    6488 cache.go:39] Caches are synced for APIServiceRegistrationController controller
Feb 23 19:30:42 truenas k3s[6488]: E0223 19:30:42.084065    6488 controller.go:163] Error removing old endpoints from kubernetes service: no master IPs were listed in storage, refusing to erase all endpoints for the kubernetes service
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.084753    6488 shared_informer.go:277] Caches are synced for crd-autoregister
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.084942    6488 aggregator.go:152] initial CRD sync complete...
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.085054    6488 shared_informer.go:277] Caches are synced for configmaps
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.085552    6488 autoregister_controller.go:141] Starting autoregister controller
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.085572    6488 cache.go:32] Waiting for caches to sync for autoregister controller
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.085602    6488 cache.go:39] Caches are synced for autoregister controller
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.116559    6488 shared_informer.go:277] Caches are synced for cluster_authentication_trust_controller
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.119716    6488 controller.go:615] quota admission added evaluator for: leases.coordination.k8s.io
Feb 23 19:30:42 truenas k3s[6488]: E0223 19:30:42.122946    6488 controller.go:189] failed to update lease, error: Operation cannot be fulfilled on leases.coordination.k8s.io "kube-apiserver-n74xodhwtbuxp6fm4ioy2qoh6m": StorageError: invalid object, Code: 4, Key: /registry/leases/kube-system/kube-apiserver-n74xodhwtbuxp6fm4ioy2qoh6m, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: deb0e877-9be6-4110-9965-6452436222f7, UID in object meta:
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Module overlay was already loaded"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Module br_netfilter was already loaded"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Set sysctl 'net/ipv4/conf/all/forwarding' to 1"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_max' to 393216"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_established' to 86400"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Set sysctl 'net/netfilter/nf_conntrack_tcp_timeout_close_wait' to 3600"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Found nvidia container runtime at /usr/bin/nvidia-container-runtime"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Logging containerd to /mnt/Test/ix-applications/k3s/agent/containerd/containerd.log"
Feb 23 19:30:42 truenas k3s[6488]: time="2024-02-23T19:30:42-08:00" level=info msg="Running containerd -c /mnt/Test/ix-applications/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /mnt/Test/ix-applications/k3s/agent/containerd"
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.752894    6488 controller.go:132] OpenAPI AggregationController: action for item k8s_internal_local_delegation_chain_0000000000: Nothing (removed from the queue).
Feb 23 19:30:42 truenas k3s[6488]: I0223 19:30:42.990344    6488 storage_scheduling.go:111] all system priority classes are created successfully or already exist.
Feb 23 19:30:43 truenas k3s[6488]: time="2024-02-23T19:30:43-08:00" level=info msg="Waiting for containerd startup: rpc error: code = Unknown desc = server is not initialized yet"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Waiting for cloud-controller-manager privileges to become available"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Kube API server is now running"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="ETCD server is now running"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="k3s is up and running"
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.022807    6488 feature_gate.go:241] Setting GA feature gate JobTrackingWithFinalizers=true. It will be removed in a future release.
Feb 23 19:30:44 truenas systemd[1]: Started k3s.service - Lightweight Kubernetes.
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Applying CRD helmcharts.helm.cattle.io"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.069123    6488 controllermanager.go:152] Version: v1.26.6+k3s-e18037a7-dirty
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Applying CRD helmchartconfigs.helm.cattle.io"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Creating service-controller event broadcaster"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Applying CRD addons.k3s.cattle.io"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Creating k3s-supervisor event broadcaster"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing static file: /mnt/Test/ix-applications/k3s/server/static/charts/traefik-10.19.300.tgz"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing static file: /mnt/Test/ix-applications/k3s/server/static/charts/traefik-crd-10.19.300.tgz"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/nfs-csi.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/rolebindings.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/coredns.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/kuberouter.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/multus-daemonset.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/zfs-operator.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/ccm.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Writing manifest: /mnt/Test/ix-applications/k3s/server/manifests/smb-csi.yaml"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Tunnel server egress proxy mode: agent"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.247923    6488 serving.go:355] Generated self-signed cert in-memory
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting k3s.cattle.io/v1, Kind=Addon controller"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Creating deploy event broadcaster"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.296951    6488 event.go:294] "Event occurred" object="kube-system/ccm" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/ccm.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting /v1, Kind=Node controller"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.330817    6488 controller.go:615] quota admission added evaluator for: rolebindings.rbac.authorization.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.332352    6488 event.go:294] "Event occurred" object="kube-system/ccm" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/ccm.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.334885    6488 controller.go:615] quota admission added evaluator for: addons.k3s.cattle.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.342766    6488 event.go:294] "Event occurred" object="kube-system/coredns" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/coredns.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting /v1, Kind=Pod controller"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.363842    6488 serving.go:355] Generated self-signed cert in-memory
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting /v1, Kind=Node controller"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting /v1, Kind=Secret controller"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=warning msg="Unable to fetch coredns config map: configmaps \"coredns\" not found"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Creating helm-controller event broadcaster"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Cluster dns configmap already exists"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Updating TLS secret for kube-system/k3s-serving (count: 11): map[listener.cattle.io/cn-0.0.0.0:0.0.0.0 listener.cattle.io/cn-127.0.0.1:127.0.0.1 listener.cattle.io/cn-172.17.0.1:172.17.0.1 listener.cattle.io/cn-192.168.0.132:192.168.0.132 listener.cattle.io/cn-__1-f16284:::1 listener.cattle.io/cn-ix-truenas:ix-truenas listener.cattle.io/cn-kubernetes:kubernetes listener.cattle.io/cn-kubernetes.default:kubernetes.default listener.cattle.io/cn-kubernetes.default.svc:kubernetes.default.svc listener.cattle.io/cn-kubernetes.default.svc.cluster.local:kubernetes.default.svc.cluster.local listener.cattle.io/cn-localhost:localhost listener.cattle.io/fingerprint:SHA1=120716F9505AA98FF2631C3560B53B581F9E3D37]"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting apps/v1, Kind=DaemonSet controller"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Labels and annotations have been set successfully on node: ix-truenas"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="containerd is now running"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Connecting to proxy" url="wss://127.0.0.1:6443/v1-k3s/connect"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/mnt/Test/ix-applications/k3s/agent/client-ca.crt --cluster-dns=172.17.0.10 --cluster-domain=cluster.local --config=/etc/rancher/k3s/kubelet_config.yaml --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ix-truenas --kubeconfig=/mnt/Test/ix-applications/k3s/agent/kubelet.kubeconfig --node-ip=0.0.0.0 --node-labels= --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/mnt/Test/ix-applications/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/etc/resolv.conf --serialize-image-pulls=false --tls-cert-file=/mnt/Test/ix-applications/k3s/agent/serving-kubelet.crt --tls-private-key-file=/mnt/Test/ix-applications/k3s/agent/serving-kubelet.key"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Handling backend connection request [ix-truenas]"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting discovery.k8s.io/v1, Kind=EndpointSlice controller"
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.538890    6488 controllermanager.go:288] "cloud-node" is disabled
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.538915    6488 controllermanager.go:288] "cloud-node-lifecycle" is disabled
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.539983    6488 controllermanager.go:311] Started "service"
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.540004    6488 controllermanager.go:288] "route" is disabled
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.541226    6488 controller.go:615] quota admission added evaluator for: deployments.apps
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.541576    6488 controller.go:227] Starting service controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.541619    6488 shared_informer.go:270] Waiting for caches to sync for service
Feb 23 19:30:44 truenas k3s[6488]: Flag --containerd has been deprecated, This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed.
Feb 23 19:30:44 truenas k3s[6488]: Flag --pod-infra-container-image has been deprecated, will be removed in 1.27. Image garbage collector will get sandbox image information from CRI.
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.542126    6488 server.go:197] "--pod-infra-container-image will not be pruned by the image garbage collector in kubelet and should also be set in the remote runtime"
Feb 23 19:30:44 truenas k3s[6488]: Flag --pod-infra-container-image has been deprecated, will be removed in 1.27. Image garbage collector will get sandbox image information from CRI.
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.550863    6488 event.go:294] "Event occurred" object="kube-system/coredns" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/coredns.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Annotations and labels have already set on node: ix-truenas"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.554626    6488 event.go:294] "Event occurred" object="kube-system/kuberouter" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/kuberouter.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.567172    6488 event.go:294] "Event occurred" object="kube-system/kuberouter" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/kuberouter.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.570932    6488 event.go:294] "Event occurred" object="kube-system/multus-daemonset" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/multus-daemonset.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting /v1, Kind=ConfigMap controller"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting /v1, Kind=ServiceAccount controller"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChart controller"
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting helm.cattle.io/v1, Kind=HelmChartConfig controller"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.595779    6488 server.go:407] "Kubelet version" kubeletVersion="v1.26.6+k3s-e18037a7-dirty"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.595798    6488 server.go:409] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding controller"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.597083    6488 dynamic_cafile_content.go:157] "Starting controller" name="client-ca-bundle::/mnt/Test/ix-applications/k3s/agent/client-ca.crt"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.601773    6488 event.go:294] "Event occurred" object="kube-system/multus-daemonset" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/multus-daemonset.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: time="2024-02-23T19:30:44-08:00" level=info msg="Starting batch/v1, Kind=Job controller"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.609613    6488 event.go:294] "Event occurred" object="kube-system/nfs-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/nfs-csi.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.610405    6488 server.go:654] "--cgroups-per-qos enabled, but --cgroup-root was not specified.  defaulting to /"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.611726    6488 container_manager_linux.go:267] "Container manager verified user specified cgroup-root exists" cgroupRoot=[]
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.611956    6488 container_manager_linux.go:272] "Creating Container Manager object based on Node Config" nodeConfig={RuntimeCgroupsName: SystemCgroupsName: KubeletCgroupsName: KubeletOOMScoreAdj:-999 ContainerRuntime: CgroupsPerQOS:true CgroupRoot:/ CgroupDriver:systemd KubeletRootDir:/var/lib/kubelet ProtectKernelDefaults:true NodeAllocatableConfig:{KubeReservedCgroupName: SystemReservedCgroupName: ReservedSystemCPUs: EnforceNodeAllocatable:map[pods:{}] KubeReserved:map[] SystemReserved:map[] HardEvictionThresholds:[{Signal:imagefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>} {Signal:nodefs.available Operator:LessThan Value:{Quantity:<nil> Percentage:0.05} GracePeriod:0s MinReclaim:<nil>}]} QOSReserved:map[] CPUManagerPolicy:none CPUManagerPolicyOptions:map[] ExperimentalTopologyManagerScope:container CPUManagerReconcilePeriod:10s ExperimentalMemoryManagerPolicy:None ExperimentalMemoryManagerReservedMemory:[] ExperimentalPodPidsLimit:-1 EnforceCPULimits:true CPUCFSQuotaPeriod:100ms ExperimentalTopologyManagerPolicy:none ExperimentalTopologyManagerPolicyOptions:map[]}
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.612462    6488 topology_manager.go:134] "Creating topology manager with policy per scope" topologyPolicyName="none" topologyScopeName="container"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.612599    6488 container_manager_linux.go:308] "Creating device plugin manager"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.626575    6488 state_mem.go:36] "Initialized new in-memory state store"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.638054    6488 kubelet.go:398] "Attempting to sync node with API server"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.638096    6488 kubelet.go:286] "Adding static pod path" path="/mnt/Test/ix-applications/k3s/agent/pod-manifests"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.638515    6488 kubelet.go:297] "Adding apiserver pod source"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.639047    6488 apiserver.go:42] "Waiting for node sync before watching apiserver pods"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.639355    6488 apiserver.go:52] "Watching apiserver"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.642288    6488 shared_informer.go:277] Caches are synced for service
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.645862    6488 kuberuntime_manager.go:244] "Container runtime initialized" containerRuntime="containerd" version="truenas/v1.7.1-k3s1" apiVersion="v1"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.663490    6488 nodeshutdown_manager_linux.go:137] "Creating node shutdown manager" shutdownGracePeriodRequested="15s" shutdownGracePeriodCriticalPods="10s" shutdownGracePeriodByPodPriority=[{Priority:0 ShutdownGracePeriodSeconds:5} {Priority:2000000000 ShutdownGracePeriodSeconds:10}]
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.663896    6488 server.go:1181] "Started kubelet"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.664108    6488 server.go:161] "Starting to listen" address="0.0.0.0" port=10250
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.664380    6488 cri_stats_provider.go:455] "Failed to get the info of the filesystem with mountpoint" err="unable to find data in memory cache" mountpoint="/mnt/Test/ix-applications/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.664410    6488 kubelet.go:1386] "Image garbage collection failed once. Stats initialization may not have completed yet" err="invalid capacity 0 on image filesystem"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.665532    6488 fs_resource_analyzer.go:67] "Starting FS ResourceAnalyzer"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.665926    6488 volume_manager.go:293] "Starting Kubelet Volume Manager"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.666043    6488 desired_state_of_world_populator.go:151] "Desired state populator starts to run"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.666252    6488 server.go:451] "Adding debug handlers to kubelet server"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.681093    6488 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"bad7441e77ecc0e73d6908055c468bad695cf065576c0a86458799e92207787c\": cni plugin not initialized" podSandboxID="bad7441e77ecc0e73d6908055c468bad695cf065576c0a86458799e92207787c"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.681146    6488 kuberuntime_gc.go:177] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"bad7441e77ecc0e73d6908055c468bad695cf065576c0a86458799e92207787c\": cni plugin not initialized" sandboxID="bad7441e77ecc0e73d6908055c468bad695cf065576c0a86458799e92207787c"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.696791    6488 cpu_manager.go:214] "Starting CPU manager" policy="none"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.696812    6488 cpu_manager.go:215] "Reconciling" reconcilePeriod="10s"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.696844    6488 state_mem.go:36] "Initialized new in-memory state store"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.697537    6488 state_mem.go:88] "Updated default CPUSet" cpuSet=""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.697552    6488 state_mem.go:96] "Updated CPUSet assignments" assignments=map[]
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.697622    6488 policy_none.go:49] "None policy: Start"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.698889    6488 memory_manager.go:169] "Starting memorymanager" policy="None"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.698953    6488 state_mem.go:35] "Initializing new in-memory state store"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.699324    6488 state_mem.go:75] "Updated machine memory state"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.743258    6488 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"4e0b463a3fc9415b61311cb6a04972c0695b32ca24b625cc8cab2345bc6b618e\": cni plugin not initialized" podSandboxID="4e0b463a3fc9415b61311cb6a04972c0695b32ca24b625cc8cab2345bc6b618e"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.743287    6488 kuberuntime_gc.go:177] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"4e0b463a3fc9415b61311cb6a04972c0695b32ca24b625cc8cab2345bc6b618e\": cni plugin not initialized" sandboxID="4e0b463a3fc9415b61311cb6a04972c0695b32ca24b625cc8cab2345bc6b618e"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.743543    6488 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"502c6bc465e2e7cbe2c3e37a1257ede44b706096d78a14c811af041faff6a834\": cni plugin not initialized" podSandboxID="502c6bc465e2e7cbe2c3e37a1257ede44b706096d78a14c811af041faff6a834"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.743570    6488 kuberuntime_gc.go:177] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"502c6bc465e2e7cbe2c3e37a1257ede44b706096d78a14c811af041faff6a834\": cni plugin not initialized" sandboxID="502c6bc465e2e7cbe2c3e37a1257ede44b706096d78a14c811af041faff6a834"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.764953    6488 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv4
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.770193    6488 controllermanager.go:182] Version: v1.26.6+k3s-e18037a7-dirty
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.770213    6488 controllermanager.go:184] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774182    6488 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774337    6488 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774201    6488 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774240    6488 secure_serving.go:210] Serving securely on 127.0.0.1:10257
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774368    6488 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774248    6488 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774347    6488 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.774350    6488 shared_informer.go:270] Waiting for caches to sync for RequestHeaderAuthRequestController
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.786110    6488 kubelet_network_linux.go:63] "Initialized iptables rules." protocol=IPv6
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.786643    6488 status_manager.go:176] "Starting to sync pod status with apiserver"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.786960    6488 kubelet.go:2113] "Starting kubelet main sync loop"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.787204    6488 kubelet.go:2137] "Skipping pod synchronization" err="[container runtime status check may not have completed yet, PLEG is not healthy: pleg has yet to be successful]"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.789996    6488 event.go:294] "Event occurred" object="kube-system/nfs-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/nfs-csi.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.790771    6488 plugin_manager.go:118] "Starting Kubelet Plugin Manager"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.792898    6488 kuberuntime_manager.go:1114] "Updating runtime config through cri with podcidr" CIDR="172.16.0.0/16"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.793240    6488 kubelet_network.go:61] "Updating Pod CIDR" originalPodCIDR="" newPodCIDR="172.16.0.0/16"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.794232    6488 event.go:294] "Event occurred" object="kube-system/rolebindings" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/rolebindings.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.803820    6488 event.go:294] "Event occurred" object="kube-system/rolebindings" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/rolebindings.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.808464    6488 event.go:294] "Event occurred" object="kube-system/smb-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/smb-csi.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.808904    6488 server.go:152] "Starting Kubernetes Scheduler" version="v1.26.6+k3s-e18037a7-dirty"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.808918    6488 server.go:154] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.810985    6488 requestheader_controller.go:169] Starting RequestHeaderAuthRequestController
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.810991    6488 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.810992    6488 configmap_cafile_content.go:202] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.811011    6488 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.811002    6488 shared_informer.go:270] Waiting for caches to sync for RequestHeaderAuthRequestController
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.811003    6488 shared_informer.go:270] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.811144    6488 secure_serving.go:210] Serving securely on 127.0.0.1:10259
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.811306    6488 tlsconfig.go:240] "Starting DynamicServingCertificateController"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.833376    6488 shared_informer.go:270] Waiting for caches to sync for tokens
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.844695    6488 controllermanager.go:622] Started "horizontalpodautoscaling"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.847304    6488 controllermanager.go:622] Started "statefulset"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.847617    6488 stateful_set.go:152] Starting stateful set controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.847630    6488 shared_informer.go:270] Waiting for caches to sync for stateful set
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.847614    6488 event.go:294] "Event occurred" object="kube-system/smb-csi" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/smb-csi.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.849214    6488 horizontal.go:181] Starting HPA controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.849230    6488 shared_informer.go:270] Waiting for caches to sync for HPA
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.849586    6488 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"f5c37793d1ce90db0abf7a0af0feb66dd90f32692441d0cad8717832aa8a449a\": cni plugin not initialized" podSandboxID="f5c37793d1ce90db0abf7a0af0feb66dd90f32692441d0cad8717832aa8a449a"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.849611    6488 kuberuntime_gc.go:177] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"f5c37793d1ce90db0abf7a0af0feb66dd90f32692441d0cad8717832aa8a449a\": cni plugin not initialized" sandboxID="f5c37793d1ce90db0abf7a0af0feb66dd90f32692441d0cad8717832aa8a449a"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.849679    6488 controllermanager.go:622] Started "attachdetach"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.849937    6488 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"e4adf9ee35d8eb96940925aa9f92d36211f0cb2e2042f6dadd7e910114080adc\": cni plugin not initialized" podSandboxID="e4adf9ee35d8eb96940925aa9f92d36211f0cb2e2042f6dadd7e910114080adc"
Feb 23 19:30:44 truenas k3s[6488]: E0223 19:30:44.850048    6488 kuberuntime_gc.go:177] "Failed to stop sandbox before removing" err="rpc error: code = Unknown desc = failed to destroy network for sandbox \"e4adf9ee35d8eb96940925aa9f92d36211f0cb2e2042f6dadd7e910114080adc\": cni plugin not initialized" sandboxID="e4adf9ee35d8eb96940925aa9f92d36211f0cb2e2042f6dadd7e910114080adc"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.850081    6488 attach_detach_controller.go:328] Starting attach detach controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.850092    6488 shared_informer.go:270] Waiting for caches to sync for attach detach
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.851796    6488 controllermanager.go:622] Started "endpoint"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.852310    6488 endpoints_controller.go:178] Starting endpoint controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.852320    6488 shared_informer.go:270] Waiting for caches to sync for endpoint
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.853881    6488 controllermanager.go:622] Started "persistentvolume-binder"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.854057    6488 pv_controller_base.go:318] Starting persistent volume controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.854067    6488 shared_informer.go:270] Waiting for caches to sync for persistent volume
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.855895    6488 controllermanager.go:622] Started "pv-protection"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.856035    6488 pv_protection_controller.go:75] Starting PV protection controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.856042    6488 shared_informer.go:270] Waiting for caches to sync for PV protection
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.857592    6488 controllermanager.go:622] Started "endpointslice"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.857738    6488 endpointslice_controller.go:257] Starting endpoint slice controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.857748    6488 shared_informer.go:270] Waiting for caches to sync for endpoint_slice
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.862436    6488 event.go:294] "Event occurred" object="kube-system/zfs-operator" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="ApplyingManifest" message="Applying manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/zfs-operator.yaml\""
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.874421    6488 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.874573    6488 shared_informer.go:277] Caches are synced for RequestHeaderAuthRequestController
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.874587    6488 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.888189    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.888813    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.888861    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889059    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889107    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889159    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889201    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889233    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889295    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889384    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889416    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889504    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889568    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="1884042593a1040affaf6362bd83f9fe55b6b7c21d91da8a37cb1e731822c033"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889579    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="c6002847aac01dc80b9a9377bf082ac17fc05158c9099395a1732832524042f5"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889589    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="f62e19c54f474ea998e889fecef15a7deaab892c7af79e606b7694576078ea8e"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889595    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="f5c37793d1ce90db0abf7a0af0feb66dd90f32692441d0cad8717832aa8a449a"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889606    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="0b4ab931394e08840b36df134c3527899b9e9b7892736fc0e3332e6ae2914495"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889614    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="e4adf9ee35d8eb96940925aa9f92d36211f0cb2e2042f6dadd7e910114080adc"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889624    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="111b5ce2e7df77db9c0bf53aff6abba6d7012a4f092bb5a60ffd283e5bece8c8"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889637    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="eec3eb70bf3ead355ece04739911c0085acc9ffb49c65c26ee99cffc650df845"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889644    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="899f85be933aa2f46e8f3fabb07eb254ef3d845aa54ebf5c9093d1c148e4e877"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889656    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="39e50236c0b07f47d1de398d3a85c0ced32f200afa3bdd115290f983653828c4"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889662    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="0629dfb943203b9be8617cd2425281a43f3f143ed5a7467b62a3ba52149d50c3"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889671    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="7115d89e6ad77a8dbb602b534fb643e205687d13c6038208902c5c88c2c829f2"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889678    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="bad7441e77ecc0e73d6908055c468bad695cf065576c0a86458799e92207787c"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889688    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="7af62abf5bdb170aeeff6eaf861041f3b8e2b8b902efc6757cd8f4fe235de3da"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889706    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="9582b7161d237ba036774d8554c4af305d3460fc76e915e6f73e6607fbba6fdd"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889712    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="da05c3f23e9c60921f757b317ed29add2046d85d8ae528c60ad0a42fc7561b6f"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889721    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="2809cf069f3ed129a3e9bafcd201a1381981d62419dc6ff0aad4369e86f3b650"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889728    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="4e0b463a3fc9415b61311cb6a04972c0695b32ca24b625cc8cab2345bc6b618e"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889739    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="d0ee38bd2c3aef30e09937cf4cb144c10a108725eae4d3087a18b40d8655615a"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889752    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="67320353dc2f7450e44acebbb62a5dba9ff90921e04888fc34d3fbf30be326dc"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.889758    6488 pod_container_deletor.go:80] "Container not found in pod's containers" containerID="502c6bc465e2e7cbe2c3e37a1257ede44b706096d78a14c811af041faff6a834"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.898504    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for limitranges
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.898549    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for deployments.apps
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.898794    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for ingresses.networking.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.898811    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for leases.coordination.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.898841    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for horizontalpodautoscalers.autoscaling
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899015    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmcharts.helm.cattle.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899034    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsnodes.zfs.openebs.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899048    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpoints
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899062    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for replicasets.apps
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899082    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for helmchartconfigs.helm.cattle.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899095    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsrestores.zfs.openebs.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899126    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for daemonsets.apps
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899161    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for jobs.batch
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899179    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for cronjobs.batch
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899197    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for poddisruptionbudgets.policy
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899217    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for rolebindings.rbac.authorization.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899236    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for roles.rbac.authorization.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899251    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for network-attachment-definitions.k8s.cni.cncf.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899264    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for volumesnapshots.snapshot.storage.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899295    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for networkpolicies.networking.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899311    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for csistoragecapacities.storage.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899329    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for addons.k3s.cattle.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899343    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsvolumes.zfs.openebs.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899355    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for statefulsets.apps
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899368    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for endpointslices.discovery.k8s.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899384    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfssnapshots.zfs.openebs.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899398    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for zfsbackups.zfs.openebs.io
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899421    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for podtemplates
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.899519    6488 shared_informer.go:548] resyncPeriod 14h35m59.153057363s is smaller than resyncCheckPeriod 20h36m56.730655629s and the informer has already started. Changing it to 20h36m56.730655629s
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899568    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for serviceaccounts
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899586    6488 resource_quota_monitor.go:218] QuotaMonitor created object count evaluator for controllerrevisions.apps
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899597    6488 controllermanager.go:622] Started "resourcequota"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899767    6488 resource_quota_controller.go:277] Starting resource quota controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899778    6488 shared_informer.go:270] Waiting for caches to sync for resource quota
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.899949    6488 resource_quota_monitor.go:295] QuotaMonitor running
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.901776    6488 controllermanager.go:622] Started "replicaset"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.902057    6488 replica_set.go:201] Starting replicaset controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.902067    6488 shared_informer.go:270] Waiting for caches to sync for ReplicaSet
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904248    6488 certificate_controller.go:112] Starting certificate controller "csrsigning-kubelet-client"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904262    6488 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-kubelet-client
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904266    6488 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/Test/ix-applications/k3s/server/tls/client-ca.nochain.crt::/mnt/Test/ix-applications/k3s/server/tls/client-ca.key"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904247    6488 certificate_controller.go:112] Starting certificate controller "csrsigning-kubelet-serving"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904289    6488 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-kubelet-serving
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904291    6488 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/Test/ix-applications/k3s/server/tls/client-ca.nochain.crt::/mnt/Test/ix-applications/k3s/server/tls/client-ca.key"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904267    6488 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/Test/ix-applications/k3s/server/tls/server-ca.nochain.crt::/mnt/Test/ix-applications/k3s/server/tls/server-ca.key"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904282    6488 certificate_controller.go:112] Starting certificate controller "csrsigning-kube-apiserver-client"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904343    6488 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-kube-apiserver-client
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904432    6488 controllermanager.go:622] Started "csrsigning"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904520    6488 certificate_controller.go:112] Starting certificate controller "csrsigning-legacy-unknown"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904527    6488 shared_informer.go:270] Waiting for caches to sync for certificate-csrsigning-legacy-unknown
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.904532    6488 dynamic_serving_content.go:132] "Starting controller" name="csr-controller::/mnt/Test/ix-applications/k3s/server/tls/server-ca.nochain.crt::/mnt/Test/ix-applications/k3s/server/tls/server-ca.key"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.906227    6488 controllermanager.go:622] Started "clusterrole-aggregation"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.906718    6488 clusterroleaggregation_controller.go:188] Starting ClusterRoleAggregator
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.906730    6488 shared_informer.go:270] Waiting for caches to sync for ClusterRoleAggregator
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.907884    6488 controllermanager.go:622] Started "pvc-protection"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.907966    6488 pvc_protection_controller.go:99] "Starting PVC protection controller"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.907984    6488 shared_informer.go:270] Waiting for caches to sync for PVC protection
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.909335    6488 controllermanager.go:622] Started "ephemeral-volume"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.909598    6488 controller.go:169] Starting ephemeral volume controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.909608    6488 shared_informer.go:270] Waiting for caches to sync for ephemeral
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.910661    6488 controllermanager.go:622] Started "podgc"
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.910670    6488 controllermanager.go:587] "bootstrapsigner" is disabled
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.910676    6488 core.go:221] configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.
Feb 23 19:30:44 truenas k3s[6488]: W0223 19:30:44.910696    6488 controllermanager.go:600] Skipping "route"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.910874    6488 gc_controller.go:102] Starting GC controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.910882    6488 shared_informer.go:270] Waiting for caches to sync for GC
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.911197    6488 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.911205    6488 shared_informer.go:277] Caches are synced for RequestHeaderAuthRequestController
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.911211    6488 shared_informer.go:277] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.912132    6488 controllermanager.go:622] Started "ttl"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.912312    6488 ttl_controller.go:120] Starting TTL controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.912323    6488 shared_informer.go:270] Waiting for caches to sync for TTL
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.914660    6488 controllermanager.go:622] Started "serviceaccount"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.914994    6488 serviceaccounts_controller.go:111] Starting service account controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.915005    6488 shared_informer.go:270] Waiting for caches to sync for service account
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.918187    6488 controllermanager.go:622] Started "disruption"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.918364    6488 disruption.go:424] Sending events to api server.
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.918393    6488 disruption.go:435] Starting disruption controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.918399    6488 shared_informer.go:270] Waiting for caches to sync for disruption
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.920535    6488 node_lifecycle_controller.go:492] Controller will reconcile labels.
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.920559    6488 controllermanager.go:622] Started "nodelifecycle"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.920662    6488 node_lifecycle_controller.go:527] Sending events to api server.
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.920683    6488 node_lifecycle_controller.go:538] Starting node controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.920688    6488 shared_informer.go:270] Waiting for caches to sync for taint
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.922312    6488 controllermanager.go:622] Started "endpointslicemirroring"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.922444    6488 endpointslicemirroring_controller.go:211] Starting EndpointSliceMirroring controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.922454    6488 shared_informer.go:270] Waiting for caches to sync for endpoint_slice_mirroring
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.923782    6488 controllermanager.go:622] Started "ttl-after-finished"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.923883    6488 ttlafterfinished_controller.go:104] Starting TTL after finished controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.923893    6488 shared_informer.go:270] Waiting for caches to sync for TTL after finished
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.925214    6488 controllermanager.go:622] Started "root-ca-cert-publisher"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.925322    6488 publisher.go:101] Starting root CA certificate configmap publisher
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.925330    6488 shared_informer.go:270] Waiting for caches to sync for crt configmap
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.926697    6488 controllermanager.go:622] Started "persistentvolume-expander"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.926804    6488 expand_controller.go:340] Starting expand controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.926812    6488 shared_informer.go:270] Waiting for caches to sync for expand
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.933999    6488 shared_informer.go:277] Caches are synced for tokens
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.940215    6488 kubelet_node_status.go:70] "Attempting to register node" node="ix-truenas"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.942129    6488 controllermanager.go:622] Started "namespace"
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.942202    6488 namespace_controller.go:195] Starting namespace controller
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.942212    6488 shared_informer.go:270] Waiting for caches to sync for namespace
Feb 23 19:30:44 truenas k3s[6488]: I0223 19:30:44.982817    6488 desired_state_of_world_populator.go:159] "Finished populating initial desired state of world"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.068508    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-config\") pod \"emby-8587ff676b-5sv24\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") " pod="ix-emby/emby-8587ff676b-5sv24"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.068942    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"emby-0\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-0\") pod \"emby-8587ff676b-5sv24\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") " pod="ix-emby/emby-8587ff676b-5sv24"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069015    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"emby-1\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-1\") pod \"emby-8587ff676b-5sv24\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") " pod="ix-emby/emby-8587ff676b-5sv24"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069074    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-frwkg\" (UniqueName: \"kubernetes.io/projected/da26666c-ae5c-4948-b4fc-1cdd12a2d5eb-kube-api-access-frwkg\") pod \"csi-smb-node-8tmnt\" (UID: \"da26666c-ae5c-4948-b4fc-1cdd12a2d5eb\") " pod="kube-system/csi-smb-node-8tmnt"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069115    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/26cdf9c8-8660-4e3a-9e85-f871458909fc-socket-dir\") pod \"csi-nfs-controller-7b74694749-qdrnn\" (UID: \"26cdf9c8-8660-4e3a-9e85-f871458909fc\") " pod="kube-system/csi-nfs-controller-7b74694749-qdrnn"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069186    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-ktndq\" (UniqueName: \"kubernetes.io/projected/26cdf9c8-8660-4e3a-9e85-f871458909fc-kube-api-access-ktndq\") pod \"csi-nfs-controller-7b74694749-qdrnn\" (UID: \"26cdf9c8-8660-4e3a-9e85-f871458909fc\") " pod="kube-system/csi-nfs-controller-7b74694749-qdrnn"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069231    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-tmp\") pod \"emby-8587ff676b-5sv24\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") " pod="ix-emby/emby-8587ff676b-5sv24"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069287    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/host-path/da26666c-ae5c-4948-b4fc-1cdd12a2d5eb-socket-dir\") pod \"csi-smb-node-8tmnt\" (UID: \"da26666c-ae5c-4948-b4fc-1cdd12a2d5eb\") " pod="kube-system/csi-smb-node-8tmnt"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069343    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"mountpoint-dir\" (UniqueName: \"kubernetes.io/host-path/da26666c-ae5c-4948-b4fc-1cdd12a2d5eb-mountpoint-dir\") pod \"csi-smb-node-8tmnt\" (UID: \"da26666c-ae5c-4948-b4fc-1cdd12a2d5eb\") " pod="kube-system/csi-smb-node-8tmnt"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069410    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/da26666c-ae5c-4948-b4fc-1cdd12a2d5eb-registration-dir\") pod \"csi-smb-node-8tmnt\" (UID: \"da26666c-ae5c-4948-b4fc-1cdd12a2d5eb\") " pod="kube-system/csi-smb-node-8tmnt"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069451    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/26cdf9c8-8660-4e3a-9e85-f871458909fc-pods-mount-dir\") pod \"csi-nfs-controller-7b74694749-qdrnn\" (UID: \"26cdf9c8-8660-4e3a-9e85-f871458909fc\") " pod="kube-system/csi-nfs-controller-7b74694749-qdrnn"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.069486    6488 reconciler.go:41] "Reconciler: start to sync state"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.086752    6488 controllermanager.go:622] Started "garbagecollector"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.086810    6488 garbagecollector.go:154] Starting garbage collector controller
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.086828    6488 shared_informer.go:270] Waiting for caches to sync for garbage collector
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.086903    6488 graph_builder.go:291] GraphBuilder running
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.135839    6488 controllermanager.go:622] Started "job"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.136223    6488 job_controller.go:191] Starting job controller
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.136236    6488 shared_informer.go:270] Waiting for caches to sync for job
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.235648    6488 controllermanager.go:622] Started "csrcleaner"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.235704    6488 cleaner.go:82] Starting CSR cleaner controller
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.336480    6488 controllermanager.go:622] Started "tokencleaner"
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.336571    6488 tokencleaner.go:111] Starting token cleaner controller
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.336585    6488 shared_informer.go:270] Waiting for caches to sync for token_cleaner
Feb 23 19:30:45 truenas k3s[6488]: I0223 19:30:45.336602    6488 shared_informer.go:277] Caches are synced for token_cleaner
Feb 23 19:30:45 truenas k3s[6488]: time="2024-02-23T19:30:45-08:00" level=info msg="Stopped tunnel to 127.0.0.1:6443"
Feb 23 19:30:45 truenas k3s[6488]: time="2024-02-23T19:30:45-08:00" level=info msg="Connecting to proxy" url="wss://192.168.0.132:6443/v1-k3s/connect"
Feb 23 19:30:45 truenas k3s[6488]: time="2024-02-23T19:30:45-08:00" level=info msg="Proxy done" err="context canceled" url="wss://127.0.0.1:6443/v1-k3s/connect"
Feb 23 19:30:45 truenas k3s[6488]: time="2024-02-23T19:30:45-08:00" level=info msg="error in remotedialer server [400]: websocket: close 1006 (abnormal closure): unexpected EOF"
Feb 23 19:30:45 truenas k3s[6488]: time="2024-02-23T19:30:45-08:00" level=info msg="Handling backend connection request [ix-truenas]"
Feb 23 19:30:45 truenas k3s[6488]: E0223 19:30:45.848491    6488 remote_runtime.go:205] "StopPodSandbox from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find sandbox \"9582b7161d237ba036774d8554c4af305d3460fc76e915e6f73e6607fbba6fdd\": not found" podSandboxID="9582b7161d237ba036774d8554c4af305d3460fc76e915e6f73e6607fbba6fdd"
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.040794    6488 kubelet_node_status.go:108] "Node was previously registered" node="ix-truenas"
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.040897    6488 kubelet_node_status.go:73] "Successfully registered node" node="ix-truenas"
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.600073    6488 event.go:294] "Event occurred" object="ix-truenas" fieldPath="" kind="Node" apiVersion="" type="Normal" reason="NodePasswordValidationComplete" message="Deferred node password secret validation complete"
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.835436    6488 event.go:294] "Event occurred" object="kube-system/zfs-operator" fieldPath="" kind="Addon" apiVersion="k3s.cattle.io/v1" type="Normal" reason="AppliedManifest" message="Applied manifest at \"/mnt/Test/ix-applications/k3s/server/manifests/zfs-operator.yaml\""
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895063    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"emby-0\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-0\") pod \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") "
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895106    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"emby-1\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-1\") pod \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") "
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895144    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-tmp\") pod \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") "
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895167    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-config\") pod \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\" (UID: \"57e9ad40-4b50-47e0-a638-3fbc82c71dfc\") "
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895487    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-config" (OuterVolumeSpecName: "config") pod "57e9ad40-4b50-47e0-a638-3fbc82c71dfc" (UID: "57e9ad40-4b50-47e0-a638-3fbc82c71dfc"). InnerVolumeSpecName "config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895484    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-1" (OuterVolumeSpecName: "emby-1") pod "57e9ad40-4b50-47e0-a638-3fbc82c71dfc" (UID: "57e9ad40-4b50-47e0-a638-3fbc82c71dfc"). InnerVolumeSpecName "emby-1". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895488    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-0" (OuterVolumeSpecName: "emby-0") pod "57e9ad40-4b50-47e0-a638-3fbc82c71dfc" (UID: "57e9ad40-4b50-47e0-a638-3fbc82c71dfc"). InnerVolumeSpecName "emby-0". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Feb 23 19:30:46 truenas k3s[6488]: W0223 19:30:46.895764    6488 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/57e9ad40-4b50-47e0-a638-3fbc82c71dfc/volumes/kubernetes.io~empty-dir/tmp: clearQuota called, but quotas disabled
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.895812    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/empty-dir/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-tmp" (OuterVolumeSpecName: "tmp") pod "57e9ad40-4b50-47e0-a638-3fbc82c71dfc" (UID: "57e9ad40-4b50-47e0-a638-3fbc82c71dfc"). InnerVolumeSpecName "tmp". PluginName "kubernetes.io/empty-dir", VolumeGidValue ""
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.995257    6488 reconciler_common.go:295] "Volume detached for volume \"config\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-config\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.995284    6488 reconciler_common.go:295] "Volume detached for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-tmp\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.995299    6488 reconciler_common.go:295] "Volume detached for volume \"emby-0\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-0\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 19:30:46 truenas k3s[6488]: I0223 19:30:46.995314    6488 reconciler_common.go:295] "Volume detached for volume \"emby-1\" (UniqueName: \"kubernetes.io/host-path/57e9ad40-4b50-47e0-a638-3fbc82c71dfc-emby-1\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 19:30:48 truenas k3s[6488]: I0223 19:30:48.724247    6488 kubelet_node_status.go:493] "Fast updating node status as it just became ready"
Feb 23 19:30:48 truenas k3s[6488]: I0223 19:30:48.955802    6488 controller.go:615] quota admission added evaluator for: daemonsets.apps
Feb 23 19:30:49 truenas k3s[6488]: time="2024-02-23T19:30:49-08:00" level=info msg="Tunnel authorizer set Kubelet Port 10250"
Feb 23 19:30:50 truenas k3s[6488]: I0223 19:30:50.868468    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/csi-smb-node-8tmnt" podStartSLOduration=13599.868150992 pod.CreationTimestamp="2024-02-23 15:44:11 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:30:50.867533231 -0800 PST m=+13.082617651" watchObservedRunningTime="2024-02-23 19:30:50.868150992 -0800 PST m=+13.083235372"
Feb 23 19:30:52 truenas k3s[6488]: I0223 19:30:52.800722    6488 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: smb.csi.k8s.io endpoint: /var/lib/kubelet/plugins/smb.csi.k8s.io/csi.sock versions: 1.0.0
Feb 23 19:30:52 truenas k3s[6488]: I0223 19:30:52.800782    6488 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: smb.csi.k8s.io at endpoint: /var/lib/kubelet/plugins/smb.csi.k8s.io/csi.sock
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.398029    6488 range_allocator.go:109] No Secondary Service CIDR provided. Skipping filtering out secondary service addresses.
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.398087    6488 controllermanager.go:622] Started "nodeipam"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.398249    6488 node_ipam_controller.go:155] Starting ipam controller
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.398264    6488 shared_informer.go:270] Waiting for caches to sync for node
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.400582    6488 core.go:207] failed to start cloud node lifecycle controller: no cloud provider provided
Feb 23 19:30:55 truenas k3s[6488]: W0223 19:30:55.400600    6488 controllermanager.go:600] Skipping "cloud-node-lifecycle"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.403094    6488 controllermanager.go:622] Started "replicationcontroller"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.403258    6488 replica_set.go:201] Starting replicationcontroller controller
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.403275    6488 shared_informer.go:270] Waiting for caches to sync for ReplicationController
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.405637    6488 controllermanager.go:622] Started "deployment"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.406258    6488 deployment_controller.go:154] "Starting controller" controller="deployment"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.406276    6488 shared_informer.go:270] Waiting for caches to sync for deployment
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.408076    6488 controllermanager.go:622] Started "cronjob"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.408681    6488 cronjob_controllerv2.go:137] "Starting cronjob controller v2"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.408949    6488 shared_informer.go:270] Waiting for caches to sync for cronjob
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.410493    6488 controllermanager.go:622] Started "csrapproving"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.410602    6488 certificate_controller.go:112] Starting certificate controller "csrapproving"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.410616    6488 shared_informer.go:270] Waiting for caches to sync for certificate-csrapproving
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.412769    6488 core.go:92] Failed to start service controller: WARNING: no cloud provider provided, services of type LoadBalancer will fail
Feb 23 19:30:55 truenas k3s[6488]: W0223 19:30:55.412788    6488 controllermanager.go:600] Skipping "service"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.415275    6488 controllermanager.go:622] Started "daemonset"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.415606    6488 daemon_controller.go:267] Starting daemon sets controller
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.415620    6488 shared_informer.go:270] Waiting for caches to sync for daemon sets
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.420836    6488 shared_informer.go:270] Waiting for caches to sync for resource quota
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.423941    6488 shared_informer.go:277] Caches are synced for TTL after finished
Feb 23 19:30:55 truenas k3s[6488]: W0223 19:30:55.428970    6488 actual_state_of_world.go:541] Failed to update statusUpdateNeeded field in actual state of world: Failed to set statusUpdateNeeded to needed true, because nodeName="ix-truenas" does not exist
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.437070    6488 shared_informer.go:277] Caches are synced for job
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.439985    6488 shared_informer.go:270] Waiting for caches to sync for garbage collector
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.448178    6488 shared_informer.go:277] Caches are synced for stateful set
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.449352    6488 shared_informer.go:277] Caches are synced for HPA
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.452407    6488 shared_informer.go:277] Caches are synced for endpoint
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.457893    6488 shared_informer.go:277] Caches are synced for endpoint_slice
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.498479    6488 shared_informer.go:277] Caches are synced for node
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.498529    6488 range_allocator.go:167] Sending events to api server.
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.498560    6488 range_allocator.go:171] Starting range CIDR allocator
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.498568    6488 shared_informer.go:270] Waiting for caches to sync for cidrallocator
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.498579    6488 shared_informer.go:277] Caches are synced for cidrallocator
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.502212    6488 shared_informer.go:277] Caches are synced for ReplicaSet
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.503368    6488 shared_informer.go:277] Caches are synced for ReplicationController
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.504321    6488 shared_informer.go:277] Caches are synced for certificate-csrsigning-kubelet-client
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.504343    6488 shared_informer.go:277] Caches are synced for certificate-csrsigning-kubelet-serving
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.504372    6488 shared_informer.go:277] Caches are synced for certificate-csrsigning-kube-apiserver-client
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.504586    6488 shared_informer.go:277] Caches are synced for certificate-csrsigning-legacy-unknown
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.506372    6488 shared_informer.go:277] Caches are synced for deployment
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.506768    6488 shared_informer.go:277] Caches are synced for ClusterRoleAggregator
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.508807    6488 shared_informer.go:277] Caches are synced for PVC protection
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.509955    6488 shared_informer.go:277] Caches are synced for cronjob
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.509981    6488 shared_informer.go:277] Caches are synced for ephemeral
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511036    6488 shared_informer.go:277] Caches are synced for certificate-csrapproving
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511059    6488 shared_informer.go:277] Caches are synced for GC
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511156    6488 gc_controller.go:207] "Garbage collecting pods" numPods=5
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511210    6488 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/coredns-59b4f5bbd5-dfmz7"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511236    6488 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/snapshot-controller-546868dfb4-srj6p"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511262    6488 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/csi-nfs-controller-7b74694749-nmlw2"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511267    6488 gc_controller.go:329] "PodGC is force deleting Pod" pod="kube-system/coredns-59b4f5bbd5-mbc85"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.511268    6488 gc_controller.go:329] "PodGC is force deleting Pod" pod="ix-emby/emby-8587ff676b-5sv24"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.512366    6488 shared_informer.go:277] Caches are synced for TTL
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.518472    6488 shared_informer.go:277] Caches are synced for disruption
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.522515    6488 shared_informer.go:277] Caches are synced for endpoint_slice_mirroring
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.525407    6488 shared_informer.go:277] Caches are synced for crt configmap
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.526889    6488 shared_informer.go:277] Caches are synced for expand
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.542274    6488 shared_informer.go:277] Caches are synced for namespace
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.554122    6488 shared_informer.go:277] Caches are synced for persistent volume
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.556258    6488 shared_informer.go:277] Caches are synced for PV protection
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.615792    6488 shared_informer.go:277] Caches are synced for service account
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.650258    6488 shared_informer.go:277] Caches are synced for attach detach
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.700465    6488 shared_informer.go:277] Caches are synced for resource quota
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.716692    6488 shared_informer.go:277] Caches are synced for daemon sets
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.718377    6488 event.go:294] "Event occurred" object="kube-system/csi-nfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Warning" reason="FailedDaemonPod" message="Found failed daemon pod kube-system/csi-nfs-node-zpnzs on node ix-truenas, will try to kill it"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.720742    6488 shared_informer.go:277] Caches are synced for taint
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.720863    6488 taint_manager.go:206] "Starting NoExecuteTaintManager"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.720910    6488 shared_informer.go:277] Caches are synced for resource quota
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.720910    6488 node_lifecycle_controller.go:1438] Initializing eviction metric for zone:
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.720948    6488 event.go:294] "Event occurred" object="ix-truenas" fieldPath="" kind="Node" apiVersion="v1" type="Normal" reason="RegisteredNode" message="Node ix-truenas event: Registered Node ix-truenas in Controller"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.720925    6488 taint_manager.go:211] "Sending events to api server"
Feb 23 19:30:55 truenas k3s[6488]: W0223 19:30:55.721096    6488 node_lifecycle_controller.go:1053] Missing timestamp for Node ix-truenas. Assuming now as a timestamp.
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.721160    6488 node_lifecycle_controller.go:1254] Controller detected that zone  is now in state Normal.
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.734546    6488 event.go:294] "Event occurred" object="kube-system/csi-nfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulDelete" message="Deleted pod: csi-nfs-node-zpnzs"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.742606    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.742929    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="openebs-zfs-plugin"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743292    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="05a1c2aa-fce3-4802-85e1-7b52b917728f" containerName="liveness-probe"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743313    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="2bbb0d62-4e01-4a2e-bd5b-8181e9716fa2" containerName="smb"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743326    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="819d7736-be19-486d-b0d3-2e08354ec56b" containerName="emby"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743341    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="2bbb0d62-4e01-4a2e-bd5b-8181e9716fa2" containerName="liveness-probe"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743352    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="1a2a6267-905e-4a69-a45e-3491c2eaed00" containerName="snapshot-controller"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743363    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="72309e28-0037-4ad0-b157-1ede67abff63" containerName="emby"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743374    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="a01c0905-172a-4ace-a5b8-5a29b8e356cc" containerName="openebs-zfs-plugin"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743385    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="a01c0905-172a-4ace-a5b8-5a29b8e356cc" containerName="csi-node-driver-registrar"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743396    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="csi-resizer"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743407    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="05a1c2aa-fce3-4802-85e1-7b52b917728f" containerName="smb"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743419    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="d529f6ba-2365-44e9-873d-03ad02f61006" containerName="node-driver-registrar"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743429    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="d5ada2a6-c0db-43fd-9f40-1513fb868f8a" containerName="coredns"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743439    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="d529f6ba-2365-44e9-873d-03ad02f61006" containerName="liveness-probe"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743450    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="snapshot-controller"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743461    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="0c1c04f3-6341-4456-9e1f-1cbee5c1a377" containerName="nvidia-device-plugin-ctr"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743471    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="csi-snapshotter"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743482    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="7fcd2129-e3cd-4eeb-9863-552b90ac9233" containerName="snapshot-controller"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743493    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="05a1c2aa-fce3-4802-85e1-7b52b917728f" containerName="csi-provisioner"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743504    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="2bbb0d62-4e01-4a2e-bd5b-8181e9716fa2" containerName="node-driver-registrar"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743515    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="d529f6ba-2365-44e9-873d-03ad02f61006" containerName="nfs"
Feb 23 19:30:55 truenas k3s[6488]: E0223 19:30:55.743526    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="csi-provisioner"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743859    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="2bbb0d62-4e01-4a2e-bd5b-8181e9716fa2" containerName="liveness-probe"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743876    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="05a1c2aa-fce3-4802-85e1-7b52b917728f" containerName="csi-provisioner"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743888    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="d529f6ba-2365-44e9-873d-03ad02f61006" containerName="node-driver-registrar"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743900    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="d5ada2a6-c0db-43fd-9f40-1513fb868f8a" containerName="coredns"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743913    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="7fcd2129-e3cd-4eeb-9863-552b90ac9233" containerName="snapshot-controller"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743925    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="72309e28-0037-4ad0-b157-1ede67abff63" containerName="emby"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743936    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="csi-resizer"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743946    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="05a1c2aa-fce3-4802-85e1-7b52b917728f" containerName="liveness-probe"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743958    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="d529f6ba-2365-44e9-873d-03ad02f61006" containerName="nfs"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743968    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="819d7736-be19-486d-b0d3-2e08354ec56b" containerName="emby"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743978    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="d529f6ba-2365-44e9-873d-03ad02f61006" containerName="liveness-probe"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743988    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="openebs-zfs-plugin"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.743999    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="0c1c04f3-6341-4456-9e1f-1cbee5c1a377" containerName="nvidia-device-plugin-ctr"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744008    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="a01c0905-172a-4ace-a5b8-5a29b8e356cc" containerName="openebs-zfs-plugin"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744019    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="a01c0905-172a-4ace-a5b8-5a29b8e356cc" containerName="csi-node-driver-registrar"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744029    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="1a2a6267-905e-4a69-a45e-3491c2eaed00" containerName="snapshot-controller"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744039    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="snapshot-controller"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744050    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="2bbb0d62-4e01-4a2e-bd5b-8181e9716fa2" containerName="smb"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744061    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="csi-provisioner"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744073    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="05a1c2aa-fce3-4802-85e1-7b52b917728f" containerName="smb"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744083    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="1010e55b-74ca-4226-8597-a8871cd5dd9e" containerName="csi-snapshotter"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744093    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="2bbb0d62-4e01-4a2e-bd5b-8181e9716fa2" containerName="node-driver-registrar"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744192    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.744899    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-gvd8d\" (UniqueName: \"kubernetes.io/projected/488a8c6f-1736-4ab4-8514-46150e2eaaf0-kube-api-access-gvd8d\") pod \"snapshot-controller-546868dfb4-nfx9b\" (UID: \"488a8c6f-1736-4ab4-8514-46150e2eaaf0\") " pod="kube-system/snapshot-controller-546868dfb4-nfx9b"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.745594    6488 event.go:294] "Event occurred" object="kube-system/nvidia-device-plugin-daemonset" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nvidia-device-plugin-daemonset-5brd8"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.745697    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.745882    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.746271    6488 event.go:294] "Event occurred" object="kube-system/openebs-zfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.747435    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.752733    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.755669    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.756518    6488 event.go:294] "Event occurred" object="kube-system/csi-nfs-node" fieldPath="" kind="DaemonSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: csi-nfs-node-p942f"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.763083    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946155    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"custom-config-volume\" (UniqueName: \"kubernetes.io/configmap/c0cc6cbb-7e93-487b-8149-55ad58fc1956-custom-config-volume\") pod \"coredns-59b4f5bbd5-8qdzp\" (UID: \"c0cc6cbb-7e93-487b-8149-55ad58fc1956\") " pod="kube-system/coredns-59b4f5bbd5-8qdzp"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946185    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-dir\" (UniqueName: \"kubernetes.io/host-path/9465c634-2aa5-41d6-8778-35ee5a18f0e7-device-dir\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946205    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-dffm4\" (UniqueName: \"kubernetes.io/projected/3aca02ed-adab-42ed-8760-cc186281c7cb-kube-api-access-dffm4\") pod \"csi-nfs-node-p942f\" (UID: \"3aca02ed-adab-42ed-8760-cc186281c7cb\") " pod="kube-system/csi-nfs-node-p942f"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946291    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/3aca02ed-adab-42ed-8760-cc186281c7cb-pods-mount-dir\") pod \"csi-nfs-node-p942f\" (UID: \"3aca02ed-adab-42ed-8760-cc186281c7cb\") " pod="kube-system/csi-nfs-node-p942f"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946345    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/3aca02ed-adab-42ed-8760-cc186281c7cb-registration-dir\") pod \"csi-nfs-node-p942f\" (UID: \"3aca02ed-adab-42ed-8760-cc186281c7cb\") " pod="kube-system/csi-nfs-node-p942f"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946376    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"encr-keys\" (UniqueName: \"kubernetes.io/host-path/9465c634-2aa5-41d6-8778-35ee5a18f0e7-encr-keys\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946400    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"registration-dir\" (UniqueName: \"kubernetes.io/host-path/9465c634-2aa5-41d6-8778-35ee5a18f0e7-registration-dir\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946469    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/35f38928-f60d-476c-b817-7feab0763635-socket-dir\") pod \"openebs-zfs-controller-0\" (UID: \"35f38928-f60d-476c-b817-7feab0763635\") " pod="kube-system/openebs-zfs-controller-0"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946536    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-mgcnh\" (UniqueName: \"kubernetes.io/projected/c0cc6cbb-7e93-487b-8149-55ad58fc1956-kube-api-access-mgcnh\") pod \"coredns-59b4f5bbd5-8qdzp\" (UID: \"c0cc6cbb-7e93-487b-8149-55ad58fc1956\") " pod="kube-system/coredns-59b4f5bbd5-8qdzp"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946585    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-dir\" (UniqueName: \"kubernetes.io/host-path/9465c634-2aa5-41d6-8778-35ee5a18f0e7-plugin-dir\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946643    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-nc2mz\" (UniqueName: \"kubernetes.io/projected/488bc48b-ecfd-47d6-963b-5fe909018eaa-kube-api-access-nc2mz\") pod \"csi-smb-controller-7fbbb8fb6f-8ngss\" (UID: \"488bc48b-ecfd-47d6-963b-5fe909018eaa\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-8ngss"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946702    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-ld8l5\" (UniqueName: \"kubernetes.io/projected/7fdca64c-43a4-4bb4-b670-0b68b6637449-kube-api-access-ld8l5\") pod \"snapshot-controller-546868dfb4-t5f78\" (UID: \"7fdca64c-43a4-4bb4-b670-0b68b6637449\") " pod="kube-system/snapshot-controller-546868dfb4-t5f78"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946743    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-bzxx2\" (UniqueName: \"kubernetes.io/projected/35f38928-f60d-476c-b817-7feab0763635-kube-api-access-bzxx2\") pod \"openebs-zfs-controller-0\" (UID: \"35f38928-f60d-476c-b817-7feab0763635\") " pod="kube-system/openebs-zfs-controller-0"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946783    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"device-plugin\" (UniqueName: \"kubernetes.io/host-path/459b9443-7cae-4289-9850-551c26f9a4ab-device-plugin\") pod \"nvidia-device-plugin-daemonset-5brd8\" (UID: \"459b9443-7cae-4289-9850-551c26f9a4ab\") " pod="kube-system/nvidia-device-plugin-daemonset-5brd8"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946816    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"host-root\" (UniqueName: \"kubernetes.io/host-path/9465c634-2aa5-41d6-8778-35ee5a18f0e7-host-root\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946862    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"pods-mount-dir\" (UniqueName: \"kubernetes.io/host-path/9465c634-2aa5-41d6-8778-35ee5a18f0e7-pods-mount-dir\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946895    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/c0cc6cbb-7e93-487b-8149-55ad58fc1956-config-volume\") pod \"coredns-59b4f5bbd5-8qdzp\" (UID: \"c0cc6cbb-7e93-487b-8149-55ad58fc1956\") " pod="kube-system/coredns-59b4f5bbd5-8qdzp"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946925    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"plugin-config\" (UniqueName: \"kubernetes.io/configmap/459b9443-7cae-4289-9850-551c26f9a4ab-plugin-config\") pod \"nvidia-device-plugin-daemonset-5brd8\" (UID: \"459b9443-7cae-4289-9850-551c26f9a4ab\") " pod="kube-system/nvidia-device-plugin-daemonset-5brd8"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.946961    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-tpb6q\" (UniqueName: \"kubernetes.io/projected/459b9443-7cae-4289-9850-551c26f9a4ab-kube-api-access-tpb6q\") pod \"nvidia-device-plugin-daemonset-5brd8\" (UID: \"459b9443-7cae-4289-9850-551c26f9a4ab\") " pod="kube-system/nvidia-device-plugin-daemonset-5brd8"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.947007    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-8ffkz\" (UniqueName: \"kubernetes.io/projected/9465c634-2aa5-41d6-8778-35ee5a18f0e7-kube-api-access-8ffkz\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.947042    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/empty-dir/488bc48b-ecfd-47d6-963b-5fe909018eaa-socket-dir\") pod \"csi-smb-controller-7fbbb8fb6f-8ngss\" (UID: \"488bc48b-ecfd-47d6-963b-5fe909018eaa\") " pod="kube-system/csi-smb-controller-7fbbb8fb6f-8ngss"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.947106    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"socket-dir\" (UniqueName: \"kubernetes.io/host-path/3aca02ed-adab-42ed-8760-cc186281c7cb-socket-dir\") pod \"csi-nfs-node-p942f\" (UID: \"3aca02ed-adab-42ed-8760-cc186281c7cb\") " pod="kube-system/csi-nfs-node-p942f"
Feb 23 19:30:55 truenas k3s[6488]: I0223 19:30:55.947149    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"chroot-zfs\" (UniqueName: \"kubernetes.io/configmap/9465c634-2aa5-41d6-8778-35ee5a18f0e7-chroot-zfs\") pod \"openebs-zfs-node-h8m2t\" (UID: \"9465c634-2aa5-41d6-8778-35ee5a18f0e7\") " pod="kube-system/openebs-zfs-node-h8m2t"
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.040399    6488 shared_informer.go:277] Caches are synced for garbage collector
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.087921    6488 shared_informer.go:277] Caches are synced for garbage collector
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.087936    6488 garbagecollector.go:163] Garbage collector: all resource monitors have synced. Proceeding to collect garbage
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.790699    6488 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=1a2a6267-905e-4a69-a45e-3491c2eaed00 path="/var/lib/kubelet/pods/1a2a6267-905e-4a69-a45e-3491c2eaed00/volumes"
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.792213    6488 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=57e9ad40-4b50-47e0-a638-3fbc82c71dfc path="/var/lib/kubelet/pods/57e9ad40-4b50-47e0-a638-3fbc82c71dfc/volumes"
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.793885    6488 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=95ebcd96-95e7-40a5-8837-416bb6c2a62d path="/var/lib/kubelet/pods/95ebcd96-95e7-40a5-8837-416bb6c2a62d/volumes"
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.862534    6488 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=d5ada2a6-c0db-43fd-9f40-1513fb868f8a path="/var/lib/kubelet/pods/d5ada2a6-c0db-43fd-9f40-1513fb868f8a/volumes"
Feb 23 19:30:56 truenas k3s[6488]: I0223 19:30:56.864181    6488 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=fdee2dbf-b26a-4b6e-bcd2-fe0c4c199ba1 path="/var/lib/kubelet/pods/fdee2dbf-b26a-4b6e-bcd2-fe0c4c199ba1/volumes"
Feb 23 19:30:57 truenas k3s[6488]: I0223 19:30:57.145666    6488 request.go:690] Waited for 1.09718837s due to client-side throttling, not priority and fairness, request: POST:https://127.0.0.1:6443/api/v1/namespaces/kube-system/serviceaccounts/csi-nfs-node-sa/token
Feb 23 19:30:59 truenas k3s[6488]: I0223 19:30:59.751251    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/snapshot-controller-546868dfb4-nfx9b" podStartSLOduration=13611.751218994 pod.CreationTimestamp="2024-02-23 15:44:08 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:30:59.751018873 -0800 PST m=+21.966103253" watchObservedRunningTime="2024-02-23 19:30:59.751218994 -0800 PST m=+21.966303364"
Feb 23 19:31:01 truenas k3s[6488]: I0223 19:31:01.902239    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/snapshot-controller-546868dfb4-t5f78" podStartSLOduration=13617.902205668 pod.CreationTimestamp="2024-02-23 15:44:04 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:01.901653447 -0800 PST m=+24.116737827" watchObservedRunningTime="2024-02-23 19:31:01.902205668 -0800 PST m=+24.117290038"
Feb 23 19:31:01 truenas k3s[6488]: I0223 19:31:01.910196    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/openebs-zfs-node-h8m2t" podStartSLOduration=6.910163489 pod.CreationTimestamp="2024-02-23 19:30:55 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:01.910003789 -0800 PST m=+24.125088169" watchObservedRunningTime="2024-02-23 19:31:01.910163489 -0800 PST m=+24.125247859"
Feb 23 19:31:02 truenas k3s[6488]: I0223 19:31:02.916074    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/coredns-59b4f5bbd5-8qdzp" podStartSLOduration=13617.916022171 pod.CreationTimestamp="2024-02-23 15:44:05 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:02.91548122 -0800 PST m=+25.130565610" watchObservedRunningTime="2024-02-23 19:31:02.916022171 -0800 PST m=+25.131106561"
Feb 23 19:31:02 truenas k3s[6488]: I0223 19:31:02.928996    6488 controller.go:615] quota admission added evaluator for: endpointslices.discovery.k8s.io
Feb 23 19:31:02 truenas k3s[6488]: I0223 19:31:02.930995    6488 controller.go:615] quota admission added evaluator for: endpoints
Feb 23 19:31:03 truenas k3s[6488]: I0223 19:31:03.914728    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/csi-smb-controller-7fbbb8fb6f-8ngss" podStartSLOduration=13616.914700953 pod.CreationTimestamp="2024-02-23 15:44:07 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:03.914367502 -0800 PST m=+26.129451882" watchObservedRunningTime="2024-02-23 19:31:03.914700953 -0800 PST m=+26.129785323"
Feb 23 19:31:03 truenas k3s[6488]: I0223 19:31:03.921779    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/nvidia-device-plugin-daemonset-5brd8" podStartSLOduration=8.921738513 pod.CreationTimestamp="2024-02-23 19:30:55 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:03.921403992 -0800 PST m=+26.136488372" watchObservedRunningTime="2024-02-23 19:31:03.921738513 -0800 PST m=+26.136822893"
Feb 23 19:31:03 truenas k3s[6488]: I0223 19:31:03.931232    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/csi-nfs-node-p942f" podStartSLOduration=8.931197778 pod.CreationTimestamp="2024-02-23 19:30:55 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:03.930671967 -0800 PST m=+26.145756367" watchObservedRunningTime="2024-02-23 19:31:03.931197778 -0800 PST m=+26.146282148"
Feb 23 19:31:04 truenas k3s[6488]: I0223 19:31:04.137946    6488 controller.go:615] quota admission added evaluator for: zfsnodes.zfs.openebs.io
Feb 23 19:31:04 truenas k3s[6488]: I0223 19:31:04.443292    6488 server.go:144] "Got registration request from device plugin with resource" resourceName="nvidia.com/gpu"
Feb 23 19:31:04 truenas k3s[6488]: I0223 19:31:04.801815    6488 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: nfs.csi.k8s.io endpoint: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock versions: 1.0.0
Feb 23 19:31:04 truenas k3s[6488]: I0223 19:31:04.801846    6488 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: nfs.csi.k8s.io at endpoint: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
Feb 23 19:31:04 truenas k3s[6488]: I0223 19:31:04.802749    6488 csi_plugin.go:99] kubernetes.io/csi: Trying to validate a new CSI Driver with name: zfs.csi.openebs.io endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock versions: 1.0.0
Feb 23 19:31:04 truenas k3s[6488]: I0223 19:31:04.802879    6488 csi_plugin.go:112] kubernetes.io/csi: Register new plugin with name: zfs.csi.openebs.io at endpoint: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
Feb 23 19:31:05 truenas k3s[6488]: I0223 19:31:05.116627    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="kube-system/openebs-zfs-controller-0" podStartSLOduration=13615.116587031 pod.CreationTimestamp="2024-02-23 15:44:10 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:05.116336531 -0800 PST m=+27.331420931" watchObservedRunningTime="2024-02-23 19:31:05.116587031 -0800 PST m=+27.331671411"
Feb 23 19:31:06 truenas k3s[6488]: I0223 19:31:06.677902    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:31:06 truenas k3s[6488]: I0223 19:31:06.809442    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"emby-1\" (UniqueName: \"kubernetes.io/host-path/24372e60-687e-4661-a757-f977d5890707-emby-1\") pod \"emby-8587ff676b-8zzvv\" (UID: \"24372e60-687e-4661-a757-f977d5890707\") " pod="ix-emby/emby-8587ff676b-8zzvv"
Feb 23 19:31:06 truenas k3s[6488]: I0223 19:31:06.809472    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"emby-0\" (UniqueName: \"kubernetes.io/host-path/24372e60-687e-4661-a757-f977d5890707-emby-0\") pod \"emby-8587ff676b-8zzvv\" (UID: \"24372e60-687e-4661-a757-f977d5890707\") " pod="ix-emby/emby-8587ff676b-8zzvv"
Feb 23 19:31:06 truenas k3s[6488]: I0223 19:31:06.809493    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/24372e60-687e-4661-a757-f977d5890707-config\") pod \"emby-8587ff676b-8zzvv\" (UID: \"24372e60-687e-4661-a757-f977d5890707\") " pod="ix-emby/emby-8587ff676b-8zzvv"
Feb 23 19:31:06 truenas k3s[6488]: I0223 19:31:06.809527    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/empty-dir/24372e60-687e-4661-a757-f977d5890707-tmp\") pod \"emby-8587ff676b-8zzvv\" (UID: \"24372e60-687e-4661-a757-f977d5890707\") " pod="ix-emby/emby-8587ff676b-8zzvv"
Feb 23 19:31:10 truenas k3s[6488]: I0223 19:31:10.945195    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-emby/emby-8587ff676b-8zzvv" podStartSLOduration=13622.94514448 pod.CreationTimestamp="2024-02-23 15:44:08 -0800 PST" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:31:10.944559339 -0800 PST m=+33.159643769" watchObservedRunningTime="2024-02-23 19:31:10.94514448 -0800 PST m=+33.160228880"
Feb 23 19:31:17 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:31:17.040-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:31:44 truenas k3s[6488]: I0223 19:31:44.966536    6488 scope.go:115] "RemoveContainer" containerID="9cacc1a7ac42958694007a1c96e03455e6e0cbad770a89fd4012ea8ef17e4353"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.049929    6488 scope.go:115] "RemoveContainer" containerID="2a02ac26d8379c2bdd9522572c95f81080c18d9379b98e1f08caba8dfcd5a50d"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.083421    6488 scope.go:115] "RemoveContainer" containerID="16b34a7d27ead7cb65927f2efc48af0f9215d4d9060f059562b7c2d6ba47cd1c"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.127985    6488 scope.go:115] "RemoveContainer" containerID="790fdc17a8199882c87664577f9a804a3d873fc08106853292cf23e82c221f4d"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.261580    6488 scope.go:115] "RemoveContainer" containerID="98f3a0e272e230809e430dc3d01be80d69ada3a0bda9870e63a8bdec3856b9f5"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.294989    6488 scope.go:115] "RemoveContainer" containerID="c1c8884c1cfea4835134ad21dbdc6dfd5808e3fe733b8e4d3abd2e354882d2b7"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.339802    6488 scope.go:115] "RemoveContainer" containerID="618c5ba37a1dcc964de142be2c8d117af27900717674bfbfd1845c460c7a0c8f"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.383055    6488 scope.go:115] "RemoveContainer" containerID="9cbb0954b252637063f01f89f65083c24c2acc19737eb852eeab8b3119eb5925"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.428934    6488 scope.go:115] "RemoveContainer" containerID="f78fa2a66a2b0fa23e0b54328d6b38cb3dae4752fc3291a4a796cf9ae6443de4"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.473642    6488 scope.go:115] "RemoveContainer" containerID="eb31478c34fa3465c68f5d185b4b96001f7ed400a6582683e175123218eadb63"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.507104    6488 scope.go:115] "RemoveContainer" containerID="5419ccee042b34abc156bccd7038bf9780438ede4b71df28eb0e0d37f1c46e1d"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.552086    6488 scope.go:115] "RemoveContainer" containerID="612ab9ebdc4e8180fd06fbbdad8abe2d2af46f6fd20c947a7511ba44e9aef4ed"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.595330    6488 scope.go:115] "RemoveContainer" containerID="08a1fbb076855e2b109005a4b40977fbde5b19cd4d2797720362dcf199592a1c"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.641292    6488 scope.go:115] "RemoveContainer" containerID="6c7cfa3387a0bc1b9ab27ecf3a401e594d65a8f4b9b6c651df33679473df4a6b"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.685885    6488 scope.go:115] "RemoveContainer" containerID="a4fda07c73be85212a1dc0fbde087b6f49afd8b50aa2ff14bda5786bd6139ae7"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.719487    6488 scope.go:115] "RemoveContainer" containerID="f4f22766fe615d9b6b84ccc72264fddb8529ee39bbb69002c5bc3528ca80fb90"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.764204    6488 scope.go:115] "RemoveContainer" containerID="bfad89d724193281f9f8ff4a4bdac6306e828a64c15d077378ad46a3dbd92849"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.798823    6488 scope.go:115] "RemoveContainer" containerID="c9e057cd01f664bbbeb009b3de88b2c86884b5b47030651541a0fb6666742cb1"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.843726    6488 scope.go:115] "RemoveContainer" containerID="6336427a79519e38a76e19d0b08d1ea66e8def0ce1195fa79ec64c9e61481ea9"
Feb 23 19:31:45 truenas k3s[6488]: I0223 19:31:45.888307    6488 scope.go:115] "RemoveContainer" containerID="d858f9b1109a425bb1c3f4e67a13944daefa800afe0c1a43bd09dcea3499b370"
Feb 23 19:31:56 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:31:56.842-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:32:28 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:32:28.403-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:32:58 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:32:58.634-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:33:41 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:33:41.726-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:34:21 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:34:21.289-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:35:03 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:35:03.109-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:35:40 truenas k3s[6488]: time="2024-02-23T19:35:40-08:00" level=info msg="COMPACT compactRev=31731 targetCompactRev=32692 currentRev=33692"
Feb 23 19:35:40 truenas k3s[6488]: time="2024-02-23T19:35:40-08:00" level=info msg="COMPACT deleted 987 rows from 961 revisions in 16.89207ms - compacted to 32692/33692"
Feb 23 19:35:40 truenas k3s[6488]: time="2024-02-23T19:35:40-08:00" level=info msg="COMPACT revision 32692 has already been compacted"
Feb 23 19:35:42 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:35:42.072-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:36:15 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:36:15.667-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:36:53 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:36:53.456-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:37:38 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:37:38.315-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:38:14 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:38:14.826-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:38:47 truenas k3s[6488]: I0223 19:38:47.759376    6488 controller.go:615] quota admission added evaluator for: namespaces
Feb 23 19:38:47 truenas k3s[6488]: I0223 19:38:47.778716    6488 controller.go:615] quota admission added evaluator for: serviceaccounts
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.345714    6488 controller.go:615] quota admission added evaluator for: roles.rbac.authorization.k8s.io
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.389942    6488 alloc.go:327] "allocated clusterIPs" service="ix-nextcloud/nextcloud" clusterIPs=map[IPv4:172.17.234.63]
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.389947    6488 alloc.go:327] "allocated clusterIPs" service="ix-nextcloud/nextcloud-postgres" clusterIPs=map[IPv4:172.17.177.179]
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.398364    6488 controller.go:615] quota admission added evaluator for: replicasets.apps
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.400288    6488 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-postgres" fieldPath="" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled up replica set nextcloud-postgres-99d8d867c to 1"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.409039    6488 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-postgres-99d8d867c" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-postgres-99d8d867c-6b4xm"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.410946    6488 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud-cf858d88" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: nextcloud-cf858d88-lvbwl"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.416575    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.417878    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.453396    6488 event.go:294] "Event occurred" object="ix-nextcloud/nextcloud" fieldPath="" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled up replica set nextcloud-cf858d88 to 1"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.575409    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"postgres-backup\" (UniqueName: \"kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-backup\") pod \"nextcloud-postgres-99d8d867c-6b4xm\" (UID: \"cd50375e-355c-44a8-bfa6-9edfa60230d5\") " pod="ix-nextcloud/nextcloud-postgres-99d8d867c-6b4xm"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.575450    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nextcloud-configuration\" (UniqueName: \"kubernetes.io/configmap/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-configuration\") pod \"nextcloud-cf858d88-lvbwl\" (UID: \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\") " pod="ix-nextcloud/nextcloud-cf858d88-lvbwl"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.575498    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-4crx6\" (UniqueName: \"kubernetes.io/projected/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-kube-api-access-4crx6\") pod \"nextcloud-cf858d88-lvbwl\" (UID: \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\") " pod="ix-nextcloud/nextcloud-cf858d88-lvbwl"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.575562    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"postgres-data\" (UniqueName: \"kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-data\") pod \"nextcloud-postgres-99d8d867c-6b4xm\" (UID: \"cd50375e-355c-44a8-bfa6-9edfa60230d5\") " pod="ix-nextcloud/nextcloud-postgres-99d8d867c-6b4xm"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.575589    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-zdk78\" (UniqueName: \"kubernetes.io/projected/cd50375e-355c-44a8-bfa6-9edfa60230d5-kube-api-access-zdk78\") pod \"nextcloud-postgres-99d8d867c-6b4xm\" (UID: \"cd50375e-355c-44a8-bfa6-9edfa60230d5\") " pod="ix-nextcloud/nextcloud-postgres-99d8d867c-6b4xm"
Feb 23 19:38:49 truenas k3s[6488]: I0223 19:38:49.575614    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"nextcloud-data\" (UniqueName: \"kubernetes.io/host-path/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-data\") pod \"nextcloud-cf858d88-lvbwl\" (UID: \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\") " pod="ix-nextcloud/nextcloud-cf858d88-lvbwl"
Feb 23 19:38:50 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:38:50.347-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0024616c0/kine.sock","attempt":0,"error":"rpc error: code = Canceled desc = context canceled"}
Feb 23 19:38:50 truenas k3s[6488]: E0223 19:38:50.347654    6488 finisher.go:175] FinishRequest: post-timeout activity - time-elapsed: 41.13µs, panicked: false, err: context canceled, panic-reason: <nil>
Feb 23 19:38:50 truenas k3s[6488]: time="2024-02-23T19:38:50-08:00" level=error msg="error in txn: context canceled"
Feb 23 19:38:50 truenas k3s[6488]: E0223 19:38:50.347836    6488 writers.go:122] apiserver was unable to write a JSON response: http: Handler timeout
Feb 23 19:38:50 truenas k3s[6488]: E0223 19:38:50.349091    6488 status.go:71] apiserver received an error that is not an metav1.Status: &errors.errorString{s:"http: Handler timeout"}: http: Handler timeout
Feb 23 19:38:50 truenas k3s[6488]: E0223 19:38:50.350007    6488 writers.go:135] apiserver was unable to write a fallback JSON response: http: Handler timeout
Feb 23 19:38:50 truenas k3s[6488]: E0223 19:38:50.351111    6488 timeout.go:142] post-timeout activity - time-elapsed: 3.637857ms, POST "/api/v1/namespaces/ix-nextcloud/events" result: <nil>
Feb 23 19:38:56 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:38:56.047-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:39:05 truenas k3s[6488]: I0223 19:39:05.909082    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-nextcloud/nextcloud-postgres-99d8d867c-6b4xm" podStartSLOduration=-9.223372019945726e+09 pod.CreationTimestamp="2024-02-23 19:38:49 -0800 PST" firstStartedPulling="2024-02-23 19:38:50.446894972 -0800 PST m=+492.661979352" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:39:05.908104357 -0800 PST m=+508.123188747" watchObservedRunningTime="2024-02-23 19:39:05.909049203 -0800 PST m=+508.124133573"
Feb 23 19:39:29 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:39:29.761-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:40:06 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:40:06.329-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:40:08 truenas k3s[6488]: I0223 19:40:08.058027    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-nextcloud/nextcloud-cf858d88-lvbwl" podStartSLOduration=-9.223371957796793e+09 pod.CreationTimestamp="2024-02-23 19:38:49 -0800 PST" firstStartedPulling="2024-02-23 19:38:50.444092303 -0800 PST m=+492.659176673" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 19:40:08.056620392 -0800 PST m=+570.271704802" watchObservedRunningTime="2024-02-23 19:40:08.057983623 -0800 PST m=+570.273068003"
Feb 23 19:40:38 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:40:38.859-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:40:40 truenas k3s[6488]: time="2024-02-23T19:40:40-08:00" level=info msg="COMPACT compactRev=32692 targetCompactRev=33284 currentRev=34284"
Feb 23 19:40:40 truenas k3s[6488]: time="2024-02-23T19:40:40-08:00" level=info msg="COMPACT deleted 389 rows from 592 revisions in 14.277085ms - compacted to 33284/34284"
Feb 23 19:41:16 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:41:16.897-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:41:53 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:41:53.968-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:42:26 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:42:26.442-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:43:04 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:43:04.474-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:43:35 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:43:35.153-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:44:11 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:44:11.553-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:44:52 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:44:52.067-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:45:25 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:45:25.618-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:45:40 truenas k3s[6488]: time="2024-02-23T19:45:40-08:00" level=info msg="COMPACT compactRev=33284 targetCompactRev=33804 currentRev=34804"
Feb 23 19:45:40 truenas k3s[6488]: time="2024-02-23T19:45:40-08:00" level=info msg="COMPACT deleted 519 rows from 520 revisions in 53.122185ms - compacted to 33804/34804"
Feb 23 19:45:58 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:45:58.418-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:46:37 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:46:37.521-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:47:15 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:47:15.858-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:47:52 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:47:52.056-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:48:24 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:48:24.825-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:49:08 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:49:08.763-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:49:53 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:49:53.510-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:50:30 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:50:30.234-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:50:40 truenas k3s[6488]: time="2024-02-23T19:50:40-08:00" level=info msg="COMPACT compactRev=33804 targetCompactRev=34319 currentRev=35319"
Feb 23 19:50:40 truenas k3s[6488]: time="2024-02-23T19:50:40-08:00" level=info msg="COMPACT deleted 470 rows from 515 revisions in 4.734843ms - compacted to 34319/35319"
Feb 23 19:51:02 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:51:02.494-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:51:45 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:51:45.058-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:52:29 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:52:29.612-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:53:05 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:53:05.652-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:53:41 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:53:41.920-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:54:22 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:54:22.418-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:55:07 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:55:07.418-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:55:38 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:55:38.518-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:55:40 truenas k3s[6488]: time="2024-02-23T19:55:40-08:00" level=info msg="COMPACT compactRev=34319 targetCompactRev=34833 currentRev=35833"
Feb 23 19:55:40 truenas k3s[6488]: time="2024-02-23T19:55:40-08:00" level=info msg="COMPACT deleted 514 rows from 514 revisions in 9.048951ms - compacted to 34833/35833"
Feb 23 19:56:23 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:56:23.081-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:56:56 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:56:56.365-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:57:29 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:57:29.802-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:58:02 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:58:02.432-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:58:33 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:58:33.206-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:59:09 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:59:09.373-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 19:59:40 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T19:59:40.170-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:00:25 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:00:25.132-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:00:40 truenas k3s[6488]: time="2024-02-23T20:00:40-08:00" level=info msg="COMPACT compactRev=34833 targetCompactRev=35348 currentRev=36348"
Feb 23 20:00:40 truenas k3s[6488]: time="2024-02-23T20:00:40-08:00" level=info msg="COMPACT deleted 515 rows from 515 revisions in 10.708915ms - compacted to 35348/36348"
Feb 23 20:01:06 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:01:06.532-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:01:48 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:01:48.936-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:01:50 truenas k3s[6488]: E0223 20:01:50.559920    6488 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to create exec \"af817220a7f5e48dca4461024df0614d4e50bd50216c4e4a691ff4eeb1ee6c2a\": task d285c4be27ee91524a3e1c1c64565a16dfe5daa8f1d892cad91f9358f6beb30f not found: not found" containerID="d285c4be27ee91524a3e1c1c64565a16dfe5daa8f1d892cad91f9358f6beb30f" cmd=[sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2; done]
Feb 23 20:01:50 truenas k3s[6488]: E0223 20:01:50.560645    6488 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d285c4be27ee91524a3e1c1c64565a16dfe5daa8f1d892cad91f9358f6beb30f not found: not found" containerID="d285c4be27ee91524a3e1c1c64565a16dfe5daa8f1d892cad91f9358f6beb30f" cmd=[sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2; done]
Feb 23 20:01:50 truenas k3s[6488]: E0223 20:01:50.561295    6488 remote_runtime.go:479] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task d285c4be27ee91524a3e1c1c64565a16dfe5daa8f1d892cad91f9358f6beb30f not found: not found" containerID="d285c4be27ee91524a3e1c1c64565a16dfe5daa8f1d892cad91f9358f6beb30f" cmd=[sh -c until pg_isready -U ${POSTGRES_USER} -h localhost; do sleep 2; done]
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.970903    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-4crx6\" (UniqueName: \"kubernetes.io/projected/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-kube-api-access-4crx6\") pod \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\" (UID: \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\") "
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.970952    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"nextcloud-data\" (UniqueName: \"kubernetes.io/host-path/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-data\") pod \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\" (UID: \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\") "
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.970991    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"nextcloud-configuration\" (UniqueName: \"kubernetes.io/configmap/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-configuration\") pod \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\" (UID: \"a6e20f75-d1cb-48f5-86f1-e142a3eaca66\") "
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.971029    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"postgres-backup\" (UniqueName: \"kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-backup\") pod \"cd50375e-355c-44a8-bfa6-9edfa60230d5\" (UID: \"cd50375e-355c-44a8-bfa6-9edfa60230d5\") "
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.971069    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"postgres-data\" (UniqueName: \"kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-data\") pod \"cd50375e-355c-44a8-bfa6-9edfa60230d5\" (UID: \"cd50375e-355c-44a8-bfa6-9edfa60230d5\") "
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.971119    6488 reconciler_common.go:169] "operationExecutor.UnmountVolume started for volume \"kube-api-access-zdk78\" (UniqueName: \"kubernetes.io/projected/cd50375e-355c-44a8-bfa6-9edfa60230d5-kube-api-access-zdk78\") pod \"cd50375e-355c-44a8-bfa6-9edfa60230d5\" (UID: \"cd50375e-355c-44a8-bfa6-9edfa60230d5\") "
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.971173    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-backup" (OuterVolumeSpecName: "postgres-backup") pod "cd50375e-355c-44a8-bfa6-9edfa60230d5" (UID: "cd50375e-355c-44a8-bfa6-9edfa60230d5"). InnerVolumeSpecName "postgres-backup". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.971192    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-data" (OuterVolumeSpecName: "postgres-data") pod "cd50375e-355c-44a8-bfa6-9edfa60230d5" (UID: "cd50375e-355c-44a8-bfa6-9edfa60230d5"). InnerVolumeSpecName "postgres-data". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.972790    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-kube-api-access-4crx6" (OuterVolumeSpecName: "kube-api-access-4crx6") pod "a6e20f75-d1cb-48f5-86f1-e142a3eaca66" (UID: "a6e20f75-d1cb-48f5-86f1-e142a3eaca66"). InnerVolumeSpecName "kube-api-access-4crx6". PluginName "kubernetes.io/projected", VolumeGidValue ""
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.973118    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/cd50375e-355c-44a8-bfa6-9edfa60230d5-kube-api-access-zdk78" (OuterVolumeSpecName: "kube-api-access-zdk78") pod "cd50375e-355c-44a8-bfa6-9edfa60230d5" (UID: "cd50375e-355c-44a8-bfa6-9edfa60230d5"). InnerVolumeSpecName "kube-api-access-zdk78". PluginName "kubernetes.io/projected", VolumeGidValue ""
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.973672    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-configuration/nextcloud/7" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.973674    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-data/nextcloud/0" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.974989    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-configuration/nextcloud/8" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.975001    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-data/nextcloud/1" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.976300    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-configuration/nextcloud/9" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.976305    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-data/nextcloud/2" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.976928    6488 empty_dir.go:525] Warning: Failed to clear quota on /var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volumes/kubernetes.io~configmap/nextcloud-configuration: clearQuota called, but quotas disabled
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.977528    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-data/nextcloud/3" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.977669    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/configmap/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-configuration" (OuterVolumeSpecName: "nextcloud-configuration") pod "a6e20f75-d1cb-48f5-86f1-e142a3eaca66" (UID: "a6e20f75-d1cb-48f5-86f1-e142a3eaca66"). InnerVolumeSpecName "nextcloud-configuration". PluginName "kubernetes.io/configmap", VolumeGidValue ""
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.978624    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-data/nextcloud/4" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.979778    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-data/nextcloud/5" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: W0223 20:01:55.981019    6488 mount_helper_common.go:142] Warning: "/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volume-subpaths/nextcloud-data/nextcloud/6" is not a mountpoint, deleting
Feb 23 20:01:55 truenas k3s[6488]: I0223 20:01:55.981357    6488 operation_generator.go:900] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-data" (OuterVolumeSpecName: "nextcloud-data") pod "a6e20f75-d1cb-48f5-86f1-e142a3eaca66" (UID: "a6e20f75-d1cb-48f5-86f1-e142a3eaca66"). InnerVolumeSpecName "nextcloud-data". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.071637    6488 reconciler_common.go:295] "Volume detached for volume \"postgres-data\" (UniqueName: \"kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-data\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.071664    6488 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-zdk78\" (UniqueName: \"kubernetes.io/projected/cd50375e-355c-44a8-bfa6-9edfa60230d5-kube-api-access-zdk78\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.071682    6488 reconciler_common.go:295] "Volume detached for volume \"kube-api-access-4crx6\" (UniqueName: \"kubernetes.io/projected/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-kube-api-access-4crx6\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.071698    6488 reconciler_common.go:295] "Volume detached for volume \"nextcloud-data\" (UniqueName: \"kubernetes.io/host-path/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-data\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.071713    6488 reconciler_common.go:295] "Volume detached for volume \"nextcloud-configuration\" (UniqueName: \"kubernetes.io/configmap/a6e20f75-d1cb-48f5-86f1-e142a3eaca66-nextcloud-configuration\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.071727    6488 reconciler_common.go:295] "Volume detached for volume \"postgres-backup\" (UniqueName: \"kubernetes.io/host-path/cd50375e-355c-44a8-bfa6-9edfa60230d5-postgres-backup\") on node \"ix-truenas\" DevicePath \"\""
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.651935    6488 scope.go:115] "RemoveContainer" containerID="7b901a535132f0af7b234673d87aadcf45fb3ecb8ef706a418d09dba4fdf3b35"
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.757907    6488 scope.go:115] "RemoveContainer" containerID="0d2d57a82ee1b5308acddbab00486393464d765a9678816556410167b4b745f7"
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.791210    6488 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=a6e20f75-d1cb-48f5-86f1-e142a3eaca66 path="/var/lib/kubelet/pods/a6e20f75-d1cb-48f5-86f1-e142a3eaca66/volumes"
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.792827    6488 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=cd50375e-355c-44a8-bfa6-9edfa60230d5 path="/var/lib/kubelet/pods/cd50375e-355c-44a8-bfa6-9edfa60230d5/volumes"
Feb 23 20:01:56 truenas k3s[6488]: I0223 20:01:56.807084    6488 scope.go:115] "RemoveContainer" containerID="d285c4be27ee91524a3e1c1c64565a16dfe5daa8f1d892cad91f9358f6beb30f"
Feb 23 20:02:12 truenas k3s[6488]: I0223 20:02:12.101582    6488 namespace_controller.go:180] Namespace has been deleted ix-nextcloud
Feb 23 20:02:26 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:02:26.538-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:03:01 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:03:01.163-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:03:36 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:03:36.364-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:04:16 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:04:16.558-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:04:49 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:04:49.300-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:05:31 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:05:31.491-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:05:40 truenas k3s[6488]: time="2024-02-23T20:05:40-08:00" level=info msg="COMPACT compactRev=35348 targetCompactRev=35928 currentRev=36928"
Feb 23 20:05:40 truenas k3s[6488]: time="2024-02-23T20:05:40-08:00" level=info msg="COMPACT deleted 580 rows from 580 revisions in 10.725432ms - compacted to 35928/36928"
Feb 23 20:06:15 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:06:15.208-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:06:45 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:06:45.535-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:07:19 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:07:19.600-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:07:56 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:07:56.508-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:08:28 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:08:28.527-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:09:10 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:09:10.954-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:09:42 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:09:42.834-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:10:18 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:10:18.178-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:10:40 truenas k3s[6488]: time="2024-02-23T20:10:40-08:00" level=info msg="COMPACT compactRev=35928 targetCompactRev=36442 currentRev=37442"
Feb 23 20:10:40 truenas k3s[6488]: time="2024-02-23T20:10:40-08:00" level=info msg="COMPACT deleted 514 rows from 514 revisions in 9.302712ms - compacted to 36442/37442"
Feb 23 20:10:49 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:10:49.054-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:11:31 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:11:31.131-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:12:05 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:12:05.120-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.245666    6488 alloc.go:327] "allocated clusterIPs" service="ix-filebrowser/filebrowser" clusterIPs=map[IPv4:172.17.113.223]
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.259288    6488 event.go:294] "Event occurred" object="ix-filebrowser/filebrowser" fieldPath="" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled up replica set filebrowser-6f4c6cf4cf to 1"
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.269398    6488 event.go:294] "Event occurred" object="ix-filebrowser/filebrowser-6f4c6cf4cf" fieldPath="" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: filebrowser-6f4c6cf4cf-9wf2j"
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.275700    6488 topology_manager.go:210] "Topology Admit Handler"
Feb 23 20:12:37 truenas k3s[6488]: E0223 20:12:37.275781    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="a6e20f75-d1cb-48f5-86f1-e142a3eaca66" containerName="nextcloud"
Feb 23 20:12:37 truenas k3s[6488]: E0223 20:12:37.275803    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="a6e20f75-d1cb-48f5-86f1-e142a3eaca66" containerName="init-postgresdb"
Feb 23 20:12:37 truenas k3s[6488]: E0223 20:12:37.275821    6488 cpu_manager.go:395] "RemoveStaleState: removing container" podUID="cd50375e-355c-44a8-bfa6-9edfa60230d5" containerName="nextcloud-postgres"
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.275886    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="a6e20f75-d1cb-48f5-86f1-e142a3eaca66" containerName="nextcloud"
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.275908    6488 memory_manager.go:346] "RemoveStaleState removing state" podUID="cd50375e-355c-44a8-bfa6-9edfa60230d5" containerName="nextcloud-postgres"
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.468281    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/9a9c47a7-a4d3-4d11-9c7a-c2e82fe86b0c-config\") pod \"filebrowser-6f4c6cf4cf-9wf2j\" (UID: \"9a9c47a7-a4d3-4d11-9c7a-c2e82fe86b0c\") " pod="ix-filebrowser/filebrowser-6f4c6cf4cf-9wf2j"
Feb 23 20:12:37 truenas k3s[6488]: I0223 20:12:37.468339    6488 reconciler_common.go:253] "operationExecutor.VerifyControllerAttachedVolume started for volume \"filebrowser-0\" (UniqueName: \"kubernetes.io/host-path/9a9c47a7-a4d3-4d11-9c7a-c2e82fe86b0c-filebrowser-0\") pod \"filebrowser-6f4c6cf4cf-9wf2j\" (UID: \"9a9c47a7-a4d3-4d11-9c7a-c2e82fe86b0c\") " pod="ix-filebrowser/filebrowser-6f4c6cf4cf-9wf2j"
Feb 23 20:12:37 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:12:37.548-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:12:53 truenas k3s[6488]: I0223 20:12:53.058274    6488 pod_startup_latency_tracker.go:102] "Observed pod startup duration" pod="ix-filebrowser/filebrowser-6f4c6cf4cf-9wf2j" podStartSLOduration=-9.223372020796549e+09 pod.CreationTimestamp="2024-02-23 20:12:37 -0800 PST" firstStartedPulling="2024-02-23 20:12:40.944197156 -0800 PST m=+2523.159281546" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2024-02-23 20:12:53.057405086 -0800 PST m=+2535.272489486" watchObservedRunningTime="2024-02-23 20:12:53.058226946 -0800 PST m=+2535.273311336"
Feb 23 20:13:17 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:13:17.332-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:13:52 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:13:52.736-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:14:36 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:14:36.152-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:15:14 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:15:14.631-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:15:40 truenas k3s[6488]: time="2024-02-23T20:15:40-08:00" level=info msg="COMPACT compactRev=36442 targetCompactRev=37007 currentRev=38007"
Feb 23 20:15:40 truenas k3s[6488]: time="2024-02-23T20:15:40-08:00" level=info msg="COMPACT deleted 610 rows from 565 revisions in 10.336904ms - compacted to 37007/38007"
Feb 23 20:15:49 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:15:49.714-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:16:20 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:16:20.122-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:16:55 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:16:55.107-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:17:37 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:17:37.724-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:18:15 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:18:15.670-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:18:56 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:18:56.842-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:19:38 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:19:38.602-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:20:20 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:20:20.302-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:20:40 truenas k3s[6488]: time="2024-02-23T20:20:40-08:00" level=info msg="COMPACT compactRev=37007 targetCompactRev=37520 currentRev=38520"
Feb 23 20:20:40 truenas k3s[6488]: time="2024-02-23T20:20:40-08:00" level=info msg="COMPACT deleted 513 rows from 513 revisions in 8.125273ms - compacted to 37520/38520"
Feb 23 20:21:03 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:21:03.839-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:21:36 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:21:36.453-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:22:10 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:22:10.848-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:22:46 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:22:46.589-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:23:26 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:23:26.515-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:24:04 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:24:04.131-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:24:37 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:24:37.836-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:25:14 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:25:14.020-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:25:40 truenas k3s[6488]: time="2024-02-23T20:25:40-08:00" level=info msg="COMPACT compactRev=37520 targetCompactRev=38035 currentRev=39035"
Feb 23 20:25:40 truenas k3s[6488]: time="2024-02-23T20:25:40-08:00" level=info msg="COMPACT deleted 489 rows from 515 revisions in 9.028857ms - compacted to 38035/39035"
Feb 23 20:25:46 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:25:46.018-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:26:22 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:26:22.736-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:26:53 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:26:53.912-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:27:34 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:27:34.966-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:28:10 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:28:10.863-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:28:42 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:28:42.607-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:29:24 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:29:24.115-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:29:58 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:29:58.550-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:30:40 truenas k3s[6488]: time="2024-02-23T20:30:40-08:00" level=info msg="COMPACT compactRev=38035 targetCompactRev=38549 currentRev=39549"
Feb 23 20:30:40 truenas k3s[6488]: time="2024-02-23T20:30:40-08:00" level=info msg="COMPACT deleted 514 rows from 514 revisions in 10.419718ms - compacted to 38549/39549"
Feb 23 20:30:40 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:30:40.554-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:31:23 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:31:23.725-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:32:00 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:32:00.221-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:32:44 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:32:44.773-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:33:28 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:33:28.602-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:34:13 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:34:13.590-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:34:49 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:34:49.573-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:35:33 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:35:33.580-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:35:40 truenas k3s[6488]: time="2024-02-23T20:35:40-08:00" level=info msg="COMPACT compactRev=38549 targetCompactRev=39549 currentRev=40833"
Feb 23 20:35:40 truenas k3s[6488]: time="2024-02-23T20:35:40-08:00" level=info msg="COMPACT deleted 1000 rows from 1000 revisions in 14.809641ms - compacted to 39549/40833"
Feb 23 20:36:15 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:36:15.779-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:36:56 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:36:56.055-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:37:32 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:37:32.818-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:38:16 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:38:16.416-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:38:49 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:38:49.202-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:39:33 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:39:33.732-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:40:12 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:40:12.606-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:40:40 truenas k3s[6488]: time="2024-02-23T20:40:40-08:00" level=info msg="COMPACT compactRev=39549 targetCompactRev=40347 currentRev=41347"
Feb 23 20:40:40 truenas k3s[6488]: time="2024-02-23T20:40:40-08:00" level=info msg="COMPACT deleted 1480 rows from 798 revisions in 21.023705ms - compacted to 40347/41347"
Feb 23 20:40:40 truenas k3s[6488]: time="2024-02-23T20:40:40-08:00" level=info msg="COMPACT revision 40347 has already been compacted"
Feb 23 20:40:45 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:40:45.017-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:41:20 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:41:20.484-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:42:02 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:42:02.522-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:42:38 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:42:38.475-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:43:12 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:43:12.149-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:43:46 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:43:46.308-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:44:28 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:44:28.825-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:45:06 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:45:06.558-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:45:40 truenas k3s[6488]: time="2024-02-23T20:45:40-08:00" level=info msg="COMPACT compactRev=40347 targetCompactRev=40861 currentRev=41861"
Feb 23 20:45:40 truenas k3s[6488]: time="2024-02-23T20:45:40-08:00" level=info msg="COMPACT deleted 604 rows from 514 revisions in 9.504187ms - compacted to 40861/41861"
Feb 23 20:45:47 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:45:47.753-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:46:19 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:46:19.534-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:46:52 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:46:52.488-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:47:27 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:47:27.554-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:48:03 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:48:03.686-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:48:48 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:48:48.255-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:49:20 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:49:20.272-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:49:53 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:49:53.070-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:50:33 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:50:33.567-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:50:40 truenas k3s[6488]: time="2024-02-23T20:50:40-08:00" level=info msg="COMPACT compactRev=40861 targetCompactRev=41377 currentRev=42377"
Feb 23 20:50:40 truenas k3s[6488]: time="2024-02-23T20:50:40-08:00" level=info msg="COMPACT deleted 516 rows from 516 revisions in 8.942316ms - compacted to 41377/42377"
Feb 23 20:51:11 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:51:11.168-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:51:51 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:51:51.534-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:52:30 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:52:30.800-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:53:14 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:53:14.948-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:53:56 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:53:56.918-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:54:33 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:54:33.345-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:55:07 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:55:07.942-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:55:40 truenas k3s[6488]: time="2024-02-23T20:55:40-08:00" level=info msg="COMPACT compactRev=41377 targetCompactRev=41890 currentRev=42890"
Feb 23 20:55:40 truenas k3s[6488]: time="2024-02-23T20:55:40-08:00" level=info msg="COMPACT deleted 513 rows from 513 revisions in 7.826296ms - compacted to 41890/42890"
Feb 23 20:55:43 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:55:43.920-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:56:17 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:56:17.818-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:56:55 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:56:55.614-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:57:33 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:57:33.947-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:58:05 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:58:05.819-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:58:42 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:58:42.285-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:59:12 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:59:12.711-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 20:59:54 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T20:59:54.992-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:00:27 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:00:27.825-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:00:40 truenas k3s[6488]: time="2024-02-23T21:00:40-08:00" level=info msg="COMPACT compactRev=41890 targetCompactRev=42405 currentRev=43405"
Feb 23 21:00:40 truenas k3s[6488]: time="2024-02-23T21:00:40-08:00" level=info msg="COMPACT deleted 515 rows from 515 revisions in 7.966727ms - compacted to 42405/43405"
Feb 23 21:00:59 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:00:59.646-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:01:42 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:01:42.445-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:02:23 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:02:23.174-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:02:53 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:02:53.803-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:03:36 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:03:36.076-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:04:20 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:04:20.093-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:05:04 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:05:04.713-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:05:36 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:05:36.405-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:05:40 truenas k3s[6488]: time="2024-02-23T21:05:40-08:00" level=info msg="COMPACT compactRev=42405 targetCompactRev=42919 currentRev=43919"
Feb 23 21:05:40 truenas k3s[6488]: time="2024-02-23T21:05:40-08:00" level=info msg="COMPACT deleted 514 rows from 514 revisions in 8.177138ms - compacted to 42919/43919"
Feb 23 21:06:15 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:06:15.994-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:06:58 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:06:58.347-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:07:36 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:07:36.253-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:08:16 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:08:16.161-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:08:49 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:08:49.667-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:09:27 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:09:27.359-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:10:09 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:10:09.889-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:10:40 truenas k3s[6488]: time="2024-02-23T21:10:40-08:00" level=info msg="COMPACT compactRev=42919 targetCompactRev=43431 currentRev=44431"
Feb 23 21:10:40 truenas k3s[6488]: time="2024-02-23T21:10:40-08:00" level=info msg="COMPACT deleted 512 rows from 512 revisions in 7.040506ms - compacted to 43431/44431"
Feb 23 21:10:41 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:10:41.653-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:11:14 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:11:14.443-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:11:53 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:11:53.880-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:12:29 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:12:29.705-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:13:12 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:13:12.674-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:13:54 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:13:54.073-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:14:38 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:14:38.495-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:15:10 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:15:10.042-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:15:40 truenas k3s[6488]: time="2024-02-23T21:15:40-08:00" level=info msg="COMPACT compactRev=43431 targetCompactRev=43960 currentRev=44960"
Feb 23 21:15:40 truenas k3s[6488]: time="2024-02-23T21:15:40-08:00" level=info msg="COMPACT deleted 529 rows from 529 revisions in 8.942868ms - compacted to 43960/44960"
Feb 23 21:15:45 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:15:45.843-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:16:22 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:16:22.286-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:16:54 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:16:54.956-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:17:25 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:17:25.297-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:18:04 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:18:04.357-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:18:43 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:18:43.091-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:19:20 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:19:20.878-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:20:04 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:20:04.732-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:20:40 truenas k3s[6488]: time="2024-02-23T21:20:40-08:00" level=info msg="COMPACT compactRev=43960 targetCompactRev=44475 currentRev=45475"
Feb 23 21:20:40 truenas k3s[6488]: time="2024-02-23T21:20:40-08:00" level=info msg="COMPACT deleted 515 rows from 515 revisions in 9.658689ms - compacted to 44475/45475"
Feb 23 21:20:48 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:20:48.649-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:21:18 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:21:18.726-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:22:02 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:22:02.080-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:22:44 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:22:44.789-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:23:24 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:23:24.639-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:24:02 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:24:02.402-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:24:44 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:24:44.798-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:25:22 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:25:22.892-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:25:40 truenas k3s[6488]: time="2024-02-23T21:25:40-08:00" level=info msg="COMPACT compactRev=44475 targetCompactRev=44989 currentRev=45989"
Feb 23 21:25:40 truenas k3s[6488]: time="2024-02-23T21:25:40-08:00" level=info msg="COMPACT deleted 528 rows from 514 revisions in 9.44828ms - compacted to 44989/45989"
Feb 23 21:26:00 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:26:00.384-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:26:35 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:26:35.026-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:27:05 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:27:05.747-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:27:36 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:27:36.550-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:28:08 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:28:08.018-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Feb 23 21:28:50 truenas k3s[6488]: {"level":"warn","ts":"2024-02-23T21:28:50.310-0800","logger":"etcd-client","caller":"v3@v3.5.7-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000405dc0/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
