+--------------------------------------------------------------------------------+
+                      Kubernetes Configuration @1654690809                      +
+--------------------------------------------------------------------------------+
{
  "id": 1,
  "pool": "Master Pool",
  "cluster_cidr": "172.16.0.0/16",
  "service_cidr": "172.17.0.0/16",
  "cluster_dns_ip": "172.17.0.10",
  "route_v4_interface": null,
  "route_v4_gateway": "",
  "route_v6_interface": null,
  "route_v6_gateway": null,
  "node_ip": "0.0.0.0",
  "configure_gpus": true,
  "servicelb": true,
  "dataset": "Master Pool/ix-applications"
}
debug finished in 0 seconds for Kubernetes Configuration


+--------------------------------------------------------------------------------+
+                     k3s kubectl describe nodes @1654690809                     +
+--------------------------------------------------------------------------------+
Name:               ix-truenas
Roles:              control-plane,master
Labels:             beta.kubernetes.io/arch=amd64
                    beta.kubernetes.io/os=linux
                    kubernetes.io/arch=amd64
                    kubernetes.io/hostname=ix-truenas
                    kubernetes.io/os=linux
                    node-role.kubernetes.io/control-plane=true
                    node-role.kubernetes.io/master=true
                    openebs.io/nodeid=ix-truenas
                    openebs.io/nodename=ix-truenas
Annotations:        csi.volume.kubernetes.io/nodeid: {"zfs.csi.openebs.io":"ix-truenas"}
                    k3s.io/node-args:
                      ["server","--cluster-cidr","172.16.0.0/16","--cluster-dns","172.17.0.10","--data-dir","/mnt/Master Pool/ix-applications/k3s","--kube-apise...
                    k3s.io/node-config-hash: LLINWCOSCTDQXZT43KVHBDE6SFAO22XLWU5XZMPXBBCETS7G3F4A====
                    k3s.io/node-env:
                      {"K3S_DATA_DIR":"/mnt/Master Pool/ix-applications/k3s/data/11c442af7d27af92e276ebd499b6ebab41fa574b9c6fafd965fe38cffa10e51d"}
                    node.alpha.kubernetes.io/ttl: 0
                    volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp:  Fri, 27 May 2022 06:36:13 -0700
Taints:             <none>
Unschedulable:      false
Lease:
  HolderIdentity:  ix-truenas
  AcquireTime:     <unset>
  RenewTime:       Wed, 08 Jun 2022 05:20:02 -0700
Conditions:
  Type             Status  LastHeartbeatTime                 LastTransitionTime                Reason                       Message
  ----             ------  -----------------                 ------------------                ------                       -------
  MemoryPressure   False   Wed, 08 Jun 2022 05:15:22 -0700   Fri, 27 May 2022 06:36:12 -0700   KubeletHasSufficientMemory   kubelet has sufficient memory available
  DiskPressure     False   Wed, 08 Jun 2022 05:15:22 -0700   Fri, 27 May 2022 06:36:12 -0700   KubeletHasNoDiskPressure     kubelet has no disk pressure
  PIDPressure      False   Wed, 08 Jun 2022 05:15:22 -0700   Fri, 27 May 2022 06:36:12 -0700   KubeletHasSufficientPID      kubelet has sufficient PID available
  Ready            True    Wed, 08 Jun 2022 05:15:22 -0700   Mon, 30 May 2022 08:59:35 -0700   KubeletReady                 kubelet is posting ready status. AppArmor enabled
Addresses:
  InternalIP:  10.12.1.5
  Hostname:    ix-truenas
Capacity:
  cpu:                 4
  ephemeral-storage:   1885715968Ki
  gpu.intel.com/i915:  1
  hugepages-1Gi:       0
  hugepages-2Mi:       0
  memory:              16258556Ki
  pods:                110
Allocatable:
  cpu:                 4
  ephemeral-storage:   1834424492232
  gpu.intel.com/i915:  1
  hugepages-1Gi:       0
  hugepages-2Mi:       0
  memory:              16258556Ki
  pods:                110
System Info:
  Machine ID:                 43f98e8a204e4d95862dc4ae8335d835
  System UUID:                94fde4c0-b05d-11e5-952e-b06ebf856766
  Boot ID:                    bf79b7fc-940b-4651-93fe-18dbc3aa2a32
  Kernel Version:             5.10.109+truenas
  OS Image:                   Debian GNU/Linux 11 (bullseye)
  Operating System:           linux
  Architecture:               amd64
  Container Runtime Version:  docker://Unknown
  Kubelet Version:            v1.23.5+k3s-937d546a-dirty
  Kube-Proxy Version:         v1.23.5+k3s-937d546a-dirty
PodCIDR:                      172.16.0.0/16
PodCIDRs:                     172.16.0.0/16
Non-terminated Pods:          (7 in total)
  Namespace                   Name                                                      CPU Requests  CPU Limits  Memory Requests  Memory Limits  Age
  ---------                   ----                                                      ------------  ----------  ---------------  -------------  ---
  kube-system                 openebs-zfs-controller-0                                  0 (0%)        0 (0%)      0 (0%)           0 (0%)         11d
  kube-system                 coredns-d76bd69b-zh8xt                                    100m (2%)     0 (0%)      70Mi (0%)        170Mi (1%)     11d
  kube-system                 intel-gpu-plugin-mksln                                    0 (0%)        0 (0%)      0 (0%)           0 (0%)         11d
  kube-system                 openebs-zfs-node-fglmx                                    0 (0%)        0 (0%)      0 (0%)           0 (0%)         11d
  ix-media-server             media-server-emby-74d544f4c-dmp2j                         0 (0%)        0 (0%)      0 (0%)           0 (0%)         8d
  ix-hass-instance            hass-instance-home-assistant-postgres-86c496c6b8-rm8tc    0 (0%)        0 (0%)      0 (0%)           0 (0%)         26m
  ix-hass-instance            hass-instance-home-assistant-7ff5685587-jhjd7             0 (0%)        0 (0%)      0 (0%)           0 (0%)         26m
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource            Requests   Limits
  --------            --------   ------
  cpu                 100m (2%)  0 (0%)
  memory              70Mi (0%)  170Mi (1%)
  ephemeral-storage   0 (0%)     0 (0%)
  hugepages-1Gi       0 (0%)     0 (0%)
  hugepages-2Mi       0 (0%)     0 (0%)
  gpu.intel.com/i915  0          0
Events:               <none>
debug finished in 2 seconds for k3s kubectl describe nodes


+--------------------------------------------------------------------------------+
+k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide @1654690811+
+--------------------------------------------------------------------------------+
NAMESPACE          NAME                                                         READY   STATUS     RESTARTS      AGE   IP            NODE         NOMINATED NODE   READINESS GATES
kube-system        pod/openebs-zfs-controller-0                                 5/5     Running    25 (8d ago)   11d   172.16.0.45   ix-truenas   <none>           <none>
kube-system        pod/coredns-d76bd69b-zh8xt                                   1/1     Running    5 (8d ago)    11d   172.16.0.43   ix-truenas   <none>           <none>
kube-system        pod/intel-gpu-plugin-mksln                                   1/1     Running    5 (8d ago)    11d   172.16.0.42   ix-truenas   <none>           <none>
kube-system        pod/openebs-zfs-node-fglmx                                   2/2     Running    11 (8d ago)   11d   10.12.1.5     ix-truenas   <none>           <none>
ix-media-server    pod/media-server-emby-74d544f4c-dmp2j                        1/1     Running    0             8d    172.16.0.46   ix-truenas   <none>           <none>
ix-hass-instance   pod/hass-instance-home-assistant-postgres-86c496c6b8-rm8tc   1/1     Running    0             26m   10.12.1.5     ix-truenas   <none>           <none>
ix-hass-instance   pod/hass-instance-home-assistant-7ff5685587-jhjd7            0/1     Init:0/2   0             26m   10.12.1.5     ix-truenas   <none>           <none>

NAMESPACE          NAME                                            TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                    AGE   SELECTOR
default            service/kubernetes                              ClusterIP   172.17.0.1       <none>        443/TCP                                    11d   <none>
kube-system        service/kube-dns                                ClusterIP   172.17.0.10      <none>        53/UDP,53/TCP,9153/TCP                     11d   k8s-app=kube-dns
ix-media-server    service/media-server-emby-udp                   ClusterIP   172.17.42.35     <none>        1900/UDP,7359/UDP                          8d    app.kubernetes.io/instance=media-server,app.kubernetes.io/name=emby
ix-media-server    service/media-server-emby-tcp                   NodePort    172.17.142.252   <none>        9096:9096/TCP,80:24529/TCP,443:37725/TCP   8d    app.kubernetes.io/instance=media-server,app.kubernetes.io/name=emby
ix-hass-instance   service/hass-instance-home-assistant-postgres   ClusterIP   172.17.61.151    <none>        5432/TCP                                   26m   app.kubernetes.io/instance=hass-instance-postgres,app.kubernetes.io/name=home-assistant-postgres,app=home-assistant,release=hass-instance
ix-hass-instance   service/hass-instance-home-assistant            NodePort    172.17.253.246   <none>        20810:20810/TCP                            26m   app.kubernetes.io/instance=hass-instance,app.kubernetes.io/name=home-assistant,app=home-assistant,release=hass-instance

NAMESPACE     NAME                              DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR              AGE   CONTAINERS                                     IMAGES                                                                             SELECTOR
kube-system   daemonset.apps/intel-gpu-plugin   1         1         1       1            1           kubernetes.io/arch=amd64   11d   intel-gpu-plugin                               intel/intel-gpu-plugin:0.19.0                                                      app=intel-gpu-plugin
kube-system   daemonset.apps/openebs-zfs-node   1         1         1       1            1           <none>                     11d   csi-node-driver-registrar,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0,openebs/zfs-driver:2.0.0   app=openebs-zfs-node

NAMESPACE          NAME                                                    READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS                IMAGES                                   SELECTOR
kube-system        deployment.apps/coredns                                 1/1     1            1           11d   coredns                   rancher/mirrored-coredns-coredns:1.9.1   k8s-app=kube-dns
ix-media-server    deployment.apps/media-server-emby                       1/1     1            1           8d    emby                      emby/embyserver:4.8.0.0                  app.kubernetes.io/instance=media-server,app.kubernetes.io/name=emby
ix-hass-instance   deployment.apps/hass-instance-home-assistant-postgres   1/1     1            1           26m   home-assistant-postgres   postgres:13.1                            app=home-assistant,app.kubernetes.io/instance=hass-instance-postgres,app.kubernetes.io/name=home-assistant-postgres,release=hass-instance
ix-hass-instance   deployment.apps/hass-instance-home-assistant            0/1     1            0           26m   home-assistant            homeassistant/home-assistant:2022.6.3    app=home-assistant,release=hass-instance

NAMESPACE     NAME                                      READY   AGE   CONTAINERS                                                                           IMAGES
kube-system   statefulset.apps/openebs-zfs-controller   1/1     11d   csi-resizer,csi-snapshotter,snapshot-controller,csi-provisioner,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-resizer:v1.2.0,k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0,k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0,k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0,openebs/zfs-driver:2.0.0

NAMESPACE   NAME                                                          PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
            storageclass.storage.k8s.io/openebs-zfspv-default (default)   zfs.csi.openebs.io   Delete          Immediate           true                   11d
            storageclass.storage.k8s.io/ix-storage-class-media-server     zfs.csi.openebs.io   Retain          Immediate           true                   8d
            storageclass.storage.k8s.io/ix-storage-class-hass-instance    zfs.csi.openebs.io   Retain          Immediate           true                   26m

NAMESPACE   NAME                         STATUS   AGE
            namespace/kube-system        Active   11d
            namespace/default            Active   11d
            namespace/kube-public        Active   11d
            namespace/kube-node-lease    Active   11d
            namespace/openebs            Active   11d
            namespace/ix-media-server    Active   8d
            namespace/ix-hass-instance   Active   26m
debug finished in 0 seconds for k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide


+--------------------------------------------------------------------------------+
+         k3s kubectl describe deployments --all-namespaces @1654690811          +
+--------------------------------------------------------------------------------+
Name:                   coredns
Namespace:              kube-system
CreationTimestamp:      Fri, 27 May 2022 06:36:14 -0700
Labels:                 k8s-app=kube-dns
                        kubernetes.io/name=CoreDNS
                        objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:            deployment.kubernetes.io/revision: 1
                        objectset.rio.cattle.io/applied:
                          H4sIAAAAAAAA/6xVUW/bNhD+K8M9S7GVtI0rYA+dna1FV8+ok74URkFTJ4szxePIkxMj0H8fTrIdu02TdtiTZfK743cfvzveg/LmE4ZoyEEOyvs42GSQwNq4AnKYoLe0rdExJFAjq0...
                        objectset.rio.cattle.io/id: 
                        objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                        objectset.rio.cattle.io/owner-name: coredns
                        objectset.rio.cattle.io/owner-namespace: kube-system
Selector:               k8s-app=kube-dns
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  1 max unavailable, 25% max surge
Pod Template:
  Labels:           k8s-app=kube-dns
  Service Account:  coredns
  Containers:
   coredns:
    Image:       rancher/mirrored-coredns-coredns:1.9.1
    Ports:       53/UDP, 53/TCP, 9153/TCP
    Host Ports:  0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
  Volumes:
   config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
   custom-config-volume:
    Type:               ConfigMap (a volume populated by a ConfigMap)
    Name:               coredns-custom
    Optional:           true
  Priority Class Name:  system-cluster-critical
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   coredns-d76bd69b (1/1 replicas created)
Events:          <none>


Name:               media-server-emby
Namespace:          ix-media-server
CreationTimestamp:  Mon, 30 May 2022 06:28:11 -0700
Labels:             app.kubernetes.io/instance=media-server
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=emby
Annotations:        deployment.kubernetes.io/revision: 4
                    meta.helm.sh/release-name: media-server
                    meta.helm.sh/release-namespace: ix-media-server
Selector:           app.kubernetes.io/instance=media-server,app.kubernetes.io/name=emby
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:       app.kubernetes.io/instance=media-server
                app.kubernetes.io/name=emby
  Annotations:  rollme: bX6rj
  Containers:
   emby:
    Image:       emby/embyserver:4.8.0.0
    Ports:       8096/TCP, 1900/UDP, 7359/UDP
    Host Ports:  0/TCP, 0/UDP, 0/UDP
    Limits:
      gpu.intel.com/i915:  0
    Liveness:              http-get http://:8096/emby/System/Ping delay=0s timeout=1s period=15s #success=1 #failure=5
    Readiness:             http-get http://:8096/emby/System/Ping delay=0s timeout=1s period=15s #success=1 #failure=5
    Startup:               http-get http://:8096/emby/System/Ping delay=5s timeout=1s period=15s #success=1 #failure=40
    Environment:
      KUBE_NAMESPACE:   (v1:metadata.namespace)
      POD_IP:           (v1:status.podIP)
    Mounts:
      /config from config (rw)
  Volumes:
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/media-server
    HostPathType:  
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   media-server-emby-74d544f4c (1/1 replicas created)
Events:          <none>


Name:               hass-instance-home-assistant-postgres
Namespace:          ix-hass-instance
CreationTimestamp:  Wed, 08 Jun 2022 04:54:05 -0700
Labels:             app=home-assistant
                    app.kubernetes.io/instance=hass-instance-postgres
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=home-assistant-postgres
                    release=hass-instance
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: hass-instance
                    meta.helm.sh/release-namespace: ix-hass-instance
Selector:           app=home-assistant,app.kubernetes.io/instance=hass-instance-postgres,app.kubernetes.io/name=home-assistant-postgres,release=hass-instance
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:       app=home-assistant
                app.kubernetes.io/instance=hass-instance-postgres
                app.kubernetes.io/name=home-assistant-postgres
                release=hass-instance
  Annotations:  rollme: H1c2R
  Containers:
   home-assistant-postgres:
    Image:      postgres:13.1
    Port:       5432/TCP
    Host Port:  5432/TCP
    Environment:
      POSTGRES_DB:        homeassistance
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:
      /postgres_backups from postgres-backup (rw)
      /var/lib/postgresql/data from postgres-data (rw)
  Volumes:
   postgres-backup:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/ix-applications/releases/hass-instance/volumes/ix_volumes/ix-postgres_backups
    HostPathType:  
   postgres-data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/ix-applications/releases/hass-instance/volumes/ix_volumes/ix-postgres_data
    HostPathType:  
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   hass-instance-home-assistant-postgres-86c496c6b8 (1/1 replicas created)
Events:
  Type    Reason             Age   From                   Message
  ----    ------             ----  ----                   -------
  Normal  ScalingReplicaSet  26m   deployment-controller  Scaled up replica set hass-instance-home-assistant-postgres-86c496c6b8 to 1


Name:               hass-instance-home-assistant
Namespace:          ix-hass-instance
CreationTimestamp:  Wed, 08 Jun 2022 04:54:05 -0700
Labels:             app=home-assistant
                    app.kubernetes.io/managed-by=Helm
                    chart=home-assistant-1.0.5
                    heritage=Helm
                    release=hass-instance
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: hass-instance
                    meta.helm.sh/release-namespace: ix-hass-instance
                    rollme: h0Id2
Selector:           app=home-assistant,release=hass-instance
Replicas:           1 desired | 1 updated | 1 total | 0 available | 1 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:       app=home-assistant
                app.kubernetes.io/instance=hass-instance
                app.kubernetes.io/name=home-assistant
                release=hass-instance
  Annotations:  rollme: SD1jv
  Init Containers:
   init-postgresdb:
    Image:      postgres:13.1
    Port:       <none>
    Host Port:  <none>
    Command:
      sh
      -c
      until pg_isready -h hass-instance-home-assistant-postgres; do echo waiting for postgres; sleep 2; done
    Environment:  <none>
    Mounts:       <none>
   init-configs:
    Image:      alpine:latest
    Port:       <none>
    Host Port:  <none>
    Command:
      sh
      /config/init/init.sh
    Environment:
      POSTGRES_HOST:      hass-instance-home-assistant-postgres:5432
      POSTGRES_DB:        homeassistance
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:
      /config from config (rw)
      /config/init from initial-config-script (rw)
      /media from media (rw)
  Containers:
   home-assistant:
    Image:      homeassistant/home-assistant:2022.6.3
    Port:       8123/TCP
    Host Port:  8123/TCP
    Environment:
      PUID:  568
      PGID:  568
      TZ:    Europe/Budapest
    Mounts:
      /config from config (rw)
      /media from media (rw)
  Volumes:
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/hass-data/configuration
    HostPathType:  
   media:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/hass-data/media
    HostPathType:  
   initial-config-script:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      home-assistance-initial-script-configmap
    Optional:  false
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      False   MinimumReplicasUnavailable
  Progressing    False   ProgressDeadlineExceeded
OldReplicaSets:  <none>
NewReplicaSet:   hass-instance-home-assistant-7ff5685587 (1/1 replicas created)
Events:
  Type    Reason             Age   From                   Message
  ----    ------             ----  ----                   -------
  Normal  ScalingReplicaSet  26m   deployment-controller  Scaled up replica set hass-instance-home-assistant-7ff5685587 to 1
debug finished in 0 seconds for k3s kubectl describe deployments --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe pods --all-namespaces @1654690811             +
+--------------------------------------------------------------------------------+
Name:                 openebs-zfs-controller-0
Namespace:            kube-system
Priority:             900000000
Priority Class Name:  openebs-zfs-csi-controller-critical
Node:                 ix-truenas/10.12.1.5
Start Time:           Fri, 27 May 2022 06:42:53 -0700
Labels:               app=openebs-zfs-controller
                      controller-revision-hash=openebs-zfs-controller-698698d48b
                      openebs.io/component-name=openebs-zfs-controller
                      openebs.io/version=ci
                      role=openebs-zfs
                      statefulset.kubernetes.io/pod-name=openebs-zfs-controller-0
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.45"
                            ],
                            "mac": "0a:65:f5:54:bb:e3",
                            "default": true,
                            "dns": {}
                        }]
                      k8s.v1.cni.cncf.io/networks-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.45"
                            ],
                            "mac": "0a:65:f5:54:bb:e3",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.0.45
IPs:
  IP:           172.16.0.45
Controlled By:  StatefulSet/openebs-zfs-controller
Containers:
  csi-resizer:
    Container ID:  docker://f6990a1cdb33581a91704504b1e3eae470753e76fc08de38634a0086e9f5f0dd
    Image:         k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Mon, 30 May 2022 08:59:32 -0700
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 30 May 2022 05:59:00 -0700
      Finished:     Mon, 30 May 2022 08:18:10 -0700
    Ready:          True
    Restart Count:  5
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-m676c (ro)
  csi-snapshotter:
    Container ID:  docker://056606e12432341bd8c0eba2209073391c437c4957978824578e84da72b19d4f
    Image:         k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-snapshotter@sha256:51f2dfde5bccac7854b3704689506aeecfb793328427b91115ba253a93e60782
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Mon, 30 May 2022 08:59:33 -0700
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 30 May 2022 05:59:00 -0700
      Finished:     Mon, 30 May 2022 08:18:10 -0700
    Ready:          True
    Restart Count:  5
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-m676c (ro)
  snapshot-controller:
    Container ID:  docker://37bd75993d31bd445f51f98b052436d2114c064e21c66204fb62df47c53c3b6d
    Image:         k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/snapshot-controller@sha256:00fcc441ea9f72899c25eed61d602272a2a58c5f0014332bdcb5ac24acef08e4
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --leader-election=true
    State:          Running
      Started:      Mon, 30 May 2022 08:59:33 -0700
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 30 May 2022 05:59:01 -0700
      Finished:     Mon, 30 May 2022 08:18:10 -0700
    Ready:          True
    Restart Count:  5
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-m676c (ro)
  csi-provisioner:
    Container ID:  docker://762b7ce84a9aad6c31c19e367b094459e229b5df823f45fde130c9f91b92908c
    Image:         k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-provisioner@sha256:6477988532358148d2e98f7c747db4e9250bbc7ad2664bf666348abf9ee1f5aa
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    State:          Running
      Started:      Mon, 30 May 2022 08:59:33 -0700
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 30 May 2022 05:59:01 -0700
      Finished:     Mon, 30 May 2022 08:18:10 -0700
    Ready:          True
    Restart Count:  5
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:  kube-system (v1:metadata.namespace)
      POD_NAME:   openebs-zfs-controller-0 (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-m676c (ro)
  openebs-zfs-plugin:
    Container ID:  docker://42f7678d36f2667b75ed1719a96563f6ec929f814007aa2e593781ae07bc7ee2
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    State:          Running
      Started:      Mon, 30 May 2022 08:59:34 -0700
    Last State:     Terminated
      Reason:       Error
      Exit Code:    137
      Started:      Mon, 30 May 2022 05:59:01 -0700
      Finished:     Mon, 30 May 2022 08:18:20 -0700
    Ready:          True
    Restart Count:  5
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-m676c (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-m676c:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:                 coredns-d76bd69b-zh8xt
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Node:                 ix-truenas/10.12.1.5
Start Time:           Fri, 27 May 2022 06:42:53 -0700
Labels:               k8s-app=kube-dns
                      pod-template-hash=d76bd69b
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.43"
                            ],
                            "mac": "72:f9:89:23:c0:71",
                            "default": true,
                            "dns": {}
                        }]
                      k8s.v1.cni.cncf.io/networks-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.0.43"
                            ],
                            "mac": "72:f9:89:23:c0:71",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.0.43
IPs:
  IP:           172.16.0.43
Controlled By:  ReplicaSet/coredns-d76bd69b
Containers:
  coredns:
    Container ID:  docker://7c6bf7105f614a5bf7116108365b54511f8ca454a80c9cf1e00926e951edc4b7
    Image:         rancher/mirrored-coredns-coredns:1.9.1
    Image ID:      docker://sha256:99376d8f35e0abb6ff9d66b50a7c81df6e6dfdb649becc5df84a691a7b4beca4
    Ports:         53/UDP, 53/TCP, 9153/TCP
    Host Ports:    0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    State:          Running
      Started:      Mon, 30 May 2022 08:59:32 -0700
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 30 May 2022 05:59:01 -0700
      Finished:     Mon, 30 May 2022 08:18:10 -0700
    Ready:          True
    Restart Count:  5
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-cxgnf (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
  custom-config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns-custom
    Optional:  true
  kube-api-access-cxgnf:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              beta.kubernetes.io/os=linux
Tolerations:                 CriticalAddonsOnly op=Exists
                             node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                             node-role.kubernetes.io/master:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason            Age                   From     Message
  ----     ------            ----                  ----     -------
  Warning  DNSConfigForming  42s (x10114 over 8d)  kubelet  Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8


Name:         intel-gpu-plugin-mksln
Namespace:    kube-system
Priority:     0
Node:         ix-truenas/10.12.1.5
Start Time:   Fri, 27 May 2022 06:42:58 -0700
Labels:       app=intel-gpu-plugin
              controller-revision-hash=759bc79459
              pod-template-generation=1
Annotations:  k8s.v1.cni.cncf.io/network-status:
                [{
                    "name": "ix-net",
                    "interface": "eth0",
                    "ips": [
                        "172.16.0.42"
                    ],
                    "mac": "12:ae:96:08:e7:14",
                    "default": true,
                    "dns": {}
                }]
              k8s.v1.cni.cncf.io/networks-status:
                [{
                    "name": "ix-net",
                    "interface": "eth0",
                    "ips": [
                        "172.16.0.42"
                    ],
                    "mac": "12:ae:96:08:e7:14",
                    "default": true,
                    "dns": {}
                }]
Status:       Running
IP:           172.16.0.42
IPs:
  IP:           172.16.0.42
Controlled By:  DaemonSet/intel-gpu-plugin
Init Containers:
  intel-gpu-initcontainer:
    Container ID:   docker://ea7fc1981d30fc79101109d1a07aa26a362bbe20d9e3e35c5142fdaa606afafa
    Image:          intel/intel-gpu-initcontainer:0.19.0
    Image ID:       docker-pullable://intel/intel-gpu-initcontainer@sha256:5f05ec6ff157841189ce7e37fd56708a0e11b4018cc94a003c31724281535fe7
    Port:           <none>
    Host Port:      <none>
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 30 May 2022 08:59:32 -0700
      Finished:     Mon, 30 May 2022 08:59:32 -0700
    Ready:          True
    Restart Count:  5
    Environment:    <none>
    Mounts:
      /etc/kubernetes/node-feature-discovery/source.d/ from nfd-source-hooks (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-55s58 (ro)
Containers:
  intel-gpu-plugin:
    Container ID:   docker://70f5907d9cb2c5f4dab2dc633be955955c709786e17eb961b5df6e2602315aa0
    Image:          intel/intel-gpu-plugin:0.19.0
    Image ID:       docker-pullable://intel/intel-gpu-plugin@sha256:eaedfe5991710e40ce3bbb530b23f47401a2b60a813ba39439d2fea999aca51c
    Port:           <none>
    Host Port:      <none>
    State:          Running
      Started:      Mon, 30 May 2022 08:59:33 -0700
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Mon, 30 May 2022 05:59:01 -0700
      Finished:     Mon, 30 May 2022 08:18:10 -0700
    Ready:          True
    Restart Count:  5
    Environment:
      NODE_NAME:   (v1:spec.nodeName)
    Mounts:
      /dev/dri from devfs (ro)
      /sys/class/drm from sysfs (ro)
      /var/lib/kubelet/device-plugins from kubeletsockets (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-55s58 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  devfs:
    Type:          HostPath (bare host directory volume)
    Path:          /dev/dri
    HostPathType:  
  sysfs:
    Type:          HostPath (bare host directory volume)
    Path:          /sys/class/drm
    HostPathType:  
  kubeletsockets:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/device-plugins
    HostPathType:  
  nfd-source-hooks:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/kubernetes/node-feature-discovery/source.d/
    HostPathType:  DirectoryOrCreate
  kube-api-access-55s58:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/arch=amd64
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:                      <none>


Name:                 openebs-zfs-node-fglmx
Namespace:            kube-system
Priority:             900001000
Priority Class Name:  openebs-zfs-csi-node-critical
Node:                 ix-truenas/10.12.1.5
Start Time:           Fri, 27 May 2022 06:42:58 -0700
Labels:               app=openebs-zfs-node
                      controller-revision-hash=57f5455f6b
                      openebs.io/component-name=openebs-zfs-node
                      openebs.io/version=ci
                      pod-template-generation=1
                      role=openebs-zfs
Annotations:          <none>
Status:               Running
IP:                   10.12.1.5
IPs:
  IP:           10.12.1.5
Controlled By:  DaemonSet/openebs-zfs-node
Containers:
  csi-node-driver-registrar:
    Container ID:  docker://785a08f6a58706e5434238209296b4b5217a5c267728ad9bb2d270f6dc3da99e
    Image:         k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:f9bcee63734b7b01555ee8fc8fb01ac2922478b2c8934bf8d468dd2916edc405
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    State:          Running
      Started:      Mon, 30 May 2022 08:59:41 -0700
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 30 May 2022 05:58:56 -0700
      Finished:     Mon, 30 May 2022 08:18:10 -0700
    Ready:          True
    Restart Count:  5
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-sttnr (ro)
  openebs-zfs-plugin:
    Container ID:  docker://138273fd582345f04307abfc64722e3387ea5cb689463a5f6a80d936502373a1
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    State:          Running
      Started:      Mon, 30 May 2022 08:59:41 -0700
    Last State:     Terminated
      Reason:       Error
      Exit Code:    137
      Started:      Mon, 30 May 2022 05:59:12 -0700
      Finished:     Mon, 30 May 2022 08:18:20 -0700
    Ready:          True
    Restart Count:  6
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-sttnr (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
  encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
  chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
  host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
  plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/
    HostPathType:  Directory
  kube-api-access-sttnr:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type     Reason            Age                     From     Message
  ----     ------            ----                    ----     -------
  Warning  DNSConfigForming  5m30s (x10119 over 8d)  kubelet  Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8


Name:         media-server-emby-74d544f4c-dmp2j
Namespace:    ix-media-server
Priority:     0
Node:         ix-truenas/10.12.1.5
Start Time:   Mon, 30 May 2022 10:04:34 -0700
Labels:       app.kubernetes.io/instance=media-server
              app.kubernetes.io/name=emby
              pod-template-hash=74d544f4c
Annotations:  k8s.v1.cni.cncf.io/network-status:
                [{
                    "name": "ix-net",
                    "interface": "eth0",
                    "ips": [
                        "172.16.0.46"
                    ],
                    "mac": "9a:df:61:ee:07:64",
                    "default": true,
                    "dns": {}
                }]
              k8s.v1.cni.cncf.io/networks-status:
                [{
                    "name": "ix-net",
                    "interface": "eth0",
                    "ips": [
                        "172.16.0.46"
                    ],
                    "mac": "9a:df:61:ee:07:64",
                    "default": true,
                    "dns": {}
                }]
              rollme: bX6rj
Status:       Running
IP:           172.16.0.46
IPs:
  IP:           172.16.0.46
Controlled By:  ReplicaSet/media-server-emby-74d544f4c
Containers:
  emby:
    Container ID:   docker://25710ca90c48f65725d3222318a609e5db622d0fb009184ffe734bf0f2ca1728
    Image:          emby/embyserver:4.8.0.0
    Image ID:       docker-pullable://emby/embyserver@sha256:30cf1df04864cec610a58e1e61c0bb43192041b2f06d12a4d0e2523a346614dc
    Ports:          8096/TCP, 1900/UDP, 7359/UDP
    Host Ports:     0/TCP, 0/UDP, 0/UDP
    State:          Running
      Started:      Mon, 30 May 2022 10:04:35 -0700
    Ready:          True
    Restart Count:  0
    Limits:
      gpu.intel.com/i915:  0
    Requests:
      gpu.intel.com/i915:  0
    Liveness:              http-get http://:8096/emby/System/Ping delay=0s timeout=1s period=15s #success=1 #failure=5
    Readiness:             http-get http://:8096/emby/System/Ping delay=0s timeout=1s period=15s #success=1 #failure=5
    Startup:               http-get http://:8096/emby/System/Ping delay=5s timeout=1s period=15s #success=1 #failure=40
    Environment:
      KUBE_NAMESPACE:  ix-media-server (v1:metadata.namespace)
      POD_IP:           (v1:status.podIP)
    Mounts:
      /config from config (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-j4gvj (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/media-server
    HostPathType:  
  kube-api-access-j4gvj:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:         hass-instance-home-assistant-postgres-86c496c6b8-rm8tc
Namespace:    ix-hass-instance
Priority:     0
Node:         ix-truenas/10.12.1.5
Start Time:   Wed, 08 Jun 2022 04:54:05 -0700
Labels:       app=home-assistant
              app.kubernetes.io/instance=hass-instance-postgres
              app.kubernetes.io/name=home-assistant-postgres
              pod-template-hash=86c496c6b8
              release=hass-instance
Annotations:  rollme: H1c2R
Status:       Running
IP:           10.12.1.5
IPs:
  IP:           10.12.1.5
Controlled By:  ReplicaSet/hass-instance-home-assistant-postgres-86c496c6b8
Containers:
  home-assistant-postgres:
    Container ID:   docker://b69b02084c242199fe9c10ffd538eb177f12747631f82b1230899404dca494ef
    Image:          postgres:13.1
    Image ID:       docker-pullable://postgres@sha256:26180996f7d804993f9c90398615d1269709a799b5eb79109292843a1c7cd851
    Port:           5432/TCP
    Host Port:      5432/TCP
    State:          Running
      Started:      Wed, 08 Jun 2022 04:54:07 -0700
    Ready:          True
    Restart Count:  0
    Environment:
      POSTGRES_DB:        homeassistance
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:
      /postgres_backups from postgres-backup (rw)
      /var/lib/postgresql/data from postgres-data (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-66nhg (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  postgres-backup:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/ix-applications/releases/hass-instance/volumes/ix_volumes/ix-postgres_backups
    HostPathType:  
  postgres-data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/ix-applications/releases/hass-instance/volumes/ix_volumes/ix-postgres_data
    HostPathType:  
  kube-api-access-66nhg:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason            Age                 From               Message
  ----     ------            ----                ----               -------
  Normal   Scheduled         26m                 default-scheduler  Successfully assigned ix-hass-instance/hass-instance-home-assistant-postgres-86c496c6b8-rm8tc to ix-truenas
  Normal   Pulled            26m                 kubelet            Container image "postgres:13.1" already present on machine
  Normal   Created           26m                 kubelet            Created container home-assistant-postgres
  Normal   Started           26m                 kubelet            Started container home-assistant-postgres
  Warning  DNSConfigForming  32s (x25 over 26m)  kubelet            Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8


Name:         hass-instance-home-assistant-7ff5685587-jhjd7
Namespace:    ix-hass-instance
Priority:     0
Node:         ix-truenas/10.12.1.5
Start Time:   Wed, 08 Jun 2022 04:54:05 -0700
Labels:       app=home-assistant
              app.kubernetes.io/instance=hass-instance
              app.kubernetes.io/name=home-assistant
              pod-template-hash=7ff5685587
              release=hass-instance
Annotations:  rollme: SD1jv
Status:       Pending
IP:           10.12.1.5
IPs:
  IP:           10.12.1.5
Controlled By:  ReplicaSet/hass-instance-home-assistant-7ff5685587
Init Containers:
  init-postgresdb:
    Container ID:  docker://a519013b4176a3e083a478053a54c685c4bfaae064a82187130fcc11f4554b74
    Image:         postgres:13.1
    Image ID:      docker-pullable://postgres@sha256:26180996f7d804993f9c90398615d1269709a799b5eb79109292843a1c7cd851
    Port:          <none>
    Host Port:     <none>
    Command:
      sh
      -c
      until pg_isready -h hass-instance-home-assistant-postgres; do echo waiting for postgres; sleep 2; done
    State:          Running
      Started:      Wed, 08 Jun 2022 04:54:07 -0700
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-727g2 (ro)
  init-configs:
    Container ID:  
    Image:         alpine:latest
    Image ID:      
    Port:          <none>
    Host Port:     <none>
    Command:
      sh
      /config/init/init.sh
    State:          Waiting
      Reason:       PodInitializing
    Ready:          False
    Restart Count:  0
    Environment:
      POSTGRES_HOST:      hass-instance-home-assistant-postgres:5432
      POSTGRES_DB:        homeassistance
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:
      /config from config (rw)
      /config/init from initial-config-script (rw)
      /media from media (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-727g2 (ro)
Containers:
  home-assistant:
    Container ID:   
    Image:          homeassistant/home-assistant:2022.6.3
    Image ID:       
    Port:           8123/TCP
    Host Port:      8123/TCP
    State:          Waiting
      Reason:       PodInitializing
    Ready:          False
    Restart Count:  0
    Environment:
      PUID:  568
      PGID:  568
      TZ:    Europe/Budapest
    Mounts:
      /config from config (rw)
      /media from media (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-727g2 (ro)
Conditions:
  Type              Status
  Initialized       False 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/hass-data/configuration
    HostPathType:  
  media:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/Master Pool/hass-data/media
    HostPathType:  
  initial-config-script:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      home-assistance-initial-script-configmap
    Optional:  false
  kube-api-access-727g2:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason            Age                 From               Message
  ----     ------            ----                ----               -------
  Normal   Scheduled         26m                 default-scheduler  Successfully assigned ix-hass-instance/hass-instance-home-assistant-7ff5685587-jhjd7 to ix-truenas
  Normal   Pulled            26m                 kubelet            Container image "postgres:13.1" already present on machine
  Normal   Created           26m                 kubelet            Created container init-postgresdb
  Normal   Started           26m                 kubelet            Started container init-postgresdb
  Warning  DNSConfigForming  64s (x23 over 26m)  kubelet            Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8
debug finished in 1 seconds for k3s kubectl describe pods --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe services --all-namespaces @1654690812           +
+--------------------------------------------------------------------------------+
Name:              kubernetes
Namespace:         default
Labels:            component=apiserver
                   provider=kubernetes
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.1
IPs:               172.17.0.1
Port:              https  443/TCP
TargetPort:        6443/TCP
Endpoints:         10.12.1.5:6443
Session Affinity:  None
Events:            <none>


Name:              kube-dns
Namespace:         kube-system
Labels:            k8s-app=kube-dns
                   kubernetes.io/cluster-service=true
                   kubernetes.io/name=CoreDNS
                   objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:       objectset.rio.cattle.io/applied:
                     H4sIAAAAAAAA/4yRwW7UMBCGXwXN2Qlxstukljig9oKQUKUCF8TBcWZZk8S2PJMgtMq7I++mYqEK9JZoPn/6558T6GA/YyTrHSiYJQjoretAwSPG2RoEASOy7jRrUCfQznnWbL2j9O...
                   objectset.rio.cattle.io/id: 
                   objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                   objectset.rio.cattle.io/owner-name: coredns
                   objectset.rio.cattle.io/owner-namespace: kube-system
                   prometheus.io/port: 9153
                   prometheus.io/scrape: true
Selector:          k8s-app=kube-dns
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.10
IPs:               172.17.0.10
Port:              dns  53/UDP
TargetPort:        53/UDP
Endpoints:         172.16.0.43:53
Port:              dns-tcp  53/TCP
TargetPort:        53/TCP
Endpoints:         172.16.0.43:53
Port:              metrics  9153/TCP
TargetPort:        9153/TCP
Endpoints:         172.16.0.43:9153
Session Affinity:  None
Events:            <none>


Name:              media-server-emby-udp
Namespace:         ix-media-server
Labels:            app.kubernetes.io/instance=media-server
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=emby
                   app.kubernetes.io/version=4.8.0.0
                   helm.sh/chart=emby-1.0.3
Annotations:       meta.helm.sh/release-name: media-server
                   meta.helm.sh/release-namespace: ix-media-server
Selector:          app.kubernetes.io/instance=media-server,app.kubernetes.io/name=emby
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.42.35
IPs:               172.17.42.35
Port:              emby-dlna  1900/UDP
TargetPort:        emby-dlna/UDP
Endpoints:         172.16.0.46:1900
Port:              emby-lnd  7359/UDP
TargetPort:        emby-lnd/UDP
Endpoints:         172.16.0.46:7359
Session Affinity:  None
Events:            <none>


Name:                     media-server-emby-tcp
Namespace:                ix-media-server
Labels:                   app.kubernetes.io/instance=media-server
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=emby
                          app.kubernetes.io/version=4.8.0.0
                          helm.sh/chart=emby-1.0.3
Annotations:              meta.helm.sh/release-name: media-server
                          meta.helm.sh/release-namespace: ix-media-server
Selector:                 app.kubernetes.io/instance=media-server,app.kubernetes.io/name=emby
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.142.252
IPs:                      172.17.142.252
Port:                     emby  9096/TCP
TargetPort:               emby/TCP
NodePort:                 emby  9096/TCP
Endpoints:                172.16.0.46:8096
Port:                     http  80/TCP
TargetPort:               emby/TCP
NodePort:                 http  24529/TCP
Endpoints:                172.16.0.46:8096
Port:                     https  443/TCP
TargetPort:               emby/TCP
NodePort:                 https  37725/TCP
Endpoints:                172.16.0.46:8096
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>


Name:              hass-instance-home-assistant-postgres
Namespace:         ix-hass-instance
Labels:            app=home-assistant
                   app.kubernetes.io/instance=hass-instance-postgres
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=home-assistant-postgres
                   app.kubernetes.io/version=2022.6.3
                   helm.sh/chart=home-assistant-1.0.5
                   release=hass-instance
Annotations:       meta.helm.sh/release-name: hass-instance
                   meta.helm.sh/release-namespace: ix-hass-instance
Selector:          app.kubernetes.io/instance=hass-instance-postgres,app.kubernetes.io/name=home-assistant-postgres,app=home-assistant,release=hass-instance
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.61.151
IPs:               172.17.61.151
Port:              postgres-tcp  5432/TCP
TargetPort:        5432/TCP
Endpoints:         10.12.1.5:5432
Session Affinity:  None
Events:            <none>


Name:                     hass-instance-home-assistant
Namespace:                ix-hass-instance
Labels:                   app=home-assistant
                          app.kubernetes.io/instance=hass-instance
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=home-assistant
                          app.kubernetes.io/version=2022.6.3
                          helm.sh/chart=home-assistant-1.0.5
                          release=hass-instance
Annotations:              meta.helm.sh/release-name: hass-instance
                          meta.helm.sh/release-namespace: ix-hass-instance
Selector:                 app.kubernetes.io/instance=hass-instance,app.kubernetes.io/name=home-assistant,app=home-assistant,release=hass-instance
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.253.246
IPs:                      172.17.253.246
Port:                     web  20810/TCP
TargetPort:               8123/TCP
NodePort:                 web  20810/TCP
Endpoints:                
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>
debug finished in 0 seconds for k3s kubectl describe services --all-namespaces


+--------------------------------------------------------------------------------+
+         k3s kubectl describe statefulset --all-namespaces @1654690812          +
+--------------------------------------------------------------------------------+
Name:               openebs-zfs-controller
Namespace:          kube-system
CreationTimestamp:  Fri, 27 May 2022 06:36:15 -0700
Selector:           app=openebs-zfs-controller,role=openebs-zfs
Labels:             objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
Annotations:        objectset.rio.cattle.io/applied:
                      H4sIAAAAAAAA/8xWXY/aOBT9KyurD60U8zEzu9ONxAMdshLqFBCgSlWFkHFughfH9to3Wego/33lBJh0Zpgvdau+ge17fM7xuY5vCDPiM1gntCIhYca4dtElAdkIFZOQzJAhJLmcAZ...
                    objectset.rio.cattle.io/id: 
                    objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                    objectset.rio.cattle.io/owner-name: zfs-operator
                    objectset.rio.cattle.io/owner-namespace: kube-system
Replicas:           1 desired | 1 total
Update Strategy:    RollingUpdate
  Partition:        0
Pods Status:        1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-controller
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-controller-sa
  Containers:
   csi-resizer:
    Image:      k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   csi-snapshotter:
    Image:      k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   snapshot-controller:
    Image:      k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --leader-election=true
    Environment:  <none>
    Mounts:       <none>
   csi-provisioner:
    Image:      k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:   (v1:metadata.namespace)
      POD_NAME:    (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
  Volumes:
   socket-dir:
    Type:               EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:             
    SizeLimit:          <unset>
  Priority Class Name:  openebs-zfs-csi-controller-critical
Volume Claims:          <none>
Events:                 <none>
debug finished in 0 seconds for k3s kubectl describe statefulset --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe job --all-namespaces @1654690812              +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 0 seconds for k3s kubectl describe job --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe cronjob --all-namespaces @1654690812            +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 0 seconds for k3s kubectl describe cronjob --all-namespaces


+--------------------------------------------------------------------------------+
+          k3s kubectl describe daemonsets --all-namespaces @1654690812          +
+--------------------------------------------------------------------------------+
Name:           intel-gpu-plugin
Selector:       app=intel-gpu-plugin
Node-Selector:  kubernetes.io/arch=amd64
Labels:         app=intel-gpu-plugin
Annotations:    deprecated.daemonset.template.generation: 1
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:  app=intel-gpu-plugin
  Init Containers:
   intel-gpu-initcontainer:
    Image:        intel/intel-gpu-initcontainer:0.19.0
    Port:         <none>
    Host Port:    <none>
    Environment:  <none>
    Mounts:
      /etc/kubernetes/node-feature-discovery/source.d/ from nfd-source-hooks (rw)
  Containers:
   intel-gpu-plugin:
    Image:      intel/intel-gpu-plugin:0.19.0
    Port:       <none>
    Host Port:  <none>
    Environment:
      NODE_NAME:   (v1:spec.nodeName)
    Mounts:
      /dev/dri from devfs (ro)
      /sys/class/drm from sysfs (ro)
      /var/lib/kubelet/device-plugins from kubeletsockets (rw)
  Volumes:
   devfs:
    Type:          HostPath (bare host directory volume)
    Path:          /dev/dri
    HostPathType:  
   sysfs:
    Type:          HostPath (bare host directory volume)
    Path:          /sys/class/drm
    HostPathType:  
   kubeletsockets:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/device-plugins
    HostPathType:  
   nfd-source-hooks:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/kubernetes/node-feature-discovery/source.d/
    HostPathType:  DirectoryOrCreate
Events:            <none>


Name:           openebs-zfs-node
Selector:       app=openebs-zfs-node
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                openebs.io/component-name=openebs-zfs-node
                openebs.io/version=ci
Annotations:    deprecated.daemonset.template.generation: 1
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIezSdnuR9oFCrl11FxBsew/VCjn2JPhw7Mh20uVW+e8nOwEChO2e1Lcknvlm5ptvJn5COGffQWkmBQoRznMdlBfIQ2smKArRGEMmxQIM8l...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                objectset.rio.cattle.io/owner-name: zfs-operator
                objectset.rio.cattle.io/owner-namespace: kube-system
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-node
                    openebs.io/component-name=openebs-zfs-node
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-node-sa
  Containers:
   csi-node-driver-registrar:
    Image:      k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
  Volumes:
   device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
   encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
   chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
   host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
   registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
   plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
   pods-mount-dir:
    Type:               HostPath (bare host directory volume)
    Path:               /var/lib/kubelet/
    HostPathType:       Directory
  Priority Class Name:  openebs-zfs-csi-node-critical
Events:                 <none>
debug finished in 1 seconds for k3s kubectl describe daemonsets --all-namespaces


+--------------------------------------------------------------------------------+
+                            docker ps -a @1654690813                            +
+--------------------------------------------------------------------------------+
CONTAINER ID   IMAGE                        COMMAND                  CREATED          STATUS                    PORTS     NAMES
a519013b4176   407cece1abff                 "sh -c 'until pg_isr…"   26 minutes ago   Up 26 minutes                       k8s_init-postgresdb_hass-instance-home-assistant-7ff5685587-jhjd7_ix-hass-instance_80a1060f-eaf5-41e5-bad6-e0e240f2cce1_0
b69b02084c24   407cece1abff                 "docker-entrypoint.s…"   26 minutes ago   Up 26 minutes                       k8s_home-assistant-postgres_hass-instance-home-assistant-postgres-86c496c6b8-rm8tc_ix-hass-instance_8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8_0
0fd917b8714d   rancher/mirrored-pause:3.6   "/pause"                 26 minutes ago   Up 26 minutes                       k8s_POD_hass-instance-home-assistant-7ff5685587-jhjd7_ix-hass-instance_80a1060f-eaf5-41e5-bad6-e0e240f2cce1_0
2a296bb5b52c   rancher/mirrored-pause:3.6   "/pause"                 26 minutes ago   Up 26 minutes                       k8s_POD_hass-instance-home-assistant-postgres-86c496c6b8-rm8tc_ix-hass-instance_8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8_0
2e65943d5652   rancher/mirrored-pause:3.6   "/pause"                 7 days ago       Exited (0) 7 days ago               k8s_POD_hass-instance-home-assistant-7967f45589-tvx5j_ix-hass-instance_bc053dc6-7d0b-4369-aa1f-748d3bea9b06_0
68402d447ede   rancher/mirrored-pause:3.6   "/pause"                 7 days ago       Exited (0) 7 days ago               k8s_POD_hass-instance-home-assistant-postgres-6ccd8596c-mh6vv_ix-hass-instance_770a50ea-bed8-4fd5-8ae5-4b3fb800e064_0
8a7f2a394fd0   rancher/mirrored-pause:3.6   "/pause"                 7 days ago       Exited (0) 7 days ago               k8s_POD_hass-instance-home-assistant-86d5d955ff-9hmvr_ix-hass-instance_929769fa-bba2-4170-9497-a2d622e138f9_0
dd1ba2aaf035   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Exited (0) 7 days ago               k8s_POD_hass-instance-home-assistant-postgres-d8475c5f9-9zctb_ix-hass-instance_4f6cd9fc-d3fc-4a2d-a9fc-b7fa567dc7a2_0
f2bc71080690   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Exited (0) 8 days ago               k8s_POD_hass-instance-home-assistant-postgres-668fbdc86b-lzqvm_ix-hass-instance_d402dd45-6e29-4b60-977e-9faebed32cda_0
25710ca90c48   efd9963380a3                 "/init"                  8 days ago       Up 8 days                           k8s_emby_media-server-emby-74d544f4c-dmp2j_ix-media-server_414dbc55-f54d-4683-91be-8dfcb704842d_0
ee75d9bddfe6   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Up 8 days                           k8s_POD_media-server-emby-74d544f4c-dmp2j_ix-media-server_414dbc55-f54d-4683-91be-8dfcb704842d_0
138273fd5823   2e17748d7c44                 "/usr/local/bin/zfs-…"   8 days ago       Up 8 days                           k8s_openebs-zfs-plugin_openebs-zfs-node-fglmx_kube-system_5571cc39-7075-45a8-af6f-01f108f8ec46_6
785a08f6a587   368ee7d6e60f                 "/csi-node-driver-re…"   8 days ago       Up 8 days                           k8s_csi-node-driver-registrar_openebs-zfs-node-fglmx_kube-system_5571cc39-7075-45a8-af6f-01f108f8ec46_5
63f0413b509a   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Up 8 days                           k8s_POD_openebs-zfs-node-fglmx_kube-system_5571cc39-7075-45a8-af6f-01f108f8ec46_5
42f7678d36f2   2e17748d7c44                 "/usr/local/bin/zfs-…"   8 days ago       Up 8 days                           k8s_openebs-zfs-plugin_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_5
762b7ce84a9a   fe0f921f3c92                 "/csi-provisioner --…"   8 days ago       Up 8 days                           k8s_csi-provisioner_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_5
37bd75993d31   f1d8a00ae690                 "/snapshot-controlle…"   8 days ago       Up 8 days                           k8s_snapshot-controller_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_5
70f5907d9cb2   f2c60b24fad8                 "/usr/local/bin/inte…"   8 days ago       Up 8 days                           k8s_intel-gpu-plugin_intel-gpu-plugin-mksln_kube-system_5aa169ba-d1c9-499e-a573-53014b3e1064_5
056606e12432   da32a49a903a                 "/csi-snapshotter --…"   8 days ago       Up 8 days                           k8s_csi-snapshotter_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_5
f6990a1cdb33   0aa9629e1508                 "/csi-resizer --v=5 …"   8 days ago       Up 8 days                           k8s_csi-resizer_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_5
7d7319c863d3   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Up 8 days                           k8s_POD_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_5
ea7fc1981d30   e026ae0f49c4                 "/usr/local/bin/gpu-…"   8 days ago       Exited (0) 8 days ago               k8s_intel-gpu-initcontainer_intel-gpu-plugin-mksln_kube-system_5aa169ba-d1c9-499e-a573-53014b3e1064_5
7c6bf7105f61   99376d8f35e0                 "/coredns -conf /etc…"   8 days ago       Up 8 days                           k8s_coredns_coredns-d76bd69b-zh8xt_kube-system_562f1e8c-7a09-412a-b161-1129b2e89f9b_5
68f3b0fdb541   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Up 8 days                           k8s_POD_intel-gpu-plugin-mksln_kube-system_5aa169ba-d1c9-499e-a573-53014b3e1064_5
80505fe80d8b   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Up 8 days                           k8s_POD_coredns-d76bd69b-zh8xt_kube-system_562f1e8c-7a09-412a-b161-1129b2e89f9b_5
ccd7e24b78de   2e17748d7c44                 "/usr/local/bin/zfs-…"   8 days ago       Exited (137) 8 days ago             k8s_openebs-zfs-plugin_openebs-zfs-node-fglmx_kube-system_5571cc39-7075-45a8-af6f-01f108f8ec46_5
ca256cb53e84   2e17748d7c44                 "/usr/local/bin/zfs-…"   8 days ago       Exited (137) 8 days ago             k8s_openebs-zfs-plugin_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_4
ab19c90a0936   f2c60b24fad8                 "/usr/local/bin/inte…"   8 days ago       Exited (2) 8 days ago               k8s_intel-gpu-plugin_intel-gpu-plugin-mksln_kube-system_5aa169ba-d1c9-499e-a573-53014b3e1064_4
eb4340d2828c   fe0f921f3c92                 "/csi-provisioner --…"   8 days ago       Exited (2) 8 days ago               k8s_csi-provisioner_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_4
cfc7437a6355   f1d8a00ae690                 "/snapshot-controlle…"   8 days ago       Exited (2) 8 days ago               k8s_snapshot-controller_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_4
0bdcf5177f8d   99376d8f35e0                 "/coredns -conf /etc…"   8 days ago       Exited (0) 8 days ago               k8s_coredns_coredns-d76bd69b-zh8xt_kube-system_562f1e8c-7a09-412a-b161-1129b2e89f9b_4
9a4c1c72c78f   da32a49a903a                 "/csi-snapshotter --…"   8 days ago       Exited (2) 8 days ago               k8s_csi-snapshotter_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_4
02ecc41c9776   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Exited (0) 8 days ago               k8s_POD_coredns-d76bd69b-zh8xt_kube-system_562f1e8c-7a09-412a-b161-1129b2e89f9b_4
0c8739409d0f   0aa9629e1508                 "/csi-resizer --v=5 …"   8 days ago       Exited (2) 8 days ago               k8s_csi-resizer_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_4
b791f9c61261   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Exited (0) 8 days ago               k8s_POD_intel-gpu-plugin-mksln_kube-system_5aa169ba-d1c9-499e-a573-53014b3e1064_4
8d41078e4aa6   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Exited (0) 8 days ago               k8s_POD_openebs-zfs-controller-0_kube-system_cfdef28f-a3b8-440e-85fb-b7902bac965c_4
452a0f1adad6   368ee7d6e60f                 "/csi-node-driver-re…"   8 days ago       Exited (0) 8 days ago               k8s_csi-node-driver-registrar_openebs-zfs-node-fglmx_kube-system_5571cc39-7075-45a8-af6f-01f108f8ec46_4
6df54178a14d   rancher/mirrored-pause:3.6   "/pause"                 8 days ago       Exited (0) 8 days ago               k8s_POD_openebs-zfs-node-fglmx_kube-system_5571cc39-7075-45a8-af6f-01f108f8ec46_4
debug finished in 0 seconds for docker ps -a


+--------------------------------------------------------------------------------+
+                          docker images -a @1654690813                          +
+--------------------------------------------------------------------------------+
REPOSITORY                                         TAG        IMAGE ID       CREATED         SIZE
emby/embyserver                                    4.8.0.0    efd9963380a3   12 days ago     571MB
alpine                                             latest     e66264b98777   2 weeks ago     5.52MB
homeassistant/home-assistant                       2022.5.5   22526c9da9de   2 weeks ago     1.2GB
rancher/mirrored-coredns-coredns                   1.9.1      99376d8f35e0   2 months ago    49.5MB
openebs/zfs-driver                                 2.0.0      2e17748d7c44   5 months ago    176MB
rancher/klipper-lb                                 v0.3.4     746788bcc27e   7 months ago    8.08MB
rancher/mirrored-pause                             3.6        6270bb605e12   9 months ago    683kB
k8s.gcr.io/sig-storage/csi-provisioner             v3.0.0     fe0f921f3c92   9 months ago    56.7MB
k8s.gcr.io/sig-storage/csi-node-driver-registrar   v2.3.0     368ee7d6e60f   10 months ago   18.7MB
k8s.gcr.io/sig-storage/csi-resizer                 v1.2.0     0aa9629e1508   13 months ago   53.9MB
postgres                                           13.1       407cece1abff   16 months ago   314MB
k8s.gcr.io/sig-storage/csi-snapshotter             v4.0.0     da32a49a903a   17 months ago   49.5MB
k8s.gcr.io/sig-storage/snapshot-controller         v4.0.0     f1d8a00ae690   17 months ago   46.6MB
intel/intel-gpu-plugin                             0.19.0     f2c60b24fad8   18 months ago   80MB
intel/intel-gpu-initcontainer                      0.19.0     e026ae0f49c4   18 months ago   69MB
debug finished in 0 seconds for docker images -a


+--------------------------------------------------------------------------------+
+                     K3s Logs (last 1000 lines) @1654690813                     +
+--------------------------------------------------------------------------------+
Jun 08 03:36:23 truenas k3s[7233]: Trace[84412210]: [908.743414ms] [908.743414ms] END
Jun 08 03:36:23 truenas k3s[7233]: I0608 03:36:23.279436    7233 trace.go:205] Trace[845540077]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:36:22.211) (total time: 1068ms):
Jun 08 03:36:23 truenas k3s[7233]: Trace[845540077]: ---"Transaction committed" 1067ms (03:36:23.279)
Jun 08 03:36:23 truenas k3s[7233]: Trace[845540077]: [1.068218984s] [1.068218984s] END
Jun 08 03:36:23 truenas k3s[7233]: I0608 03:36:23.279672    7233 trace.go:205] Trace[971843475]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:db4ae049-58c9-4796-9ffe-d5488e66b604,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:36:22.210) (total time: 1068ms):
Jun 08 03:36:23 truenas k3s[7233]: Trace[971843475]: ---"Object stored in database" 1068ms (03:36:23.279)
Jun 08 03:36:23 truenas k3s[7233]: Trace[971843475]: [1.068883435s] [1.068883435s] END
Jun 08 03:36:43 truenas k3s[7233]: E0608 03:36:43.886846    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:36:53 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:36:53.605-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:36:56 truenas k3s[7233]: E0608 03:36:56.887726    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:37:33 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:37:33.811-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:37:34 truenas k3s[7233]: I0608 03:37:34.399552    7233 trace.go:205] Trace[1436956165]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:7a34679f-2e32-426d-bafe-73e68e7ad908,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:37:33.784) (total time: 614ms):
Jun 08 03:37:34 truenas k3s[7233]: Trace[1436956165]: ---"About to write a response" 614ms (03:37:34.399)
Jun 08 03:37:34 truenas k3s[7233]: Trace[1436956165]: [614.706157ms] [614.706157ms] END
Jun 08 03:37:45 truenas k3s[7233]: E0608 03:37:45.888178    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:37:59 truenas k3s[7233]: E0608 03:37:59.887614    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:38:12 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:38:12.714-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:38:47 truenas k3s[7233]: E0608 03:38:47.887538    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:38:56 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:38:56.600-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:39:07 truenas k3s[7233]: E0608 03:39:07.887345    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:39:27 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:39:27.372-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:40:02 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:40:02.289-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:40:14 truenas k3s[7233]: E0608 03:40:14.887239    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:40:37 truenas k3s[7233]: E0608 03:40:37.886736    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:40:44 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:40:44.765-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:41:04 truenas k3s[7233]: I0608 03:41:04.633479    7233 trace.go:205] Trace[2141592800]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:43252900-0b4d-46e2-a71f-f4946a04ac3a,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:41:03.715) (total time: 918ms):
Jun 08 03:41:04 truenas k3s[7233]: Trace[2141592800]: ---"About to write a response" 918ms (03:41:04.633)
Jun 08 03:41:04 truenas k3s[7233]: Trace[2141592800]: [918.122024ms] [918.122024ms] END
Jun 08 03:41:16 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:41:16.058-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:41:41 truenas k3s[7233]: E0608 03:41:41.886830    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:41:51 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:41:51.343-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:42:04 truenas k3s[7233]: E0608 03:42:04.887187    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:42:32 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:42:32.433-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:42:36 truenas k3s[7233]: I0608 03:42:36.693676    7233 trace.go:205] Trace[1728132545]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:42:35.899) (total time: 794ms):
Jun 08 03:42:36 truenas k3s[7233]: Trace[1728132545]: ---"Transaction committed" 793ms (03:42:36.693)
Jun 08 03:42:36 truenas k3s[7233]: Trace[1728132545]: [794.200021ms] [794.200021ms] END
Jun 08 03:42:36 truenas k3s[7233]: I0608 03:42:36.695410    7233 trace.go:205] Trace[1238408092]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:f89215b8-00d8-4dd9-bc66-c105a4913760,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:42:35.898) (total time: 796ms):
Jun 08 03:42:36 truenas k3s[7233]: Trace[1238408092]: ---"Object stored in database" 795ms (03:42:36.695)
Jun 08 03:42:36 truenas k3s[7233]: Trace[1238408092]: [796.404108ms] [796.404108ms] END
Jun 08 03:42:36 truenas k3s[7233]: I0608 03:42:36.715130    7233 trace.go:205] Trace[2044010594]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:84aa0c63-bafe-487c-aba7-135961cf233b,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:42:36.005) (total time: 709ms):
Jun 08 03:42:36 truenas k3s[7233]: Trace[2044010594]: ---"About to write a response" 709ms (03:42:36.714)
Jun 08 03:42:36 truenas k3s[7233]: Trace[2044010594]: [709.416632ms] [709.416632ms] END
Jun 08 03:42:36 truenas k3s[7233]: I0608 03:42:36.715524    7233 trace.go:205] Trace[1478093992]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:4f106f26-fd51-4521-96ab-b6b4981ac9e8,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:42:36.137) (total time: 577ms):
Jun 08 03:42:36 truenas k3s[7233]: Trace[1478093992]: ---"About to write a response" 577ms (03:42:36.715)
Jun 08 03:42:36 truenas k3s[7233]: Trace[1478093992]: [577.983641ms] [577.983641ms] END
Jun 08 03:42:36 truenas k3s[7233]: I0608 03:42:36.842112    7233 trace.go:205] Trace[1776965328]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:42:35.906) (total time: 935ms):
Jun 08 03:42:36 truenas k3s[7233]: Trace[1776965328]: ---"Transaction committed" 934ms (03:42:36.842)
Jun 08 03:42:36 truenas k3s[7233]: Trace[1776965328]: [935.75287ms] [935.75287ms] END
Jun 08 03:42:36 truenas k3s[7233]: I0608 03:42:36.842353    7233 trace.go:205] Trace[1831908546]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:83994150-b71c-49c3-aadb-b310c94dcc03,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:42:35.905) (total time: 936ms):
Jun 08 03:42:36 truenas k3s[7233]: Trace[1831908546]: ---"Object stored in database" 935ms (03:42:36.842)
Jun 08 03:42:36 truenas k3s[7233]: Trace[1831908546]: [936.445721ms] [936.445721ms] END
Jun 08 03:42:53 truenas k3s[7233]: E0608 03:42:53.887319    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:43:07 truenas k3s[7233]: I0608 03:43:07.946590    7233 trace.go:205] Trace[1821723096]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:43:07.171) (total time: 775ms):
Jun 08 03:43:07 truenas k3s[7233]: Trace[1821723096]: ---"Transaction committed" 774ms (03:43:07.946)
Jun 08 03:43:07 truenas k3s[7233]: Trace[1821723096]: [775.213632ms] [775.213632ms] END
Jun 08 03:43:07 truenas k3s[7233]: I0608 03:43:07.946836    7233 trace.go:205] Trace[251450025]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:c4e81ca8-6d1a-49f7-bcea-0bed3200c0c0,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:43:07.171) (total time: 775ms):
Jun 08 03:43:07 truenas k3s[7233]: Trace[251450025]: ---"Object stored in database" 775ms (03:43:07.946)
Jun 08 03:43:07 truenas k3s[7233]: Trace[251450025]: [775.715362ms] [775.715362ms] END
Jun 08 03:43:07 truenas k3s[7233]: I0608 03:43:07.948158    7233 trace.go:205] Trace[35506277]: "Get" url:/api/v1/namespaces/default,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:9ad95d3c-dedf-41c3-98ab-31e0f9b95d5a,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 03:43:07.149) (total time: 798ms):
Jun 08 03:43:07 truenas k3s[7233]: Trace[35506277]: ---"About to write a response" 798ms (03:43:07.948)
Jun 08 03:43:07 truenas k3s[7233]: Trace[35506277]: [798.660147ms] [798.660147ms] END
Jun 08 03:43:07 truenas k3s[7233]: I0608 03:43:07.949732    7233 trace.go:205] Trace[1712078863]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:1607398b-b90d-47b1-ab3a-0203f6fef802,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:43:07.412) (total time: 537ms):
Jun 08 03:43:07 truenas k3s[7233]: Trace[1712078863]: ---"About to write a response" 537ms (03:43:07.949)
Jun 08 03:43:07 truenas k3s[7233]: Trace[1712078863]: [537.494306ms] [537.494306ms] END
Jun 08 03:43:07 truenas k3s[7233]: I0608 03:43:07.951342    7233 trace.go:205] Trace[1688341338]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:9f949889-55bc-4605-b428-81ccdbf1f554,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:43:07.236) (total time: 714ms):
Jun 08 03:43:07 truenas k3s[7233]: Trace[1688341338]: ---"About to write a response" 714ms (03:43:07.951)
Jun 08 03:43:07 truenas k3s[7233]: Trace[1688341338]: [714.882173ms] [714.882173ms] END
Jun 08 03:43:07 truenas k3s[7233]: I0608 03:43:07.951513    7233 trace.go:205] Trace[1015325883]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:d8b17288-3ed2-4e9b-a4fd-c118653aba79,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:43:07.247) (total time: 703ms):
Jun 08 03:43:07 truenas k3s[7233]: Trace[1015325883]: ---"About to write a response" 703ms (03:43:07.951)
Jun 08 03:43:07 truenas k3s[7233]: Trace[1015325883]: [703.824475ms] [703.824475ms] END
Jun 08 03:43:16 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:43:16.500-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:43:26 truenas k3s[7233]: E0608 03:43:26.887609    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:43:37 truenas k3s[7233]: I0608 03:43:37.949432    7233 trace.go:205] Trace[528688114]: "GuaranteedUpdate etcd3" type:*v1.Endpoints (08-Jun-2022 03:43:37.162) (total time: 786ms):
Jun 08 03:43:37 truenas k3s[7233]: Trace[528688114]: ---"initial value restored" 137ms (03:43:37.300)
Jun 08 03:43:37 truenas k3s[7233]: Trace[528688114]: ---"Transaction committed" 648ms (03:43:37.949)
Jun 08 03:43:37 truenas k3s[7233]: Trace[528688114]: [786.395471ms] [786.395471ms] END
Jun 08 03:43:49 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:43:49.064-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:44:04 truenas k3s[7233]: E0608 03:44:04.886674    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:44:28 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:44:28.129-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:44:35 truenas k3s[7233]: E0608 03:44:35.888434    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:45:04 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:45:04.184-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:45:21 truenas k3s[7233]: E0608 03:45:21.887004    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:45:36 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:45:36.725-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:46:01 truenas k3s[7233]: E0608 03:46:01.886845    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:46:09 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:46:09.007-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:46:44 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:46:44.197-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:46:51 truenas k3s[7233]: E0608 03:46:51.887099    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:47:26 truenas k3s[7233]: E0608 03:47:26.887524    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:47:26 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:47:26.908-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:47:53 truenas k3s[7233]: E0608 03:47:53.887809    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:48:03 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:48:03.206-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:48:40 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:48:40.806-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:48:42 truenas k3s[7233]: E0608 03:48:42.887660    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:49:20 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:49:20.081-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:49:23 truenas k3s[7233]: E0608 03:49:23.887323    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:49:46 truenas k3s[7233]: I0608 03:49:46.669750    7233 trace.go:205] Trace[421277837]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:e7b2adae-acde-4a28-93a7-a0c21260f819,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:49:46.094) (total time: 575ms):
Jun 08 03:49:46 truenas k3s[7233]: Trace[421277837]: ---"About to write a response" 575ms (03:49:46.669)
Jun 08 03:49:46 truenas k3s[7233]: Trace[421277837]: [575.15625ms] [575.15625ms] END
Jun 08 03:49:46 truenas k3s[7233]: I0608 03:49:46.707984    7233 trace.go:205] Trace[17639224]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:49:46.100) (total time: 607ms):
Jun 08 03:49:46 truenas k3s[7233]: Trace[17639224]: ---"Transaction committed" 606ms (03:49:46.707)
Jun 08 03:49:46 truenas k3s[7233]: Trace[17639224]: [607.428716ms] [607.428716ms] END
Jun 08 03:49:46 truenas k3s[7233]: I0608 03:49:46.708739    7233 trace.go:205] Trace[1551374122]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:9f4281a7-7826-4781-9271-1fdc8c9a3f92,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:49:46.100) (total time: 608ms):
Jun 08 03:49:46 truenas k3s[7233]: Trace[1551374122]: ---"Object stored in database" 608ms (03:49:46.708)
Jun 08 03:49:46 truenas k3s[7233]: Trace[1551374122]: [608.419751ms] [608.419751ms] END
Jun 08 03:49:46 truenas k3s[7233]: I0608 03:49:46.716394    7233 trace.go:205] Trace[616846681]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:0f222365-3861-413d-8dfc-24a6d75a2c08,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:49:46.094) (total time: 621ms):
Jun 08 03:49:46 truenas k3s[7233]: Trace[616846681]: ---"About to write a response" 621ms (03:49:46.716)
Jun 08 03:49:46 truenas k3s[7233]: Trace[616846681]: [621.443243ms] [621.443243ms] END
Jun 08 03:49:46 truenas k3s[7233]: I0608 03:49:46.716399    7233 trace.go:205] Trace[189353697]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:20076000-cae7-4863-b670-d9e05049a498,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:49:46.095) (total time: 620ms):
Jun 08 03:49:46 truenas k3s[7233]: Trace[189353697]: ---"About to write a response" 620ms (03:49:46.716)
Jun 08 03:49:46 truenas k3s[7233]: Trace[189353697]: [620.520916ms] [620.520916ms] END
Jun 08 03:49:50 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:49:50.107-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:50:10 truenas k3s[7233]: E0608 03:50:10.887544    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:50:28 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:50:28.644-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:50:36 truenas k3s[7233]: E0608 03:50:36.887288    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:51:03 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:51:03.826-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:51:34 truenas k3s[7233]: E0608 03:51:34.887220    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:51:38 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:51:38.723-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:51:43 truenas k3s[7233]: E0608 03:51:43.886798    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:52:13 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:52:13.922-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:52:57 truenas k3s[7233]: E0608 03:52:57.887634    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:52:58 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:52:58.849-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:53:01 truenas k3s[7233]: E0608 03:53:01.887588    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:53:40 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:53:40.871-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:54:10 truenas k3s[7233]: E0608 03:54:10.886418    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:54:13 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:54:13.791-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:54:15 truenas k3s[7233]: E0608 03:54:15.888302    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:54:56 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:54:56.852-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:55:09 truenas k3s[7233]: I0608 03:55:09.453072    7233 trace.go:205] Trace[59911946]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:63a7a8fd-7116-4f80-8e21-1d1423c8815f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:55:08.810) (total time: 642ms):
Jun 08 03:55:09 truenas k3s[7233]: Trace[59911946]: ---"About to write a response" 642ms (03:55:09.452)
Jun 08 03:55:09 truenas k3s[7233]: Trace[59911946]: [642.674473ms] [642.674473ms] END
Jun 08 03:55:28 truenas k3s[7233]: E0608 03:55:28.887063    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:55:30 truenas k3s[7233]: E0608 03:55:30.886995    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:55:40 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:55:40.634-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:55:50 truenas k3s[7233]: I0608 03:55:50.419280    7233 trace.go:205] Trace[1529249659]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:55:49.848) (total time: 570ms):
Jun 08 03:55:50 truenas k3s[7233]: Trace[1529249659]: ---"Transaction committed" 569ms (03:55:50.419)
Jun 08 03:55:50 truenas k3s[7233]: Trace[1529249659]: [570.419117ms] [570.419117ms] END
Jun 08 03:55:50 truenas k3s[7233]: I0608 03:55:50.419535    7233 trace.go:205] Trace[956354362]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:2660a3ef-b3cf-41c5-91fb-d979607546af,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:55:49.848) (total time: 571ms):
Jun 08 03:55:50 truenas k3s[7233]: Trace[956354362]: ---"Object stored in database" 570ms (03:55:50.419)
Jun 08 03:55:50 truenas k3s[7233]: Trace[956354362]: [571.142089ms] [571.142089ms] END
Jun 08 03:56:11 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:56:11.663-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:56:26 truenas k3s[7233]: I0608 03:56:26.104795    7233 trace.go:205] Trace[1929965610]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:56:25.598) (total time: 505ms):
Jun 08 03:56:26 truenas k3s[7233]: Trace[1929965610]: ---"Transaction committed" 505ms (03:56:26.104)
Jun 08 03:56:26 truenas k3s[7233]: Trace[1929965610]: [505.988394ms] [505.988394ms] END
Jun 08 03:56:26 truenas k3s[7233]: I0608 03:56:26.105054    7233 trace.go:205] Trace[568818565]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:a39067e6-eb17-4647-a305-f2964728b927,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:56:25.598) (total time: 506ms):
Jun 08 03:56:26 truenas k3s[7233]: Trace[568818565]: ---"Object stored in database" 506ms (03:56:26.104)
Jun 08 03:56:26 truenas k3s[7233]: Trace[568818565]: [506.394031ms] [506.394031ms] END
Jun 08 03:56:26 truenas k3s[7233]: I0608 03:56:26.137760    7233 trace.go:205] Trace[130903218]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:56:25.599) (total time: 538ms):
Jun 08 03:56:26 truenas k3s[7233]: Trace[130903218]: ---"Transaction committed" 538ms (03:56:26.137)
Jun 08 03:56:26 truenas k3s[7233]: Trace[130903218]: [538.614801ms] [538.614801ms] END
Jun 08 03:56:26 truenas k3s[7233]: I0608 03:56:26.138010    7233 trace.go:205] Trace[492478668]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:e9cb675b-920e-4fde-9076-b2e6963f08ba,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:56:25.598) (total time: 538ms):
Jun 08 03:56:26 truenas k3s[7233]: Trace[492478668]: ---"Object stored in database" 538ms (03:56:26.137)
Jun 08 03:56:26 truenas k3s[7233]: Trace[492478668]: [538.967575ms] [538.967575ms] END
Jun 08 03:56:26 truenas k3s[7233]: I0608 03:56:26.167866    7233 trace.go:205] Trace[353538767]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:56:25.629) (total time: 537ms):
Jun 08 03:56:26 truenas k3s[7233]: Trace[353538767]: ---"Transaction committed" 537ms (03:56:26.167)
Jun 08 03:56:26 truenas k3s[7233]: Trace[353538767]: [537.856493ms] [537.856493ms] END
Jun 08 03:56:26 truenas k3s[7233]: I0608 03:56:26.168104    7233 trace.go:205] Trace[562144851]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:312be46f-c8bd-4fb1-8713-21c918aede98,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:56:25.629) (total time: 538ms):
Jun 08 03:56:26 truenas k3s[7233]: Trace[562144851]: ---"Object stored in database" 538ms (03:56:26.167)
Jun 08 03:56:26 truenas k3s[7233]: Trace[562144851]: [538.315869ms] [538.315869ms] END
Jun 08 03:56:46 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:56:46.286-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:56:47 truenas k3s[7233]: E0608 03:56:47.887736    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:56:57 truenas k3s[7233]: E0608 03:56:57.887561    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:57:07 truenas k3s[7233]: I0608 03:57:07.099508    7233 trace.go:205] Trace[1720304585]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:57:06.448) (total time: 651ms):
Jun 08 03:57:07 truenas k3s[7233]: Trace[1720304585]: ---"Transaction committed" 651ms (03:57:07.099)
Jun 08 03:57:07 truenas k3s[7233]: Trace[1720304585]: [651.430453ms] [651.430453ms] END
Jun 08 03:57:07 truenas k3s[7233]: I0608 03:57:07.099764    7233 trace.go:205] Trace[1735981104]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:52e011db-1a98-40ab-a1d9-ac81438b4bde,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:57:06.447) (total time: 651ms):
Jun 08 03:57:07 truenas k3s[7233]: Trace[1735981104]: ---"Object stored in database" 651ms (03:57:07.099)
Jun 08 03:57:07 truenas k3s[7233]: Trace[1735981104]: [651.809911ms] [651.809911ms] END
Jun 08 03:57:07 truenas k3s[7233]: I0608 03:57:07.105394    7233 trace.go:205] Trace[1821997277]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:57:06.448) (total time: 656ms):
Jun 08 03:57:07 truenas k3s[7233]: Trace[1821997277]: ---"Transaction committed" 656ms (03:57:07.105)
Jun 08 03:57:07 truenas k3s[7233]: Trace[1821997277]: [656.590269ms] [656.590269ms] END
Jun 08 03:57:07 truenas k3s[7233]: I0608 03:57:07.105665    7233 trace.go:205] Trace[1111170384]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:93aaf37c-17fe-41a9-a083-4060f81dbd58,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:57:06.448) (total time: 656ms):
Jun 08 03:57:07 truenas k3s[7233]: Trace[1111170384]: ---"Object stored in database" 656ms (03:57:07.105)
Jun 08 03:57:07 truenas k3s[7233]: Trace[1111170384]: [656.9625ms] [656.9625ms] END
Jun 08 03:57:11 truenas k3s[7233]: I0608 03:57:11.977047    7233 trace.go:205] Trace[1576560401]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:08fe792b-ee08-4c3a-afd0-4cfc84a283c5,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:57:11.459) (total time: 517ms):
Jun 08 03:57:11 truenas k3s[7233]: Trace[1576560401]: ---"About to write a response" 517ms (03:57:11.976)
Jun 08 03:57:11 truenas k3s[7233]: Trace[1576560401]: [517.616827ms] [517.616827ms] END
Jun 08 03:57:17 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:57:17.343-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:57:47 truenas k3s[7233]: I0608 03:57:47.749257    7233 trace.go:205] Trace[765311312]: "Get" url:/api/v1/namespaces/default,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:cd9e5f91-4cc9-419d-825a-404cf1192bb9,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 03:57:47.207) (total time: 541ms):
Jun 08 03:57:47 truenas k3s[7233]: Trace[765311312]: ---"About to write a response" 541ms (03:57:47.749)
Jun 08 03:57:47 truenas k3s[7233]: Trace[765311312]: [541.659365ms] [541.659365ms] END
Jun 08 03:57:56 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:57:56.569-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:58:11 truenas k3s[7233]: E0608 03:58:11.886024    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:58:13 truenas k3s[7233]: E0608 03:58:13.887285    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:58:32 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:58:32.266-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:59:04 truenas k3s[7233]: I0608 03:59:04.868607    7233 trace.go:205] Trace[217533491]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:59:04.367) (total time: 501ms):
Jun 08 03:59:04 truenas k3s[7233]: Trace[217533491]: ---"Transaction committed" 500ms (03:59:04.868)
Jun 08 03:59:04 truenas k3s[7233]: Trace[217533491]: [501.229795ms] [501.229795ms] END
Jun 08 03:59:04 truenas k3s[7233]: I0608 03:59:04.869464    7233 trace.go:205] Trace[360579013]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:e4c8c210-6516-4d0a-9910-53a72f7e6158,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:59:04.366) (total time: 502ms):
Jun 08 03:59:04 truenas k3s[7233]: Trace[360579013]: ---"Object stored in database" 501ms (03:59:04.869)
Jun 08 03:59:04 truenas k3s[7233]: Trace[360579013]: [502.471308ms] [502.471308ms] END
Jun 08 03:59:08 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:59:08.418-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 03:59:14 truenas k3s[7233]: I0608 03:59:14.913131    7233 trace.go:205] Trace[1006935836]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 03:59:14.401) (total time: 511ms):
Jun 08 03:59:14 truenas k3s[7233]: Trace[1006935836]: ---"Transaction committed" 511ms (03:59:14.913)
Jun 08 03:59:14 truenas k3s[7233]: Trace[1006935836]: [511.543604ms] [511.543604ms] END
Jun 08 03:59:14 truenas k3s[7233]: I0608 03:59:14.913370    7233 trace.go:205] Trace[860786310]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:829e8f95-0c87-4311-a13d-c7f0fb4ebb15,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:59:14.401) (total time: 511ms):
Jun 08 03:59:14 truenas k3s[7233]: Trace[860786310]: ---"Object stored in database" 511ms (03:59:14.913)
Jun 08 03:59:14 truenas k3s[7233]: Trace[860786310]: [511.926814ms] [511.926814ms] END
Jun 08 03:59:25 truenas k3s[7233]: E0608 03:59:25.888056    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:59:36 truenas k3s[7233]: E0608 03:59:36.887158    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 03:59:45 truenas k3s[7233]: I0608 03:59:45.995151    7233 trace.go:205] Trace[716372869]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:53047d70-f2f3-494e-b770-80163480f633,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:59:45.426) (total time: 568ms):
Jun 08 03:59:45 truenas k3s[7233]: Trace[716372869]: ---"About to write a response" 568ms (03:59:45.994)
Jun 08 03:59:45 truenas k3s[7233]: Trace[716372869]: [568.925059ms] [568.925059ms] END
Jun 08 03:59:45 truenas k3s[7233]: I0608 03:59:45.997378    7233 trace.go:205] Trace[1159617047]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:5d0db1ed-78e2-4b93-bd4b-251568615b16,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:59:45.451) (total time: 545ms):
Jun 08 03:59:45 truenas k3s[7233]: Trace[1159617047]: ---"About to write a response" 545ms (03:59:45.997)
Jun 08 03:59:45 truenas k3s[7233]: Trace[1159617047]: [545.641456ms] [545.641456ms] END
Jun 08 03:59:45 truenas k3s[7233]: I0608 03:59:45.998564    7233 trace.go:205] Trace[285431505]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:0a72fe95-8b87-45a6-bf5f-7ba674b75fd2,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 03:59:45.448) (total time: 550ms):
Jun 08 03:59:45 truenas k3s[7233]: Trace[285431505]: ---"About to write a response" 549ms (03:59:45.998)
Jun 08 03:59:45 truenas k3s[7233]: Trace[285431505]: [550.128818ms] [550.128818ms] END
Jun 08 03:59:51 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T03:59:51.489-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:00:26 truenas k3s[7233]: I0608 04:00:26.816322    7233 trace.go:205] Trace[285976518]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:e31abaa8-4d8b-42fc-a4f8-5c75b429cd8f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:00:26.163) (total time: 652ms):
Jun 08 04:00:26 truenas k3s[7233]: Trace[285976518]: ---"About to write a response" 652ms (04:00:26.816)
Jun 08 04:00:26 truenas k3s[7233]: Trace[285976518]: [652.288383ms] [652.288383ms] END
Jun 08 04:00:34 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:00:34.688-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:00:42 truenas k3s[7233]: E0608 04:00:42.887636    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:00:50 truenas k3s[7233]: E0608 04:00:50.887218    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:00:52 truenas k3s[7233]: I0608 04:00:52.356353    7233 trace.go:205] Trace[1447372101]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:00:51.490) (total time: 866ms):
Jun 08 04:00:52 truenas k3s[7233]: Trace[1447372101]: ---"Transaction committed" 865ms (04:00:52.356)
Jun 08 04:00:52 truenas k3s[7233]: Trace[1447372101]: [866.105851ms] [866.105851ms] END
Jun 08 04:00:52 truenas k3s[7233]: I0608 04:00:52.356590    7233 trace.go:205] Trace[1680668225]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:57467cf2-57d0-4fab-b554-7129397e1b88,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:00:51.489) (total time: 866ms):
Jun 08 04:00:52 truenas k3s[7233]: Trace[1680668225]: ---"Object stored in database" 866ms (04:00:52.356)
Jun 08 04:00:52 truenas k3s[7233]: Trace[1680668225]: [866.80088ms] [866.80088ms] END
Jun 08 04:01:08 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:01:08.215-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:01:40 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:01:40.036-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:01:43 truenas k3s[7233]: I0608 04:01:43.673340    7233 trace.go:205] Trace[790650259]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:01:42.717) (total time: 956ms):
Jun 08 04:01:43 truenas k3s[7233]: Trace[790650259]: ---"Transaction committed" 955ms (04:01:43.673)
Jun 08 04:01:43 truenas k3s[7233]: Trace[790650259]: [956.145699ms] [956.145699ms] END
Jun 08 04:01:43 truenas k3s[7233]: I0608 04:01:43.673635    7233 trace.go:205] Trace[1422298749]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:4b2bd933-1869-4e71-abf9-58debe967ebe,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:01:42.716) (total time: 956ms):
Jun 08 04:01:43 truenas k3s[7233]: Trace[1422298749]: ---"Object stored in database" 956ms (04:01:43.673)
Jun 08 04:01:43 truenas k3s[7233]: Trace[1422298749]: [956.908572ms] [956.908572ms] END
Jun 08 04:01:43 truenas k3s[7233]: I0608 04:01:43.715738    7233 trace.go:205] Trace[2055471873]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:01:42.878) (total time: 837ms):
Jun 08 04:01:43 truenas k3s[7233]: Trace[2055471873]: ---"Transaction committed" 836ms (04:01:43.715)
Jun 08 04:01:43 truenas k3s[7233]: Trace[2055471873]: [837.651038ms] [837.651038ms] END
Jun 08 04:01:43 truenas k3s[7233]: I0608 04:01:43.715986    7233 trace.go:205] Trace[94458529]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:bd20c082-5c8a-431a-a462-ca4b281e53d1,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:01:42.877) (total time: 838ms):
Jun 08 04:01:43 truenas k3s[7233]: Trace[94458529]: ---"Object stored in database" 837ms (04:01:43.715)
Jun 08 04:01:43 truenas k3s[7233]: Trace[94458529]: [838.360697ms] [838.360697ms] END
Jun 08 04:01:52 truenas k3s[7233]: E0608 04:01:52.887935    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:02:10 truenas k3s[7233]: E0608 04:02:10.887626    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:02:14 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:02:14.936-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:02:24 truenas k3s[7233]: I0608 04:02:24.405042    7233 trace.go:205] Trace[1844823456]: "Get" url:/api/v1/namespaces/kube-system,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:ca66ad66-d48b-43cb-be0d-d156f1532c19,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 04:02:23.800) (total time: 604ms):
Jun 08 04:02:24 truenas k3s[7233]: Trace[1844823456]: ---"About to write a response" 604ms (04:02:24.404)
Jun 08 04:02:24 truenas k3s[7233]: Trace[1844823456]: [604.846519ms] [604.846519ms] END
Jun 08 04:02:52 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:02:52.969-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:03:20 truenas k3s[7233]: E0608 04:03:20.887209    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:03:24 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:03:24.404-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:03:27 truenas k3s[7233]: E0608 04:03:27.886796    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:03:58 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:03:58.241-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:04:27 truenas k3s[7233]: E0608 04:04:27.887686    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:04:34 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:04:34.798-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:04:44 truenas k3s[7233]: E0608 04:04:44.887298    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:05:17 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:05:17.744-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:05:40 truenas k3s[7233]: E0608 04:05:40.887774    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:05:49 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:05:49.270-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:05:52 truenas k3s[7233]: E0608 04:05:52.887472    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:06:33 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:06:33.289-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:07:03 truenas k3s[7233]: E0608 04:07:03.888043    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:07:12 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:07:12.649-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:07:13 truenas k3s[7233]: E0608 04:07:13.887649    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:07:44 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:07:44.835-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:08:05 truenas k3s[7233]: I0608 04:08:05.752402    7233 trace.go:205] Trace[1123643943]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:08:04.931) (total time: 820ms):
Jun 08 04:08:05 truenas k3s[7233]: Trace[1123643943]: ---"Transaction committed" 820ms (04:08:05.752)
Jun 08 04:08:05 truenas k3s[7233]: Trace[1123643943]: [820.523275ms] [820.523275ms] END
Jun 08 04:08:05 truenas k3s[7233]: I0608 04:08:05.752656    7233 trace.go:205] Trace[1772822526]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:b1a70332-ea5d-4a45-8219-7c0a946ade12,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:08:04.931) (total time: 820ms):
Jun 08 04:08:05 truenas k3s[7233]: Trace[1772822526]: ---"Object stored in database" 820ms (04:08:05.752)
Jun 08 04:08:05 truenas k3s[7233]: Trace[1772822526]: [820.915418ms] [820.915418ms] END
Jun 08 04:08:19 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:08:19.879-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:08:21 truenas k3s[7233]: E0608 04:08:21.887475    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:08:26 truenas k3s[7233]: E0608 04:08:26.887643    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:08:52 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:08:52.176-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:09:31 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:09:31.767-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:09:32 truenas k3s[7233]: E0608 04:09:32.887703    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:09:45 truenas k3s[7233]: E0608 04:09:45.888360    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:10:10 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:10:10.412-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:10:47 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:10:47.834-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:10:52 truenas k3s[7233]: E0608 04:10:52.888188    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:11:01 truenas k3s[7233]: E0608 04:11:01.887575    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:11:21 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:11:21.992-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:11:22 truenas k3s[7233]: I0608 04:11:22.116662    7233 trace.go:205] Trace[1043041490]: "Get" url:/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas/system-node-high,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:699b8351-246d-4593-b645-2251452af2b8,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 04:11:21.454) (total time: 662ms):
Jun 08 04:11:22 truenas k3s[7233]: Trace[1043041490]: ---"About to write a response" 662ms (04:11:22.116)
Jun 08 04:11:22 truenas k3s[7233]: Trace[1043041490]: [662.58353ms] [662.58353ms] END
Jun 08 04:12:00 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:12:00.948-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:12:13 truenas k3s[7233]: E0608 04:12:13.887370    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:12:18 truenas k3s[7233]: E0608 04:12:18.886915    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:12:33 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:12:33.163-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:13:14 truenas k3s[7233]: I0608 04:13:14.688096    7233 trace.go:205] Trace[2070721174]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:7c49dcd8-f20b-4be6-ae5e-917cdf9b7e46,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:13:13.939) (total time: 748ms):
Jun 08 04:13:14 truenas k3s[7233]: Trace[2070721174]: ---"About to write a response" 748ms (04:13:14.687)
Jun 08 04:13:14 truenas k3s[7233]: Trace[2070721174]: [748.250352ms] [748.250352ms] END
Jun 08 04:13:14 truenas k3s[7233]: I0608 04:13:14.688542    7233 trace.go:205] Trace[987615682]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:0b5daa54-fc69-454f-9294-cbd6593e575b,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:13:14.004) (total time: 683ms):
Jun 08 04:13:14 truenas k3s[7233]: Trace[987615682]: ---"About to write a response" 683ms (04:13:14.688)
Jun 08 04:13:14 truenas k3s[7233]: Trace[987615682]: [683.731075ms] [683.731075ms] END
Jun 08 04:13:14 truenas k3s[7233]: I0608 04:13:14.692901    7233 trace.go:205] Trace[671122144]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:98db683e-1fbd-434d-aaac-53d135816bf5,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:13:14.059) (total time: 633ms):
Jun 08 04:13:14 truenas k3s[7233]: Trace[671122144]: ---"About to write a response" 633ms (04:13:14.692)
Jun 08 04:13:14 truenas k3s[7233]: Trace[671122144]: [633.783146ms] [633.783146ms] END
Jun 08 04:13:14 truenas k3s[7233]: I0608 04:13:14.715450    7233 trace.go:205] Trace[816606018]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:13:13.836) (total time: 879ms):
Jun 08 04:13:14 truenas k3s[7233]: Trace[816606018]: ---"Transaction committed" 878ms (04:13:14.715)
Jun 08 04:13:14 truenas k3s[7233]: Trace[816606018]: [879.051061ms] [879.051061ms] END
Jun 08 04:13:14 truenas k3s[7233]: I0608 04:13:14.716190    7233 trace.go:205] Trace[56408074]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:4f7a762f-84ab-4a52-889e-9ac1cc5215dd,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:13:13.836) (total time: 879ms):
Jun 08 04:13:14 truenas k3s[7233]: Trace[56408074]: ---"Object stored in database" 879ms (04:13:14.715)
Jun 08 04:13:14 truenas k3s[7233]: Trace[56408074]: [879.979747ms] [879.979747ms] END
Jun 08 04:13:16 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:13:16.640-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:13:24 truenas k3s[7233]: E0608 04:13:24.887394    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:13:41 truenas k3s[7233]: E0608 04:13:41.887323    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:13:56 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:13:56.142-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:13:56 truenas k3s[7233]: I0608 04:13:56.182579    7233 trace.go:205] Trace[85626896]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:13:55.052) (total time: 1130ms):
Jun 08 04:13:56 truenas k3s[7233]: Trace[85626896]: ---"Transaction committed" 1129ms (04:13:56.182)
Jun 08 04:13:56 truenas k3s[7233]: Trace[85626896]: [1.130073807s] [1.130073807s] END
Jun 08 04:13:56 truenas k3s[7233]: I0608 04:13:56.182810    7233 trace.go:205] Trace[1362420178]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:855c5c0f-90b2-4e0d-ab48-22b4c2b85426,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:13:55.052) (total time: 1130ms):
Jun 08 04:13:56 truenas k3s[7233]: Trace[1362420178]: ---"Object stored in database" 1130ms (04:13:56.182)
Jun 08 04:13:56 truenas k3s[7233]: Trace[1362420178]: [1.130418569s] [1.130418569s] END
Jun 08 04:13:56 truenas k3s[7233]: I0608 04:13:56.195184    7233 trace.go:205] Trace[2077887287]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:6df44f53-fdd2-481d-bcff-9cc3c7cfdd78,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:13:55.127) (total time: 1067ms):
Jun 08 04:13:56 truenas k3s[7233]: Trace[2077887287]: ---"About to write a response" 1067ms (04:13:56.194)
Jun 08 04:13:56 truenas k3s[7233]: Trace[2077887287]: [1.067778486s] [1.067778486s] END
Jun 08 04:13:56 truenas k3s[7233]: I0608 04:13:56.195369    7233 trace.go:205] Trace[546407527]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:defafe29-5701-40f1-909b-40db5d0ea886,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:13:55.105) (total time: 1089ms):
Jun 08 04:13:56 truenas k3s[7233]: Trace[546407527]: ---"About to write a response" 1089ms (04:13:56.195)
Jun 08 04:13:56 truenas k3s[7233]: Trace[546407527]: [1.089582398s] [1.089582398s] END
Jun 08 04:14:31 truenas k3s[7233]: I0608 04:14:31.752470    7233 trace.go:205] Trace[1389761771]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:14:30.927) (total time: 824ms):
Jun 08 04:14:31 truenas k3s[7233]: Trace[1389761771]: ---"Transaction committed" 824ms (04:14:31.752)
Jun 08 04:14:31 truenas k3s[7233]: Trace[1389761771]: [824.902642ms] [824.902642ms] END
Jun 08 04:14:31 truenas k3s[7233]: I0608 04:14:31.752714    7233 trace.go:205] Trace[453331233]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:7714263b-1a4b-4f5e-a994-0232ff19f519,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:14:30.927) (total time: 825ms):
Jun 08 04:14:31 truenas k3s[7233]: Trace[453331233]: ---"Object stored in database" 825ms (04:14:31.752)
Jun 08 04:14:31 truenas k3s[7233]: Trace[453331233]: [825.283178ms] [825.283178ms] END
Jun 08 04:14:31 truenas k3s[7233]: I0608 04:14:31.854600    7233 trace.go:205] Trace[1655526719]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:14:31.316) (total time: 538ms):
Jun 08 04:14:31 truenas k3s[7233]: Trace[1655526719]: ---"Transaction committed" 537ms (04:14:31.854)
Jun 08 04:14:31 truenas k3s[7233]: Trace[1655526719]: [538.201055ms] [538.201055ms] END
Jun 08 04:14:31 truenas k3s[7233]: I0608 04:14:31.854831    7233 trace.go:205] Trace[2094332034]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:4e2d5f7b-ecb5-4927-add9-56b7ae423814,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:14:31.315) (total time: 538ms):
Jun 08 04:14:31 truenas k3s[7233]: Trace[2094332034]: ---"Object stored in database" 538ms (04:14:31.854)
Jun 08 04:14:31 truenas k3s[7233]: Trace[2094332034]: [538.932087ms] [538.932087ms] END
Jun 08 04:14:36 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:14:36.200-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:14:43 truenas k3s[7233]: E0608 04:14:43.887852    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:14:48 truenas k3s[7233]: E0608 04:14:48.886963    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:15:12 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:15:12.849-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:15:28 truenas k3s[7233]: I0608 04:15:28.030416    7233 trace.go:205] Trace[264395869]: "Get" url:/api/v1/namespaces/default,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:69fd927a-47c0-43b5-bdc3-920a48e5abca,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 04:15:27.273) (total time: 757ms):
Jun 08 04:15:28 truenas k3s[7233]: Trace[264395869]: ---"About to write a response" 757ms (04:15:28.030)
Jun 08 04:15:28 truenas k3s[7233]: Trace[264395869]: [757.301143ms] [757.301143ms] END
Jun 08 04:15:28 truenas k3s[7233]: I0608 04:15:28.091059    7233 trace.go:205] Trace[599373540]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:3d086dc2-3c26-4358-a177-acccf8eb4da0,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:15:27.515) (total time: 575ms):
Jun 08 04:15:28 truenas k3s[7233]: Trace[599373540]: ---"About to write a response" 575ms (04:15:28.090)
Jun 08 04:15:28 truenas k3s[7233]: Trace[599373540]: [575.758527ms] [575.758527ms] END
Jun 08 04:15:28 truenas k3s[7233]: I0608 04:15:28.091888    7233 trace.go:205] Trace[1436871137]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:15:27.522) (total time: 569ms):
Jun 08 04:15:28 truenas k3s[7233]: Trace[1436871137]: ---"Transaction committed" 569ms (04:15:28.091)
Jun 08 04:15:28 truenas k3s[7233]: Trace[1436871137]: [569.691109ms] [569.691109ms] END
Jun 08 04:15:28 truenas k3s[7233]: I0608 04:15:28.092106    7233 trace.go:205] Trace[1566662728]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:91c8ab92-897d-4e08-b412-525df9eee332,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:15:27.521) (total time: 570ms):
Jun 08 04:15:28 truenas k3s[7233]: Trace[1566662728]: ---"Object stored in database" 569ms (04:15:28.091)
Jun 08 04:15:28 truenas k3s[7233]: Trace[1566662728]: [570.063806ms] [570.063806ms] END
Jun 08 04:15:38 truenas k3s[7233]: I0608 04:15:38.094032    7233 trace.go:205] Trace[117436132]: "GuaranteedUpdate etcd3" type:*v1.Endpoints (08-Jun-2022 04:15:37.277) (total time: 816ms):
Jun 08 04:15:38 truenas k3s[7233]: Trace[117436132]: ---"Transaction committed" 816ms (04:15:38.093)
Jun 08 04:15:38 truenas k3s[7233]: Trace[117436132]: [816.920892ms] [816.920892ms] END
Jun 08 04:15:38 truenas k3s[7233]: I0608 04:15:38.186197    7233 trace.go:205] Trace[280223826]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:15:37.555) (total time: 630ms):
Jun 08 04:15:38 truenas k3s[7233]: Trace[280223826]: ---"Transaction committed" 630ms (04:15:38.186)
Jun 08 04:15:38 truenas k3s[7233]: Trace[280223826]: [630.316693ms] [630.316693ms] END
Jun 08 04:15:38 truenas k3s[7233]: I0608 04:15:38.186435    7233 trace.go:205] Trace[1657886568]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:5cb02190-017c-48ce-bcea-8f25d53a5ac0,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:15:37.555) (total time: 630ms):
Jun 08 04:15:38 truenas k3s[7233]: Trace[1657886568]: ---"Object stored in database" 630ms (04:15:38.186)
Jun 08 04:15:38 truenas k3s[7233]: Trace[1657886568]: [630.709495ms] [630.709495ms] END
Jun 08 04:15:38 truenas k3s[7233]: I0608 04:15:38.306698    7233 trace.go:205] Trace[1623412043]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:15:37.554) (total time: 752ms):
Jun 08 04:15:38 truenas k3s[7233]: Trace[1623412043]: ---"Transaction committed" 752ms (04:15:38.306)
Jun 08 04:15:38 truenas k3s[7233]: Trace[1623412043]: [752.481465ms] [752.481465ms] END
Jun 08 04:15:38 truenas k3s[7233]: I0608 04:15:38.306942    7233 trace.go:205] Trace[1615324490]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:658a93f2-69d8-450e-b362-bc55ad148590,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:15:37.554) (total time: 752ms):
Jun 08 04:15:38 truenas k3s[7233]: Trace[1615324490]: ---"Object stored in database" 752ms (04:15:38.306)
Jun 08 04:15:38 truenas k3s[7233]: Trace[1615324490]: [752.880361ms] [752.880361ms] END
Jun 08 04:15:50 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:15:50.552-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:15:51 truenas k3s[7233]: E0608 04:15:51.886838    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:16:03 truenas k3s[7233]: I0608 04:16:03.807919    7233 trace.go:205] Trace[1822776085]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:f1e22937-fc54-43ef-a940-1f3e9ac890f8,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:16:03.234) (total time: 573ms):
Jun 08 04:16:03 truenas k3s[7233]: Trace[1822776085]: ---"About to write a response" 573ms (04:16:03.807)
Jun 08 04:16:03 truenas k3s[7233]: Trace[1822776085]: [573.229519ms] [573.229519ms] END
Jun 08 04:16:03 truenas k3s[7233]: I0608 04:16:03.812511    7233 trace.go:205] Trace[1044393515]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:16:03.235) (total time: 576ms):
Jun 08 04:16:03 truenas k3s[7233]: Trace[1044393515]: ---"Transaction committed" 576ms (04:16:03.812)
Jun 08 04:16:03 truenas k3s[7233]: Trace[1044393515]: [576.514908ms] [576.514908ms] END
Jun 08 04:16:03 truenas k3s[7233]: I0608 04:16:03.812900    7233 trace.go:205] Trace[2107176727]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:abfe027e-911d-41f4-b436-e387c0283db7,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:16:03.235) (total time: 577ms):
Jun 08 04:16:03 truenas k3s[7233]: Trace[2107176727]: ---"Object stored in database" 576ms (04:16:03.812)
Jun 08 04:16:03 truenas k3s[7233]: Trace[2107176727]: [577.033604ms] [577.033604ms] END
Jun 08 04:16:03 truenas k3s[7233]: I0608 04:16:03.865951    7233 trace.go:205] Trace[1548185507]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:16:03.322) (total time: 543ms):
Jun 08 04:16:03 truenas k3s[7233]: Trace[1548185507]: ---"Transaction committed" 542ms (04:16:03.865)
Jun 08 04:16:03 truenas k3s[7233]: Trace[1548185507]: [543.337044ms] [543.337044ms] END
Jun 08 04:16:03 truenas k3s[7233]: I0608 04:16:03.866200    7233 trace.go:205] Trace[1970476141]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:9e9531c0-231f-4e10-a888-fcd03a30528e,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:16:03.322) (total time: 544ms):
Jun 08 04:16:03 truenas k3s[7233]: Trace[1970476141]: ---"Object stored in database" 543ms (04:16:03.866)
Jun 08 04:16:03 truenas k3s[7233]: Trace[1970476141]: [544.045787ms] [544.045787ms] END
Jun 08 04:16:04 truenas k3s[7233]: E0608 04:16:04.888030    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:16:24 truenas k3s[7233]: I0608 04:16:24.568997    7233 trace.go:205] Trace[1593806651]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:16:24.045) (total time: 523ms):
Jun 08 04:16:24 truenas k3s[7233]: Trace[1593806651]: ---"Transaction committed" 522ms (04:16:24.568)
Jun 08 04:16:24 truenas k3s[7233]: Trace[1593806651]: [523.057372ms] [523.057372ms] END
Jun 08 04:16:24 truenas k3s[7233]: I0608 04:16:24.569838    7233 trace.go:205] Trace[433851798]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:385aed76-7cdf-4f7f-a485-2d3dbe90e1f6,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:16:24.045) (total time: 524ms):
Jun 08 04:16:24 truenas k3s[7233]: Trace[433851798]: ---"Object stored in database" 523ms (04:16:24.569)
Jun 08 04:16:24 truenas k3s[7233]: Trace[433851798]: [524.399764ms] [524.399764ms] END
Jun 08 04:16:24 truenas k3s[7233]: I0608 04:16:24.603220    7233 trace.go:205] Trace[1529951137]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:4e9313ff-ad0f-4ac2-b203-36be68c87e8e,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:16:23.952) (total time: 650ms):
Jun 08 04:16:24 truenas k3s[7233]: Trace[1529951137]: ---"About to write a response" 650ms (04:16:24.603)
Jun 08 04:16:24 truenas k3s[7233]: Trace[1529951137]: [650.414404ms] [650.414404ms] END
Jun 08 04:16:24 truenas k3s[7233]: I0608 04:16:24.609536    7233 trace.go:205] Trace[1566629310]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:16:23.927) (total time: 681ms):
Jun 08 04:16:24 truenas k3s[7233]: Trace[1566629310]: ---"Transaction committed" 681ms (04:16:24.609)
Jun 08 04:16:24 truenas k3s[7233]: Trace[1566629310]: [681.79694ms] [681.79694ms] END
Jun 08 04:16:24 truenas k3s[7233]: I0608 04:16:24.609652    7233 trace.go:205] Trace[2136197336]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ix-truenas,user-agent:Go-http-client/2.0,audit-id:da2af53e-e107-4419-8cb0-b18292b76444,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf,application/json,protocol:HTTP/2.0 (08-Jun-2022 04:16:23.927) (total time: 681ms):
Jun 08 04:16:24 truenas k3s[7233]: Trace[2136197336]: ---"Object stored in database" 681ms (04:16:24.609)
Jun 08 04:16:24 truenas k3s[7233]: Trace[2136197336]: [681.98989ms] [681.98989ms] END
Jun 08 04:16:30 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:16:30.053-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:17:08 truenas k3s[7233]: E0608 04:17:08.886901    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:17:09 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:17:09.215-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:17:20 truenas k3s[7233]: E0608 04:17:20.887052    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:17:40 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:17:40.074-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:18:01 truenas k3s[7233]: I0608 04:18:01.808427    7233 trace.go:205] Trace[293017740]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:18:00.633) (total time: 1175ms):
Jun 08 04:18:01 truenas k3s[7233]: Trace[293017740]: ---"Transaction committed" 1174ms (04:18:01.808)
Jun 08 04:18:01 truenas k3s[7233]: Trace[293017740]: [1.175192526s] [1.175192526s] END
Jun 08 04:18:01 truenas k3s[7233]: I0608 04:18:01.810041    7233 trace.go:205] Trace[254505770]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:a218ef54-9094-4380-a8d3-17b4ac5869f9,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:18:00.633) (total time: 1176ms):
Jun 08 04:18:01 truenas k3s[7233]: Trace[254505770]: ---"Object stored in database" 1176ms (04:18:01.809)
Jun 08 04:18:01 truenas k3s[7233]: Trace[254505770]: [1.176943091s] [1.176943091s] END
Jun 08 04:18:01 truenas k3s[7233]: I0608 04:18:01.809458    7233 trace.go:205] Trace[765745939]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:8326f84d-2187-4bdb-8e59-0e9d58ec456a,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:18:00.767) (total time: 1041ms):
Jun 08 04:18:01 truenas k3s[7233]: Trace[765745939]: ---"About to write a response" 1041ms (04:18:01.809)
Jun 08 04:18:01 truenas k3s[7233]: Trace[765745939]: [1.041994414s] [1.041994414s] END
Jun 08 04:18:06 truenas k3s[7233]: I0608 04:18:06.295044    7233 trace.go:205] Trace[727008151]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:af3955b5-c65d-4fad-b8a9-721a7ccc11c6,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:18:05.787) (total time: 507ms):
Jun 08 04:18:06 truenas k3s[7233]: Trace[727008151]: ---"About to write a response" 507ms (04:18:06.294)
Jun 08 04:18:06 truenas k3s[7233]: Trace[727008151]: [507.492422ms] [507.492422ms] END
Jun 08 04:18:06 truenas k3s[7233]: I0608 04:18:06.354290    7233 trace.go:205] Trace[359481870]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:18:05.644) (total time: 709ms):
Jun 08 04:18:06 truenas k3s[7233]: Trace[359481870]: ---"Transaction committed" 709ms (04:18:06.354)
Jun 08 04:18:06 truenas k3s[7233]: Trace[359481870]: [709.775358ms] [709.775358ms] END
Jun 08 04:18:06 truenas k3s[7233]: I0608 04:18:06.354542    7233 trace.go:205] Trace[1391913567]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:9197e924-feaf-4f5d-b67c-2bfea6f9e82d,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:18:05.644) (total time: 710ms):
Jun 08 04:18:06 truenas k3s[7233]: Trace[1391913567]: ---"Object stored in database" 709ms (04:18:06.354)
Jun 08 04:18:06 truenas k3s[7233]: Trace[1391913567]: [710.204558ms] [710.204558ms] END
Jun 08 04:18:17 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:18:17.711-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:18:25 truenas k3s[7233]: E0608 04:18:25.887274    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:18:32 truenas k3s[7233]: I0608 04:18:32.771823    7233 trace.go:205] Trace[462976354]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:c0aaccd7-ab0b-4464-b09d-d070057be9ae,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:18:32.264) (total time: 506ms):
Jun 08 04:18:32 truenas k3s[7233]: Trace[462976354]: ---"About to write a response" 506ms (04:18:32.771)
Jun 08 04:18:32 truenas k3s[7233]: Trace[462976354]: [506.95673ms] [506.95673ms] END
Jun 08 04:18:43 truenas k3s[7233]: I0608 04:18:43.228097    7233 trace.go:205] Trace[2111602883]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:aa42b544-6ff3-48eb-8c90-35a304494d51,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:18:42.329) (total time: 898ms):
Jun 08 04:18:43 truenas k3s[7233]: Trace[2111602883]: ---"About to write a response" 897ms (04:18:43.227)
Jun 08 04:18:43 truenas k3s[7233]: Trace[2111602883]: [898.078653ms] [898.078653ms] END
Jun 08 04:18:48 truenas k3s[7233]: E0608 04:18:48.887836    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:19:00 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:19:00.606-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:19:29 truenas k3s[7233]: E0608 04:19:29.887547    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:19:43 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:19:43.636-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:20:08 truenas k3s[7233]: E0608 04:20:08.887069    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:20:13 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:20:13.799-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:20:44 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:20:44.627-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:20:49 truenas k3s[7233]: E0608 04:20:49.887302    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:21:14 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:21:14.994-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:21:29 truenas k3s[7233]: E0608 04:21:29.887114    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:21:50 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:21:50.200-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:21:58 truenas k3s[7233]: E0608 04:21:58.886854    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:22:31 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:22:31.639-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:22:51 truenas k3s[7233]: E0608 04:22:51.887884    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:23:04 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:23:04.082-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:23:22 truenas k3s[7233]: E0608 04:23:22.887261    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:23:34 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:23:34.212-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:24:15 truenas k3s[7233]: E0608 04:24:15.887638    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:24:17 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:24:17.988-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:24:27 truenas k3s[7233]: E0608 04:24:27.886745    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:24:48 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:24:48.401-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:25:06 truenas k3s[7233]: I0608 04:25:06.873412    7233 trace.go:205] Trace[910538561]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:25:06.168) (total time: 704ms):
Jun 08 04:25:06 truenas k3s[7233]: Trace[910538561]: ---"Transaction committed" 703ms (04:25:06.873)
Jun 08 04:25:06 truenas k3s[7233]: Trace[910538561]: [704.800764ms] [704.800764ms] END
Jun 08 04:25:06 truenas k3s[7233]: I0608 04:25:06.873756    7233 trace.go:205] Trace[819673468]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ix-truenas,user-agent:Go-http-client/2.0,audit-id:d181012e-30a0-4669-b9f1-8f95365c4e08,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf,application/json,protocol:HTTP/2.0 (08-Jun-2022 04:25:06.168) (total time: 705ms):
Jun 08 04:25:06 truenas k3s[7233]: Trace[819673468]: ---"Object stored in database" 704ms (04:25:06.873)
Jun 08 04:25:06 truenas k3s[7233]: Trace[819673468]: [705.336382ms] [705.336382ms] END
Jun 08 04:25:17 truenas k3s[7233]: I0608 04:25:17.566568    7233 trace.go:205] Trace[2122422757]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:25:16.914) (total time: 652ms):
Jun 08 04:25:17 truenas k3s[7233]: Trace[2122422757]: ---"Transaction committed" 651ms (04:25:17.566)
Jun 08 04:25:17 truenas k3s[7233]: Trace[2122422757]: [652.118941ms] [652.118941ms] END
Jun 08 04:25:17 truenas k3s[7233]: I0608 04:25:17.567148    7233 trace.go:205] Trace[1572494355]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ix-truenas,user-agent:Go-http-client/2.0,audit-id:42cd21ba-99d1-4e4f-8f7a-9eadeec86b16,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf,application/json,protocol:HTTP/2.0 (08-Jun-2022 04:25:16.914) (total time: 652ms):
Jun 08 04:25:17 truenas k3s[7233]: Trace[1572494355]: ---"Object stored in database" 652ms (04:25:17.567)
Jun 08 04:25:17 truenas k3s[7233]: Trace[1572494355]: [652.775513ms] [652.775513ms] END
Jun 08 04:25:17 truenas k3s[7233]: I0608 04:25:17.574914    7233 trace.go:205] Trace[1850407372]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:c4469acf-3462-4511-9c19-eff8eb3c9052,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:25:17.064) (total time: 510ms):
Jun 08 04:25:17 truenas k3s[7233]: Trace[1850407372]: ---"About to write a response" 510ms (04:25:17.574)
Jun 08 04:25:17 truenas k3s[7233]: Trace[1850407372]: [510.300528ms] [510.300528ms] END
Jun 08 04:25:31 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:25:31.096-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:25:31 truenas k3s[7233]: E0608 04:25:31.887163    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:25:48 truenas k3s[7233]: E0608 04:25:48.887604    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:26:03 truenas k3s[7233]: I0608 04:26:03.329275    7233 trace.go:205] Trace[926771947]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:26:02.777) (total time: 552ms):
Jun 08 04:26:03 truenas k3s[7233]: Trace[926771947]: ---"Transaction committed" 551ms (04:26:03.329)
Jun 08 04:26:03 truenas k3s[7233]: Trace[926771947]: [552.022045ms] [552.022045ms] END
Jun 08 04:26:03 truenas k3s[7233]: I0608 04:26:03.329544    7233 trace.go:205] Trace[353973879]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:6816a674-d1d5-4bfb-9d3f-78f90e49dbc0,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:26:02.777) (total time: 552ms):
Jun 08 04:26:03 truenas k3s[7233]: Trace[353973879]: ---"Object stored in database" 552ms (04:26:03.329)
Jun 08 04:26:03 truenas k3s[7233]: Trace[353973879]: [552.406814ms] [552.406814ms] END
Jun 08 04:26:03 truenas k3s[7233]: I0608 04:26:03.335235    7233 trace.go:205] Trace[862273754]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:2c1ced98-7929-4398-b024-2570e1cf5726,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:26:02.771) (total time: 563ms):
Jun 08 04:26:03 truenas k3s[7233]: Trace[862273754]: ---"About to write a response" 563ms (04:26:03.335)
Jun 08 04:26:03 truenas k3s[7233]: Trace[862273754]: [563.909904ms] [563.909904ms] END
Jun 08 04:26:09 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:26:09.608-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:26:39 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:26:39.868-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:26:57 truenas k3s[7233]: E0608 04:26:57.887708    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:27:04 truenas k3s[7233]: I0608 04:27:04.722754    7233 trace.go:205] Trace[1555140001]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:27:04.055) (total time: 667ms):
Jun 08 04:27:04 truenas k3s[7233]: Trace[1555140001]: ---"Transaction committed" 667ms (04:27:04.722)
Jun 08 04:27:04 truenas k3s[7233]: Trace[1555140001]: [667.448539ms] [667.448539ms] END
Jun 08 04:27:04 truenas k3s[7233]: I0608 04:27:04.723011    7233 trace.go:205] Trace[805042589]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:1476ee28-f2c4-4f0c-8eaf-3c00e9abbc8d,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:27:04.055) (total time: 667ms):
Jun 08 04:27:04 truenas k3s[7233]: Trace[805042589]: ---"Object stored in database" 667ms (04:27:04.722)
Jun 08 04:27:04 truenas k3s[7233]: Trace[805042589]: [667.800295ms] [667.800295ms] END
Jun 08 04:27:04 truenas k3s[7233]: I0608 04:27:04.728086    7233 trace.go:205] Trace[46224002]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:27:04.091) (total time: 636ms):
Jun 08 04:27:04 truenas k3s[7233]: Trace[46224002]: ---"Transaction committed" 635ms (04:27:04.727)
Jun 08 04:27:04 truenas k3s[7233]: Trace[46224002]: [636.651332ms] [636.651332ms] END
Jun 08 04:27:04 truenas k3s[7233]: I0608 04:27:04.728315    7233 trace.go:205] Trace[323189963]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:7aa4c86a-9970-4134-8eb5-13d655fc6f8b,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:27:04.090) (total time: 637ms):
Jun 08 04:27:04 truenas k3s[7233]: Trace[323189963]: ---"Object stored in database" 636ms (04:27:04.728)
Jun 08 04:27:04 truenas k3s[7233]: Trace[323189963]: [637.341496ms] [637.341496ms] END
Jun 08 04:27:04 truenas k3s[7233]: I0608 04:27:04.738058    7233 trace.go:205] Trace[1796610851]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:27:04.056) (total time: 681ms):
Jun 08 04:27:04 truenas k3s[7233]: Trace[1796610851]: ---"Transaction committed" 680ms (04:27:04.738)
Jun 08 04:27:04 truenas k3s[7233]: Trace[1796610851]: [681.240565ms] [681.240565ms] END
Jun 08 04:27:04 truenas k3s[7233]: I0608 04:27:04.738331    7233 trace.go:205] Trace[1543261162]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:5d77a6b9-ecec-47b4-a910-0161e37cfbb8,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:27:04.056) (total time: 681ms):
Jun 08 04:27:04 truenas k3s[7233]: Trace[1543261162]: ---"Object stored in database" 681ms (04:27:04.738)
Jun 08 04:27:04 truenas k3s[7233]: Trace[1543261162]: [681.647315ms] [681.647315ms] END
Jun 08 04:27:04 truenas k3s[7233]: E0608 04:27:04.887333    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.249907    7233 trace.go:205] Trace[1047458323]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:27:14.139) (total time: 1109ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[1047458323]: ---"Transaction committed" 1109ms (04:27:15.249)
Jun 08 04:27:15 truenas k3s[7233]: Trace[1047458323]: [1.109889645s] [1.109889645s] END
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.250161    7233 trace.go:205] Trace[1583898125]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:dcad0bc9-dcb8-44e8-8977-42b6b3c9e9aa,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:27:14.139) (total time: 1110ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[1583898125]: ---"Object stored in database" 1110ms (04:27:15.249)
Jun 08 04:27:15 truenas k3s[7233]: Trace[1583898125]: [1.110352341s] [1.110352341s] END
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.262491    7233 trace.go:205] Trace[784710533]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:27:14.750) (total time: 511ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[784710533]: ---"Transaction committed" 510ms (04:27:15.262)
Jun 08 04:27:15 truenas k3s[7233]: Trace[784710533]: [511.741777ms] [511.741777ms] END
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.263215    7233 trace.go:205] Trace[1865465363]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:4c69c31c-4a9f-48e9-b269-92996f0d213f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:27:14.750) (total time: 512ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[1865465363]: ---"Object stored in database" 512ms (04:27:15.263)
Jun 08 04:27:15 truenas k3s[7233]: Trace[1865465363]: [512.890663ms] [512.890663ms] END
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.270375    7233 trace.go:205] Trace[284035133]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:27:14.754) (total time: 515ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[284035133]: ---"Transaction committed" 515ms (04:27:15.270)
Jun 08 04:27:15 truenas k3s[7233]: Trace[284035133]: [515.434906ms] [515.434906ms] END
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.270604    7233 trace.go:205] Trace[23923004]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:17264e4f-d223-4cc2-b90b-b4c92700b8ac,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:27:14.754) (total time: 515ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[23923004]: ---"Object stored in database" 515ms (04:27:15.270)
Jun 08 04:27:15 truenas k3s[7233]: Trace[23923004]: [515.839529ms] [515.839529ms] END
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.292721    7233 trace.go:205] Trace[761068225]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:27:14.751) (total time: 541ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[761068225]: ---"Transaction committed" 540ms (04:27:15.292)
Jun 08 04:27:15 truenas k3s[7233]: Trace[761068225]: [541.348154ms] [541.348154ms] END
Jun 08 04:27:15 truenas k3s[7233]: I0608 04:27:15.292962    7233 trace.go:205] Trace[440804907]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:68640079-3039-4449-81f7-fb4d39651c92,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:27:14.750) (total time: 541ms):
Jun 08 04:27:15 truenas k3s[7233]: Trace[440804907]: ---"Object stored in database" 541ms (04:27:15.292)
Jun 08 04:27:15 truenas k3s[7233]: Trace[440804907]: [541.980718ms] [541.980718ms] END
Jun 08 04:27:17 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:27:17.443-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:27:55 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:27:55.571-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:28:16 truenas k3s[7233]: E0608 04:28:16.887203    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:28:21 truenas k3s[7233]: I0608 04:28:21.792842    7233 trace.go:205] Trace[1648411491]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:28:21.049) (total time: 743ms):
Jun 08 04:28:21 truenas k3s[7233]: Trace[1648411491]: ---"Transaction committed" 742ms (04:28:21.792)
Jun 08 04:28:21 truenas k3s[7233]: Trace[1648411491]: [743.046947ms] [743.046947ms] END
Jun 08 04:28:21 truenas k3s[7233]: I0608 04:28:21.794475    7233 trace.go:205] Trace[1467096887]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:bf5e9dc7-7c15-4871-9fcd-a00210e6002f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:28:21.049) (total time: 744ms):
Jun 08 04:28:21 truenas k3s[7233]: Trace[1467096887]: ---"Object stored in database" 744ms (04:28:21.794)
Jun 08 04:28:21 truenas k3s[7233]: Trace[1467096887]: [744.837228ms] [744.837228ms] END
Jun 08 04:28:21 truenas k3s[7233]: I0608 04:28:21.809029    7233 trace.go:205] Trace[780625826]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:ff7ed19f-27de-45cb-885a-9be2621f591f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:28:21.058) (total time: 750ms):
Jun 08 04:28:21 truenas k3s[7233]: Trace[780625826]: ---"About to write a response" 750ms (04:28:21.808)
Jun 08 04:28:21 truenas k3s[7233]: Trace[780625826]: [750.83566ms] [750.83566ms] END
Jun 08 04:28:21 truenas k3s[7233]: I0608 04:28:21.809056    7233 trace.go:205] Trace[236474774]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:29475584-69a3-40ce-9dbd-aa310a209d41,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:28:21.090) (total time: 718ms):
Jun 08 04:28:21 truenas k3s[7233]: Trace[236474774]: ---"About to write a response" 718ms (04:28:21.808)
Jun 08 04:28:21 truenas k3s[7233]: Trace[236474774]: [718.152196ms] [718.152196ms] END
Jun 08 04:28:21 truenas k3s[7233]: I0608 04:28:21.813322    7233 trace.go:205] Trace[1220410611]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:740ee62d-ddbc-41fe-9975-dd51fee7bb29,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:28:21.085) (total time: 727ms):
Jun 08 04:28:21 truenas k3s[7233]: Trace[1220410611]: ---"About to write a response" 727ms (04:28:21.813)
Jun 08 04:28:21 truenas k3s[7233]: Trace[1220410611]: [727.94198ms] [727.94198ms] END
Jun 08 04:28:28 truenas k3s[7233]: E0608 04:28:28.887460    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:28:39 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:28:39.956-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:29:15 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:29:15.889-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:29:33 truenas k3s[7233]: I0608 04:29:33.015200    7233 trace.go:205] Trace[98457917]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:c1a9ffe6-f9cb-4e07-b1ee-9a5479ad397d,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:29:32.435) (total time: 580ms):
Jun 08 04:29:33 truenas k3s[7233]: Trace[98457917]: ---"About to write a response" 579ms (04:29:33.014)
Jun 08 04:29:33 truenas k3s[7233]: Trace[98457917]: [580.001474ms] [580.001474ms] END
Jun 08 04:29:33 truenas k3s[7233]: I0608 04:29:33.018352    7233 trace.go:205] Trace[309045478]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:420f878a-5c88-43fd-a7ab-281efbb9b7a1,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:29:32.434) (total time: 583ms):
Jun 08 04:29:33 truenas k3s[7233]: Trace[309045478]: ---"About to write a response" 583ms (04:29:33.018)
Jun 08 04:29:33 truenas k3s[7233]: Trace[309045478]: [583.345141ms] [583.345141ms] END
Jun 08 04:29:33 truenas k3s[7233]: I0608 04:29:33.020423    7233 trace.go:205] Trace[921249361]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:29:32.450) (total time: 569ms):
Jun 08 04:29:33 truenas k3s[7233]: Trace[921249361]: ---"Transaction committed" 568ms (04:29:33.020)
Jun 08 04:29:33 truenas k3s[7233]: Trace[921249361]: [569.625985ms] [569.625985ms] END
Jun 08 04:29:33 truenas k3s[7233]: I0608 04:29:33.020571    7233 trace.go:205] Trace[314569595]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:ea4d0017-4764-40a3-a905-648b1d0da1a4,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:29:32.450) (total time: 570ms):
Jun 08 04:29:33 truenas k3s[7233]: Trace[314569595]: ---"Object stored in database" 569ms (04:29:33.020)
Jun 08 04:29:33 truenas k3s[7233]: Trace[314569595]: [570.066912ms] [570.066912ms] END
Jun 08 04:29:42 truenas k3s[7233]: E0608 04:29:42.887179    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:29:47 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:29:47.312-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:29:56 truenas k3s[7233]: E0608 04:29:56.887250    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:30:27 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:30:27.967-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:30:34 truenas k3s[7233]: I0608 04:30:34.332201    7233 trace.go:205] Trace[1132750101]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:30:33.795) (total time: 536ms):
Jun 08 04:30:34 truenas k3s[7233]: Trace[1132750101]: ---"Transaction committed" 535ms (04:30:34.332)
Jun 08 04:30:34 truenas k3s[7233]: Trace[1132750101]: [536.83043ms] [536.83043ms] END
Jun 08 04:30:34 truenas k3s[7233]: I0608 04:30:34.332452    7233 trace.go:205] Trace[218335606]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:5ba4e2ab-5f00-419a-98a4-0939a9064c8f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:30:33.794) (total time: 537ms):
Jun 08 04:30:34 truenas k3s[7233]: Trace[218335606]: ---"Object stored in database" 537ms (04:30:34.332)
Jun 08 04:30:34 truenas k3s[7233]: Trace[218335606]: [537.528128ms] [537.528128ms] END
Jun 08 04:30:34 truenas k3s[7233]: I0608 04:30:34.351639    7233 trace.go:205] Trace[26100412]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:11091448-4248-497d-a690-67359de3be7f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:30:33.769) (total time: 581ms):
Jun 08 04:30:34 truenas k3s[7233]: Trace[26100412]: ---"About to write a response" 581ms (04:30:34.351)
Jun 08 04:30:34 truenas k3s[7233]: Trace[26100412]: [581.686509ms] [581.686509ms] END
Jun 08 04:31:03 truenas k3s[7233]: E0608 04:31:03.887125    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:31:05 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:31:05.041-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:31:21 truenas k3s[7233]: E0608 04:31:21.887102    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:31:31 truenas k3s[7233]: I0608 04:31:31.060941    7233 trace.go:205] Trace[224347763]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:31:30.105) (total time: 955ms):
Jun 08 04:31:31 truenas k3s[7233]: Trace[224347763]: ---"Transaction committed" 954ms (04:31:31.060)
Jun 08 04:31:31 truenas k3s[7233]: Trace[224347763]: [955.282206ms] [955.282206ms] END
Jun 08 04:31:31 truenas k3s[7233]: I0608 04:31:31.061196    7233 trace.go:205] Trace[1617531267]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:16d5151a-395f-45d0-aaa7-1d9074a4c123,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:31:30.104) (total time: 956ms):
Jun 08 04:31:31 truenas k3s[7233]: Trace[1617531267]: ---"Object stored in database" 955ms (04:31:31.061)
Jun 08 04:31:31 truenas k3s[7233]: Trace[1617531267]: [956.244864ms] [956.244864ms] END
Jun 08 04:31:39 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:31:39.020-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:32:15 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:32:15.501-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:32:15 truenas k3s[7233]: E0608 04:32:15.887687    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:32:36 truenas k3s[7233]: E0608 04:32:36.887004    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:32:37 truenas k3s[7233]: I0608 04:32:37.210673    7233 trace.go:205] Trace[518217915]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:b386b788-7554-4967-a33e-ec3b38df2474,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:32:36.668) (total time: 541ms):
Jun 08 04:32:37 truenas k3s[7233]: Trace[518217915]: ---"About to write a response" 541ms (04:32:37.210)
Jun 08 04:32:37 truenas k3s[7233]: Trace[518217915]: [541.953437ms] [541.953437ms] END
Jun 08 04:32:52 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:32:52.327-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:33:26 truenas k3s[7233]: E0608 04:33:26.888158    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:33:32 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:33:32.572-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:33:39 truenas k3s[7233]: E0608 04:33:39.887423    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:34:09 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:34:09.189-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:34:47 truenas k3s[7233]: E0608 04:34:47.887678    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:34:53 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:34:53.198-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:35:07 truenas k3s[7233]: E0608 04:35:07.887152    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:35:30 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:35:30.640-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:35:50 truenas k3s[7233]: E0608 04:35:50.887946    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:36:14 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:36:14.381-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:36:33 truenas k3s[7233]: E0608 04:36:33.886821    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:36:56 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:36:56.451-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:37:02 truenas k3s[7233]: E0608 04:37:02.886124    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:37:28 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:37:28.494-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:37:38 truenas k3s[7233]: E0608 04:37:38.886685    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:38:07 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:38:07.418-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:38:09 truenas k3s[7233]: E0608 04:38:09.888124    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:38:15 truenas k3s[7233]: I0608 04:38:15.356405    7233 trace.go:205] Trace[1827328460]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:38:14.757) (total time: 599ms):
Jun 08 04:38:15 truenas k3s[7233]: Trace[1827328460]: ---"Transaction committed" 598ms (04:38:15.356)
Jun 08 04:38:15 truenas k3s[7233]: Trace[1827328460]: [599.225827ms] [599.225827ms] END
Jun 08 04:38:15 truenas k3s[7233]: I0608 04:38:15.356692    7233 trace.go:205] Trace[1975030845]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ix-truenas,user-agent:Go-http-client/2.0,audit-id:64bf9aa5-a46e-4cdf-bd2b-012eb73ea741,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf,application/json,protocol:HTTP/2.0 (08-Jun-2022 04:38:14.757) (total time: 599ms):
Jun 08 04:38:15 truenas k3s[7233]: Trace[1975030845]: ---"Object stored in database" 599ms (04:38:15.356)
Jun 08 04:38:15 truenas k3s[7233]: Trace[1975030845]: [599.583589ms] [599.583589ms] END
Jun 08 04:38:47 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:38:47.139-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:38:55 truenas k3s[7233]: E0608 04:38:55.886568    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:39:19 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:39:19.873-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:39:25 truenas k3s[7233]: I0608 04:39:25.447335    7233 trace.go:205] Trace[1037388351]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:39:24.770) (total time: 676ms):
Jun 08 04:39:25 truenas k3s[7233]: Trace[1037388351]: ---"Transaction committed" 676ms (04:39:25.447)
Jun 08 04:39:25 truenas k3s[7233]: Trace[1037388351]: [676.496445ms] [676.496445ms] END
Jun 08 04:39:25 truenas k3s[7233]: I0608 04:39:25.447589    7233 trace.go:205] Trace[1052320100]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:2741f60c-8fe4-4adf-92ff-2656391036c7,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:39:24.770) (total time: 676ms):
Jun 08 04:39:25 truenas k3s[7233]: Trace[1052320100]: ---"Object stored in database" 676ms (04:39:25.447)
Jun 08 04:39:25 truenas k3s[7233]: Trace[1052320100]: [676.942409ms] [676.942409ms] END
Jun 08 04:39:25 truenas k3s[7233]: I0608 04:39:25.492371    7233 trace.go:205] Trace[2047288289]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:cb112e0e-eb8c-4aed-910c-e480cd02157f,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:39:24.879) (total time: 612ms):
Jun 08 04:39:25 truenas k3s[7233]: Trace[2047288289]: ---"About to write a response" 612ms (04:39:25.492)
Jun 08 04:39:25 truenas k3s[7233]: Trace[2047288289]: [612.875648ms] [612.875648ms] END
Jun 08 04:39:25 truenas k3s[7233]: I0608 04:39:25.492432    7233 trace.go:205] Trace[1116958778]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:86efb68f-85f6-40dd-bdf3-c4e554feadfc,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:39:24.849) (total time: 642ms):
Jun 08 04:39:25 truenas k3s[7233]: Trace[1116958778]: ---"About to write a response" 642ms (04:39:25.492)
Jun 08 04:39:25 truenas k3s[7233]: Trace[1116958778]: [642.634804ms] [642.634804ms] END
Jun 08 04:39:25 truenas k3s[7233]: I0608 04:39:25.492376    7233 trace.go:205] Trace[133522864]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:a2f874bd-67c9-4ed4-aad1-7916e0b22012,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:39:24.865) (total time: 627ms):
Jun 08 04:39:25 truenas k3s[7233]: Trace[133522864]: ---"About to write a response" 627ms (04:39:25.492)
Jun 08 04:39:25 truenas k3s[7233]: Trace[133522864]: [627.187442ms] [627.187442ms] END
Jun 08 04:39:39 truenas k3s[7233]: E0608 04:39:39.888012    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:39:53 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:39:53.848-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:40:22 truenas k3s[7233]: E0608 04:40:22.887286    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:40:26 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:40:26.258-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:41:03 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:41:03.674-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:41:03 truenas k3s[7233]: E0608 04:41:03.887986    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:41:26 truenas k3s[7233]: E0608 04:41:26.886700    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:41:44 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:41:44.571-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:41:45 truenas k3s[7233]: I0608 04:41:45.264324    7233 trace.go:205] Trace[1662935454]: "GuaranteedUpdate etcd3" type:*unstructured.Unstructured (08-Jun-2022 04:41:44.654) (total time: 609ms):
Jun 08 04:41:45 truenas k3s[7233]: Trace[1662935454]: ---"Transaction committed" 608ms (04:41:45.264)
Jun 08 04:41:45 truenas k3s[7233]: Trace[1662935454]: [609.304064ms] [609.304064ms] END
Jun 08 04:41:45 truenas k3s[7233]: I0608 04:41:45.264455    7233 trace.go:205] Trace[2117782817]: "Update" url:/apis/zfs.openebs.io/v1/namespaces/openebs/zfsnodes/ix-truenas,user-agent:zfs-driver/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:45e00e9b-4c8a-4542-a25f-4b864a495285,client:10.12.1.5,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:41:44.654) (total time: 609ms):
Jun 08 04:41:45 truenas k3s[7233]: Trace[2117782817]: ---"Object stored in database" 609ms (04:41:45.264)
Jun 08 04:41:45 truenas k3s[7233]: Trace[2117782817]: [609.742217ms] [609.742217ms] END
Jun 08 04:42:14 truenas k3s[7233]: E0608 04:42:14.887173    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:42:21 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:42:21.534-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:42:35 truenas k3s[7233]: E0608 04:42:35.887746    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:42:54 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:42:54.005-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:43:38 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:43:38.089-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:43:40 truenas k3s[7233]: E0608 04:43:40.887655    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:44:00 truenas k3s[7233]: E0608 04:44:00.886272    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:44:12 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:44:12.809-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:44:45 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:44:45.464-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:45:07 truenas k3s[7233]: E0608 04:45:07.887870    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:45:10 truenas k3s[7233]: E0608 04:45:10.887660    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:45:18 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:45:18.787-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:45:50 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:45:50.767-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:46:24 truenas k3s[7233]: E0608 04:46:24.887201    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:46:26 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:46:26.451-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:46:35 truenas k3s[7233]: E0608 04:46:35.887786    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:47:11 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:47:11.315-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:47:49 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:47:49.323-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:47:51 truenas k3s[7233]: I0608 04:47:51.714878    7233 namespace_controller.go:185] Namespace has been deleted ix-hass-instance
Jun 08 04:47:51 truenas k3s[7233]: I0608 04:47:51.991223    7233 trace.go:205] Trace[977217294]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:787e74c2-a7f9-4016-a3ce-e77ad2570322,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:47:51.414) (total time: 576ms):
Jun 08 04:47:51 truenas k3s[7233]: Trace[977217294]: ---"About to write a response" 576ms (04:47:51.991)
Jun 08 04:47:51 truenas k3s[7233]: Trace[977217294]: [576.936954ms] [576.936954ms] END
Jun 08 04:47:52 truenas k3s[7233]: I0608 04:47:52.042433    7233 trace.go:205] Trace[362939074]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:47:51.300) (total time: 741ms):
Jun 08 04:47:52 truenas k3s[7233]: Trace[362939074]: ---"Transaction committed" 741ms (04:47:52.042)
Jun 08 04:47:52 truenas k3s[7233]: Trace[362939074]: [741.881582ms] [741.881582ms] END
Jun 08 04:47:52 truenas k3s[7233]: I0608 04:47:52.042670    7233 trace.go:205] Trace[2119855664]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:e35c43d6-1fd0-4651-ac37-d9f2998bc059,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:47:51.300) (total time: 742ms):
Jun 08 04:47:52 truenas k3s[7233]: Trace[2119855664]: ---"Object stored in database" 742ms (04:47:52.042)
Jun 08 04:47:52 truenas k3s[7233]: Trace[2119855664]: [742.300835ms] [742.300835ms] END
Jun 08 04:47:52 truenas k3s[7233]: I0608 04:47:52.042815    7233 trace.go:205] Trace[315801614]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:dc7f6f75-f46c-4f45-b51b-15b90812a135,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:47:51.502) (total time: 540ms):
Jun 08 04:47:52 truenas k3s[7233]: Trace[315801614]: ---"About to write a response" 540ms (04:47:52.042)
Jun 08 04:47:52 truenas k3s[7233]: Trace[315801614]: [540.293793ms] [540.293793ms] END
Jun 08 04:47:54 truenas k3s[7233]: E0608 04:47:54.887257    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:47:56 truenas k3s[7233]: E0608 04:47:56.887641    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:48:30 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:48:30.193-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:48:38 truenas k3s[7233]: I0608 04:48:38.318148    7233 trace.go:205] Trace[949070666]: "List etcd3" key:/masterleases/,resourceVersion:0,resourceVersionMatch:NotOlderThan,limit:0,continue: (08-Jun-2022 04:48:37.420) (total time: 897ms):
Jun 08 04:48:38 truenas k3s[7233]: Trace[949070666]: [897.928072ms] [897.928072ms] END
Jun 08 04:49:09 truenas k3s[7233]: E0608 04:49:09.887435    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:49:12 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:49:12.569-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:49:23 truenas k3s[7233]: E0608 04:49:23.887267    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:49:51 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:49:51.103-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:50:19 truenas k3s[7233]: E0608 04:50:19.887556    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:50:22 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:50:22.451-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:50:35 truenas k3s[7233]: E0608 04:50:35.888063    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:51:02 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:51:02.586-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:51:22 truenas k3s[7233]: I0608 04:51:22.046770    7233 trace.go:205] Trace[1976383286]: "Get" url:/apis/flowcontrol.apiserver.k8s.io/v1beta2/flowschemas/system-nodes,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:4255ac0f-b822-4392-9ffe-bf26cab29c93,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 04:51:21.448) (total time: 598ms):
Jun 08 04:51:22 truenas k3s[7233]: Trace[1976383286]: ---"About to write a response" 598ms (04:51:22.046)
Jun 08 04:51:22 truenas k3s[7233]: Trace[1976383286]: [598.253199ms] [598.253199ms] END
Jun 08 04:51:26 truenas k3s[7233]: I0608 04:51:26.898621    7233 trace.go:205] Trace[1440060931]: "Get" url:/api/v1/namespaces/kube-system,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:43a7f5a8-2c32-4b9d-a12c-aa4cf6abc304,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 04:51:26.343) (total time: 555ms):
Jun 08 04:51:26 truenas k3s[7233]: Trace[1440060931]: ---"About to write a response" 555ms (04:51:26.898)
Jun 08 04:51:26 truenas k3s[7233]: Trace[1440060931]: [555.539294ms] [555.539294ms] END
Jun 08 04:51:44 truenas k3s[7233]: E0608 04:51:44.887138    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:51:45 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:51:45.762-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:51:47 truenas k3s[7233]: E0608 04:51:47.887785    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:52:08 truenas k3s[7233]: I0608 04:52:08.178900    7233 trace.go:205] Trace[216219731]: "Get" url:/api/v1/namespaces/default,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:22c7201d-8990-404d-8da1-5f59a0d0061e,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf, */*,protocol:HTTP/2.0 (08-Jun-2022 04:52:07.398) (total time: 780ms):
Jun 08 04:52:08 truenas k3s[7233]: Trace[216219731]: ---"About to write a response" 780ms (04:52:08.178)
Jun 08 04:52:08 truenas k3s[7233]: Trace[216219731]: [780.353139ms] [780.353139ms] END
Jun 08 04:52:22 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:52:22.047-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:52:58 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:52:58.457-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:53:03 truenas k3s[7233]: E0608 04:53:03.887454    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:53:08 truenas k3s[7233]: I0608 04:53:08.230407    7233 trace.go:205] Trace[1589861642]: "GuaranteedUpdate etcd3" type:*v1.Endpoints (08-Jun-2022 04:53:07.401) (total time: 829ms):
Jun 08 04:53:08 truenas k3s[7233]: Trace[1589861642]: ---"Transaction committed" 828ms (04:53:08.230)
Jun 08 04:53:08 truenas k3s[7233]: Trace[1589861642]: [829.188154ms] [829.188154ms] END
Jun 08 04:53:14 truenas k3s[7233]: E0608 04:53:14.887315    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:53:14 truenas k3s[7233]: I0608 04:53:14.900944    7233 trace.go:205] Trace[374589059]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:53:14.284) (total time: 616ms):
Jun 08 04:53:14 truenas k3s[7233]: Trace[374589059]: ---"Transaction committed" 616ms (04:53:14.900)
Jun 08 04:53:14 truenas k3s[7233]: Trace[374589059]: [616.847464ms] [616.847464ms] END
Jun 08 04:53:14 truenas k3s[7233]: I0608 04:53:14.901207    7233 trace.go:205] Trace[991020694]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:9b6f747c-373c-4c3b-b940-6926aa65923a,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:53:14.283) (total time: 617ms):
Jun 08 04:53:14 truenas k3s[7233]: Trace[991020694]: ---"Object stored in database" 616ms (04:53:14.900)
Jun 08 04:53:14 truenas k3s[7233]: Trace[991020694]: [617.240954ms] [617.240954ms] END
Jun 08 04:53:14 truenas k3s[7233]: I0608 04:53:14.909231    7233 trace.go:205] Trace[1617195145]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:80b592b3-2a17-4884-8984-fc48ec508286,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:53:14.350) (total time: 558ms):
Jun 08 04:53:14 truenas k3s[7233]: Trace[1617195145]: ---"About to write a response" 558ms (04:53:14.909)
Jun 08 04:53:14 truenas k3s[7233]: Trace[1617195145]: [558.94647ms] [558.94647ms] END
Jun 08 04:53:14 truenas k3s[7233]: I0608 04:53:14.963573    7233 trace.go:205] Trace[299847260]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:53:14.327) (total time: 636ms):
Jun 08 04:53:14 truenas k3s[7233]: Trace[299847260]: ---"Transaction committed" 635ms (04:53:14.963)
Jun 08 04:53:14 truenas k3s[7233]: Trace[299847260]: [636.24315ms] [636.24315ms] END
Jun 08 04:53:14 truenas k3s[7233]: I0608 04:53:14.963823    7233 trace.go:205] Trace[2077906877]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:240f743d-f04d-455a-9fe9-bccb93d15c6c,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:53:14.326) (total time: 636ms):
Jun 08 04:53:14 truenas k3s[7233]: Trace[2077906877]: ---"Object stored in database" 636ms (04:53:14.963)
Jun 08 04:53:14 truenas k3s[7233]: Trace[2077906877]: [636.926827ms] [636.926827ms] END
Jun 08 04:53:30 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:53:30.487-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:53:35 truenas k3s[7233]: I0608 04:53:35.208634    7233 trace.go:205] Trace[1311537690]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:309584ad-41d4-4ba9-b1fb-84db9b39d4d0,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:53:34.426) (total time: 782ms):
Jun 08 04:53:35 truenas k3s[7233]: Trace[1311537690]: ---"About to write a response" 781ms (04:53:35.208)
Jun 08 04:53:35 truenas k3s[7233]: Trace[1311537690]: [782.129777ms] [782.129777ms] END
Jun 08 04:53:45 truenas k3s[7233]: I0608 04:53:45.193609    7233 trace.go:205] Trace[1170195754]: "GuaranteedUpdate etcd3" type:*unstructured.Unstructured (08-Jun-2022 04:53:44.662) (total time: 531ms):
Jun 08 04:53:45 truenas k3s[7233]: Trace[1170195754]: ---"Transaction committed" 530ms (04:53:45.192)
Jun 08 04:53:45 truenas k3s[7233]: Trace[1170195754]: [531.396433ms] [531.396433ms] END
Jun 08 04:53:45 truenas k3s[7233]: I0608 04:53:45.193952    7233 trace.go:205] Trace[2131748078]: "Update" url:/apis/zfs.openebs.io/v1/namespaces/openebs/zfsnodes/ix-truenas,user-agent:zfs-driver/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:e5027dbc-5d11-4cd2-ba8f-a90175715fab,client:10.12.1.5,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:53:44.661) (total time: 532ms):
Jun 08 04:53:45 truenas k3s[7233]: Trace[2131748078]: ---"Object stored in database" 531ms (04:53:45.193)
Jun 08 04:53:45 truenas k3s[7233]: Trace[2131748078]: [532.056689ms] [532.056689ms] END
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.403885    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.422914    7233 event.go:294] "Event occurred" object="ix-hass-instance/hass-instance-home-assistant-preinstall-job" kind="Job" apiVersion="batch/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: hass-instance-home-assistant-preinstall-job-ngrzw"
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.423854    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.431336    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:01 truenas k3s[7233]: E0608 04:54:01.433288    7233 job_controller.go:488] syncing job: tracking status: adding uncounted pods to status: Operation cannot be fulfilled on jobs.batch "hass-instance-home-assistant-preinstall-job": the object has been modified; please apply your changes to the latest version and try again
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.436291    7233 topology_manager.go:200] "Topology Admit Handler"
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.437271    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.455705    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.573742    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-config\") pod \"hass-instance-home-assistant-preinstall-job-ngrzw\" (UID: \"11243742-c209-4a0d-ae94-b609082c9cc1\") " pod="ix-hass-instance/hass-instance-home-assistant-preinstall-job-ngrzw"
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.573861    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"media\" (UniqueName: \"kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-media\") pod \"hass-instance-home-assistant-preinstall-job-ngrzw\" (UID: \"11243742-c209-4a0d-ae94-b609082c9cc1\") " pod="ix-hass-instance/hass-instance-home-assistant-preinstall-job-ngrzw"
Jun 08 04:54:01 truenas k3s[7233]: I0608 04:54:01.573941    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-hgwjf\" (UniqueName: \"kubernetes.io/projected/11243742-c209-4a0d-ae94-b609082c9cc1-kube-api-access-hgwjf\") pod \"hass-instance-home-assistant-preinstall-job-ngrzw\" (UID: \"11243742-c209-4a0d-ae94-b609082c9cc1\") " pod="ix-hass-instance/hass-instance-home-assistant-preinstall-job-ngrzw"
Jun 08 04:54:02 truenas k3s[7233]: I0608 04:54:02.625624    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:02 truenas k3s[7233]: I0608 04:54:02.617369  772395 event.go:278] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ix-hass-instance", Name:"hass-instance-home-assistant-preinstall-job-ngrzw", UID:"11243742-c209-4a0d-ae94-b609082c9cc1", APIVersion:"v1", ResourceVersion:"807539", FieldPath:""}): type: 'Normal' reason: 'AddedInterface' Add eth0 [172.16.0.59/16] from ix-net
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.186564    7233 scope.go:110] "RemoveContainer" containerID="981edeb51863ecbe2655c942f2c3b9d796aeee721d375e16af4ae1f923a0b272"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.219280    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.235063    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.241087    7233 event.go:294] "Event occurred" object="ix-hass-instance/hass-instance-home-assistant-preinstall-job" kind="Job" apiVersion="batch/v1" type="Normal" reason="Completed" message="Job completed"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.242936    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.245621    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:05 truenas k3s[7233]: E0608 04:54:05.247382    7233 job_controller.go:488] syncing job: tracking status: adding uncounted pods to status: Operation cannot be fulfilled on jobs.batch "hass-instance-home-assistant-preinstall-job": the object has been modified; please apply your changes to the latest version and try again
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.260237    7233 job_controller.go:453] enqueueing job ix-hass-instance/hass-instance-home-assistant-preinstall-job
Jun 08 04:54:05 truenas k3s[7233]: E0608 04:54:05.260275    7233 tracking_utils.go:109] "deleting tracking annotation UID expectations" err="couldn't create key for object ix-hass-instance/hass-instance-home-assistant-preinstall-job: could not find key for obj \"ix-hass-instance/hass-instance-home-assistant-preinstall-job\"" job="ix-hass-instance/hass-instance-home-assistant-preinstall-job"
Jun 08 04:54:05 truenas k3s[7233]: E0608 04:54:05.300295    7233 projected.go:293] Couldn't get configMap ix-hass-instance/kube-root-ca.crt: object "ix-hass-instance"/"kube-root-ca.crt" not registered
Jun 08 04:54:05 truenas k3s[7233]: E0608 04:54:05.300360    7233 projected.go:199] Error preparing data for projected volume kube-api-access-hgwjf for pod ix-hass-instance/hass-instance-home-assistant-preinstall-job-ngrzw: object "ix-hass-instance"/"kube-root-ca.crt" not registered
Jun 08 04:54:05 truenas k3s[7233]: E0608 04:54:05.300464    7233 nestedpendingoperations.go:335] Operation for "{volumeName:kubernetes.io/projected/11243742-c209-4a0d-ae94-b609082c9cc1-kube-api-access-hgwjf podName:11243742-c209-4a0d-ae94-b609082c9cc1 nodeName:}" failed. No retries permitted until 2022-06-08 04:54:05.800425424 -0700 PDT m=+762886.569711758 (durationBeforeRetry 500ms). Error: MountVolume.SetUp failed for volume "kube-api-access-hgwjf" (UniqueName: "kubernetes.io/projected/11243742-c209-4a0d-ae94-b609082c9cc1-kube-api-access-hgwjf") pod "hass-instance-home-assistant-preinstall-job-ngrzw" (UID: "11243742-c209-4a0d-ae94-b609082c9cc1") : object "ix-hass-instance"/"kube-root-ca.crt" not registered
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.365036    7233 alloc.go:329] "allocated clusterIPs" service="ix-hass-instance/hass-instance-home-assistant-postgres" clusterIPs=map[IPv4:172.17.61.151]
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.407191    7233 alloc.go:329] "allocated clusterIPs" service="ix-hass-instance/hass-instance-home-assistant" clusterIPs=map[IPv4:172.17.253.246]
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.525406    7233 event.go:294] "Event occurred" object="ix-hass-instance/hass-instance-home-assistant-postgres" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled up replica set hass-instance-home-assistant-postgres-86c496c6b8 to 1"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.538212    7233 event.go:294] "Event occurred" object="ix-hass-instance/hass-instance-home-assistant-postgres-86c496c6b8" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: hass-instance-home-assistant-postgres-86c496c6b8-rm8tc"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.542580    7233 event.go:294] "Event occurred" object="ix-hass-instance/hass-instance-home-assistant" kind="Deployment" apiVersion="apps/v1" type="Normal" reason="ScalingReplicaSet" message="Scaled up replica set hass-instance-home-assistant-7ff5685587 to 1"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.634352    7233 topology_manager.go:200] "Topology Admit Handler"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.660656    7233 event.go:294] "Event occurred" object="ix-hass-instance/hass-instance-home-assistant-7ff5685587" kind="ReplicaSet" apiVersion="apps/v1" type="Normal" reason="SuccessfulCreate" message="Created pod: hass-instance-home-assistant-7ff5685587-jhjd7"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.685293    7233 topology_manager.go:200] "Topology Admit Handler"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.701403    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"postgres-backup\" (UniqueName: \"kubernetes.io/host-path/8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8-postgres-backup\") pod \"hass-instance-home-assistant-postgres-86c496c6b8-rm8tc\" (UID: \"8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8\") " pod="ix-hass-instance/hass-instance-home-assistant-postgres-86c496c6b8-rm8tc"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.701604    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"postgres-data\" (UniqueName: \"kubernetes.io/host-path/8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8-postgres-data\") pod \"hass-instance-home-assistant-postgres-86c496c6b8-rm8tc\" (UID: \"8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8\") " pod="ix-hass-instance/hass-instance-home-assistant-postgres-86c496c6b8-rm8tc"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.701689    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-66nhg\" (UniqueName: \"kubernetes.io/projected/8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8-kube-api-access-66nhg\") pod \"hass-instance-home-assistant-postgres-86c496c6b8-rm8tc\" (UID: \"8b4ef4e7-6e19-4892-bd6d-675a24e9aeb8\") " pod="ix-hass-instance/hass-instance-home-assistant-postgres-86c496c6b8-rm8tc"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.801979    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/80a1060f-eaf5-41e5-bad6-e0e240f2cce1-config\") pod \"hass-instance-home-assistant-7ff5685587-jhjd7\" (UID: \"80a1060f-eaf5-41e5-bad6-e0e240f2cce1\") " pod="ix-hass-instance/hass-instance-home-assistant-7ff5685587-jhjd7"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.802024    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-727g2\" (UniqueName: \"kubernetes.io/projected/80a1060f-eaf5-41e5-bad6-e0e240f2cce1-kube-api-access-727g2\") pod \"hass-instance-home-assistant-7ff5685587-jhjd7\" (UID: \"80a1060f-eaf5-41e5-bad6-e0e240f2cce1\") " pod="ix-hass-instance/hass-instance-home-assistant-7ff5685587-jhjd7"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.802043    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"media\" (UniqueName: \"kubernetes.io/host-path/80a1060f-eaf5-41e5-bad6-e0e240f2cce1-media\") pod \"hass-instance-home-assistant-7ff5685587-jhjd7\" (UID: \"80a1060f-eaf5-41e5-bad6-e0e240f2cce1\") " pod="ix-hass-instance/hass-instance-home-assistant-7ff5685587-jhjd7"
Jun 08 04:54:05 truenas k3s[7233]: I0608 04:54:05.802094    7233 reconciler.go:221] "operationExecutor.VerifyControllerAttachedVolume started for volume \"initial-config-script\" (UniqueName: \"kubernetes.io/configmap/80a1060f-eaf5-41e5-bad6-e0e240f2cce1-initial-config-script\") pod \"hass-instance-home-assistant-7ff5685587-jhjd7\" (UID: \"80a1060f-eaf5-41e5-bad6-e0e240f2cce1\") " pod="ix-hass-instance/hass-instance-home-assistant-7ff5685587-jhjd7"
Jun 08 04:54:05 truenas k3s[7233]: E0608 04:54:05.958710    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:05 truenas k3s[7233]: E0608 04:54:05.989314    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.199201    7233 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="672c9d871956817632ba3ae73f4ef4110d80c6897118fc5fbbb9ae8c3459adfe"
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.307341    7233 reconciler.go:192] "operationExecutor.UnmountVolume started for volume \"kube-api-access-hgwjf\" (UniqueName: \"kubernetes.io/projected/11243742-c209-4a0d-ae94-b609082c9cc1-kube-api-access-hgwjf\") pod \"11243742-c209-4a0d-ae94-b609082c9cc1\" (UID: \"11243742-c209-4a0d-ae94-b609082c9cc1\") "
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.307372    7233 reconciler.go:192] "operationExecutor.UnmountVolume started for volume \"config\" (UniqueName: \"kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-config\") pod \"11243742-c209-4a0d-ae94-b609082c9cc1\" (UID: \"11243742-c209-4a0d-ae94-b609082c9cc1\") "
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.307386    7233 reconciler.go:192] "operationExecutor.UnmountVolume started for volume \"media\" (UniqueName: \"kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-media\") pod \"11243742-c209-4a0d-ae94-b609082c9cc1\" (UID: \"11243742-c209-4a0d-ae94-b609082c9cc1\") "
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.307461    7233 operation_generator.go:910] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-media" (OuterVolumeSpecName: "media") pod "11243742-c209-4a0d-ae94-b609082c9cc1" (UID: "11243742-c209-4a0d-ae94-b609082c9cc1"). InnerVolumeSpecName "media". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.307625    7233 operation_generator.go:910] UnmountVolume.TearDown succeeded for volume "kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-config" (OuterVolumeSpecName: "config") pod "11243742-c209-4a0d-ae94-b609082c9cc1" (UID: "11243742-c209-4a0d-ae94-b609082c9cc1"). InnerVolumeSpecName "config". PluginName "kubernetes.io/host-path", VolumeGidValue ""
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.309755    7233 operation_generator.go:910] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/11243742-c209-4a0d-ae94-b609082c9cc1-kube-api-access-hgwjf" (OuterVolumeSpecName: "kube-api-access-hgwjf") pod "11243742-c209-4a0d-ae94-b609082c9cc1" (UID: "11243742-c209-4a0d-ae94-b609082c9cc1"). InnerVolumeSpecName "kube-api-access-hgwjf". PluginName "kubernetes.io/projected", VolumeGidValue ""
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.408530    7233 reconciler.go:300] "Volume detached for volume \"kube-api-access-hgwjf\" (UniqueName: \"kubernetes.io/projected/11243742-c209-4a0d-ae94-b609082c9cc1-kube-api-access-hgwjf\") on node \"ix-truenas\" DevicePath \"\""
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.408554    7233 reconciler.go:300] "Volume detached for volume \"media\" (UniqueName: \"kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-media\") on node \"ix-truenas\" DevicePath \"\""
Jun 08 04:54:06 truenas k3s[7233]: I0608 04:54:06.408562    7233 reconciler.go:300] "Volume detached for volume \"config\" (UniqueName: \"kubernetes.io/host-path/11243742-c209-4a0d-ae94-b609082c9cc1-config\") on node \"ix-truenas\" DevicePath \"\""
Jun 08 04:54:06 truenas k3s[7233]: E0608 04:54:06.599542    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:06 truenas k3s[7233]: E0608 04:54:06.601823    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:07 truenas k3s[7233]: I0608 04:54:07.904340    7233 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=11243742-c209-4a0d-ae94-b609082c9cc1 path="/var/lib/kubelet/pods/11243742-c209-4a0d-ae94-b609082c9cc1/volumes"
Jun 08 04:54:08 truenas k3s[7233]: E0608 04:54:08.417888    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:08 truenas k3s[7233]: E0608 04:54:08.426159    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:09 truenas k3s[7233]: E0608 04:54:09.435829    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:09 truenas k3s[7233]: E0608 04:54:09.435869    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:13 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:54:13.192-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:54:22 truenas k3s[7233]: E0608 04:54:22.886729    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:27 truenas k3s[7233]: E0608 04:54:27.887213    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:54:43 truenas k3s[7233]: I0608 04:54:43.401944    7233 scope.go:110] "RemoveContainer" containerID="981edeb51863ecbe2655c942f2c3b9d796aeee721d375e16af4ae1f923a0b272"
Jun 08 04:54:44 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:54:44.558-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:55:10 truenas k3s[7233]: E0608 04:55:10.886557    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:55:27 truenas k3s[7233]: E0608 04:55:27.887793    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:55:29 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:55:29.326-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:55:35 truenas k3s[7233]: E0608 04:55:35.888195    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:55:54 truenas k3s[7233]: E0608 04:55:54.887683    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:56:12 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:56:12.239-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:56:27 truenas k3s[7233]: E0608 04:56:27.886827    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:56:47 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:56:47.953-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:56:49 truenas k3s[7233]: E0608 04:56:49.887114    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:56:56 truenas k3s[7233]: E0608 04:56:56.886934    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:57:18 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:57:18.142-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:57:21 truenas k3s[7233]: E0608 04:57:21.887341    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:57:23 truenas k3s[7233]: I0608 04:57:23.200466    7233 trace.go:205] Trace[1838777696]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:819adcb7-e525-4bff-9940-cec485aa200b,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:57:22.491) (total time: 709ms):
Jun 08 04:57:23 truenas k3s[7233]: Trace[1838777696]: ---"About to write a response" 708ms (04:57:23.200)
Jun 08 04:57:23 truenas k3s[7233]: Trace[1838777696]: [709.110528ms] [709.110528ms] END
Jun 08 04:57:33 truenas k3s[7233]: I0608 04:57:33.794924    7233 trace.go:205] Trace[952034509]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:bf5b8810-d15c-4871-b448-f6da8ff2cfce,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:57:33.268) (total time: 526ms):
Jun 08 04:57:33 truenas k3s[7233]: Trace[952034509]: ---"About to write a response" 526ms (04:57:33.794)
Jun 08 04:57:33 truenas k3s[7233]: Trace[952034509]: [526.641146ms] [526.641146ms] END
Jun 08 04:57:55 truenas k3s[7233]: E0608 04:57:55.887718    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:58:00 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:58:00.912-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:58:03 truenas k3s[7233]: E0608 04:58:03.886825    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:58:08 truenas k3s[7233]: E0608 04:58:08.885754    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:58:27 truenas k3s[7233]: E0608 04:58:27.887908    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:58:44 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:58:44.495-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:59:12 truenas k3s[7233]: E0608 04:59:12.885605    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:59:13 truenas k3s[7233]: E0608 04:59:13.886754    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:59:20 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T04:59:20.244-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 04:59:23 truenas k3s[7233]: I0608 04:59:23.964907    7233 trace.go:205] Trace[658545752]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:59:23.115) (total time: 849ms):
Jun 08 04:59:23 truenas k3s[7233]: Trace[658545752]: ---"Transaction committed" 849ms (04:59:23.964)
Jun 08 04:59:23 truenas k3s[7233]: Trace[658545752]: [849.765555ms] [849.765555ms] END
Jun 08 04:59:23 truenas k3s[7233]: I0608 04:59:23.965154    7233 trace.go:205] Trace[650698577]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:1ea723ac-f8e5-4683-9a60-68a6ec45aa33,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:59:23.114) (total time: 850ms):
Jun 08 04:59:23 truenas k3s[7233]: Trace[650698577]: ---"Object stored in database" 849ms (04:59:23.964)
Jun 08 04:59:23 truenas k3s[7233]: Trace[650698577]: [850.137306ms] [850.137306ms] END
Jun 08 04:59:24 truenas k3s[7233]: I0608 04:59:24.001491    7233 trace.go:205] Trace[1325731176]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:59:23.469) (total time: 531ms):
Jun 08 04:59:24 truenas k3s[7233]: Trace[1325731176]: ---"Transaction committed" 531ms (04:59:24.001)
Jun 08 04:59:24 truenas k3s[7233]: Trace[1325731176]: [531.529025ms] [531.529025ms] END
Jun 08 04:59:24 truenas k3s[7233]: I0608 04:59:24.002200    7233 trace.go:205] Trace[724080807]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:60d60f79-fb5d-40b6-afb8-6186a5d4ccaf,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:59:23.469) (total time: 532ms):
Jun 08 04:59:24 truenas k3s[7233]: Trace[724080807]: ---"Object stored in database" 532ms (04:59:24.002)
Jun 08 04:59:24 truenas k3s[7233]: Trace[724080807]: [532.9683ms] [532.9683ms] END
Jun 08 04:59:24 truenas k3s[7233]: I0608 04:59:24.048839    7233 trace.go:205] Trace[402125450]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 04:59:23.116) (total time: 932ms):
Jun 08 04:59:24 truenas k3s[7233]: Trace[402125450]: ---"Transaction committed" 932ms (04:59:24.048)
Jun 08 04:59:24 truenas k3s[7233]: Trace[402125450]: [932.775337ms] [932.775337ms] END
Jun 08 04:59:24 truenas k3s[7233]: I0608 04:59:24.048950    7233 trace.go:205] Trace[2068091426]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:2272fab3-7fdb-4019-9abe-ea71150ce510,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 04:59:23.115) (total time: 932ms):
Jun 08 04:59:24 truenas k3s[7233]: Trace[2068091426]: ---"Object stored in database" 932ms (04:59:24.048)
Jun 08 04:59:24 truenas k3s[7233]: Trace[2068091426]: [932.997253ms] [932.997253ms] END
Jun 08 04:59:36 truenas k3s[7233]: E0608 04:59:36.887437    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 04:59:42 truenas k3s[7233]: E0608 04:59:42.887049    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:00:02 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:00:02.261-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:00:31 truenas k3s[7233]: E0608 05:00:31.886615    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:00:37 truenas k3s[7233]: E0608 05:00:37.887152    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:00:46 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:00:46.482-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:00:51 truenas k3s[7233]: E0608 05:00:51.887344    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:00:51 truenas k3s[7233]: E0608 05:00:51.887364    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:01:19 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:01:19.265-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:01:39 truenas k3s[7233]: E0608 05:01:39.887060    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:01:56 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:01:56.742-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:02:05 truenas k3s[7233]: E0608 05:02:05.887152    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:02:07 truenas k3s[7233]: E0608 05:02:07.886483    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:02:18 truenas k3s[7233]: E0608 05:02:18.887206    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:02:27 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:02:27.238-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:02:58 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:02:58.203-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:03:00 truenas k3s[7233]: E0608 05:03:00.886708    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:03:07 truenas k3s[7233]: E0608 05:03:07.886005    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:03:26 truenas k3s[7233]: I0608 05:03:26.747935    7233 trace.go:205] Trace[560456741]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:03:26.036) (total time: 711ms):
Jun 08 05:03:26 truenas k3s[7233]: Trace[560456741]: ---"Transaction committed" 710ms (05:03:26.747)
Jun 08 05:03:26 truenas k3s[7233]: Trace[560456741]: [711.527001ms] [711.527001ms] END
Jun 08 05:03:26 truenas k3s[7233]: I0608 05:03:26.748188    7233 trace.go:205] Trace[97417759]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:5121aaf3-b7c6-490d-90cf-86b5d84801ec,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:03:26.035) (total time: 712ms):
Jun 08 05:03:26 truenas k3s[7233]: Trace[97417759]: ---"Object stored in database" 711ms (05:03:26.747)
Jun 08 05:03:26 truenas k3s[7233]: Trace[97417759]: [712.350145ms] [712.350145ms] END
Jun 08 05:03:26 truenas k3s[7233]: I0608 05:03:26.767610    7233 trace.go:205] Trace[1191862505]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:03:25.869) (total time: 897ms):
Jun 08 05:03:26 truenas k3s[7233]: Trace[1191862505]: ---"Transaction committed" 896ms (05:03:26.767)
Jun 08 05:03:26 truenas k3s[7233]: Trace[1191862505]: [897.827793ms] [897.827793ms] END
Jun 08 05:03:26 truenas k3s[7233]: I0608 05:03:26.767845    7233 trace.go:205] Trace[978648214]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:3c8de589-decf-4317-af06-39dcad41f8c2,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:03:25.869) (total time: 898ms):
Jun 08 05:03:26 truenas k3s[7233]: Trace[978648214]: ---"Object stored in database" 898ms (05:03:26.767)
Jun 08 05:03:26 truenas k3s[7233]: Trace[978648214]: [898.531173ms] [898.531173ms] END
Jun 08 05:03:33 truenas k3s[7233]: E0608 05:03:33.886997    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:03:34 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:03:34.327-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:03:36 truenas k3s[7233]: E0608 05:03:36.887763    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:04:08 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:04:08.870-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:04:14 truenas k3s[7233]: E0608 05:04:14.887153    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:04:29 truenas k3s[7233]: E0608 05:04:29.886993    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:04:38 truenas k3s[7233]: E0608 05:04:38.887733    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:04:40 truenas k3s[7233]: E0608 05:04:40.887664    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:04:50 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:04:50.522-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:05:23 truenas k3s[7233]: E0608 05:05:23.887037    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:05:24 truenas k3s[7233]: I0608 05:05:24.753631    7233 trace.go:205] Trace[1449215330]: "GuaranteedUpdate etcd3" type:*core.Event (08-Jun-2022 05:05:23.895) (total time: 858ms):
Jun 08 05:05:24 truenas k3s[7233]: Trace[1449215330]: ---"initial value restored" 848ms (05:05:24.744)
Jun 08 05:05:24 truenas k3s[7233]: Trace[1449215330]: [858.027675ms] [858.027675ms] END
Jun 08 05:05:24 truenas k3s[7233]: I0608 05:05:24.753920    7233 trace.go:205] Trace[275004810]: "Patch" url:/api/v1/namespaces/ix-hass-instance/events/hass-instance-home-assistant-postgres-86c496c6b8-rm8tc.16f6a31a2f06bc30,user-agent:k3s/v1.23.5+k3s (linux/amd64) kubernetes/937d546,audit-id:d2fa6ffa-bc07-478a-990d-c49454c5dd1a,client:127.0.0.1,accept:application/vnd.kubernetes.protobuf,application/json,protocol:HTTP/2.0 (08-Jun-2022 05:05:23.895) (total time: 858ms):
Jun 08 05:05:24 truenas k3s[7233]: Trace[275004810]: ---"About to apply patch" 848ms (05:05:24.744)
Jun 08 05:05:24 truenas k3s[7233]: Trace[275004810]: [858.382425ms] [858.382425ms] END
Jun 08 05:05:34 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:05:34.495-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:05:39 truenas k3s[7233]: E0608 05:05:39.886940    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:05:44 truenas k3s[7233]: E0608 05:05:44.885979    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:05:58 truenas k3s[7233]: E0608 05:05:58.888185    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:06:08 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:06:08.657-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:06:40 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:06:40.577-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:06:40 truenas k3s[7233]: E0608 05:06:40.886415    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:06:44 truenas k3s[7233]: E0608 05:06:44.886434    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:06:54 truenas k3s[7233]: E0608 05:06:54.887262    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:07:09 truenas k3s[7233]: E0608 05:07:09.888064    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:07:13 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:07:13.493-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:07:52 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:07:52.387-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:08:07 truenas k3s[7233]: E0608 05:08:07.885866    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:08:13 truenas k3s[7233]: E0608 05:08:13.886553    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:08:15 truenas k3s[7233]: E0608 05:08:15.887537    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:08:16 truenas k3s[7233]: E0608 05:08:16.887110    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:08:29 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:08:29.714-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:09:04 truenas k3s[7233]: I0608 05:09:04.864082    7233 trace.go:205] Trace[230339632]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:53cacb6a-e56c-4bda-9bf1-cc5db85daeff,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:09:03.981) (total time: 882ms):
Jun 08 05:09:04 truenas k3s[7233]: Trace[230339632]: ---"About to write a response" 882ms (05:09:04.863)
Jun 08 05:09:04 truenas k3s[7233]: Trace[230339632]: [882.801087ms] [882.801087ms] END
Jun 08 05:09:04 truenas k3s[7233]: I0608 05:09:04.889323    7233 trace.go:205] Trace[1876816679]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:09:04.035) (total time: 853ms):
Jun 08 05:09:04 truenas k3s[7233]: Trace[1876816679]: ---"Transaction committed" 852ms (05:09:04.889)
Jun 08 05:09:04 truenas k3s[7233]: Trace[1876816679]: [853.598985ms] [853.598985ms] END
Jun 08 05:09:04 truenas k3s[7233]: I0608 05:09:04.889593    7233 trace.go:205] Trace[702852311]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-snapshotter-leader-zfs-csi-openebs-io,user-agent:csi-snapshotter/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:aabc60d4-7e92-4094-9679-00a1654d34dc,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:09:04.035) (total time: 854ms):
Jun 08 05:09:04 truenas k3s[7233]: Trace[702852311]: ---"Object stored in database" 853ms (05:09:04.889)
Jun 08 05:09:04 truenas k3s[7233]: Trace[702852311]: [854.296249ms] [854.296249ms] END
Jun 08 05:09:04 truenas k3s[7233]: I0608 05:09:04.938783    7233 trace.go:205] Trace[1024647532]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:5735c356-8fff-4205-98d5-24ba92cf721d,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:09:04.378) (total time: 560ms):
Jun 08 05:09:04 truenas k3s[7233]: Trace[1024647532]: ---"About to write a response" 560ms (05:09:04.938)
Jun 08 05:09:04 truenas k3s[7233]: Trace[1024647532]: [560.562463ms] [560.562463ms] END
Jun 08 05:09:04 truenas k3s[7233]: I0608 05:09:04.943655    7233 trace.go:205] Trace[863820135]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:09:04.036) (total time: 907ms):
Jun 08 05:09:04 truenas k3s[7233]: Trace[863820135]: ---"Transaction committed" 906ms (05:09:04.943)
Jun 08 05:09:04 truenas k3s[7233]: Trace[863820135]: [907.161723ms] [907.161723ms] END
Jun 08 05:09:04 truenas k3s[7233]: I0608 05:09:04.943881    7233 trace.go:205] Trace[1780157367]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:993d7bff-bf43-4d1b-a390-e254adafcc68,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:09:04.036) (total time: 907ms):
Jun 08 05:09:04 truenas k3s[7233]: Trace[1780157367]: ---"Object stored in database" 907ms (05:09:04.943)
Jun 08 05:09:04 truenas k3s[7233]: Trace[1780157367]: [907.79255ms] [907.79255ms] END
Jun 08 05:09:09 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:09:09.819-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:09:11 truenas k3s[7233]: E0608 05:09:11.887338    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:09:36 truenas k3s[7233]: E0608 05:09:36.886873    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:09:37 truenas k3s[7233]: E0608 05:09:37.888137    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:09:41 truenas k3s[7233]: E0608 05:09:41.887017    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:09:50 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:09:50.022-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:10:12 truenas k3s[7233]: E0608 05:10:12.887179    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:10:28 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:10:28.446-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:10:39 truenas k3s[7233]: E0608 05:10:39.887440    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:11:00 truenas k3s[7233]: E0608 05:11:00.887277    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:11:01 truenas k3s[7233]: E0608 05:11:01.885976    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:11:09 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:11:09.979-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:11:19 truenas k3s[7233]: E0608 05:11:19.887218    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:11:49 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:11:49.478-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:12:03 truenas k3s[7233]: E0608 05:12:03.888001    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:12:06 truenas k3s[7233]: E0608 05:12:06.886557    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:12:20 truenas k3s[7233]: E0608 05:12:20.886593    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:12:21 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:12:21.306-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:12:21 truenas k3s[7233]: E0608 05:12:21.887293    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:12:55 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:12:55.371-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:13:03 truenas k3s[7233]: I0608 05:13:03.198043    7233 trace.go:205] Trace[208656211]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:0a196447-fa62-4978-ad1b-0502c159a327,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:13:02.441) (total time: 756ms):
Jun 08 05:13:03 truenas k3s[7233]: Trace[208656211]: ---"About to write a response" 756ms (05:13:03.197)
Jun 08 05:13:03 truenas k3s[7233]: Trace[208656211]: [756.657361ms] [756.657361ms] END
Jun 08 05:13:03 truenas k3s[7233]: I0608 05:13:03.198510    7233 trace.go:205] Trace[332886567]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:13:02.344) (total time: 853ms):
Jun 08 05:13:03 truenas k3s[7233]: Trace[332886567]: ---"Transaction committed" 852ms (05:13:03.198)
Jun 08 05:13:03 truenas k3s[7233]: Trace[332886567]: [853.500691ms] [853.500691ms] END
Jun 08 05:13:03 truenas k3s[7233]: I0608 05:13:03.198716    7233 trace.go:205] Trace[286352584]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:d64626ed-3454-49e8-a2fa-5957e3ed66f8,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:13:02.344) (total time: 853ms):
Jun 08 05:13:03 truenas k3s[7233]: Trace[286352584]: ---"Object stored in database" 853ms (05:13:03.198)
Jun 08 05:13:03 truenas k3s[7233]: Trace[286352584]: [853.980021ms] [853.980021ms] END
Jun 08 05:13:03 truenas k3s[7233]: I0608 05:13:03.203482    7233 trace.go:205] Trace[1539074141]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:6a1dfefb-fe5e-4f07-932e-275f0680a62d,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:13:02.531) (total time: 672ms):
Jun 08 05:13:03 truenas k3s[7233]: Trace[1539074141]: ---"About to write a response" 672ms (05:13:03.203)
Jun 08 05:13:03 truenas k3s[7233]: Trace[1539074141]: [672.299313ms] [672.299313ms] END
Jun 08 05:13:21 truenas k3s[7233]: E0608 05:13:21.887036    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:13:22 truenas k3s[7233]: E0608 05:13:22.886985    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:13:34 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:13:34.050-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:13:34 truenas k3s[7233]: E0608 05:13:34.886515    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:13:47 truenas k3s[7233]: E0608 05:13:47.887248    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:14:06 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:14:06.171-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:14:31 truenas k3s[7233]: E0608 05:14:31.887692    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:14:37 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:14:37.175-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:14:42 truenas k3s[7233]: E0608 05:14:42.887605    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:14:55 truenas k3s[7233]: E0608 05:14:55.887878    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:14:59 truenas k3s[7233]: E0608 05:14:59.886582    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:15:13 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:15:13.071-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:15:45 truenas k3s[7233]: E0608 05:15:45.887852    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:15:49 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:15:49.032-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:15:58 truenas k3s[7233]: E0608 05:15:58.886231    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:16:03 truenas k3s[7233]: E0608 05:16:03.887032    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:16:04 truenas k3s[7233]: I0608 05:16:04.994997    7233 trace.go:205] Trace[546141280]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:16:04.100) (total time: 894ms):
Jun 08 05:16:04 truenas k3s[7233]: Trace[546141280]: ---"Transaction committed" 893ms (05:16:04.994)
Jun 08 05:16:04 truenas k3s[7233]: Trace[546141280]: [894.826892ms] [894.826892ms] END
Jun 08 05:16:04 truenas k3s[7233]: I0608 05:16:04.995243    7233 trace.go:205] Trace[1177650482]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/snapshot-controller-leader,user-agent:snapshot-controller/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:75a38611-41a8-4a34-95bc-dc51b7f6785e,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:16:04.099) (total time: 895ms):
Jun 08 05:16:04 truenas k3s[7233]: Trace[1177650482]: ---"Object stored in database" 895ms (05:16:04.995)
Jun 08 05:16:04 truenas k3s[7233]: Trace[1177650482]: [895.526744ms] [895.526744ms] END
Jun 08 05:16:15 truenas k3s[7233]: I0608 05:16:15.120870    7233 trace.go:205] Trace[2011085972]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:16:14.154) (total time: 966ms):
Jun 08 05:16:15 truenas k3s[7233]: Trace[2011085972]: ---"Transaction committed" 965ms (05:16:15.120)
Jun 08 05:16:15 truenas k3s[7233]: Trace[2011085972]: [966.687313ms] [966.687313ms] END
Jun 08 05:16:15 truenas k3s[7233]: I0608 05:16:15.121112    7233 trace.go:205] Trace[2058949516]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:7c0be3bc-096b-48c0-9795-7cbea2c63a48,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:16:14.153) (total time: 967ms):
Jun 08 05:16:15 truenas k3s[7233]: Trace[2058949516]: ---"Object stored in database" 966ms (05:16:15.120)
Jun 08 05:16:15 truenas k3s[7233]: Trace[2058949516]: [967.374767ms] [967.374767ms] END
Jun 08 05:16:15 truenas k3s[7233]: I0608 05:16:15.146651    7233 trace.go:205] Trace[1351275567]: "Get" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/zfs-csi-openebs-io,user-agent:csi-provisioner/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:49f507de-13e0-4f75-b6d2-597f20e4dd54,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:16:14.418) (total time: 727ms):
Jun 08 05:16:15 truenas k3s[7233]: Trace[1351275567]: ---"About to write a response" 727ms (05:16:15.146)
Jun 08 05:16:15 truenas k3s[7233]: Trace[1351275567]: [727.81807ms] [727.81807ms] END
Jun 08 05:16:15 truenas k3s[7233]: E0608 05:16:15.886820    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:16:22 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:16:22.565-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:16:55 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:16:55.666-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:17:10 truenas k3s[7233]: E0608 05:17:10.887044    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:17:10 truenas k3s[7233]: E0608 05:17:10.887398    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:17:12 truenas k3s[7233]: E0608 05:17:12.886973    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:17:16 truenas k3s[7233]: I0608 05:17:16.116490    7233 trace.go:205] Trace[1892531929]: "GuaranteedUpdate etcd3" type:*coordination.Lease (08-Jun-2022 05:17:15.405) (total time: 710ms):
Jun 08 05:17:16 truenas k3s[7233]: Trace[1892531929]: ---"Transaction committed" 710ms (05:17:16.116)
Jun 08 05:17:16 truenas k3s[7233]: Trace[1892531929]: [710.889819ms] [710.889819ms] END
Jun 08 05:17:16 truenas k3s[7233]: I0608 05:17:16.116734    7233 trace.go:205] Trace[1881916624]: "Update" url:/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/external-resizer-zfs-csi-openebs-io,user-agent:csi-resizer/v0.0.0 (linux/amd64) kubernetes/$Format,audit-id:969a3bad-5b4c-4ca8-8bae-da61e024fb46,client:172.16.0.45,accept:application/json, */*,protocol:HTTP/2.0 (08-Jun-2022 05:17:15.405) (total time: 711ms):
Jun 08 05:17:16 truenas k3s[7233]: Trace[1881916624]: ---"Object stored in database" 711ms (05:17:16.116)
Jun 08 05:17:16 truenas k3s[7233]: Trace[1881916624]: [711.320649ms] [711.320649ms] END
Jun 08 05:17:32 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:17:32.335-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:17:40 truenas k3s[7233]: E0608 05:17:40.885891    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:18:13 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:18:13.530-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:18:17 truenas k3s[7233]: E0608 05:18:17.887113    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:18:23 truenas k3s[7233]: E0608 05:18:23.887602    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:18:37 truenas k3s[7233]: E0608 05:18:37.886600    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:18:55 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:18:55.228-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:19:08 truenas k3s[7233]: E0608 05:19:08.886470    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:19:26 truenas k3s[7233]: E0608 05:19:26.887950    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:19:28 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:19:28.163-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:19:30 truenas k3s[7233]: E0608 05:19:30.887564    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:19:40 truenas k3s[7233]: E0608 05:19:40.886618    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
Jun 08 05:19:58 truenas k3s[7233]: {"level":"warn","ts":"2022-06-08T05:19:58.994-0700","logger":"etcd-client","caller":"v3@v3.5.1-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e0a700/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jun 08 05:20:12 truenas k3s[7233]: E0608 05:20:12.885928    7233 dns.go:157] "Nameserver limits exceeded" err="Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: 8.8.4.4 1.0.0.1 8.8.8.8"
debug finished in 0 seconds for K3s Logs (last 1000 lines)


