+--------------------------------------------------------------------------------+
+                      Kubernetes Configuration @1674165672                      +
+--------------------------------------------------------------------------------+
{
  "id": 1,
  "pool": "apps",
  "cluster_cidr": "172.16.0.0/16",
  "service_cidr": "172.17.0.0/16",
  "cluster_dns_ip": "172.17.0.10",
  "route_v4_interface": "enp9s0",
  "route_v4_gateway": "172.23.32.1",
  "route_v6_interface": null,
  "route_v6_gateway": null,
  "node_ip": "0.0.0.0",
  "configure_gpus": true,
  "servicelb": true,
  "validate_host_path": false,
  "dataset": "apps/ix-applications"
}
debug finished in 0 seconds for Kubernetes Configuration


+--------------------------------------------------------------------------------+
+                     k3s kubectl describe nodes @1674165673                     +
+--------------------------------------------------------------------------------+
Name:               ix-truenas
Roles:              control-plane,master
Labels:             beta.kubernetes.io/arch=amd64
                    beta.kubernetes.io/os=linux
                    egress.k3s.io/cluster=true
                    kubernetes.io/arch=amd64
                    kubernetes.io/hostname=ix-truenas
                    kubernetes.io/os=linux
                    node-role.kubernetes.io/control-plane=true
                    node-role.kubernetes.io/master=true
                    openebs.io/nodeid=ix-truenas
                    openebs.io/nodename=ix-truenas
Annotations:        csi.volume.kubernetes.io/nodeid: {"zfs.csi.openebs.io":"ix-truenas"}
                    k3s.io/node-args:
                      ["server","--cluster-cidr","172.16.0.0/16","--cluster-dns","172.17.0.10","--data-dir","/mnt/apps/ix-applications/k3s","--kube-apiserver-ar...
                    k3s.io/node-config-hash: FOCEDLUKJVNQRHJLV7LTFE7YK7AA2R2KE5TOBXOTIG2RZAGSGT7Q====
                    k3s.io/node-env: {"K3S_DATA_DIR":"/mnt/apps/ix-applications/k3s/data/3622b68605bf9a49243ca44f328acbb9e681a95150de9d9f94baaba1d5511603"}
                    node.alpha.kubernetes.io/ttl: 0
                    volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp:  Sun, 01 Jan 2023 20:59:05 +0100
Taints:             <none>
Unschedulable:      false
Lease:
  HolderIdentity:  ix-truenas
  AcquireTime:     <unset>
  RenewTime:       Thu, 19 Jan 2023 23:01:05 +0100
Conditions:
  Type             Status  LastHeartbeatTime                 LastTransitionTime                Reason                       Message
  ----             ------  -----------------                 ------------------                ------                       -------
  MemoryPressure   False   Thu, 19 Jan 2023 22:57:46 +0100   Sat, 07 Jan 2023 20:15:32 +0100   KubeletHasSufficientMemory   kubelet has sufficient memory available
  DiskPressure     False   Thu, 19 Jan 2023 22:57:46 +0100   Sat, 07 Jan 2023 20:15:32 +0100   KubeletHasNoDiskPressure     kubelet has no disk pressure
  PIDPressure      False   Thu, 19 Jan 2023 22:57:46 +0100   Sat, 07 Jan 2023 20:15:32 +0100   KubeletHasSufficientPID      kubelet has sufficient PID available
  Ready            True    Thu, 19 Jan 2023 22:57:46 +0100   Wed, 18 Jan 2023 06:03:41 +0100   KubeletReady                 kubelet is posting ready status. AppArmor enabled
Addresses:
  InternalIP:  172.23.32.10
  Hostname:    ix-truenas
Capacity:
  cpu:                20
  ephemeral-storage:  463919744Ki
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  memory:             49017736Ki
  pods:               250
Allocatable:
  cpu:                20
  ephemeral-storage:  451301126610
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  memory:             49017736Ki
  pods:               250
System Info:
  Machine ID:                 494a737de33f47dba67bcbb8538278a6
  System UUID:                335c4b74-7376-ab13-6ef4-1831bfcdc785
  Boot ID:                    56050c79-0d7c-4e9d-abd6-f595634118cd
  Kernel Version:             5.15.79+truenas
  OS Image:                   Debian GNU/Linux 11 (bullseye)
  Operating System:           linux
  Architecture:               amd64
  Container Runtime Version:  docker://Unknown
  Kubelet Version:            v1.25.3+k3s-9afcd6b9-dirty
  Kube-Proxy Version:         v1.25.3+k3s-9afcd6b9-dirty
PodCIDR:                      172.16.0.0/16
PodCIDRs:                     172.16.0.0/16
Non-terminated Pods:          (17 in total)
  Namespace                   Name                                    CPU Requests  CPU Limits  Memory Requests  Memory Limits  Age
  ---------                   ----                                    ------------  ----------  ---------------  -------------  ---
  kube-system                 openebs-zfs-node-ggcjx                  0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  ix-podgrab                  podgrab-6c95994565-6dvgp                10m (0%)      4 (20%)     50Mi (0%)        8Gi (17%)      40h
  metallb-system              speaker-5rvgc                           0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  kube-system                 openebs-zfs-controller-0                0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  prometheus-operator         prometheus-operator-5dcffb7cb8-bnp7n    100m (0%)     200m (1%)   100Mi (0%)       200Mi (0%)     40h
  kube-system                 svclb-jellyfin-255633bc-m2wcn           0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  kube-system                 svclb-podgrab-67e271ac-5jfjq            0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  kube-system                 svclb-jellyfin2-fd08da81-5vlr9          0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  ix-syncthing                syncthing-6c5594b84-lqwgb               0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  ix-jellyfin2                jellyfin2-7b88f7d76f-pr64f              10m (0%)      4 (20%)     50Mi (0%)        8Gi (17%)      40h
  ix-jellyfin                 jellyfin-7f97d55865-p8m7x               10m (0%)      4 (20%)     50Mi (0%)        8Gi (17%)      40h
  kube-system                 coredns-75fc8f8fff-jkzv8                100m (0%)     0 (0%)      70Mi (0%)        170Mi (0%)     40h
  metallb-system              controller-84d6d4db45-9glps             0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
  ix-tailscale                tailscale-6d84b7f475-m9498              10m (0%)      4 (20%)     50Mi (0%)        8Gi (17%)      40h
  ix-nextcloud                nextcloud-postgres-5777466b99-vpdzg     0 (0%)        0 (0%)      0 (0%)           0 (0%)         7h49m
  ix-nextcloud                nextcloud-cbf457787-mks7l               0 (0%)        0 (0%)      0 (0%)           0 (0%)         7h49m
  kube-system                 nvidia-device-plugin-daemonset-f8pn2    0 (0%)        0 (0%)      0 (0%)           0 (0%)         40h
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource           Requests    Limits
  --------           --------    ------
  cpu                240m (1%)   16200m (81%)
  memory             370Mi (0%)  33138Mi (69%)
  ephemeral-storage  0 (0%)      0 (0%)
  hugepages-1Gi      0 (0%)      0 (0%)
  hugepages-2Mi      0 (0%)      0 (0%)
Events:              <none>
debug finished in 0 seconds for k3s kubectl describe nodes


+--------------------------------------------------------------------------------+
+k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide @1674165673+
+--------------------------------------------------------------------------------+
NAMESPACE             NAME                                       READY   STATUS             RESTARTS         AGE     IP             NODE         NOMINATED NODE   READINESS GATES
ix-syncthing          pod/syncthing-6c9d7b7bc5-74pld             0/1     Error              0                3d7h    <none>         ix-truenas   <none>           <none>
prometheus-operator   pod/prometheus-operator-5dcffb7cb8-4rphd   0/1     Error              0                3d7h    <none>         ix-truenas   <none>           <none>
kube-system           pod/openebs-zfs-node-ggcjx                 2/2     Running            0                40h     172.23.32.10   ix-truenas   <none>           <none>
metallb-system        pod/controller-84d6d4db45-m7cr5            0/1     Error              1                3d7h    <none>         ix-truenas   <none>           <none>
ix-podgrab            pod/podgrab-6c95994565-6dvgp               1/1     Running            0                40h     172.16.2.26    ix-truenas   <none>           <none>
metallb-system        pod/speaker-5rvgc                          1/1     Running            0                40h     172.23.32.10   ix-truenas   <none>           <none>
kube-system           pod/openebs-zfs-controller-0               5/5     Running            0                40h     172.16.2.22    ix-truenas   <none>           <none>
prometheus-operator   pod/prometheus-operator-5dcffb7cb8-bnp7n   1/1     Running            0                40h     172.16.2.29    ix-truenas   <none>           <none>
kube-system           pod/svclb-jellyfin-255633bc-m2wcn          1/1     Running            0                40h     172.16.2.32    ix-truenas   <none>           <none>
kube-system           pod/svclb-podgrab-67e271ac-5jfjq           1/1     Running            0                40h     172.16.2.34    ix-truenas   <none>           <none>
kube-system           pod/svclb-jellyfin2-fd08da81-5vlr9         1/1     Running            0                40h     172.16.2.36    ix-truenas   <none>           <none>
ix-syncthing          pod/syncthing-6c5594b84-lqwgb              1/1     Running            0                40h     172.16.2.37    ix-truenas   <none>           <none>
ix-jellyfin2          pod/jellyfin2-7b88f7d76f-pr64f             1/1     Running            0                40h     172.16.2.23    ix-truenas   <none>           <none>
ix-jellyfin           pod/jellyfin-7f97d55865-p8m7x              1/1     Running            0                40h     172.16.2.25    ix-truenas   <none>           <none>
kube-system           pod/coredns-75fc8f8fff-jkzv8               1/1     Running            0                40h     172.16.2.21    ix-truenas   <none>           <none>
metallb-system        pod/controller-84d6d4db45-9glps            1/1     Running            1 (40h ago)      40h     172.16.2.30    ix-truenas   <none>           <none>
ix-tailscale          pod/tailscale-6d84b7f475-m9498             1/1     Running            2 (40h ago)      40h     172.23.32.10   ix-truenas   <none>           <none>
ix-nextcloud          pod/nextcloud-postgres-5777466b99-vpdzg    1/1     Running            0                7h49m   172.16.2.43    ix-truenas   <none>           <none>
ix-nextcloud          pod/nextcloud-cbf457787-mks7l              2/2     Running            0                7h49m   172.16.2.42    ix-truenas   <none>           <none>
kube-system           pod/nvidia-device-plugin-daemonset-f8pn2   0/1     CrashLoopBackOff   484 (5m6s ago)   40h     172.16.2.20    ix-truenas   <none>           <none>

NAMESPACE             NAME                          TYPE           CLUSTER-IP       EXTERNAL-IP    PORT(S)                                           AGE     SELECTOR
default               service/kubernetes            ClusterIP      172.17.0.1       <none>         443/TCP                                           18d     <none>
kube-system           service/kube-dns              ClusterIP      172.17.0.10      <none>         53/UDP,53/TCP,9153/TCP                            18d     k8s-app=kube-dns
metallb-system        service/webhook-service       ClusterIP      172.17.238.220   <none>         443/TCP                                           15d     component=controller
prometheus-operator   service/prometheus-operator   ClusterIP      None             <none>         8080/TCP                                          15d     app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
kube-system           service/kubelet               ClusterIP      None             <none>         10250/TCP,10255/TCP,4194/TCP                      15d     <none>
ix-syncthing          service/syncthing             NodePort       172.17.79.0      <none>         20910:20910/TCP,20978:20978/TCP,20979:20979/UDP   9d      app.kubernetes.io/instance=syncthing,app.kubernetes.io/name=syncthing,app=syncthing,release=syncthing
ix-jellyfin           service/jellyfin              LoadBalancer   172.17.225.233   172.23.32.10   8096:43695/TCP                                    40h     app.kubernetes.io/instance=jellyfin,app.kubernetes.io/name=jellyfin
ix-podgrab            service/podgrab-tcp           ClusterIP      172.17.168.247   <none>         51080/TCP                                         14d     app.kubernetes.io/instance=podgrab,app.kubernetes.io/name=podgrab
ix-podgrab            service/podgrab               LoadBalancer   172.17.188.203   172.23.32.10   10093:47234/TCP                                   40h     app.kubernetes.io/instance=podgrab,app.kubernetes.io/name=podgrab
ix-jellyfin2          service/jellyfin2             LoadBalancer   172.17.197.21    172.23.32.10   8097:19593/TCP                                    40h     app.kubernetes.io/instance=jellyfin2,app.kubernetes.io/name=jellyfin
ix-nextcloud          service/nextcloud-postgres    ClusterIP      172.17.20.160    <none>         5432/TCP                                          4d21h   app.kubernetes.io/instance=nextcloud-postgres,app.kubernetes.io/name=nextcloud-postgres
ix-nextcloud          service/nextcloud             NodePort       172.17.245.170   <none>         443:9001/TCP                                      4d21h   app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud

NAMESPACE        NAME                                            DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE   CONTAINERS                                     IMAGES                                                                             SELECTOR
kube-system      daemonset.apps/nvidia-device-plugin-daemonset   1         1         0       1            0           <none>                   16d   nvidia-device-plugin-ctr                       nvidia/k8s-device-plugin:v0.10.0                                                   name=nvidia-device-plugin-ds
kube-system      daemonset.apps/openebs-zfs-node                 1         1         1       1            1           <none>                   17d   csi-node-driver-registrar,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0,openebs/zfs-driver:2.0.0   app=openebs-zfs-node
metallb-system   daemonset.apps/speaker                          1         1         1       1            1           kubernetes.io/os=linux   15d   speaker                                        quay.io/metallb/speaker:v0.13.7                                                    app=metallb,component=speaker
kube-system      daemonset.apps/svclb-jellyfin-255633bc          1         1         1       1            1           <none>                   40h   lb-tcp-8096                                    rancher/klipper-lb:v0.3.5                                                          app=svclb-jellyfin-255633bc
kube-system      daemonset.apps/svclb-podgrab-67e271ac           1         1         1       1            1           <none>                   40h   lb-tcp-10093                                   rancher/klipper-lb:v0.3.5                                                          app=svclb-podgrab-67e271ac
kube-system      daemonset.apps/svclb-jellyfin2-fd08da81         1         1         1       1            1           <none>                   40h   lb-tcp-8097                                    rancher/klipper-lb:v0.3.5                                                          app=svclb-jellyfin2-fd08da81

NAMESPACE             NAME                                  READY   UP-TO-DATE   AVAILABLE   AGE     CONTAINERS            IMAGES                                                                                                       SELECTOR
prometheus-operator   deployment.apps/prometheus-operator   1/1     1            1           15d     prometheus-operator   quay.io/prometheus-operator/prometheus-operator:v0.60.1                                                      app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
ix-podgrab            deployment.apps/podgrab               1/1     1            1           15d     podgrab               tccr.io/truecharts/podgrab:v1.0.0@sha256:4f12ed1458991f2918990a7041a8d4e4ff84232e5dc0bf57d14acb70debb0103    app.kubernetes.io/instance=podgrab,app.kubernetes.io/name=podgrab
ix-syncthing          deployment.apps/syncthing             1/1     1            1           11d     syncthing             syncthing/syncthing:1.23.0                                                                                   app=syncthing,release=syncthing
ix-jellyfin2          deployment.apps/jellyfin2             1/1     1            1           15d     jellyfin2             tccr.io/truecharts/jellyfin:10.8.8@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5   app.kubernetes.io/instance=jellyfin2,app.kubernetes.io/name=jellyfin
ix-jellyfin           deployment.apps/jellyfin              1/1     1            1           15d     jellyfin              tccr.io/truecharts/jellyfin:10.8.8@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5   app.kubernetes.io/instance=jellyfin,app.kubernetes.io/name=jellyfin
kube-system           deployment.apps/coredns               1/1     1            1           18d     coredns               rancher/mirrored-coredns-coredns:1.9.1                                                                       k8s-app=kube-dns
metallb-system        deployment.apps/controller            1/1     1            1           15d     controller            quay.io/metallb/controller:v0.13.7                                                                           app=metallb,component=controller
ix-tailscale          deployment.apps/tailscale             1/1     1            1           15d     tailscale             tailscale/tailscale:v1.34.2@sha256:08d4203e1664be5f51238bb32f0b93524b497056719ce3f51bde431ef2dbee54          app.kubernetes.io/instance=tailscale,app.kubernetes.io/name=tailscale
ix-nextcloud          deployment.apps/nextcloud-postgres    1/1     1            1           7h49m   nextcloud-postgres    postgres:13.1                                                                                                app.kubernetes.io/instance=nextcloud-postgres,app.kubernetes.io/name=nextcloud-postgres
ix-nextcloud          deployment.apps/nextcloud             1/1     1            1           7h49m   nginx,nextcloud       nginx:1.23.3,nextcloud:25.0.3                                                                                app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud

NAMESPACE     NAME                                      READY   AGE   CONTAINERS                                                                           IMAGES
kube-system   statefulset.apps/openebs-zfs-controller   1/1     17d   csi-resizer,csi-snapshotter,snapshot-controller,csi-provisioner,openebs-zfs-plugin   k8s.gcr.io/sig-storage/csi-resizer:v1.2.0,k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0,k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0,k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0,openebs/zfs-driver:2.0.0

NAMESPACE   NAME                                                          PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
            storageclass.storage.k8s.io/ix-storage-class-tailscale        zfs.csi.openebs.io   Retain          Immediate           true                   15d
            storageclass.storage.k8s.io/ix-storage-class-jellyfin         zfs.csi.openebs.io   Retain          Immediate           true                   15d
            storageclass.storage.k8s.io/ix-storage-class-podgrab          zfs.csi.openebs.io   Retain          Immediate           true                   15d
            storageclass.storage.k8s.io/ix-storage-class-jellyfin2        zfs.csi.openebs.io   Retain          Immediate           true                   15d
            storageclass.storage.k8s.io/ix-storage-class-nextcloud        zfs.csi.openebs.io   Retain          Immediate           true                   15d
            storageclass.storage.k8s.io/ix-storage-class-syncthing        zfs.csi.openebs.io   Retain          Immediate           true                   11d
            storageclass.storage.k8s.io/openebs-zfspv-default (default)   zfs.csi.openebs.io   Delete          Immediate           true                   40h

NAMESPACE      NAME                                        STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS                 AGE   VOLUMEMODE
ix-jellyfin    persistentvolumeclaim/jellyfin-transcode    Bound    pvc-06992fd2-0215-42c9-84c9-db84cdeaa7ff   256Gi      RWO            ix-storage-class-jellyfin    15d   Filesystem
ix-jellyfin2   persistentvolumeclaim/jellyfin2-transcode   Bound    pvc-51ec3873-82b8-4fbd-99a2-15a7761a007e   256Gi      RWO            ix-storage-class-jellyfin2   15d   Filesystem

NAMESPACE   NAME                            STATUS   AGE
            namespace/default               Active   18d
            namespace/kube-system           Active   18d
            namespace/kube-public           Active   18d
            namespace/kube-node-lease       Active   18d
            namespace/openebs               Active   18d
            namespace/ix-tailscale          Active   15d
            namespace/cnpg-system           Active   15d
            namespace/metallb-system        Active   15d
            namespace/prometheus-operator   Active   15d
            namespace/tc-system             Active   15d
            namespace/ix-jellyfin           Active   15d
            namespace/ix-podgrab            Active   15d
            namespace/ix-jellyfin2          Active   15d
            namespace/ix-nextcloud          Active   15d
            namespace/ix-syncthing          Active   11d
debug finished in 0 seconds for k3s kubectl get pods,svc,daemonsets,deployments,statefulset,sc,pvc,ns,job --all-namespaces -o wide


+--------------------------------------------------------------------------------+
+         k3s kubectl describe deployments --all-namespaces @1674165673          +
+--------------------------------------------------------------------------------+
Name:                   prometheus-operator
Namespace:              prometheus-operator
CreationTimestamp:      Wed, 04 Jan 2023 15:43:07 +0100
Labels:                 app.kubernetes.io/component=controller
                        app.kubernetes.io/name=prometheus-operator
                        app.kubernetes.io/version=0.60.1
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app.kubernetes.io/component=controller
                    app.kubernetes.io/name=prometheus-operator
                    app.kubernetes.io/version=0.60.1
  Annotations:      kubectl.kubernetes.io/default-container: prometheus-operator
  Service Account:  prometheus-operator
  Containers:
   prometheus-operator:
    Image:      quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Port:       8080/TCP
    Host Port:  0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:       <none>
  Volumes:        <none>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   prometheus-operator-5dcffb7cb8 (1/1 replicas created)
Events:          <none>


Name:               podgrab
Namespace:          ix-podgrab
CreationTimestamp:  Wed, 04 Jan 2023 16:00:37 +0100
Labels:             app.kubernetes.io/instance=podgrab
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=podgrab
                    app.kubernetes.io/version=1.0.0
                    helm-revision=17
                    helm.sh/chart=podgrab-12.0.7
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: podgrab
                    meta.helm.sh/release-namespace: ix-podgrab
                    rollme: ZqBNi
Selector:           app.kubernetes.io/instance=podgrab,app.kubernetes.io/name=podgrab
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=podgrab
                    app.kubernetes.io/name=podgrab
  Service Account:  default
  Containers:
   podgrab:
    Image:       tccr.io/truecharts/podgrab:v1.0.0@sha256:4f12ed1458991f2918990a7041a8d4e4ff84232e5dc0bf57d14acb70debb0103
    Ports:       8080/TCP, 51080/TCP
    Host Ports:  0/TCP, 0/TCP
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment Variables from:
      podgrab  Secret  Optional: false
    Environment:
      UMASK:                   2
      UMASK_SET:               2
      S6_READ_ONLY_ROOT:       1
      NVIDIA_VISIBLE_DEVICES:  void
      TZ:                      Europe/Berlin
      CHECK_FREQUENCY:         240
    Mounts:
      /assets from list-0 (rw)
      /config from config (rw)
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
  Volumes:
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/podgrab-config
    HostPathType:  
   list-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/media/Podcasts
    HostPathType:  
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   podgrab-6c95994565 (1/1 replicas created)
Events:          <none>


Name:               syncthing
Namespace:          ix-syncthing
CreationTimestamp:  Sun, 08 Jan 2023 21:46:19 +0100
Labels:             app=syncthing
                    app.kubernetes.io/managed-by=Helm
                    chart=syncthing-1.0.16
                    heritage=Helm
                    release=syncthing
Annotations:        deployment.kubernetes.io/revision: 13
                    meta.helm.sh/release-name: syncthing
                    meta.helm.sh/release-namespace: ix-syncthing
                    rollme: 4JDYo
Selector:           app=syncthing,release=syncthing
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:       app=syncthing
                app.kubernetes.io/instance=syncthing
                app.kubernetes.io/name=syncthing
                release=syncthing
  Annotations:  rollme: HO11v
  Containers:
   syncthing:
    Image:       syncthing/syncthing:1.23.0
    Ports:       8384/TCP, 22000/TCP, 22000/UDP
    Host Ports:  0/TCP, 0/TCP, 0/UDP
    Environment:
      PUID:          568
      PGID:          568
      STGUIADDRESS:  0.0.0.0:8384
    Mounts:
      /var/syncthing from config (rw)
      /vault/media from extrappvolume-0 (rw)
      /vault/vault from extrappvolume-1 (rw)
  Volumes:
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/syncthing-config
    HostPathType:  
   extrappvolume-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/media
    HostPathType:  
   extrappvolume-1:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/vault
    HostPathType:  
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   syncthing-6c5594b84 (1/1 replicas created)
Events:          <none>


Name:               jellyfin2
Namespace:          ix-jellyfin2
CreationTimestamp:  Wed, 04 Jan 2023 16:08:03 +0100
Labels:             app.kubernetes.io/instance=jellyfin2
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=jellyfin
                    app.kubernetes.io/version=10.8.8
                    helm-revision=18
                    helm.sh/chart=jellyfin-13.0.9
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: jellyfin2
                    meta.helm.sh/release-namespace: ix-jellyfin2
                    rollme: zE1lW
Selector:           app.kubernetes.io/instance=jellyfin2,app.kubernetes.io/name=jellyfin
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=jellyfin2
                    app.kubernetes.io/name=jellyfin
  Service Account:  default
  Containers:
   jellyfin2:
    Image:      tccr.io/truecharts/jellyfin:10.8.8@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5
    Port:       8096/TCP
    Host Port:  0/TCP
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    tcp-socket :8096 delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                        2
      UMASK_SET:                    2
      S6_READ_ONLY_ROOT:            1
      NVIDIA_VISIBLE_DEVICES:       void
      TZ:                           Europe/Berlin
      JELLYFIN_PublishedServerUrl:  
    Mounts:
      /cache from cache (rw)
      /config from config (rw)
      /config/transcodes from transcode (rw)
      /dev/shm from shm (rw)
      /privat from list-0 (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
  Volumes:
   cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/jellyfin2-config
    HostPathType:  
   list-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/privat
    HostPathType:  
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   transcode:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  jellyfin2-transcode
    ReadOnly:   false
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   jellyfin2-7b88f7d76f (1/1 replicas created)
Events:          <none>


Name:               jellyfin
Namespace:          ix-jellyfin
CreationTimestamp:  Wed, 04 Jan 2023 15:55:42 +0100
Labels:             app.kubernetes.io/instance=jellyfin
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=jellyfin
                    app.kubernetes.io/version=10.8.8
                    helm-revision=17
                    helm.sh/chart=jellyfin-13.0.9
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: jellyfin
                    meta.helm.sh/release-namespace: ix-jellyfin
                    rollme: fj3yf
Selector:           app.kubernetes.io/instance=jellyfin,app.kubernetes.io/name=jellyfin
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=jellyfin
                    app.kubernetes.io/name=jellyfin
  Service Account:  default
  Containers:
   jellyfin:
    Image:      tccr.io/truecharts/jellyfin:10.8.8@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5
    Port:       8096/TCP
    Host Port:  0/TCP
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    tcp-socket :8096 delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                        2
      UMASK_SET:                    2
      S6_READ_ONLY_ROOT:            1
      NVIDIA_VISIBLE_DEVICES:       void
      TZ:                           Europe/Berlin
      JELLYFIN_PublishedServerUrl:  
    Mounts:
      /cache from cache (rw)
      /config from config (rw)
      /config/transcodes from transcode (rw)
      /dev/shm from shm (rw)
      /media from list-0 (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
  Volumes:
   cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/jellyfin-config
    HostPathType:  
   list-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/media
    HostPathType:  
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   transcode:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  jellyfin-transcode
    ReadOnly:   false
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   jellyfin-7f97d55865 (1/1 replicas created)
Events:          <none>


Name:                   coredns
Namespace:              kube-system
CreationTimestamp:      Sun, 01 Jan 2023 21:03:29 +0100
Labels:                 k8s-app=kube-dns
                        kubernetes.io/name=CoreDNS
                        objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:            deployment.kubernetes.io/revision: 1
                        objectset.rio.cattle.io/applied:
                          H4sIAAAAAAAA/6xVUW/bNhD+K8M9S7GVtI0rYA+dna1FV8+ok74URkFTJ4szxePIkxMj0H8fTrIdu02TdtiTZfK743cfvzveg/LmE4ZoyEEOyvs42GSQwNq4AnKYoLe0rdExJFAjq0...
                        objectset.rio.cattle.io/id: 
                        objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                        objectset.rio.cattle.io/owner-name: coredns
                        objectset.rio.cattle.io/owner-namespace: kube-system
Selector:               k8s-app=kube-dns
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  1 max unavailable, 25% max surge
Pod Template:
  Labels:           k8s-app=kube-dns
  Service Account:  coredns
  Containers:
   coredns:
    Image:       rancher/mirrored-coredns-coredns:1.9.1
    Ports:       53/UDP, 53/TCP, 9153/TCP
    Host Ports:  0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
  Volumes:
   config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
   custom-config-volume:
    Type:                       ConfigMap (a volume populated by a ConfigMap)
    Name:                       coredns-custom
    Optional:                   true
  Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
  Priority Class Name:          system-cluster-critical
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   coredns-75fc8f8fff (1/1 replicas created)
Events:          <none>


Name:                   controller
Namespace:              metallb-system
CreationTimestamp:      Wed, 04 Jan 2023 15:43:07 +0100
Labels:                 app=metallb
                        component=controller
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app=metallb,component=controller
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app=metallb
                    component=controller
  Annotations:      prometheus.io/port: 7472
                    prometheus.io/scrape: true
  Service Account:  controller
  Containers:
   controller:
    Image:       quay.io/metallb/controller:v0.13.7
    Ports:       7472/TCP, 9443/TCP
    Host Ports:  0/TCP, 0/TCP
    Args:
      --port=7472
      --log-level=info
    Liveness:   http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_ML_SECRET_NAME:  memberlist
      METALLB_DEPLOYMENT:      controller
    Mounts:
      /tmp/k8s-webhook-server/serving-certs from cert (ro)
  Volumes:
   cert:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  webhook-server-cert
    Optional:    false
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   controller-84d6d4db45 (1/1 replicas created)
Events:          <none>


Name:               tailscale
Namespace:          ix-tailscale
CreationTimestamp:  Wed, 04 Jan 2023 15:43:10 +0100
Labels:             app.kubernetes.io/instance=tailscale
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=tailscale
                    app.kubernetes.io/version=1.34.2
                    helm-revision=18
                    helm.sh/chart=tailscale-3.0.12
Annotations:        deployment.kubernetes.io/revision: 2
                    meta.helm.sh/release-name: tailscale
                    meta.helm.sh/release-namespace: ix-tailscale
                    rollme: 9K8VQ
Selector:           app.kubernetes.io/instance=tailscale,app.kubernetes.io/name=tailscale
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:           app.kubernetes.io/instance=tailscale
                    app.kubernetes.io/name=tailscale
  Service Account:  tailscale
  Containers:
   tailscale:
    Image:      tailscale/tailscale:v1.34.2@sha256:08d4203e1664be5f51238bb32f0b93524b497056719ce3f51bde431ef2dbee54
    Port:       <none>
    Host Port:  <none>
    Command:
      /usr/local/bin/containerboot
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment Variables from:
      tailscale-tailscale-config  ConfigMap  Optional: false
    Environment:
      PUID:                    568
      USER_ID:                 568
      UID:                     568
      UMASK:                   2
      UMASK_SET:               2
      PGID:                    568
      GROUP_ID:                568
      GID:                     568
      NVIDIA_VISIBLE_DEVICES:  void
      TZ:                      Europe/Berlin
    Mounts:
      /dev/net/tun from tun (rw)
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
  Volumes:
   shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
   temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   tun:
    Type:          HostPath (bare host directory volume)
    Path:          /dev/net/tun
    HostPathType:  
   varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
   varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Progressing    True    NewReplicaSetAvailable
  Available      True    MinimumReplicasAvailable
OldReplicaSets:  <none>
NewReplicaSet:   tailscale-6d84b7f475 (1/1 replicas created)
Events:          <none>


Name:               nextcloud-postgres
Namespace:          ix-nextcloud
CreationTimestamp:  Thu, 19 Jan 2023 15:11:40 +0100
Labels:             app.kubernetes.io/instance=nextcloud-postgres
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=nextcloud-postgres
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: nextcloud
                    meta.helm.sh/release-namespace: ix-nextcloud
Selector:           app.kubernetes.io/instance=nextcloud-postgres,app.kubernetes.io/name=nextcloud-postgres
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:       app.kubernetes.io/instance=nextcloud-postgres
                app.kubernetes.io/name=nextcloud-postgres
  Annotations:  rollme: efVSg
  Containers:
   nextcloud-postgres:
    Image:      postgres:13.1
    Port:       5432/TCP
    Host Port:  0/TCP
    Environment:
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:
      /postgres_backups from postgres-backup (rw)
      /var/lib/postgresql/data from postgres-data (rw)
  Volumes:
   postgres-backup:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/apps/ix-applications/releases/nextcloud/volumes/ix_volumes/ix-postgres_backups
    HostPathType:  
   postgres-data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/apps/ix-applications/releases/nextcloud/volumes/ix_volumes/ix-postgres_data
    HostPathType:  
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   nextcloud-postgres-5777466b99 (1/1 replicas created)
Events:          <none>


Name:               nextcloud
Namespace:          ix-nextcloud
CreationTimestamp:  Thu, 19 Jan 2023 15:11:40 +0100
Labels:             app.kubernetes.io/instance=nextcloud
                    app.kubernetes.io/managed-by=Helm
                    app.kubernetes.io/name=nextcloud
Annotations:        deployment.kubernetes.io/revision: 1
                    meta.helm.sh/release-name: nextcloud
                    meta.helm.sh/release-namespace: ix-nextcloud
Selector:           app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud
Replicas:           1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:       Recreate
MinReadySeconds:    0
Pod Template:
  Labels:       app.kubernetes.io/instance=nextcloud
                app.kubernetes.io/name=nextcloud
  Annotations:  rollme: w2iDv
  Init Containers:
   init-postgresdb:
    Image:      postgres:13.1
    Port:       <none>
    Host Port:  <none>
    Command:
      sh
      -c
      until pg_isready -U $POSTGRES_USER -d nextcloud -h nextcloud-postgres; do echo waiting for postgres; sleep 2; done
    Environment:
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:               <none>
  Containers:
   nginx:
    Image:        nginx:1.23.3
    Ports:        8000/TCP, 443/TCP
    Host Ports:   0/TCP, 0/TCP
    Environment:  <none>
    Mounts:
      /etc/nginx-certs from cert-secret-volume (rw)
      /etc/nginx/nginx.conf from nginx-configuration (rw,path="nginx.conf")
   nextcloud:
    Image:      nextcloud:25.0.3
    Port:       80/TCP
    Host Port:  0/TCP
    Environment:
      POSTGRES_USER:              <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:          <set to the key 'db-password' in secret 'db-details'>  Optional: false
      POSTGRES_HOST:              nextcloud-postgres:5432
      POSTGRES_DB:                nextcloud
      NEXTCLOUD_DATA_DIR:         /var/www/html/data
      APACHE_DISABLE_REWRITE_IP:  1
      OVERWRITEHOST:              172.23.32.10:9001
      OVERWRITEPROTOCOL:          https
      TRUSTED_PROXIES:            127.0.0.1
      NEXTCLOUD_TRUSTED_DOMAINS:  172.23.32.10
      NEXTCLOUD_ADMIN_USER:       <set to the key 'nextcloud-username' in secret 'nextcloud'>  Optional: false
      NEXTCLOUD_ADMIN_PASSWORD:   <set to the key 'nextcloud-password' in secret 'nextcloud'>  Optional: false
    Mounts:
      /var/www/ from nextcloud-data (rw,path="root")
      /var/www/html from nextcloud-data (rw,path="html")
      /var/www/html/config from nextcloud-data (rw,path="config")
      /var/www/html/custom_apps from nextcloud-data (rw,path="custom_apps")
      /var/www/html/data from nextcloud-data (rw,path="data")
      /var/www/html/themes from nextcloud-data (rw,path="themes")
      /var/www/tmp from nextcloud-data (rw,path="tmp")
  Volumes:
   nginx-configuration:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nginx-configuration
    Optional:  false
   cert-secret-volume:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  nginx-secret
    Optional:    false
   nextcloud-data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/nextcloud-data
    HostPathType:  
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   nextcloud-cbf457787 (1/1 replicas created)
Events:          <none>
debug finished in 1 seconds for k3s kubectl describe deployments --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe pods --all-namespaces @1674165674             +
+--------------------------------------------------------------------------------+
Name:             syncthing-6c9d7b7bc5-74pld
Namespace:        ix-syncthing
Priority:         0
Service Account:  default
Node:             ix-truenas/172.23.32.10
Start Time:       Mon, 16 Jan 2023 15:07:21 +0100
Labels:           app=syncthing
                  app.kubernetes.io/instance=syncthing
                  app.kubernetes.io/name=syncthing
                  pod-template-hash=6c9d7b7bc5
                  release=syncthing
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.16"
                        ],
                        "mac": "42:dc:90:b4:8b:a2",
                        "default": true,
                        "dns": {}
                    }]
                  rollme: Y4pgn
Status:           Failed
Reason:           TaintToleration
Message:          Pod Predicate TaintToleration failed
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/syncthing-6c9d7b7bc5
Containers:
  syncthing:
    Container ID:   docker://83402af657a6587d5dce1be2f39f00fda483c5508de6bdc4764878e13f401d11
    Image:          syncthing/syncthing:1.23.0
    Image ID:       docker-pullable://syncthing/syncthing@sha256:57699adea5bae30edd2c4f83d5c311969e3255c3d23e7de747638dffbc3a8bc2
    Ports:          8384/TCP, 22000/TCP, 22000/UDP
    Host Ports:     0/TCP, 0/TCP, 0/UDP
    State:          Terminated
      Reason:       Error
      Exit Code:    255
      Started:      Mon, 16 Jan 2023 15:07:22 +0100
      Finished:     Wed, 18 Jan 2023 06:03:23 +0100
    Ready:          False
    Restart Count:  0
    Environment:
      PUID:          568
      PGID:          568
      STGUIADDRESS:  0.0.0.0:8384
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-vcrs8 (ro)
      /var/syncthing from config (rw)
      /vault/media from extrappvolume-0 (rw)
      /vault/vault from extrappvolume-1 (rw)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/syncthing-config
    HostPathType:  
  extrappvolume-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/media
    HostPathType:  
  extrappvolume-1:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/vault
    HostPathType:  
  kube-api-access-vcrs8:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             prometheus-operator-5dcffb7cb8-4rphd
Namespace:        prometheus-operator
Priority:         0
Service Account:  prometheus-operator
Node:             ix-truenas/172.23.32.10
Start Time:       Mon, 16 Jan 2023 15:06:30 +0100
Labels:           app.kubernetes.io/component=controller
                  app.kubernetes.io/name=prometheus-operator
                  app.kubernetes.io/version=0.60.1
                  pod-template-hash=5dcffb7cb8
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.8"
                        ],
                        "mac": "da:e7:52:26:1c:d1",
                        "default": true,
                        "dns": {}
                    }]
                  kubectl.kubernetes.io/default-container: prometheus-operator
Status:           Failed
Reason:           TaintToleration
Message:          Pod Predicate TaintToleration failed
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/prometheus-operator-5dcffb7cb8
Containers:
  prometheus-operator:
    Container ID:  docker://44f2e5808794b8557afea83e6ef0d9b2d973b8780eae0bedcbbc949594aeb409
    Image:         quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Image ID:      docker-pullable://quay.io/prometheus-operator/prometheus-operator@sha256:63d4d0f8fb817313ec13c2960b72d1b4341bea269eb97ec7bf44c585a04b1e2e
    Port:          8080/TCP
    Host Port:     0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    State:          Terminated
      Reason:       Error
      Exit Code:    255
      Started:      Mon, 16 Jan 2023 15:06:31 +0100
      Finished:     Wed, 18 Jan 2023 06:03:23 +0100
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-rf7vl (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  kube-api-access-rf7vl:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:                 openebs-zfs-node-ggcjx
Namespace:            kube-system
Priority:             900001000
Priority Class Name:  openebs-zfs-csi-node-critical
Service Account:      openebs-zfs-node-sa
Node:                 ix-truenas/172.23.32.10
Start Time:           Wed, 18 Jan 2023 06:03:42 +0100
Labels:               app=openebs-zfs-node
                      controller-revision-hash=66bcbb6975
                      openebs.io/component-name=openebs-zfs-node
                      openebs.io/version=ci
                      pod-template-generation=1
                      role=openebs-zfs
Annotations:          <none>
Status:               Running
IP:                   172.23.32.10
IPs:
  IP:           172.23.32.10
Controlled By:  DaemonSet/openebs-zfs-node
Containers:
  csi-node-driver-registrar:
    Container ID:  docker://a1eecce077e1ee6f2d6400b5dffde1723ccba56cf26fb7f0a05a75f31d214cf9
    Image:         k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:f9bcee63734b7b01555ee8fc8fb01ac2922478b2c8934bf8d468dd2916edc405
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:44 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-rcksd (ro)
  openebs-zfs-plugin:
    Container ID:  docker://4f21eedf761f3e0adc4568ea44e2906046dac77cec2e2e5bc7abf39c5350ae74
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:45 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-rcksd (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
  encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
  chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
  host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
  plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
  pods-mount-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/
    HostPathType:  Directory
  kube-api-access-rcksd:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:                      <none>


Name:             controller-84d6d4db45-m7cr5
Namespace:        metallb-system
Priority:         0
Service Account:  controller
Node:             ix-truenas/172.23.32.10
Start Time:       Mon, 16 Jan 2023 15:06:31 +0100
Labels:           app=metallb
                  component=controller
                  pod-template-hash=84d6d4db45
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.9"
                        ],
                        "mac": "b2:f6:c8:31:0e:fe",
                        "default": true,
                        "dns": {}
                    }]
                  prometheus.io/port: 7472
                  prometheus.io/scrape: true
Status:           Failed
Reason:           TaintToleration
Message:          Pod Predicate TaintToleration failed
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/controller-84d6d4db45
Containers:
  controller:
    Container ID:  docker://c793231f526338e9cc86f995e9f71c3f5db55eac4bd2678732e30021672b1d6b
    Image:         quay.io/metallb/controller:v0.13.7
    Image ID:      docker-pullable://quay.io/metallb/controller@sha256:0e486995b27d62433e728f7b266fb8b7b7f4fd65e1b751bef17fe56e76653bf1
    Ports:         7472/TCP, 9443/TCP
    Host Ports:    0/TCP, 0/TCP
    Args:
      --port=7472
      --log-level=info
    State:          Terminated
      Reason:       Error
      Exit Code:    255
      Started:      Mon, 16 Jan 2023 15:07:13 +0100
      Finished:     Wed, 18 Jan 2023 06:03:23 +0100
    Ready:          False
    Restart Count:  1
    Liveness:       http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:      http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_ML_SECRET_NAME:  memberlist
      METALLB_DEPLOYMENT:      controller
    Mounts:
      /tmp/k8s-webhook-server/serving-certs from cert (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-7s446 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  cert:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  webhook-server-cert
    Optional:    false
  kube-api-access-7s446:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             podgrab-6c95994565-6dvgp
Namespace:        ix-podgrab
Priority:         0
Service Account:  default
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:03:56 +0100
Labels:           app.kubernetes.io/instance=podgrab
                  app.kubernetes.io/name=podgrab
                  pod-template-hash=6c95994565
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.26"
                        ],
                        "mac": "fe:fa:e4:e4:c0:7d",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.2.26
IPs:
  IP:           172.16.2.26
Controlled By:  ReplicaSet/podgrab-6c95994565
Containers:
  podgrab:
    Container ID:   docker://5dd375746ec45f46a10b774fc8e8884ba0d38e69d43cc70e68c74823f900c7f7
    Image:          tccr.io/truecharts/podgrab:v1.0.0@sha256:4f12ed1458991f2918990a7041a8d4e4ff84232e5dc0bf57d14acb70debb0103
    Image ID:       docker-pullable://tccr.io/truecharts/podgrab@sha256:4f12ed1458991f2918990a7041a8d4e4ff84232e5dc0bf57d14acb70debb0103
    Ports:          8080/TCP, 51080/TCP
    Host Ports:     0/TCP, 0/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 06:04:01 +0100
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment Variables from:
      podgrab  Secret  Optional: false
    Environment:
      UMASK:                   2
      UMASK_SET:               2
      S6_READ_ONLY_ROOT:       1
      NVIDIA_VISIBLE_DEVICES:  void
      TZ:                      Europe/Berlin
      CHECK_FREQUENCY:         240
    Mounts:
      /assets from list-0 (rw)
      /config from config (rw)
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-rdvzm (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/podgrab-config
    HostPathType:  
  list-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/media/Podcasts
    HostPathType:  
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-rdvzm:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             speaker-5rvgc
Namespace:        metallb-system
Priority:         0
Service Account:  speaker
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:03:42 +0100
Labels:           app=metallb
                  component=speaker
                  controller-revision-hash=67d677d8b6
                  pod-template-generation=1
Annotations:      prometheus.io/port: 7472
                  prometheus.io/scrape: true
Status:           Running
IP:               172.23.32.10
IPs:
  IP:           172.23.32.10
Controlled By:  DaemonSet/speaker
Containers:
  speaker:
    Container ID:  docker://8671efec8f9a51e379133a885f80bed70cc7f517fe526f041545f254a5bd56a0
    Image:         quay.io/metallb/speaker:v0.13.7
    Image ID:      docker-pullable://quay.io/metallb/speaker@sha256:aeab5aeeb25c7ca31a1c468a35b1db4513cd6199bf376bdf19ff5689d2ebad28
    Ports:         7472/TCP, 7946/TCP, 7946/UDP
    Host Ports:    7472/TCP, 7946/TCP, 7946/UDP
    Args:
      --port=7472
      --log-level=info
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:44 +0100
    Ready:          True
    Restart Count:  0
    Liveness:       http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:      http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_NODE_NAME:       (v1:spec.nodeName)
      METALLB_HOST:            (v1:status.hostIP)
      METALLB_ML_BIND_ADDR:    (v1:status.podIP)
      METALLB_ML_LABELS:      app=metallb,component=speaker
      METALLB_ML_SECRET_KEY:  <set to the key 'secretkey' in secret 'memberlist'>  Optional: false
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-g6bkx (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  kube-api-access-g6bkx:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                             node-role.kubernetes.io/master:NoSchedule op=Exists
                             node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/network-unavailable:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:                      <none>


Name:                 openebs-zfs-controller-0
Namespace:            kube-system
Priority:             900000000
Priority Class Name:  openebs-zfs-csi-controller-critical
Service Account:      openebs-zfs-controller-sa
Node:                 ix-truenas/172.23.32.10
Start Time:           Wed, 18 Jan 2023 06:03:51 +0100
Labels:               app=openebs-zfs-controller
                      controller-revision-hash=openebs-zfs-controller-698698d48b
                      openebs.io/component-name=openebs-zfs-controller
                      openebs.io/version=ci
                      role=openebs-zfs
                      statefulset.kubernetes.io/pod-name=openebs-zfs-controller-0
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.2.22"
                            ],
                            "mac": "0a:9d:04:bb:4a:d5",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.2.22
IPs:
  IP:           172.16.2.22
Controlled By:  StatefulSet/openebs-zfs-controller
Containers:
  csi-resizer:
    Container ID:  docker://65ffa7d22a5eb5cb3ac2e6fbce4f8b0421b1556efc63cfba54974581543f45f5
    Image:         k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:53 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-t4mjr (ro)
  csi-snapshotter:
    Container ID:  docker://9c03978f0375794e7869e7f7ea8d162abb45c0fa762ecf97fd047ef226893347
    Image:         k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-snapshotter@sha256:51f2dfde5bccac7854b3704689506aeecfb793328427b91115ba253a93e60782
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:54 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-t4mjr (ro)
  snapshot-controller:
    Container ID:  docker://c7eb4772e750e82788b7b02bbcb599d6ced71dcf83ed91a717a0f3efe7d2dfd8
    Image:         k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/snapshot-controller@sha256:00fcc441ea9f72899c25eed61d602272a2a58c5f0014332bdcb5ac24acef08e4
    Port:          <none>
    Host Port:     <none>
    Args:
      --v=5
      --leader-election=true
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:54 +0100
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-t4mjr (ro)
  csi-provisioner:
    Container ID:  docker://6f5dd36c5c2aa12fd4543d1c3dda01e153ddd0e815c11f572f9510484259c9ca
    Image:         k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Image ID:      docker-pullable://k8s.gcr.io/sig-storage/csi-provisioner@sha256:6477988532358148d2e98f7c747db4e9250bbc7ad2664bf666348abf9ee1f5aa
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:54 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:  kube-system (v1:metadata.namespace)
      POD_NAME:   openebs-zfs-controller-0 (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-t4mjr (ro)
  openebs-zfs-plugin:
    Container ID:  docker://2aecae1bb7fe774baebf0f156123b4e7c5790ad3658bada2a8d335cfc9879634
    Image:         openebs/zfs-driver:2.0.0
    Image ID:      docker-pullable://openebs/zfs-driver@sha256:472dc7ca43f9780246cc6d2d478ff4b08ead904d757dbb263f796fb613dbe04d
    Port:          <none>
    Host Port:     <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:54 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-t4mjr (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-t4mjr:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             prometheus-operator-5dcffb7cb8-bnp7n
Namespace:        prometheus-operator
Priority:         0
Service Account:  prometheus-operator
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:04:02 +0100
Labels:           app.kubernetes.io/component=controller
                  app.kubernetes.io/name=prometheus-operator
                  app.kubernetes.io/version=0.60.1
                  pod-template-hash=5dcffb7cb8
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.29"
                        ],
                        "mac": "1a:6b:e7:33:8f:fb",
                        "default": true,
                        "dns": {}
                    }]
                  kubectl.kubernetes.io/default-container: prometheus-operator
Status:           Running
IP:               172.16.2.29
IPs:
  IP:           172.16.2.29
Controlled By:  ReplicaSet/prometheus-operator-5dcffb7cb8
Containers:
  prometheus-operator:
    Container ID:  docker://6064e856750cbcb5761bee771641a6dc935dbe1e477f9305be58484324ec45bb
    Image:         quay.io/prometheus-operator/prometheus-operator:v0.60.1
    Image ID:      docker-pullable://quay.io/prometheus-operator/prometheus-operator@sha256:63d4d0f8fb817313ec13c2960b72d1b4341bea269eb97ec7bf44c585a04b1e2e
    Port:          8080/TCP
    Host Port:     0/TCP
    Args:
      --kubelet-service=kube-system/kubelet
      --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.60.1
    State:          Running
      Started:      Wed, 18 Jan 2023 06:04:04 +0100
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     200m
      memory:  200Mi
    Requests:
      cpu:        100m
      memory:     100Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-nzdmn (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  kube-api-access-nzdmn:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             svclb-jellyfin-255633bc-m2wcn
Namespace:        kube-system
Priority:         0
Service Account:  svclb
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:04:24 +0100
Labels:           app=svclb-jellyfin-255633bc
                  controller-revision-hash=7cd999c45c
                  pod-template-generation=1
                  svccontroller.k3s.cattle.io/svcname=jellyfin
                  svccontroller.k3s.cattle.io/svcnamespace=ix-jellyfin
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.32"
                        ],
                        "mac": "0e:e8:03:7e:44:3c",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.2.32
IPs:
  IP:           172.16.2.32
Controlled By:  DaemonSet/svclb-jellyfin-255633bc
Containers:
  lb-tcp-8096:
    Container ID:   docker://a20c4b53d3dfb651b8d009ef988b3d48297ae8f147ab68367b0fe1c4ba096ddf
    Image:          rancher/klipper-lb:v0.3.5
    Image ID:       docker://sha256:dbd43b6716a084c929553d435c5f86f17b331739fe4652eb95e6bdc12df38a10
    Port:           8096/TCP
    Host Port:      8096/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 06:04:25 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      SRC_PORT:    8096
      SRC_RANGES:  0.0.0.0/0
      DEST_PROTO:  TCP
      DEST_PORT:   8096
      DEST_IPS:    172.17.225.233
    Mounts:        <none>
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:            <none>
QoS Class:          BestEffort
Node-Selectors:     <none>
Tolerations:        CriticalAddonsOnly op=Exists
                    node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                    node-role.kubernetes.io/master:NoSchedule op=Exists
                    node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                    node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                    node.kubernetes.io/not-ready:NoExecute op=Exists
                    node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                    node.kubernetes.io/unreachable:NoExecute op=Exists
                    node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:             <none>


Name:             svclb-podgrab-67e271ac-5jfjq
Namespace:        kube-system
Priority:         0
Service Account:  svclb
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:04:40 +0100
Labels:           app=svclb-podgrab-67e271ac
                  controller-revision-hash=85b567fcf7
                  pod-template-generation=1
                  svccontroller.k3s.cattle.io/svcname=podgrab
                  svccontroller.k3s.cattle.io/svcnamespace=ix-podgrab
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.34"
                        ],
                        "mac": "62:1b:3e:e7:bd:df",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.2.34
IPs:
  IP:           172.16.2.34
Controlled By:  DaemonSet/svclb-podgrab-67e271ac
Containers:
  lb-tcp-10093:
    Container ID:   docker://0d736603a0fef0b8478e877a1d9b8a863116f9f5c684ed75156d1a4fc4be70c0
    Image:          rancher/klipper-lb:v0.3.5
    Image ID:       docker://sha256:dbd43b6716a084c929553d435c5f86f17b331739fe4652eb95e6bdc12df38a10
    Port:           10093/TCP
    Host Port:      10093/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 06:04:41 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      SRC_PORT:    10093
      SRC_RANGES:  0.0.0.0/0
      DEST_PROTO:  TCP
      DEST_PORT:   10093
      DEST_IPS:    172.17.188.203
    Mounts:        <none>
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:            <none>
QoS Class:          BestEffort
Node-Selectors:     <none>
Tolerations:        CriticalAddonsOnly op=Exists
                    node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                    node-role.kubernetes.io/master:NoSchedule op=Exists
                    node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                    node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                    node.kubernetes.io/not-ready:NoExecute op=Exists
                    node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                    node.kubernetes.io/unreachable:NoExecute op=Exists
                    node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:             <none>


Name:             svclb-jellyfin2-fd08da81-5vlr9
Namespace:        kube-system
Priority:         0
Service Account:  svclb
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:04:56 +0100
Labels:           app=svclb-jellyfin2-fd08da81
                  controller-revision-hash=5dcf8888d9
                  pod-template-generation=1
                  svccontroller.k3s.cattle.io/svcname=jellyfin2
                  svccontroller.k3s.cattle.io/svcnamespace=ix-jellyfin2
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.36"
                        ],
                        "mac": "b2:a1:64:ca:6d:9e",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.2.36
IPs:
  IP:           172.16.2.36
Controlled By:  DaemonSet/svclb-jellyfin2-fd08da81
Containers:
  lb-tcp-8097:
    Container ID:   docker://a4b9911b6fec2e71d8126640caec91e7ddc3d63af97ddcfe49d722675b712a00
    Image:          rancher/klipper-lb:v0.3.5
    Image ID:       docker://sha256:dbd43b6716a084c929553d435c5f86f17b331739fe4652eb95e6bdc12df38a10
    Port:           8097/TCP
    Host Port:      8097/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 06:04:57 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      SRC_PORT:    8097
      SRC_RANGES:  0.0.0.0/0
      DEST_PROTO:  TCP
      DEST_PORT:   8097
      DEST_IPS:    172.17.197.21
    Mounts:        <none>
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:            <none>
QoS Class:          BestEffort
Node-Selectors:     <none>
Tolerations:        CriticalAddonsOnly op=Exists
                    node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                    node-role.kubernetes.io/master:NoSchedule op=Exists
                    node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                    node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                    node.kubernetes.io/not-ready:NoExecute op=Exists
                    node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                    node.kubernetes.io/unreachable:NoExecute op=Exists
                    node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:             <none>


Name:             syncthing-6c5594b84-lqwgb
Namespace:        ix-syncthing
Priority:         0
Service Account:  default
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:05:02 +0100
Labels:           app=syncthing
                  app.kubernetes.io/instance=syncthing
                  app.kubernetes.io/name=syncthing
                  pod-template-hash=6c5594b84
                  release=syncthing
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.37"
                        ],
                        "mac": "ba:91:68:c0:27:6a",
                        "default": true,
                        "dns": {}
                    }]
                  rollme: HO11v
Status:           Running
IP:               172.16.2.37
IPs:
  IP:           172.16.2.37
Controlled By:  ReplicaSet/syncthing-6c5594b84
Containers:
  syncthing:
    Container ID:   docker://e85657019089f948b4168a15582024022acca01273c5c41062db84f269e1caca
    Image:          syncthing/syncthing:1.23.0
    Image ID:       docker-pullable://syncthing/syncthing@sha256:57699adea5bae30edd2c4f83d5c311969e3255c3d23e7de747638dffbc3a8bc2
    Ports:          8384/TCP, 22000/TCP, 22000/UDP
    Host Ports:     0/TCP, 0/TCP, 0/UDP
    State:          Running
      Started:      Wed, 18 Jan 2023 06:05:03 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      PUID:          568
      PGID:          568
      STGUIADDRESS:  0.0.0.0:8384
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-2qqn4 (ro)
      /var/syncthing from config (rw)
      /vault/media from extrappvolume-0 (rw)
      /vault/vault from extrappvolume-1 (rw)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/syncthing-config
    HostPathType:  
  extrappvolume-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/media
    HostPathType:  
  extrappvolume-1:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/vault
    HostPathType:  
  kube-api-access-2qqn4:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             jellyfin2-7b88f7d76f-pr64f
Namespace:        ix-jellyfin2
Priority:         0
Service Account:  default
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:03:49 +0100
Labels:           app.kubernetes.io/instance=jellyfin2
                  app.kubernetes.io/name=jellyfin
                  pod-template-hash=7b88f7d76f
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.23"
                        ],
                        "mac": "36:19:41:ca:31:10",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.2.23
IPs:
  IP:           172.16.2.23
Controlled By:  ReplicaSet/jellyfin2-7b88f7d76f
Containers:
  jellyfin2:
    Container ID:   docker://602a4d0989ef1edfe87cd83d52119ce5cc539207890ec628aaae4cf5acf27cbe
    Image:          tccr.io/truecharts/jellyfin:10.8.8@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5
    Image ID:       docker-pullable://tccr.io/truecharts/jellyfin@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5
    Port:           8096/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:53 +0100
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    tcp-socket :8096 delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                        2
      UMASK_SET:                    2
      S6_READ_ONLY_ROOT:            1
      NVIDIA_VISIBLE_DEVICES:       void
      TZ:                           Europe/Berlin
      JELLYFIN_PublishedServerUrl:  
    Mounts:
      /cache from cache (rw)
      /config from config (rw)
      /config/transcodes from transcode (rw)
      /dev/shm from shm (rw)
      /privat from list-0 (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-pk5df (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/jellyfin2-config
    HostPathType:  
  list-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/privat
    HostPathType:  
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  transcode:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  jellyfin2-transcode
    ReadOnly:   false
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-pk5df:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             jellyfin-7f97d55865-p8m7x
Namespace:        ix-jellyfin
Priority:         0
Service Account:  default
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:03:55 +0100
Labels:           app.kubernetes.io/instance=jellyfin
                  app.kubernetes.io/name=jellyfin
                  pod-template-hash=7f97d55865
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.25"
                        ],
                        "mac": "6e:c8:a9:ad:6e:8c",
                        "default": true,
                        "dns": {}
                    }]
Status:           Running
IP:               172.16.2.25
IPs:
  IP:           172.16.2.25
Controlled By:  ReplicaSet/jellyfin-7f97d55865
Containers:
  jellyfin:
    Container ID:   docker://b04a7824e972ebc239c2e6c2e14c41b7c0481e38df69c8b7654acef0cdc93844
    Image:          tccr.io/truecharts/jellyfin:10.8.8@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5
    Image ID:       docker-pullable://tccr.io/truecharts/jellyfin@sha256:c66fa2d0c2e94f33e6659ce0be4ef715083a3188ce9ddfafd2bc405d3389cbe5
    Port:           8096/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:59 +0100
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:      10m
      memory:   50Mi
    Liveness:   tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Readiness:  tcp-socket :8096 delay=10s timeout=5s period=10s #success=1 #failure=5
    Startup:    tcp-socket :8096 delay=10s timeout=2s period=5s #success=1 #failure=60
    Environment:
      UMASK:                        2
      UMASK_SET:                    2
      S6_READ_ONLY_ROOT:            1
      NVIDIA_VISIBLE_DEVICES:       void
      TZ:                           Europe/Berlin
      JELLYFIN_PublishedServerUrl:  
    Mounts:
      /cache from cache (rw)
      /config from config (rw)
      /config/transcodes from transcode (rw)
      /dev/shm from shm (rw)
      /media from list-0 (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6f8mm (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  cache:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  config:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/jellyfin-config
    HostPathType:  
  list-0:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/vault/media
    HostPathType:  
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  transcode:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  jellyfin-transcode
    ReadOnly:   false
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  kube-api-access-6f8mm:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:                 coredns-75fc8f8fff-jkzv8
Namespace:            kube-system
Priority:             2000000000
Priority Class Name:  system-cluster-critical
Service Account:      coredns
Node:                 ix-truenas/172.23.32.10
Start Time:           Wed, 18 Jan 2023 06:03:46 +0100
Labels:               k8s-app=kube-dns
                      pod-template-hash=75fc8f8fff
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.2.21"
                            ],
                            "mac": "12:82:5a:96:44:de",
                            "default": true,
                            "dns": {}
                        }]
Status:               Running
IP:                   172.16.2.21
IPs:
  IP:           172.16.2.21
Controlled By:  ReplicaSet/coredns-75fc8f8fff
Containers:
  coredns:
    Container ID:  docker://b2042bb7b5951be6be0f49468b6142b8b7ff4e1542ddddae7c983aad85ed6dbd
    Image:         rancher/mirrored-coredns-coredns:1.9.1
    Image ID:      docker://sha256:99376d8f35e0abb6ff9d66b50a7c81df6e6dfdb649becc5df84a691a7b4beca4
    Ports:         53/UDP, 53/TCP, 9153/TCP
    Host Ports:    0/UDP, 0/TCP, 0/TCP
    Args:
      -conf
      /etc/coredns/Corefile
    State:          Running
      Started:      Wed, 18 Jan 2023 06:03:48 +0100
    Ready:          True
    Restart Count:  0
    Limits:
      memory:  170Mi
    Requests:
      cpu:        100m
      memory:     70Mi
    Liveness:     http-get http://:8080/health delay=60s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:8181/ready delay=0s timeout=1s period=2s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/coredns from config-volume (ro)
      /etc/coredns/custom from custom-config-volume (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-fbzj6 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns
    Optional:  false
  custom-config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      coredns-custom
    Optional:  true
  kube-api-access-fbzj6:
    Type:                     Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:   3607
    ConfigMapName:            kube-root-ca.crt
    ConfigMapOptional:        <nil>
    DownwardAPI:              true
QoS Class:                    Burstable
Node-Selectors:               beta.kubernetes.io/os=linux
Tolerations:                  CriticalAddonsOnly op=Exists
                              node-role.kubernetes.io/control-plane:NoSchedule op=Exists
                              node-role.kubernetes.io/master:NoSchedule op=Exists
                              node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                              node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Topology Spread Constraints:  kubernetes.io/hostname:DoNotSchedule when max skew 1 is exceeded for selector k8s-app=kube-dns
Events:                       <none>


Name:             controller-84d6d4db45-9glps
Namespace:        metallb-system
Priority:         0
Service Account:  controller
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:04:05 +0100
Labels:           app=metallb
                  component=controller
                  pod-template-hash=84d6d4db45
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.30"
                        ],
                        "mac": "02:60:41:c7:05:76",
                        "default": true,
                        "dns": {}
                    }]
                  prometheus.io/port: 7472
                  prometheus.io/scrape: true
Status:           Running
IP:               172.16.2.30
IPs:
  IP:           172.16.2.30
Controlled By:  ReplicaSet/controller-84d6d4db45
Containers:
  controller:
    Container ID:  docker://e0fa96f0b7c38071e79b23333fb307916cf8a50e478d6f6d74d60a8cf7d04d66
    Image:         quay.io/metallb/controller:v0.13.7
    Image ID:      docker-pullable://quay.io/metallb/controller@sha256:0e486995b27d62433e728f7b266fb8b7b7f4fd65e1b751bef17fe56e76653bf1
    Ports:         7472/TCP, 9443/TCP
    Host Ports:    0/TCP, 0/TCP
    Args:
      --port=7472
      --log-level=info
    State:          Running
      Started:      Wed, 18 Jan 2023 06:04:47 +0100
    Last State:     Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Wed, 18 Jan 2023 06:04:07 +0100
      Finished:     Wed, 18 Jan 2023 06:04:47 +0100
    Ready:          True
    Restart Count:  1
    Liveness:       http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:      http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_ML_SECRET_NAME:  memberlist
      METALLB_DEPLOYMENT:      controller
    Mounts:
      /tmp/k8s-webhook-server/serving-certs from cert (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-glc7t (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  cert:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  webhook-server-cert
    Optional:    false
  kube-api-access-glc7t:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             tailscale-6d84b7f475-m9498
Namespace:        ix-tailscale
Priority:         0
Service Account:  tailscale
Node:             ix-truenas/172.23.32.10
Start Time:       Wed, 18 Jan 2023 06:03:42 +0100
Labels:           app.kubernetes.io/instance=tailscale
                  app.kubernetes.io/name=tailscale
                  pod-template-hash=6d84b7f475
Annotations:      <none>
Status:           Running
IP:               172.23.32.10
IPs:
  IP:           172.23.32.10
Controlled By:  ReplicaSet/tailscale-6d84b7f475
Containers:
  tailscale:
    Container ID:  docker://76a81876f5a8e8f8343777f08b8d95a250ed9fe4321475206c3ffaa23c5e2ca2
    Image:         tailscale/tailscale:v1.34.2@sha256:08d4203e1664be5f51238bb32f0b93524b497056719ce3f51bde431ef2dbee54
    Image ID:      docker-pullable://tailscale/tailscale@sha256:08d4203e1664be5f51238bb32f0b93524b497056719ce3f51bde431ef2dbee54
    Port:          <none>
    Host Port:     <none>
    Command:
      /usr/local/bin/containerboot
    State:          Running
      Started:      Wed, 18 Jan 2023 06:06:02 +0100
    Last State:     Terminated
      Reason:       Error
      Exit Code:    1
      Started:      Wed, 18 Jan 2023 06:04:46 +0100
      Finished:     Wed, 18 Jan 2023 06:05:46 +0100
    Ready:          True
    Restart Count:  2
    Limits:
      cpu:     4
      memory:  8Gi
    Requests:
      cpu:     10m
      memory:  50Mi
    Environment Variables from:
      tailscale-tailscale-config  ConfigMap  Optional: false
    Environment:
      PUID:                    568
      USER_ID:                 568
      UID:                     568
      UMASK:                   2
      UMASK_SET:               2
      PGID:                    568
      GROUP_ID:                568
      GID:                     568
      NVIDIA_VISIBLE_DEVICES:  void
      TZ:                      Europe/Berlin
    Mounts:
      /dev/net/tun from tun (rw)
      /dev/shm from shm (rw)
      /shared from shared (rw)
      /tmp from temp (rw)
      /var/logs from varlogs (rw)
      /var/run from varrun (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-bsqh9 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  shared:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  shm:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  temp:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  tun:
    Type:          HostPath (bare host directory volume)
    Path:          /dev/net/tun
    HostPathType:  
  varlogs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     
    SizeLimit:  <unset>
  varrun:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  kube-api-access-bsqh9:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             nextcloud-postgres-5777466b99-vpdzg
Namespace:        ix-nextcloud
Priority:         0
Service Account:  default
Node:             ix-truenas/172.23.32.10
Start Time:       Thu, 19 Jan 2023 15:11:40 +0100
Labels:           app.kubernetes.io/instance=nextcloud-postgres
                  app.kubernetes.io/name=nextcloud-postgres
                  pod-template-hash=5777466b99
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.43"
                        ],
                        "mac": "7e:38:83:96:c6:84",
                        "default": true,
                        "dns": {}
                    }]
                  rollme: efVSg
Status:           Running
IP:               172.16.2.43
IPs:
  IP:           172.16.2.43
Controlled By:  ReplicaSet/nextcloud-postgres-5777466b99
Containers:
  nextcloud-postgres:
    Container ID:   docker://373eba60baabb1bc5def0ceb669a819519e844675f0e9f5741bf217da0ca7138
    Image:          postgres:13.1
    Image ID:       docker-pullable://postgres@sha256:26180996f7d804993f9c90398615d1269709a799b5eb79109292843a1c7cd851
    Port:           5432/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Thu, 19 Jan 2023 15:11:41 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:
      /postgres_backups from postgres-backup (rw)
      /var/lib/postgresql/data from postgres-data (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-lm59g (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  postgres-backup:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/apps/ix-applications/releases/nextcloud/volumes/ix_volumes/ix-postgres_backups
    HostPathType:  
  postgres-data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/apps/ix-applications/releases/nextcloud/volumes/ix_volumes/ix-postgres_data
    HostPathType:  
  kube-api-access-lm59g:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:             nextcloud-cbf457787-mks7l
Namespace:        ix-nextcloud
Priority:         0
Service Account:  default
Node:             ix-truenas/172.23.32.10
Start Time:       Thu, 19 Jan 2023 15:11:40 +0100
Labels:           app.kubernetes.io/instance=nextcloud
                  app.kubernetes.io/name=nextcloud
                  pod-template-hash=cbf457787
Annotations:      k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ix-net",
                        "interface": "eth0",
                        "ips": [
                            "172.16.2.42"
                        ],
                        "mac": "5e:02:50:a8:a8:94",
                        "default": true,
                        "dns": {}
                    }]
                  rollme: w2iDv
Status:           Running
IP:               172.16.2.42
IPs:
  IP:           172.16.2.42
Controlled By:  ReplicaSet/nextcloud-cbf457787
Init Containers:
  init-postgresdb:
    Container ID:  docker://863e2ed6c6aaee33b5667ef9383b60a9a723803268701e72f46318c62b40da7a
    Image:         postgres:13.1
    Image ID:      docker-pullable://postgres@sha256:26180996f7d804993f9c90398615d1269709a799b5eb79109292843a1c7cd851
    Port:          <none>
    Host Port:     <none>
    Command:
      sh
      -c
      until pg_isready -U $POSTGRES_USER -d nextcloud -h nextcloud-postgres; do echo waiting for postgres; sleep 2; done
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Thu, 19 Jan 2023 15:11:41 +0100
      Finished:     Thu, 19 Jan 2023 15:11:43 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      POSTGRES_USER:      <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:  <set to the key 'db-password' in secret 'db-details'>  Optional: false
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-8gxhh (ro)
Containers:
  nginx:
    Container ID:   docker://c40e39a47de35864696d9fc9cff0dd4f6632a9da263a4035d28ac060182b665d
    Image:          nginx:1.23.3
    Image ID:       docker-pullable://nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e
    Ports:          8000/TCP, 443/TCP
    Host Ports:     0/TCP, 0/TCP
    State:          Running
      Started:      Thu, 19 Jan 2023 15:11:44 +0100
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /etc/nginx-certs from cert-secret-volume (rw)
      /etc/nginx/nginx.conf from nginx-configuration (rw,path="nginx.conf")
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-8gxhh (ro)
  nextcloud:
    Container ID:   docker://336a3752f086295e2096c35cbaa3f85814081a0dd4796fe20dbe5aac20a417b5
    Image:          nextcloud:25.0.3
    Image ID:       docker-pullable://nextcloud@sha256:aa43a87b0b6acd52c1989dc343d1702f6a9d3ae1be263e62a9d0f99ec10f29a7
    Port:           80/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Thu, 19 Jan 2023 15:12:02 +0100
    Ready:          True
    Restart Count:  0
    Environment:
      POSTGRES_USER:              <set to the key 'db-user' in secret 'db-details'>      Optional: false
      POSTGRES_PASSWORD:          <set to the key 'db-password' in secret 'db-details'>  Optional: false
      POSTGRES_HOST:              nextcloud-postgres:5432
      POSTGRES_DB:                nextcloud
      NEXTCLOUD_DATA_DIR:         /var/www/html/data
      APACHE_DISABLE_REWRITE_IP:  1
      OVERWRITEHOST:              172.23.32.10:9001
      OVERWRITEPROTOCOL:          https
      TRUSTED_PROXIES:            127.0.0.1
      NEXTCLOUD_TRUSTED_DOMAINS:  172.23.32.10
      NEXTCLOUD_ADMIN_USER:       <set to the key 'nextcloud-username' in secret 'nextcloud'>  Optional: false
      NEXTCLOUD_ADMIN_PASSWORD:   <set to the key 'nextcloud-password' in secret 'nextcloud'>  Optional: false
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-8gxhh (ro)
      /var/www/ from nextcloud-data (rw,path="root")
      /var/www/html from nextcloud-data (rw,path="html")
      /var/www/html/config from nextcloud-data (rw,path="config")
      /var/www/html/custom_apps from nextcloud-data (rw,path="custom_apps")
      /var/www/html/data from nextcloud-data (rw,path="data")
      /var/www/html/themes from nextcloud-data (rw,path="themes")
      /var/www/tmp from nextcloud-data (rw,path="tmp")
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  nginx-configuration:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      nginx-configuration
    Optional:  false
  cert-secret-volume:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  nginx-secret
    Optional:    false
  nextcloud-data:
    Type:          HostPath (bare host directory volume)
    Path:          /mnt/app-data/nextcloud-data
    HostPathType:  
  kube-api-access-8gxhh:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>


Name:                 nvidia-device-plugin-daemonset-f8pn2
Namespace:            kube-system
Priority:             2000001000
Priority Class Name:  system-node-critical
Service Account:      default
Node:                 ix-truenas/172.23.32.10
Start Time:           Wed, 18 Jan 2023 06:03:45 +0100
Labels:               controller-revision-hash=57d78cc674
                      name=nvidia-device-plugin-ds
                      pod-template-generation=1
Annotations:          k8s.v1.cni.cncf.io/network-status:
                        [{
                            "name": "ix-net",
                            "interface": "eth0",
                            "ips": [
                                "172.16.2.20"
                            ],
                            "mac": "02:50:2c:3f:54:c2",
                            "default": true,
                            "dns": {}
                        }]
                      scheduler.alpha.kubernetes.io/critical-pod: 
Status:               Running
IP:                   172.16.2.20
IPs:
  IP:           172.16.2.20
Controlled By:  DaemonSet/nvidia-device-plugin-daemonset
Containers:
  nvidia-device-plugin-ctr:
    Container ID:  docker://2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4
    Image:         nvidia/k8s-device-plugin:v0.10.0
    Image ID:      docker-pullable://nvidia/k8s-device-plugin@sha256:5b967b3e92900797a74e0f7cd71005747fa3154503986676f372b9d82fe1d898
    Port:          <none>
    Host Port:     <none>
    State:         Waiting
      Reason:      CrashLoopBackOff
    Last State:    Terminated
      Reason:      ContainerCannotRun
      Message:     failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
      Exit Code:    128
      Started:      Thu, 19 Jan 2023 22:56:07 +0100
      Finished:     Thu, 19 Jan 2023 22:56:07 +0100
    Ready:          False
    Restart Count:  484
    Environment:    <none>
    Mounts:
      /var/lib/kubelet/device-plugins from device-plugin (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6l7hd (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  device-plugin:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/device-plugins
    HostPathType:  
  kube-api-access-6l7hd:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 CriticalAddonsOnly op=Exists
                             node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
                             nvidia.com/gpu:NoSchedule op=Exists
Events:
  Type     Reason   Age                      From     Message
  ----     ------   ----                     ----     -------
  Warning  BackOff  2m17s (x11081 over 40h)  kubelet  Back-off restarting failed container
debug finished in 0 seconds for k3s kubectl describe pods --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe services --all-namespaces @1674165674           +
+--------------------------------------------------------------------------------+
Name:              kubernetes
Namespace:         default
Labels:            component=apiserver
                   provider=kubernetes
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.1
IPs:               172.17.0.1
Port:              https  443/TCP
TargetPort:        6443/TCP
Endpoints:         172.23.32.10:6443
Session Affinity:  None
Events:            <none>


Name:              kube-dns
Namespace:         kube-system
Labels:            k8s-app=kube-dns
                   kubernetes.io/cluster-service=true
                   kubernetes.io/name=CoreDNS
                   objectset.rio.cattle.io/hash=bce283298811743a0386ab510f2f67ef74240c57
Annotations:       objectset.rio.cattle.io/applied:
                     H4sIAAAAAAAA/4yRwW7UMBCGXwXN2Qlxstukljig9oKQUKUCF8TBcWZZk8S2PJMgtMq7I++mYqEK9JZoPn/6558T6GA/YyTrHSiYJQjoretAwSPG2RoEASOy7jRrUCfQznnWbL2j9O...
                   objectset.rio.cattle.io/id: 
                   objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                   objectset.rio.cattle.io/owner-name: coredns
                   objectset.rio.cattle.io/owner-namespace: kube-system
                   prometheus.io/port: 9153
                   prometheus.io/scrape: true
Selector:          k8s-app=kube-dns
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.0.10
IPs:               172.17.0.10
Port:              dns  53/UDP
TargetPort:        53/UDP
Endpoints:         172.16.2.21:53
Port:              dns-tcp  53/TCP
TargetPort:        53/TCP
Endpoints:         172.16.2.21:53
Port:              metrics  9153/TCP
TargetPort:        9153/TCP
Endpoints:         172.16.2.21:9153
Session Affinity:  None
Events:            <none>


Name:              webhook-service
Namespace:         metallb-system
Labels:            <none>
Annotations:       <none>
Selector:          component=controller
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.238.220
IPs:               172.17.238.220
Port:              <unset>  443/TCP
TargetPort:        9443/TCP
Endpoints:         172.16.2.30:9443
Session Affinity:  None
Events:            <none>


Name:              prometheus-operator
Namespace:         prometheus-operator
Labels:            app.kubernetes.io/component=controller
                   app.kubernetes.io/name=prometheus-operator
                   app.kubernetes.io/version=0.60.1
Annotations:       <none>
Selector:          app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                None
IPs:               None
Port:              http  8080/TCP
TargetPort:        http/TCP
Endpoints:         172.16.2.29:8080
Session Affinity:  None
Events:            <none>


Name:              kubelet
Namespace:         kube-system
Labels:            app.kubernetes.io/managed-by=prometheus-operator
                   app.kubernetes.io/name=kubelet
                   k8s-app=kubelet
Annotations:       <none>
Selector:          <none>
Type:              ClusterIP
IP Family Policy:  RequireDualStack
IP Families:       IPv4,IPv6
IP:                None
IPs:               None
Port:              https-metrics  10250/TCP
TargetPort:        10250/TCP
Endpoints:         172.23.32.10:10250
Port:              http-metrics  10255/TCP
TargetPort:        10255/TCP
Endpoints:         172.23.32.10:10255
Port:              cadvisor  4194/TCP
TargetPort:        4194/TCP
Endpoints:         172.23.32.10:4194
Session Affinity:  None
Events:            <none>


Name:                     syncthing
Namespace:                ix-syncthing
Labels:                   app=syncthing
                          app.kubernetes.io/instance=syncthing
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=syncthing
                          app.kubernetes.io/version=1.23.0
                          helm.sh/chart=syncthing-1.0.16
                          release=syncthing
Annotations:              meta.helm.sh/release-name: syncthing
                          meta.helm.sh/release-namespace: ix-syncthing
Selector:                 app.kubernetes.io/instance=syncthing,app.kubernetes.io/name=syncthing,app=syncthing,release=syncthing
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.79.0
IPs:                      172.17.79.0
Port:                     web  20910/TCP
TargetPort:               8384/TCP
NodePort:                 web  20910/TCP
Endpoints:                172.16.2.37:8384
Port:                     tcp  20978/TCP
TargetPort:               22000/TCP
NodePort:                 tcp  20978/TCP
Endpoints:                172.16.2.37:22000
Port:                     udp  20979/UDP
TargetPort:               22000/UDP
NodePort:                 udp  20979/UDP
Endpoints:                172.16.2.37:22000
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>


Name:                     jellyfin
Namespace:                ix-jellyfin
Labels:                   app.kubernetes.io/instance=jellyfin
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=jellyfin
                          app.kubernetes.io/version=10.8.8
                          helm-revision=17
                          helm.sh/chart=jellyfin-13.0.9
Annotations:              meta.helm.sh/release-name: jellyfin
                          meta.helm.sh/release-namespace: ix-jellyfin
                          metallb.universe.tf/allow-shared-ip: jellyfin
Selector:                 app.kubernetes.io/instance=jellyfin,app.kubernetes.io/name=jellyfin
Type:                     LoadBalancer
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.225.233
IPs:                      172.17.225.233
LoadBalancer Ingress:     172.23.32.10
Port:                     main  8096/TCP
TargetPort:               8096/TCP
NodePort:                 main  43695/TCP
Endpoints:                172.16.2.25:8096
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>


Name:              podgrab-tcp
Namespace:         ix-podgrab
Labels:            app.kubernetes.io/instance=podgrab
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=podgrab
                   app.kubernetes.io/version=1.0.0
                   helm-revision=17
                   helm.sh/chart=podgrab-12.0.7
Annotations:       meta.helm.sh/release-name: podgrab
                   meta.helm.sh/release-namespace: ix-podgrab
Selector:          app.kubernetes.io/instance=podgrab,app.kubernetes.io/name=podgrab
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.168.247
IPs:               172.17.168.247
Port:              tcp  51080/TCP
TargetPort:        51080/TCP
Endpoints:         172.16.2.26:51080
Session Affinity:  None
Events:            <none>


Name:                     podgrab
Namespace:                ix-podgrab
Labels:                   app.kubernetes.io/instance=podgrab
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=podgrab
                          app.kubernetes.io/version=1.0.0
                          helm-revision=17
                          helm.sh/chart=podgrab-12.0.7
Annotations:              meta.helm.sh/release-name: podgrab
                          meta.helm.sh/release-namespace: ix-podgrab
                          metallb.universe.tf/allow-shared-ip: podgrab
Selector:                 app.kubernetes.io/instance=podgrab,app.kubernetes.io/name=podgrab
Type:                     LoadBalancer
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.188.203
IPs:                      172.17.188.203
LoadBalancer Ingress:     172.23.32.10
Port:                     main  10093/TCP
TargetPort:               8080/TCP
NodePort:                 main  47234/TCP
Endpoints:                172.16.2.26:8080
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>


Name:                     jellyfin2
Namespace:                ix-jellyfin2
Labels:                   app.kubernetes.io/instance=jellyfin2
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=jellyfin
                          app.kubernetes.io/version=10.8.8
                          helm-revision=18
                          helm.sh/chart=jellyfin-13.0.9
Annotations:              meta.helm.sh/release-name: jellyfin2
                          meta.helm.sh/release-namespace: ix-jellyfin2
                          metallb.universe.tf/allow-shared-ip: jellyfin2
Selector:                 app.kubernetes.io/instance=jellyfin2,app.kubernetes.io/name=jellyfin
Type:                     LoadBalancer
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.197.21
IPs:                      172.17.197.21
LoadBalancer Ingress:     172.23.32.10
Port:                     main  8097/TCP
TargetPort:               8096/TCP
NodePort:                 main  19593/TCP
Endpoints:                172.16.2.23:8096
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>


Name:              nextcloud-postgres
Namespace:         ix-nextcloud
Labels:            app.kubernetes.io/instance=nextcloud-postgres
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=nextcloud-postgres
                   app.kubernetes.io/version=25.0.3
                   helm.sh/chart=nextcloud-1.6.16
Annotations:       meta.helm.sh/release-name: nextcloud
                   meta.helm.sh/release-namespace: ix-nextcloud
Selector:          app.kubernetes.io/instance=nextcloud-postgres,app.kubernetes.io/name=nextcloud-postgres
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                172.17.20.160
IPs:               172.17.20.160
Port:              postgres-tcp  5432/TCP
TargetPort:        5432/TCP
Endpoints:         172.16.2.43:5432
Session Affinity:  None
Events:            <none>


Name:                     nextcloud
Namespace:                ix-nextcloud
Labels:                   app.kubernetes.io/instance=nextcloud
                          app.kubernetes.io/managed-by=Helm
                          app.kubernetes.io/name=nextcloud
                          app.kubernetes.io/version=25.0.3
                          helm.sh/chart=nextcloud-1.6.16
Annotations:              meta.helm.sh/release-name: nextcloud
                          meta.helm.sh/release-namespace: ix-nextcloud
Selector:                 app.kubernetes.io/instance=nextcloud,app.kubernetes.io/name=nextcloud
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       172.17.245.170
IPs:                      172.17.245.170
Port:                     nginx-https  443/TCP
TargetPort:               443/TCP
NodePort:                 nginx-https  9001/TCP
Endpoints:                172.16.2.42:443
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>
debug finished in 0 seconds for k3s kubectl describe services --all-namespaces


+--------------------------------------------------------------------------------+
+         k3s kubectl describe statefulset --all-namespaces @1674165674          +
+--------------------------------------------------------------------------------+
Name:               openebs-zfs-controller
Namespace:          kube-system
CreationTimestamp:  Mon, 02 Jan 2023 00:28:44 +0100
Selector:           app=openebs-zfs-controller,role=openebs-zfs
Labels:             objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
Annotations:        objectset.rio.cattle.io/applied:
                      H4sIAAAAAAAA/8xWXY/aOBT9KyurD60U8zEzu9ONxAMdshLqFBCgSlWFkHFughfH9to3Wego/33lBJh0Zpgvdau+ge17fM7xuY5vCDPiM1gntCIhYca4dtElAdkIFZOQzJAhJLmcAZ...
                    objectset.rio.cattle.io/id: 
                    objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                    objectset.rio.cattle.io/owner-name: zfs-operator
                    objectset.rio.cattle.io/owner-namespace: kube-system
Replicas:           1 desired | 1 total
Update Strategy:    RollingUpdate
  Partition:        0
Pods Status:        1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-controller
                    openebs.io/component-name=openebs-zfs-controller
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-controller-sa
  Containers:
   csi-resizer:
    Image:      k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   csi-snapshotter:
    Image:      k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --leader-election
    Environment:
      ADDRESS:  /var/lib/csi/sockets/pluginproxy/csi.sock
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   snapshot-controller:
    Image:      k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --leader-election=true
    Environment:  <none>
    Mounts:       <none>
   csi-provisioner:
    Image:      k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --csi-address=$(ADDRESS)
      --v=5
      --feature-gates=Topology=true
      --strict-topology
      --leader-election
      --extra-create-metadata=true
      --enable-capacity=true
      --default-fstype=ext4
    Environment:
      ADDRESS:    /var/lib/csi/sockets/pluginproxy/csi.sock
      NAMESPACE:   (v1:metadata.namespace)
      POD_NAME:    (v1:metadata.name)
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_CONTROLLER_DRIVER)
    Environment:
      OPENEBS_CONTROLLER_DRIVER:    controller
      OPENEBS_CSI_ENDPOINT:         unix:///var/lib/csi/sockets/pluginproxy/csi.sock
      OPENEBS_NAMESPACE:            openebs
      OPENEBS_IO_INSTALLER_TYPE:    zfs-operator
      OPENEBS_IO_ENABLE_ANALYTICS:  true
    Mounts:
      /var/lib/csi/sockets/pluginproxy/ from socket-dir (rw)
  Volumes:
   socket-dir:
    Type:               EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:             
    SizeLimit:          <unset>
  Priority Class Name:  openebs-zfs-csi-controller-critical
Volume Claims:          <none>
Events:                 <none>
debug finished in 0 seconds for k3s kubectl describe statefulset --all-namespaces


+--------------------------------------------------------------------------------+
+             k3s kubectl describe job --all-namespaces @1674165674              +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 1 seconds for k3s kubectl describe job --all-namespaces


+--------------------------------------------------------------------------------+
+           k3s kubectl describe cronjob --all-namespaces @1674165675            +
+--------------------------------------------------------------------------------+
No resources found
debug finished in 0 seconds for k3s kubectl describe cronjob --all-namespaces


+--------------------------------------------------------------------------------+
+          k3s kubectl describe daemonsets --all-namespaces @1674165675          +
+--------------------------------------------------------------------------------+
Name:           nvidia-device-plugin-daemonset
Selector:       name=nvidia-device-plugin-ds
Node-Selector:  <none>
Labels:         <none>
Annotations:    deprecated.daemonset.template.generation: 1
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 0
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:       name=nvidia-device-plugin-ds
  Annotations:  scheduler.alpha.kubernetes.io/critical-pod: 
  Containers:
   nvidia-device-plugin-ctr:
    Image:        nvidia/k8s-device-plugin:v0.10.0
    Port:         <none>
    Host Port:    <none>
    Environment:  <none>
    Mounts:
      /var/lib/kubelet/device-plugins from device-plugin (rw)
  Volumes:
   device-plugin:
    Type:               HostPath (bare host directory volume)
    Path:               /var/lib/kubelet/device-plugins
    HostPathType:       
  Priority Class Name:  system-node-critical
Events:                 <none>


Name:           openebs-zfs-node
Selector:       app=openebs-zfs-node
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=4b1efd47d7f37713706abe9f8b0de3f0adf02e2c
                openebs.io/component-name=openebs-zfs-node
                openebs.io/version=ci
Annotations:    deprecated.daemonset.template.generation: 1
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/6xWUY/aOBD+K5XVk1opIezSdnuR9oFCrl11FxBsew/VCjn2JPhw7Mh20uVW+e8nOwEChO2e1Lcknvlm5ptvJn5COGffQWkmBQoRznMdlBfIQ2smKArRGEMmxQIM8l...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: k3s.cattle.io/v1, Kind=Addon
                objectset.rio.cattle.io/owner-name: zfs-operator
                objectset.rio.cattle.io/owner-namespace: kube-system
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=openebs-zfs-node
                    openebs.io/component-name=openebs-zfs-node
                    openebs.io/version=ci
                    role=openebs-zfs
  Service Account:  openebs-zfs-node-sa
  Containers:
   csi-node-driver-registrar:
    Image:      k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --v=5
      --csi-address=$(ADDRESS)
      --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
    Environment:
      ADDRESS:               /plugin/csi.sock
      DRIVER_REG_SOCK_PATH:  /var/lib/kubelet/plugins/zfs-localpv/csi.sock
      KUBE_NODE_NAME:         (v1:spec.nodeName)
      NODE_DRIVER:           openebs-zfs
    Mounts:
      /plugin from plugin-dir (rw)
      /registration from registration-dir (rw)
   openebs-zfs-plugin:
    Image:      openebs/zfs-driver:2.0.0
    Port:       <none>
    Host Port:  <none>
    Args:
      --nodename=$(OPENEBS_NODE_NAME)
      --endpoint=$(OPENEBS_CSI_ENDPOINT)
      --plugin=$(OPENEBS_NODE_DRIVER)
    Environment:
      OPENEBS_NODE_NAME:      (v1:spec.nodeName)
      OPENEBS_CSI_ENDPOINT:  unix:///plugin/csi.sock
      OPENEBS_NODE_DRIVER:   agent
      OPENEBS_NAMESPACE:     openebs
      ALLOWED_TOPOLOGIES:    All
    Mounts:
      /dev from device-dir (rw)
      /home/keys from encr-keys (rw)
      /host from host-root (ro)
      /plugin from plugin-dir (rw)
      /sbin/zfs from chroot-zfs (rw,path="zfs")
      /var/lib/kubelet/ from pods-mount-dir (rw)
  Volumes:
   device-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  Directory
   encr-keys:
    Type:          HostPath (bare host directory volume)
    Path:          /home/keys
    HostPathType:  DirectoryOrCreate
   chroot-zfs:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      openebs-zfspv-bin
    Optional:  false
   host-root:
    Type:          HostPath (bare host directory volume)
    Path:          /
    HostPathType:  Directory
   registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  DirectoryOrCreate
   plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/zfs-localpv/
    HostPathType:  DirectoryOrCreate
   pods-mount-dir:
    Type:               HostPath (bare host directory volume)
    Path:               /var/lib/kubelet/
    HostPathType:       Directory
  Priority Class Name:  openebs-zfs-csi-node-critical
Events:                 <none>


Name:           speaker
Selector:       app=metallb,component=speaker
Node-Selector:  kubernetes.io/os=linux
Labels:         app=metallb
                component=speaker
Annotations:    deprecated.daemonset.template.generation: 1
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=metallb
                    component=speaker
  Annotations:      prometheus.io/port: 7472
                    prometheus.io/scrape: true
  Service Account:  speaker
  Containers:
   speaker:
    Image:       quay.io/metallb/speaker:v0.13.7
    Ports:       7472/TCP, 7946/TCP, 7946/UDP
    Host Ports:  7472/TCP, 7946/TCP, 7946/UDP
    Args:
      --port=7472
      --log-level=info
    Liveness:   http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Readiness:  http-get http://:monitoring/metrics delay=10s timeout=1s period=10s #success=1 #failure=3
    Environment:
      METALLB_NODE_NAME:       (v1:spec.nodeName)
      METALLB_HOST:            (v1:status.hostIP)
      METALLB_ML_BIND_ADDR:    (v1:status.podIP)
      METALLB_ML_LABELS:      app=metallb,component=speaker
      METALLB_ML_SECRET_KEY:  <set to the key 'secretkey' in secret 'memberlist'>  Optional: false
    Mounts:                   <none>
  Volumes:                    <none>
Events:                       <none>


Name:           svclb-jellyfin-255633bc
Selector:       app=svclb-jellyfin-255633bc
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=2f5c5bec0b5ebea63e85e7b1a5bb337ec0085a38
                svccontroller.k3s.cattle.io/nodeselector=false
                svccontroller.k3s.cattle.io/svcname=jellyfin
                svccontroller.k3s.cattle.io/svcnamespace=ix-jellyfin
Annotations:    deprecated.daemonset.template.generation: 1
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/7xUTW/jNhD9K8WcZflDVZIS6CFIgiJo1zFsby8LIxhR45hriiTIkTZCoP9eUHZcLdbJBj0UOgjkvPdIvvl4AXTqb/JBWQMC0LkwbqaQwF6ZEgTcIlXWrIghgYoYS2...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: /v1, Kind=Service
                objectset.rio.cattle.io/owner-name: jellyfin
                objectset.rio.cattle.io/owner-namespace: ix-jellyfin
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=svclb-jellyfin-255633bc
                    svccontroller.k3s.cattle.io/svcname=jellyfin
                    svccontroller.k3s.cattle.io/svcnamespace=ix-jellyfin
  Service Account:  svclb
  Containers:
   lb-tcp-8096:
    Image:      rancher/klipper-lb:v0.3.5
    Port:       8096/TCP
    Host Port:  8096/TCP
    Environment:
      SRC_PORT:    8096
      SRC_RANGES:  0.0.0.0/0
      DEST_PROTO:  TCP
      DEST_PORT:   8096
      DEST_IPS:    172.17.225.233
    Mounts:        <none>
  Volumes:         <none>
Events:            <none>


Name:           svclb-podgrab-67e271ac
Selector:       app=svclb-podgrab-67e271ac
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=09f097367bdab67e323365eaae190c682d5c6a61
                svccontroller.k3s.cattle.io/nodeselector=false
                svccontroller.k3s.cattle.io/svcname=podgrab
                svccontroller.k3s.cattle.io/svcnamespace=ix-podgrab
Annotations:    deprecated.daemonset.template.generation: 1
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/7xUwW7jNhD9lWLOsiLZjZ0I6CFIgiJo1zFsby8LIxiRY5s1RRLkSBsj0L8XlO1UwTqboIfCB0Mzbx7JN2/mBdCpv8gHZQ0UgM6FiyaHBHbKSCjgDqmyZkEMCVTEKJ...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: /v1, Kind=Service
                objectset.rio.cattle.io/owner-name: podgrab
                objectset.rio.cattle.io/owner-namespace: ix-podgrab
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=svclb-podgrab-67e271ac
                    svccontroller.k3s.cattle.io/svcname=podgrab
                    svccontroller.k3s.cattle.io/svcnamespace=ix-podgrab
  Service Account:  svclb
  Containers:
   lb-tcp-10093:
    Image:      rancher/klipper-lb:v0.3.5
    Port:       10093/TCP
    Host Port:  10093/TCP
    Environment:
      SRC_PORT:    10093
      SRC_RANGES:  0.0.0.0/0
      DEST_PROTO:  TCP
      DEST_PORT:   10093
      DEST_IPS:    172.17.188.203
    Mounts:        <none>
  Volumes:         <none>
Events:            <none>


Name:           svclb-jellyfin2-fd08da81
Selector:       app=svclb-jellyfin2-fd08da81
Node-Selector:  <none>
Labels:         objectset.rio.cattle.io/hash=4ef10c522ec561b40b69a3bac98209526d56a503
                svccontroller.k3s.cattle.io/nodeselector=false
                svccontroller.k3s.cattle.io/svcname=jellyfin2
                svccontroller.k3s.cattle.io/svcnamespace=ix-jellyfin2
Annotations:    deprecated.daemonset.template.generation: 1
                objectset.rio.cattle.io/applied:
                  H4sIAAAAAAAA/7xUTW/jNhD9K8WcZVl21klMoIcgCYqgXcewvb0sjGBEjmOuKZIgR9oYgf57QcVJtKjzgR4KHQRy5j2Sb97MI6DXf1OI2lkQgN7HYTOCDHbaKhBwhVQ5uySGDCpiVM...
                objectset.rio.cattle.io/id: 
                objectset.rio.cattle.io/owner-gvk: /v1, Kind=Service
                objectset.rio.cattle.io/owner-name: jellyfin2
                objectset.rio.cattle.io/owner-namespace: ix-jellyfin2
Desired Number of Nodes Scheduled: 1
Current Number of Nodes Scheduled: 1
Number of Nodes Scheduled with Up-to-date Pods: 1
Number of Nodes Scheduled with Available Pods: 1
Number of Nodes Misscheduled: 0
Pods Status:  1 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=svclb-jellyfin2-fd08da81
                    svccontroller.k3s.cattle.io/svcname=jellyfin2
                    svccontroller.k3s.cattle.io/svcnamespace=ix-jellyfin2
  Service Account:  svclb
  Containers:
   lb-tcp-8097:
    Image:      rancher/klipper-lb:v0.3.5
    Port:       8097/TCP
    Host Port:  8097/TCP
    Environment:
      SRC_PORT:    8097
      SRC_RANGES:  0.0.0.0/0
      DEST_PROTO:  TCP
      DEST_PORT:   8097
      DEST_IPS:    172.17.197.21
    Mounts:        <none>
  Volumes:         <none>
Events:            <none>
debug finished in 0 seconds for k3s kubectl describe daemonsets --all-namespaces


+--------------------------------------------------------------------------------+
+                            docker ps -a @1674165675                            +
+--------------------------------------------------------------------------------+
CONTAINER ID   IMAGE                        COMMAND                  CREATED         STATUS                      PORTS     NAMES
2a54ee86c739   1e326d146757                 "nvidia-device-plugin"   5 minutes ago   Created                               k8s_nvidia-device-plugin-ctr_nvidia-device-plugin-daemonset-f8pn2_kube-system_3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8_484
336a3752f086   nextcloud                    "/entrypoint.sh apac…"   8 hours ago     Up 8 hours                            k8s_nextcloud_nextcloud-cbf457787-mks7l_ix-nextcloud_0ccda19e-0228-4c05-a8ce-105c444eb9bc_0
c40e39a47de3   a99a39d070bf                 "/docker-entrypoint.…"   8 hours ago     Up 8 hours                            k8s_nginx_nextcloud-cbf457787-mks7l_ix-nextcloud_0ccda19e-0228-4c05-a8ce-105c444eb9bc_0
373eba60baab   407cece1abff                 "docker-entrypoint.s…"   8 hours ago     Up 8 hours                            k8s_nextcloud-postgres_nextcloud-postgres-5777466b99-vpdzg_ix-nextcloud_57f4cf00-a4e7-445d-b79e-9ddaa320589c_0
863e2ed6c6aa   407cece1abff                 "sh -c 'until pg_isr…"   8 hours ago     Exited (0) 8 hours ago                k8s_init-postgresdb_nextcloud-cbf457787-mks7l_ix-nextcloud_0ccda19e-0228-4c05-a8ce-105c444eb9bc_0
a50f4a9c7fbd   rancher/mirrored-pause:3.6   "/pause"                 8 hours ago     Up 8 hours                            k8s_POD_nextcloud-cbf457787-mks7l_ix-nextcloud_0ccda19e-0228-4c05-a8ce-105c444eb9bc_0
4aaf6674807d   rancher/mirrored-pause:3.6   "/pause"                 8 hours ago     Up 8 hours                            k8s_POD_nextcloud-postgres-5777466b99-vpdzg_ix-nextcloud_57f4cf00-a4e7-445d-b79e-9ddaa320589c_0
76a81876f5a8   00ea7395a689                 "/usr/local/bin/cont…"   41 hours ago    Up 41 hours                           k8s_tailscale_tailscale-6d84b7f475-m9498_ix-tailscale_42335f73-cda3-4980-87e6-9a0db07ba057_2
e85657019089   a8397fb2cf84                 "/bin/entrypoint.sh …"   41 hours ago    Up 41 hours                           k8s_syncthing_syncthing-6c5594b84-lqwgb_ix-syncthing_32759f4d-80d1-4207-af6a-40e1292dadcd_0
ef5b6515f5fd   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_syncthing-6c5594b84-lqwgb_ix-syncthing_32759f4d-80d1-4207-af6a-40e1292dadcd_0
a4b9911b6fec   dbd43b6716a0                 "entry"                  41 hours ago    Up 41 hours                           k8s_lb-tcp-8097_svclb-jellyfin2-fd08da81-5vlr9_kube-system_f54f208d-5d9f-45ce-b163-79f82b4bf6b5_0
82de93e8ad77   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_svclb-jellyfin2-fd08da81-5vlr9_kube-system_f54f208d-5d9f-45ce-b163-79f82b4bf6b5_0
e0fa96f0b7c3   e73361dabfb8                 "/controller --port=…"   41 hours ago    Up 41 hours                           k8s_controller_controller-84d6d4db45-9glps_metallb-system_058bf5a0-0882-47fa-a4c1-3d0dad5c0f25_1
e9fd5afca030   00ea7395a689                 "/usr/local/bin/cont…"   41 hours ago    Exited (1) 41 hours ago               k8s_tailscale_tailscale-6d84b7f475-m9498_ix-tailscale_42335f73-cda3-4980-87e6-9a0db07ba057_1
0d736603a0fe   dbd43b6716a0                 "entry"                  41 hours ago    Up 41 hours                           k8s_lb-tcp-10093_svclb-podgrab-67e271ac-5jfjq_kube-system_d548871e-65ed-43fa-a6a0-fe6ed732d43f_0
aa4462a681e8   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_svclb-podgrab-67e271ac-5jfjq_kube-system_d548871e-65ed-43fa-a6a0-fe6ed732d43f_0
a20c4b53d3df   dbd43b6716a0                 "entry"                  41 hours ago    Up 41 hours                           k8s_lb-tcp-8096_svclb-jellyfin-255633bc-m2wcn_kube-system_89ae21b9-6e85-4e97-b8be-1894670a840e_0
13eb817bca56   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_svclb-jellyfin-255633bc-m2wcn_kube-system_89ae21b9-6e85-4e97-b8be-1894670a840e_0
0ab8a67f9ef8   e73361dabfb8                 "/controller --port=…"   41 hours ago    Exited (0) 41 hours ago               k8s_controller_controller-84d6d4db45-9glps_metallb-system_058bf5a0-0882-47fa-a4c1-3d0dad5c0f25_0
cb9bcb438f1c   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_controller-84d6d4db45-9glps_metallb-system_058bf5a0-0882-47fa-a4c1-3d0dad5c0f25_0
6064e856750c   df968278084b                 "/bin/operator --kub…"   41 hours ago    Up 41 hours                           k8s_prometheus-operator_prometheus-operator-5dcffb7cb8-bnp7n_prometheus-operator_4c53ef8f-9829-4540-b0d2-a050a445f1f5_0
8f221e00b64d   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_prometheus-operator-5dcffb7cb8-bnp7n_prometheus-operator_4c53ef8f-9829-4540-b0d2-a050a445f1f5_0
5dd375746ec4   tccr.io/truecharts/podgrab   "./app"                  41 hours ago    Up 41 hours                           k8s_podgrab_podgrab-6c95994565-6dvgp_ix-podgrab_acdaa2fa-cfe1-4bca-b6cb-ccf19d6d463e_0
fd8e44147a87   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_podgrab-6c95994565-6dvgp_ix-podgrab_acdaa2fa-cfe1-4bca-b6cb-ccf19d6d463e_0
b04a7824e972   62af791121c8                 "/jellyfin/jellyfin"     41 hours ago    Up 41 hours                           k8s_jellyfin_jellyfin-7f97d55865-p8m7x_ix-jellyfin_780186d0-e307-47b6-8509-a7c39e53e989_0
14da10afe005   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_jellyfin-7f97d55865-p8m7x_ix-jellyfin_780186d0-e307-47b6-8509-a7c39e53e989_0
2aecae1bb7fe   2e17748d7c44                 "/usr/local/bin/zfs-…"   41 hours ago    Up 41 hours                           k8s_openebs-zfs-plugin_openebs-zfs-controller-0_kube-system_5d000c2c-b53b-47b0-977a-26befd3267b0_0
6f5dd36c5c2a   fe0f921f3c92                 "/csi-provisioner --…"   41 hours ago    Up 41 hours                           k8s_csi-provisioner_openebs-zfs-controller-0_kube-system_5d000c2c-b53b-47b0-977a-26befd3267b0_0
c7eb4772e750   f1d8a00ae690                 "/snapshot-controlle…"   41 hours ago    Up 41 hours                           k8s_snapshot-controller_openebs-zfs-controller-0_kube-system_5d000c2c-b53b-47b0-977a-26befd3267b0_0
9c03978f0375   da32a49a903a                 "/csi-snapshotter --…"   41 hours ago    Up 41 hours                           k8s_csi-snapshotter_openebs-zfs-controller-0_kube-system_5d000c2c-b53b-47b0-977a-26befd3267b0_0
602a4d0989ef   62af791121c8                 "/jellyfin/jellyfin"     41 hours ago    Up 41 hours                           k8s_jellyfin2_jellyfin2-7b88f7d76f-pr64f_ix-jellyfin2_62bcc024-1999-4150-be7c-3046c889b41b_0
65ffa7d22a5e   0aa9629e1508                 "/csi-resizer --v=5 …"   41 hours ago    Up 41 hours                           k8s_csi-resizer_openebs-zfs-controller-0_kube-system_5d000c2c-b53b-47b0-977a-26befd3267b0_0
f77a874af280   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_jellyfin2-7b88f7d76f-pr64f_ix-jellyfin2_62bcc024-1999-4150-be7c-3046c889b41b_0
4ed6048f746d   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_openebs-zfs-controller-0_kube-system_5d000c2c-b53b-47b0-977a-26befd3267b0_0
b2042bb7b595   99376d8f35e0                 "/coredns -conf /etc…"   41 hours ago    Up 41 hours                           k8s_coredns_coredns-75fc8f8fff-jkzv8_kube-system_91c7effd-29b4-4efe-968d-460e84595fc2_0
9d7dad17cee3   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_coredns-75fc8f8fff-jkzv8_kube-system_91c7effd-29b4-4efe-968d-460e84595fc2_0
1be73d5540ac   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_nvidia-device-plugin-daemonset-f8pn2_kube-system_3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8_0
ad9e1589fa26   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_tailscale-6d84b7f475-m9498_ix-tailscale_42335f73-cda3-4980-87e6-9a0db07ba057_0
4f21eedf761f   2e17748d7c44                 "/usr/local/bin/zfs-…"   41 hours ago    Up 41 hours                           k8s_openebs-zfs-plugin_openebs-zfs-node-ggcjx_kube-system_c5d0fbe9-d5b2-44fb-9a4c-c5fdabeddaf3_0
8671efec8f9a   738c5d221d60                 "/speaker --port=747…"   41 hours ago    Up 41 hours                           k8s_speaker_speaker-5rvgc_metallb-system_e669156f-514a-4715-a4f5-7ef49050c306_0
a1eecce077e1   368ee7d6e60f                 "/csi-node-driver-re…"   41 hours ago    Up 41 hours                           k8s_csi-node-driver-registrar_openebs-zfs-node-ggcjx_kube-system_c5d0fbe9-d5b2-44fb-9a4c-c5fdabeddaf3_0
f479e423c7b0   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_speaker-5rvgc_metallb-system_e669156f-514a-4715-a4f5-7ef49050c306_0
02df80a147f6   rancher/mirrored-pause:3.6   "/pause"                 41 hours ago    Up 41 hours                           k8s_POD_openebs-zfs-node-ggcjx_kube-system_c5d0fbe9-d5b2-44fb-9a4c-c5fdabeddaf3_0
83402af657a6   a8397fb2cf84                 "/bin/entrypoint.sh …"   3 days ago      Exited (255) 41 hours ago             k8s_syncthing_syncthing-6c9d7b7bc5-74pld_ix-syncthing_aa0799ca-9055-4c21-a6a3-5b04ecad9e5f_0
67a78f14308e   rancher/mirrored-pause:3.6   "/pause"                 3 days ago      Exited (255) 41 hours ago             k8s_POD_syncthing-6c9d7b7bc5-74pld_ix-syncthing_aa0799ca-9055-4c21-a6a3-5b04ecad9e5f_0
c793231f5263   e73361dabfb8                 "/controller --port=…"   3 days ago      Exited (255) 41 hours ago             k8s_controller_controller-84d6d4db45-m7cr5_metallb-system_296041ea-4066-4148-b956-0e1d37b27f7f_1
fac11553831c   rancher/mirrored-pause:3.6   "/pause"                 3 days ago      Exited (255) 41 hours ago             k8s_POD_controller-84d6d4db45-m7cr5_metallb-system_296041ea-4066-4148-b956-0e1d37b27f7f_0
44f2e5808794   df968278084b                 "/bin/operator --kub…"   3 days ago      Exited (255) 41 hours ago             k8s_prometheus-operator_prometheus-operator-5dcffb7cb8-4rphd_prometheus-operator_04e40082-21f7-4a21-b879-de6dba611e80_0
d41e84c1a139   rancher/mirrored-pause:3.6   "/pause"                 3 days ago      Exited (255) 41 hours ago             k8s_POD_prometheus-operator-5dcffb7cb8-4rphd_prometheus-operator_04e40082-21f7-4a21-b879-de6dba611e80_0
debug finished in 0 seconds for docker ps -a


+--------------------------------------------------------------------------------+
+                          docker images -a @1674165675                          +
+--------------------------------------------------------------------------------+
REPOSITORY                                         TAG       IMAGE ID       CREATED         SIZE
nextcloud                                          25.0.3    3539f97df93a   2 days ago      1.02GB
nextcloud                                          25.0.2    88161e9426f1   7 days ago      1.02GB
nginx                                              1.23.3    a99a39d070bf   8 days ago      142MB
nextcloud                                          <none>    3592d045a8e8   13 days ago     1.02GB
syncthing/syncthing                                1.23.0    a8397fb2cf84   2 weeks ago     34.6MB
tccr.io/truecharts/kubectl                         <none>    672c44cd9781   3 weeks ago     117MB
nextcloud                                          <none>    8020120ba36c   4 weeks ago     1.02GB
nginx                                              <none>    1403e55ab369   4 weeks ago     142MB
tailscale/tailscale                                <none>    c96e24b0a6c0   5 weeks ago     60MB
tailscale/tailscale                                <none>    00ea7395a689   5 weeks ago     60.1MB
rancher/mirrored-library-busybox                   1.34.1    334e4a014c81   6 weeks ago     4.86MB
tccr.io/truecharts/jellyfin                        <none>    62af791121c8   7 weeks ago     1.06GB
alpine                                             latest    49176f190c7e   8 weeks ago     7.05MB
quay.io/metallb/speaker                            v0.13.7   738c5d221d60   3 months ago    106MB
quay.io/metallb/controller                         v0.13.7   e73361dabfb8   3 months ago    58.7MB
quay.io/prometheus-operator/prometheus-operator    v0.60.1   df968278084b   3 months ago    50.3MB
tccr.io/truecharts/podgrab                         <none>    a0d4de19ab53   4 months ago    32.4MB
rancher/mirrored-coredns-coredns                   1.9.1     99376d8f35e0   10 months ago   49.5MB
rancher/klipper-lb                                 v0.3.5    dbd43b6716a0   12 months ago   8.09MB
openebs/zfs-driver                                 2.0.0     2e17748d7c44   12 months ago   176MB
bitnami/kubectl                                    1.19      dafb7f32e309   13 months ago   129MB
nvidia/k8s-device-plugin                           v0.10.0   1e326d146757   14 months ago   136MB
rancher/mirrored-pause                             3.6       6270bb605e12   17 months ago   683kB
k8s.gcr.io/sig-storage/csi-provisioner             v3.0.0    fe0f921f3c92   17 months ago   56.7MB
k8s.gcr.io/sig-storage/csi-node-driver-registrar   v2.3.0    368ee7d6e60f   17 months ago   18.7MB
k8s.gcr.io/sig-storage/csi-resizer                 v1.2.0    0aa9629e1508   20 months ago   54MB
postgres                                           13.1      407cece1abff   23 months ago   314MB
k8s.gcr.io/sig-storage/csi-snapshotter             v4.0.0    da32a49a903a   2 years ago     49.6MB
k8s.gcr.io/sig-storage/snapshot-controller         v4.0.0    f1d8a00ae690   2 years ago     46.6MB
debug finished in 0 seconds for docker images -a


+--------------------------------------------------------------------------------+
+                     K3s Logs (last 1000 lines) @1674165675                     +
+--------------------------------------------------------------------------------+
Jan 19 22:20:22 truenas k3s[29115]: I0119 22:20:22.803326   29115 scope.go:115] "RemoveContainer" containerID="070bb857ae9504ec8b2149f4cb6a5dd8a91e25699b9e4adf9551315177cbf839"
Jan 19 22:20:23 truenas k3s[29115]: E0119 22:20:23.199326   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:20:23 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:20:23 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:20:23 truenas k3s[29115]:  > containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:20:23 truenas k3s[29115]: E0119 22:20:23.199450   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:20:23 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:20:23 truenas k3s[29115]: E0119 22:20:23.199525   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:20:27 truenas k3s[29115]: time="2023-01-19T22:20:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:20:32 truenas k3s[29115]: time="2023-01-19T22:20:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:20:33 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:20:33.300+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:20:35 truenas k3s[29115]: I0119 22:20:35.803792   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:20:35 truenas k3s[29115]: E0119 22:20:35.804169   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:20:37 truenas k3s[29115]: time="2023-01-19T22:20:37+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:20:42 truenas k3s[29115]: time="2023-01-19T22:20:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:20:47 truenas k3s[29115]: time="2023-01-19T22:20:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:20:49 truenas k3s[29115]: I0119 22:20:49.804135   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:20:49 truenas k3s[29115]: E0119 22:20:49.804526   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:20:52 truenas k3s[29115]: time="2023-01-19T22:20:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:20:55 truenas k3s[29115]: I0119 22:20:55.913712   29115 scope.go:115] "RemoveContainer" containerID="070bb857ae9504ec8b2149f4cb6a5dd8a91e25699b9e4adf9551315177cbf839"
Jan 19 22:20:56 truenas k3s[29115]: I0119 22:20:56.790688   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="070bb857ae9504ec8b2149f4cb6a5dd8a91e25699b9e4adf9551315177cbf839"
Jan 19 22:20:56 truenas k3s[29115]: I0119 22:20:56.791117   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:20:56 truenas k3s[29115]: E0119 22:20:56.791470   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:20:57 truenas k3s[29115]: time="2023-01-19T22:20:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:02 truenas k3s[29115]: time="2023-01-19T22:21:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:06 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:21:06.437+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:21:07 truenas k3s[29115]: time="2023-01-19T22:21:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:09 truenas k3s[29115]: I0119 22:21:09.803720   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:21:09 truenas k3s[29115]: E0119 22:21:09.804183   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:21:12 truenas k3s[29115]: time="2023-01-19T22:21:12+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:17 truenas k3s[29115]: time="2023-01-19T22:21:17+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:22 truenas k3s[29115]: time="2023-01-19T22:21:22+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:23 truenas k3s[29115]: I0119 22:21:23.803641   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:21:23 truenas k3s[29115]: E0119 22:21:23.804076   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:21:27 truenas k3s[29115]: time="2023-01-19T22:21:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:32 truenas k3s[29115]: time="2023-01-19T22:21:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:37 truenas k3s[29115]: time="2023-01-19T22:21:37+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:38 truenas k3s[29115]: I0119 22:21:38.803769   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:21:38 truenas k3s[29115]: E0119 22:21:38.804176   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:21:42 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:21:42.532+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:21:42 truenas k3s[29115]: time="2023-01-19T22:21:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:47 truenas k3s[29115]: time="2023-01-19T22:21:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:49 truenas k3s[29115]: I0119 22:21:49.803668   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:21:49 truenas k3s[29115]: E0119 22:21:49.804039   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:21:52 truenas k3s[29115]: time="2023-01-19T22:21:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:21:57 truenas k3s[29115]: time="2023-01-19T22:21:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:00 truenas k3s[29115]: I0119 22:22:00.803730   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:22:00 truenas k3s[29115]: E0119 22:22:00.804098   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:22:02 truenas k3s[29115]: time="2023-01-19T22:22:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:07 truenas k3s[29115]: time="2023-01-19T22:22:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:12 truenas k3s[29115]: time="2023-01-19T22:22:12+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:14 truenas k3s[29115]: I0119 22:22:14.803035   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:22:14 truenas k3s[29115]: E0119 22:22:14.803658   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:22:16 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:22:16.892+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:22:17 truenas k3s[29115]: time="2023-01-19T22:22:17+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:22 truenas k3s[29115]: time="2023-01-19T22:22:22+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:27 truenas k3s[29115]: time="2023-01-19T22:22:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:29 truenas k3s[29115]: I0119 22:22:29.803939   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:22:29 truenas k3s[29115]: E0119 22:22:29.804314   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:22:32 truenas k3s[29115]: time="2023-01-19T22:22:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:37 truenas k3s[29115]: time="2023-01-19T22:22:37+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:42 truenas k3s[29115]: time="2023-01-19T22:22:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:42 truenas k3s[29115]: I0119 22:22:42.803502   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:22:42 truenas k3s[29115]: E0119 22:22:42.803887   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:22:47 truenas k3s[29115]: time="2023-01-19T22:22:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:52 truenas k3s[29115]: time="2023-01-19T22:22:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:22:57 truenas k3s[29115]: I0119 22:22:57.803655   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:22:57 truenas k3s[29115]: E0119 22:22:57.804042   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:22:57 truenas k3s[29115]: time="2023-01-19T22:22:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:01 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:23:01.460+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:23:02 truenas k3s[29115]: time="2023-01-19T22:23:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:07 truenas k3s[29115]: time="2023-01-19T22:23:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:08 truenas k3s[29115]: I0119 22:23:08.803512   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:23:08 truenas k3s[29115]: E0119 22:23:08.803890   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:23:12 truenas k3s[29115]: time="2023-01-19T22:23:12+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:17 truenas k3s[29115]: time="2023-01-19T22:23:17+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:21 truenas k3s[29115]: I0119 22:23:21.803934   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:23:21 truenas k3s[29115]: E0119 22:23:21.804352   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:23:22 truenas k3s[29115]: time="2023-01-19T22:23:22+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:27 truenas k3s[29115]: time="2023-01-19T22:23:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:32 truenas k3s[29115]: time="2023-01-19T22:23:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:34 truenas k3s[29115]: I0119 22:23:34.804034   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:23:34 truenas k3s[29115]: E0119 22:23:34.804492   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:23:38 truenas k3s[29115]: time="2023-01-19T22:23:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:39 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:23:39.115+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:23:43 truenas k3s[29115]: time="2023-01-19T22:23:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:48 truenas k3s[29115]: time="2023-01-19T22:23:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:49 truenas k3s[29115]: I0119 22:23:49.803547   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:23:49 truenas k3s[29115]: E0119 22:23:49.803951   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:23:53 truenas k3s[29115]: time="2023-01-19T22:23:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:23:58 truenas k3s[29115]: time="2023-01-19T22:23:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:02 truenas k3s[29115]: I0119 22:24:02.803591   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:24:02 truenas k3s[29115]: E0119 22:24:02.803984   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:24:03 truenas k3s[29115]: time="2023-01-19T22:24:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:08 truenas k3s[29115]: time="2023-01-19T22:24:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:12 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:24:12.649+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:24:13 truenas k3s[29115]: time="2023-01-19T22:24:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:14 truenas k3s[29115]: I0119 22:24:14.803910   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:24:14 truenas k3s[29115]: E0119 22:24:14.804298   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:24:18 truenas k3s[29115]: time="2023-01-19T22:24:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:23 truenas k3s[29115]: time="2023-01-19T22:24:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:27 truenas k3s[29115]: I0119 22:24:27.803071   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:24:27 truenas k3s[29115]: E0119 22:24:27.803335   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:24:28 truenas k3s[29115]: time="2023-01-19T22:24:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:33 truenas k3s[29115]: time="2023-01-19T22:24:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:38 truenas k3s[29115]: time="2023-01-19T22:24:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:39 truenas k3s[29115]: I0119 22:24:39.803378   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:24:39 truenas k3s[29115]: E0119 22:24:39.803808   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:24:43 truenas k3s[29115]: time="2023-01-19T22:24:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:48 truenas k3s[29115]: time="2023-01-19T22:24:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:53 truenas k3s[29115]: time="2023-01-19T22:24:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:24:54 truenas k3s[29115]: I0119 22:24:54.803181   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:24:54 truenas k3s[29115]: E0119 22:24:54.803585   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:24:55 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:24:55.770+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:24:58 truenas k3s[29115]: time="2023-01-19T22:24:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:03 truenas k3s[29115]: time="2023-01-19T22:25:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:08 truenas k3s[29115]: time="2023-01-19T22:25:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:08 truenas k3s[29115]: I0119 22:25:08.803749   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:25:08 truenas k3s[29115]: E0119 22:25:08.804122   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:25:13 truenas k3s[29115]: time="2023-01-19T22:25:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:18 truenas k3s[29115]: time="2023-01-19T22:25:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:23 truenas k3s[29115]: time="2023-01-19T22:25:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:23 truenas k3s[29115]: I0119 22:25:23.803127   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:25:24 truenas k3s[29115]: E0119 22:25:24.170641   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:25:24 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:25:24 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:25:24 truenas k3s[29115]:  > containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:25:24 truenas k3s[29115]: E0119 22:25:24.170768   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:25:24 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:25:24 truenas k3s[29115]: E0119 22:25:24.170844   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:25:27 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:25:27.309+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:25:28 truenas k3s[29115]: time="2023-01-19T22:25:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:33 truenas k3s[29115]: time="2023-01-19T22:25:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:37 truenas k3s[29115]: I0119 22:25:37.803433   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:25:37 truenas k3s[29115]: E0119 22:25:37.803821   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:25:38 truenas k3s[29115]: time="2023-01-19T22:25:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:43 truenas k3s[29115]: time="2023-01-19T22:25:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:48 truenas k3s[29115]: time="2023-01-19T22:25:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:48 truenas k3s[29115]: I0119 22:25:48.802882   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:25:48 truenas k3s[29115]: E0119 22:25:48.803223   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:25:53 truenas k3s[29115]: time="2023-01-19T22:25:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:25:56 truenas k3s[29115]: I0119 22:25:56.086013   29115 scope.go:115] "RemoveContainer" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:25:56 truenas k3s[29115]: I0119 22:25:56.833818   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="147d25623131b1084bf86c9e527a9bcf44d400c9032b79b8a76d00f182d2acd2"
Jan 19 22:25:56 truenas k3s[29115]: I0119 22:25:56.834373   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:25:56 truenas k3s[29115]: E0119 22:25:56.834743   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:25:57 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:25:57.971+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:25:58 truenas k3s[29115]: time="2023-01-19T22:25:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:03 truenas k3s[29115]: time="2023-01-19T22:26:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:07 truenas k3s[29115]: I0119 22:26:07.803031   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:26:07 truenas k3s[29115]: E0119 22:26:07.803279   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:26:08 truenas k3s[29115]: time="2023-01-19T22:26:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:13 truenas k3s[29115]: time="2023-01-19T22:26:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:18 truenas k3s[29115]: time="2023-01-19T22:26:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:19 truenas k3s[29115]: I0119 22:26:19.803577   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:26:19 truenas k3s[29115]: E0119 22:26:19.803960   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:26:23 truenas k3s[29115]: time="2023-01-19T22:26:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:28 truenas k3s[29115]: time="2023-01-19T22:26:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:29 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:26:29.142+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:26:33 truenas k3s[29115]: time="2023-01-19T22:26:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:34 truenas k3s[29115]: I0119 22:26:34.803188   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:26:34 truenas k3s[29115]: E0119 22:26:34.803561   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:26:38 truenas k3s[29115]: time="2023-01-19T22:26:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:43 truenas k3s[29115]: time="2023-01-19T22:26:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:48 truenas k3s[29115]: time="2023-01-19T22:26:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:49 truenas k3s[29115]: I0119 22:26:49.803476   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:26:49 truenas k3s[29115]: E0119 22:26:49.803847   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:26:53 truenas k3s[29115]: time="2023-01-19T22:26:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:26:58 truenas k3s[29115]: time="2023-01-19T22:26:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:00 truenas k3s[29115]: I0119 22:27:00.803177   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:27:00 truenas k3s[29115]: E0119 22:27:00.803560   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:27:03 truenas k3s[29115]: time="2023-01-19T22:27:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:08 truenas k3s[29115]: time="2023-01-19T22:27:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:11 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:27:11.797+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:27:13 truenas k3s[29115]: time="2023-01-19T22:27:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:14 truenas k3s[29115]: I0119 22:27:14.803393   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:27:14 truenas k3s[29115]: E0119 22:27:14.803762   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:27:18 truenas k3s[29115]: time="2023-01-19T22:27:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:23 truenas k3s[29115]: time="2023-01-19T22:27:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:27 truenas k3s[29115]: I0119 22:27:27.803839   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:27:27 truenas k3s[29115]: E0119 22:27:27.804217   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:27:28 truenas k3s[29115]: time="2023-01-19T22:27:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:33 truenas k3s[29115]: time="2023-01-19T22:27:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:38 truenas k3s[29115]: time="2023-01-19T22:27:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:39 truenas k3s[29115]: I0119 22:27:39.803737   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:27:39 truenas k3s[29115]: E0119 22:27:39.804142   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:27:43 truenas k3s[29115]: time="2023-01-19T22:27:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:48 truenas k3s[29115]: time="2023-01-19T22:27:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:53 truenas k3s[29115]: time="2023-01-19T22:27:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:27:54 truenas k3s[29115]: I0119 22:27:54.803401   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:27:54 truenas k3s[29115]: E0119 22:27:54.803773   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:27:55 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:27:55.566+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:27:58 truenas k3s[29115]: time="2023-01-19T22:27:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:03 truenas k3s[29115]: time="2023-01-19T22:28:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:07 truenas k3s[29115]: I0119 22:28:07.803072   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:28:07 truenas k3s[29115]: E0119 22:28:07.803451   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:28:08 truenas k3s[29115]: time="2023-01-19T22:28:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:14 truenas k3s[29115]: time="2023-01-19T22:28:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:19 truenas k3s[29115]: time="2023-01-19T22:28:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:21 truenas k3s[29115]: I0119 22:28:21.804163   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:28:21 truenas k3s[29115]: E0119 22:28:21.804569   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:28:24 truenas k3s[29115]: time="2023-01-19T22:28:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:29 truenas k3s[29115]: time="2023-01-19T22:28:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:30 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:28:30.276+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:28:34 truenas k3s[29115]: time="2023-01-19T22:28:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:34 truenas k3s[29115]: I0119 22:28:34.803530   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:28:34 truenas k3s[29115]: E0119 22:28:34.803903   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:28:39 truenas k3s[29115]: time="2023-01-19T22:28:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:44 truenas k3s[29115]: time="2023-01-19T22:28:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:48 truenas k3s[29115]: I0119 22:28:48.803567   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:28:48 truenas k3s[29115]: E0119 22:28:48.803946   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:28:49 truenas k3s[29115]: time="2023-01-19T22:28:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:54 truenas k3s[29115]: time="2023-01-19T22:28:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:28:59 truenas k3s[29115]: time="2023-01-19T22:28:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:01 truenas k3s[29115]: I0119 22:29:01.803720   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:29:01 truenas k3s[29115]: E0119 22:29:01.804096   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:29:04 truenas k3s[29115]: time="2023-01-19T22:29:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:05 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:29:05.354+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:29:09 truenas k3s[29115]: time="2023-01-19T22:29:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:13 truenas k3s[29115]: I0119 22:29:13.803627   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:29:13 truenas k3s[29115]: E0119 22:29:13.804000   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:29:14 truenas k3s[29115]: time="2023-01-19T22:29:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:19 truenas k3s[29115]: time="2023-01-19T22:29:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:24 truenas k3s[29115]: time="2023-01-19T22:29:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:25 truenas k3s[29115]: I0119 22:29:25.804039   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:29:25 truenas k3s[29115]: E0119 22:29:25.804489   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:29:29 truenas k3s[29115]: time="2023-01-19T22:29:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:34 truenas k3s[29115]: time="2023-01-19T22:29:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:37 truenas k3s[29115]: I0119 22:29:37.803165   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:29:37 truenas k3s[29115]: E0119 22:29:37.803593   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:29:39 truenas k3s[29115]: time="2023-01-19T22:29:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:44 truenas k3s[29115]: time="2023-01-19T22:29:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:49 truenas k3s[29115]: time="2023-01-19T22:29:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:50 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:29:50.106+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:29:51 truenas k3s[29115]: I0119 22:29:51.805161   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:29:51 truenas k3s[29115]: E0119 22:29:51.805768   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:29:54 truenas k3s[29115]: time="2023-01-19T22:29:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:29:59 truenas k3s[29115]: time="2023-01-19T22:29:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:03 truenas k3s[29115]: I0119 22:30:03.803637   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:30:03 truenas k3s[29115]: E0119 22:30:03.804019   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:30:04 truenas k3s[29115]: time="2023-01-19T22:30:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:09 truenas k3s[29115]: time="2023-01-19T22:30:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:14 truenas k3s[29115]: time="2023-01-19T22:30:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:14 truenas k3s[29115]: I0119 22:30:14.803614   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:30:14 truenas k3s[29115]: E0119 22:30:14.804003   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:30:19 truenas k3s[29115]: time="2023-01-19T22:30:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:24 truenas k3s[29115]: time="2023-01-19T22:30:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:26 truenas k3s[29115]: I0119 22:30:26.802974   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:30:27 truenas k3s[29115]: E0119 22:30:27.183245   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:30:27 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:30:27 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:30:27 truenas k3s[29115]:  > containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:30:27 truenas k3s[29115]: E0119 22:30:27.183375   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:30:27 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:30:27 truenas k3s[29115]: E0119 22:30:27.183452   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:30:27 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:30:27.235+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:30:29 truenas k3s[29115]: time="2023-01-19T22:30:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:34 truenas k3s[29115]: time="2023-01-19T22:30:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:39 truenas k3s[29115]: time="2023-01-19T22:30:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:41 truenas k3s[29115]: I0119 22:30:41.804395   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:30:41 truenas k3s[29115]: E0119 22:30:41.804799   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:30:44 truenas k3s[29115]: time="2023-01-19T22:30:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:49 truenas k3s[29115]: time="2023-01-19T22:30:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:53 truenas k3s[29115]: I0119 22:30:53.802971   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:30:53 truenas k3s[29115]: E0119 22:30:53.803304   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:30:54 truenas k3s[29115]: time="2023-01-19T22:30:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:30:56 truenas k3s[29115]: I0119 22:30:56.242010   29115 scope.go:115] "RemoveContainer" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:30:56 truenas k3s[29115]: I0119 22:30:56.903079   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="dd9d12fe0589be22eca18afd84aa9a46493540d725d4f10b27d6ef6b67dc356a"
Jan 19 22:30:56 truenas k3s[29115]: I0119 22:30:56.903412   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:30:56 truenas k3s[29115]: E0119 22:30:56.903697   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:30:59 truenas k3s[29115]: time="2023-01-19T22:30:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:00 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:31:00.427+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:31:04 truenas k3s[29115]: time="2023-01-19T22:31:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:09 truenas k3s[29115]: time="2023-01-19T22:31:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:09 truenas k3s[29115]: I0119 22:31:09.803170   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:31:09 truenas k3s[29115]: E0119 22:31:09.803549   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:31:14 truenas k3s[29115]: time="2023-01-19T22:31:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:19 truenas k3s[29115]: time="2023-01-19T22:31:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:22 truenas k3s[29115]: I0119 22:31:22.803686   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:31:22 truenas k3s[29115]: E0119 22:31:22.804077   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:31:24 truenas k3s[29115]: time="2023-01-19T22:31:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:29 truenas k3s[29115]: time="2023-01-19T22:31:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:33 truenas k3s[29115]: I0119 22:31:33.803997   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:31:33 truenas k3s[29115]: E0119 22:31:33.804369   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:31:34 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:31:34.294+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:31:34 truenas k3s[29115]: time="2023-01-19T22:31:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:39 truenas k3s[29115]: time="2023-01-19T22:31:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:44 truenas k3s[29115]: time="2023-01-19T22:31:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:48 truenas k3s[29115]: I0119 22:31:48.804010   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:31:48 truenas k3s[29115]: E0119 22:31:48.804378   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:31:49 truenas k3s[29115]: time="2023-01-19T22:31:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:54 truenas k3s[29115]: time="2023-01-19T22:31:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:31:59 truenas k3s[29115]: time="2023-01-19T22:31:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:01 truenas k3s[29115]: I0119 22:32:01.803579   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:32:01 truenas k3s[29115]: E0119 22:32:01.803957   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:32:04 truenas k3s[29115]: time="2023-01-19T22:32:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:07 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:32:07.744+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:32:09 truenas k3s[29115]: time="2023-01-19T22:32:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:13 truenas k3s[29115]: I0119 22:32:13.803714   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:32:13 truenas k3s[29115]: E0119 22:32:13.804098   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:32:14 truenas k3s[29115]: time="2023-01-19T22:32:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:19 truenas k3s[29115]: time="2023-01-19T22:32:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:24 truenas k3s[29115]: time="2023-01-19T22:32:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:27 truenas k3s[29115]: I0119 22:32:27.803894   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:32:27 truenas k3s[29115]: E0119 22:32:27.804266   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:32:29 truenas k3s[29115]: time="2023-01-19T22:32:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:34 truenas k3s[29115]: time="2023-01-19T22:32:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:39 truenas k3s[29115]: time="2023-01-19T22:32:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:42 truenas k3s[29115]: I0119 22:32:42.803541   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:32:42 truenas k3s[29115]: E0119 22:32:42.803951   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:32:45 truenas k3s[29115]: time="2023-01-19T22:32:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:46 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:32:46.734+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:32:50 truenas k3s[29115]: time="2023-01-19T22:32:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:55 truenas k3s[29115]: time="2023-01-19T22:32:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:32:56 truenas k3s[29115]: I0119 22:32:56.803517   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:32:56 truenas k3s[29115]: E0119 22:32:56.803892   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:33:00 truenas k3s[29115]: time="2023-01-19T22:33:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:05 truenas k3s[29115]: time="2023-01-19T22:33:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:07 truenas k3s[29115]: I0119 22:33:07.804110   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:33:07 truenas k3s[29115]: E0119 22:33:07.804498   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:33:10 truenas k3s[29115]: time="2023-01-19T22:33:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:15 truenas k3s[29115]: time="2023-01-19T22:33:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:20 truenas k3s[29115]: time="2023-01-19T22:33:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:22 truenas k3s[29115]: I0119 22:33:22.803464   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:33:22 truenas k3s[29115]: E0119 22:33:22.803839   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:33:25 truenas k3s[29115]: time="2023-01-19T22:33:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:29 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:33:29.174+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:33:30 truenas k3s[29115]: time="2023-01-19T22:33:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:33 truenas k3s[29115]: I0119 22:33:33.803097   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:33:33 truenas k3s[29115]: E0119 22:33:33.803461   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:33:35 truenas k3s[29115]: time="2023-01-19T22:33:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:40 truenas k3s[29115]: time="2023-01-19T22:33:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:45 truenas k3s[29115]: time="2023-01-19T22:33:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:46 truenas k3s[29115]: I0119 22:33:46.803506   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:33:46 truenas k3s[29115]: E0119 22:33:46.804198   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:33:50 truenas k3s[29115]: time="2023-01-19T22:33:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:33:55 truenas k3s[29115]: time="2023-01-19T22:33:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:00 truenas k3s[29115]: time="2023-01-19T22:34:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:00 truenas k3s[29115]: I0119 22:34:00.803061   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:34:00 truenas k3s[29115]: E0119 22:34:00.803438   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:34:05 truenas k3s[29115]: time="2023-01-19T22:34:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:07 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:34:07.557+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:34:10 truenas k3s[29115]: time="2023-01-19T22:34:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:15 truenas k3s[29115]: time="2023-01-19T22:34:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:15 truenas k3s[29115]: I0119 22:34:15.803383   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:34:15 truenas k3s[29115]: E0119 22:34:15.803763   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:34:20 truenas k3s[29115]: time="2023-01-19T22:34:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:25 truenas k3s[29115]: time="2023-01-19T22:34:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:27 truenas k3s[29115]: I0119 22:34:27.803321   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:34:27 truenas k3s[29115]: E0119 22:34:27.803728   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:34:30 truenas k3s[29115]: time="2023-01-19T22:34:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:35 truenas k3s[29115]: time="2023-01-19T22:34:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:40 truenas k3s[29115]: time="2023-01-19T22:34:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:42 truenas k3s[29115]: I0119 22:34:42.803691   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:34:42 truenas k3s[29115]: E0119 22:34:42.804054   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:34:45 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:34:45.067+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:34:45 truenas k3s[29115]: time="2023-01-19T22:34:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:50 truenas k3s[29115]: time="2023-01-19T22:34:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:34:54 truenas k3s[29115]: I0119 22:34:54.803279   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:34:54 truenas k3s[29115]: E0119 22:34:54.803674   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:34:55 truenas k3s[29115]: time="2023-01-19T22:34:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:00 truenas k3s[29115]: time="2023-01-19T22:35:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:05 truenas k3s[29115]: time="2023-01-19T22:35:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:05 truenas k3s[29115]: I0119 22:35:05.803662   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:35:05 truenas k3s[29115]: E0119 22:35:05.804038   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:35:10 truenas k3s[29115]: time="2023-01-19T22:35:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:15 truenas k3s[29115]: time="2023-01-19T22:35:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:17 truenas k3s[29115]: I0119 22:35:17.803923   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:35:17 truenas k3s[29115]: E0119 22:35:17.804300   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:35:20 truenas k3s[29115]: time="2023-01-19T22:35:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:25 truenas k3s[29115]: time="2023-01-19T22:35:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:26 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:35:26.739+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:35:29 truenas k3s[29115]: I0119 22:35:29.803988   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:35:30 truenas k3s[29115]: E0119 22:35:30.182629   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:35:30 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:35:30 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:35:30 truenas k3s[29115]:  > containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:35:30 truenas k3s[29115]: E0119 22:35:30.182753   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:35:30 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:35:30 truenas k3s[29115]: E0119 22:35:30.182830   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:35:30 truenas k3s[29115]: time="2023-01-19T22:35:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:35 truenas k3s[29115]: time="2023-01-19T22:35:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:40 truenas k3s[29115]: time="2023-01-19T22:35:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:44 truenas k3s[29115]: I0119 22:35:44.803237   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:35:44 truenas k3s[29115]: E0119 22:35:44.803609   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:35:45 truenas k3s[29115]: time="2023-01-19T22:35:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:50 truenas k3s[29115]: time="2023-01-19T22:35:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:55 truenas k3s[29115]: time="2023-01-19T22:35:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:35:56 truenas k3s[29115]: I0119 22:35:56.395693   29115 scope.go:115] "RemoveContainer" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:35:56 truenas k3s[29115]: I0119 22:35:56.972779   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="f83f6e0c90664e2bda97b5135f6049807ba70ad58713a4d930fd4a2774cd9e1f"
Jan 19 22:35:56 truenas k3s[29115]: I0119 22:35:56.973256   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:35:56 truenas k3s[29115]: E0119 22:35:56.973796   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:36:00 truenas k3s[29115]: time="2023-01-19T22:36:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:04 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:36:04.451+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:36:05 truenas k3s[29115]: time="2023-01-19T22:36:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:10 truenas k3s[29115]: time="2023-01-19T22:36:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:10 truenas k3s[29115]: I0119 22:36:10.803948   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:36:10 truenas k3s[29115]: E0119 22:36:10.804328   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:36:15 truenas k3s[29115]: time="2023-01-19T22:36:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:20 truenas k3s[29115]: time="2023-01-19T22:36:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:23 truenas k3s[29115]: I0119 22:36:23.803932   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:36:23 truenas k3s[29115]: E0119 22:36:23.804320   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:36:25 truenas k3s[29115]: time="2023-01-19T22:36:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:30 truenas k3s[29115]: time="2023-01-19T22:36:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:35 truenas k3s[29115]: time="2023-01-19T22:36:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:37 truenas k3s[29115]: I0119 22:36:37.803923   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:36:37 truenas k3s[29115]: E0119 22:36:37.804227   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:36:40 truenas k3s[29115]: time="2023-01-19T22:36:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:45 truenas k3s[29115]: time="2023-01-19T22:36:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:46 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:36:46.082+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:36:50 truenas k3s[29115]: time="2023-01-19T22:36:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:36:52 truenas k3s[29115]: I0119 22:36:52.803028   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:36:52 truenas k3s[29115]: E0119 22:36:52.803404   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:36:55 truenas k3s[29115]: time="2023-01-19T22:36:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:00 truenas k3s[29115]: time="2023-01-19T22:37:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:05 truenas k3s[29115]: time="2023-01-19T22:37:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:06 truenas k3s[29115]: I0119 22:37:06.803568   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:37:06 truenas k3s[29115]: E0119 22:37:06.803943   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:37:11 truenas k3s[29115]: time="2023-01-19T22:37:11+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:16 truenas k3s[29115]: time="2023-01-19T22:37:16+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:18 truenas k3s[29115]: I0119 22:37:18.803363   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:37:18 truenas k3s[29115]: E0119 22:37:18.803748   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:37:21 truenas k3s[29115]: time="2023-01-19T22:37:21+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:21 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:37:21.526+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:37:26 truenas k3s[29115]: time="2023-01-19T22:37:26+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:31 truenas k3s[29115]: time="2023-01-19T22:37:31+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:32 truenas k3s[29115]: I0119 22:37:32.802970   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:37:32 truenas k3s[29115]: E0119 22:37:32.803339   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:37:36 truenas k3s[29115]: time="2023-01-19T22:37:36+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:41 truenas k3s[29115]: time="2023-01-19T22:37:41+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:46 truenas k3s[29115]: time="2023-01-19T22:37:46+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:47 truenas k3s[29115]: I0119 22:37:47.802990   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:37:47 truenas k3s[29115]: E0119 22:37:47.803364   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:37:51 truenas k3s[29115]: time="2023-01-19T22:37:51+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:37:56 truenas k3s[29115]: time="2023-01-19T22:37:56+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:00 truenas k3s[29115]: I0119 22:38:00.803167   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:38:00 truenas k3s[29115]: E0119 22:38:00.803539   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:38:01 truenas k3s[29115]: time="2023-01-19T22:38:01+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:02 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:38:02.914+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:38:06 truenas k3s[29115]: time="2023-01-19T22:38:06+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:11 truenas k3s[29115]: time="2023-01-19T22:38:11+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:12 truenas k3s[29115]: I0119 22:38:12.803121   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:38:12 truenas k3s[29115]: E0119 22:38:12.803492   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:38:16 truenas k3s[29115]: time="2023-01-19T22:38:16+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:21 truenas k3s[29115]: time="2023-01-19T22:38:21+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:24 truenas k3s[29115]: I0119 22:38:24.803309   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:38:24 truenas k3s[29115]: E0119 22:38:24.803683   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:38:26 truenas k3s[29115]: time="2023-01-19T22:38:26+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:31 truenas k3s[29115]: time="2023-01-19T22:38:31+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:36 truenas k3s[29115]: time="2023-01-19T22:38:36+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:37 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:38:37.125+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:38:38 truenas k3s[29115]: I0119 22:38:38.803884   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:38:38 truenas k3s[29115]: E0119 22:38:38.804261   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:38:41 truenas k3s[29115]: time="2023-01-19T22:38:41+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:46 truenas k3s[29115]: time="2023-01-19T22:38:46+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:51 truenas k3s[29115]: time="2023-01-19T22:38:51+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:38:52 truenas k3s[29115]: I0119 22:38:52.803521   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:38:52 truenas k3s[29115]: E0119 22:38:52.803895   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:38:56 truenas k3s[29115]: time="2023-01-19T22:38:56+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:01 truenas k3s[29115]: time="2023-01-19T22:39:01+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:06 truenas k3s[29115]: time="2023-01-19T22:39:06+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:07 truenas k3s[29115]: I0119 22:39:07.803893   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:39:07 truenas k3s[29115]: E0119 22:39:07.804282   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:39:11 truenas k3s[29115]: time="2023-01-19T22:39:11+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:16 truenas k3s[29115]: time="2023-01-19T22:39:16+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:19 truenas k3s[29115]: I0119 22:39:19.803782   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:39:19 truenas k3s[29115]: E0119 22:39:19.804184   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:39:20 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:39:20.778+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:39:21 truenas k3s[29115]: time="2023-01-19T22:39:21+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:26 truenas k3s[29115]: time="2023-01-19T22:39:26+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:31 truenas k3s[29115]: time="2023-01-19T22:39:31+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:31 truenas k3s[29115]: I0119 22:39:31.804151   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:39:31 truenas k3s[29115]: E0119 22:39:31.804553   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:39:36 truenas k3s[29115]: time="2023-01-19T22:39:36+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:41 truenas k3s[29115]: time="2023-01-19T22:39:41+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:43 truenas k3s[29115]: I0119 22:39:43.803076   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:39:43 truenas k3s[29115]: E0119 22:39:43.803456   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:39:46 truenas k3s[29115]: time="2023-01-19T22:39:46+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:51 truenas k3s[29115]: time="2023-01-19T22:39:51+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:54 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:39:54.926+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:39:56 truenas k3s[29115]: time="2023-01-19T22:39:56+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:39:56 truenas k3s[29115]: I0119 22:39:56.803654   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:39:56 truenas k3s[29115]: E0119 22:39:56.804026   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:40:01 truenas k3s[29115]: time="2023-01-19T22:40:01+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:06 truenas k3s[29115]: time="2023-01-19T22:40:06+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:07 truenas k3s[29115]: I0119 22:40:07.804079   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:40:07 truenas k3s[29115]: E0119 22:40:07.804462   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:40:11 truenas k3s[29115]: time="2023-01-19T22:40:11+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:16 truenas k3s[29115]: time="2023-01-19T22:40:16+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:21 truenas k3s[29115]: time="2023-01-19T22:40:21+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:21 truenas k3s[29115]: I0119 22:40:21.803596   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:40:21 truenas k3s[29115]: E0119 22:40:21.803970   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:40:26 truenas k3s[29115]: time="2023-01-19T22:40:26+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:31 truenas k3s[29115]: time="2023-01-19T22:40:31+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:36 truenas k3s[29115]: time="2023-01-19T22:40:36+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:36 truenas k3s[29115]: I0119 22:40:36.803470   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:40:37 truenas k3s[29115]: E0119 22:40:37.182369   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:40:37 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:40:37 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:40:37 truenas k3s[29115]:  > containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:40:37 truenas k3s[29115]: E0119 22:40:37.182486   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:40:37 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:40:37 truenas k3s[29115]: E0119 22:40:37.182562   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:40:39 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:40:39.888+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:40:41 truenas k3s[29115]: time="2023-01-19T22:40:41+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:46 truenas k3s[29115]: time="2023-01-19T22:40:46+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:48 truenas k3s[29115]: I0119 22:40:48.803987   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:40:48 truenas k3s[29115]: E0119 22:40:48.804374   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:40:51 truenas k3s[29115]: time="2023-01-19T22:40:51+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:56 truenas k3s[29115]: I0119 22:40:56.565151   29115 scope.go:115] "RemoveContainer" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:40:56 truenas k3s[29115]: time="2023-01-19T22:40:56+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:40:57 truenas k3s[29115]: I0119 22:40:57.074549   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="351bb1bc4dde27672e2d76767eacf45d69856c8aab8b9351aed8f7418a8862bc"
Jan 19 22:40:57 truenas k3s[29115]: I0119 22:40:57.074959   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:40:57 truenas k3s[29115]: E0119 22:40:57.075311   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:41:01 truenas k3s[29115]: time="2023-01-19T22:41:01+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:06 truenas k3s[29115]: time="2023-01-19T22:41:06+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:08 truenas k3s[29115]: I0119 22:41:08.803189   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:41:08 truenas k3s[29115]: E0119 22:41:08.803570   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:41:11 truenas k3s[29115]: time="2023-01-19T22:41:11+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:16 truenas k3s[29115]: time="2023-01-19T22:41:16+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:17 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:41:17.917+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:41:21 truenas k3s[29115]: I0119 22:41:21.803835   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:41:21 truenas k3s[29115]: E0119 22:41:21.804203   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:41:21 truenas k3s[29115]: time="2023-01-19T22:41:21+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:26 truenas k3s[29115]: time="2023-01-19T22:41:26+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:31 truenas k3s[29115]: time="2023-01-19T22:41:31+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:32 truenas k3s[29115]: I0119 22:41:32.803002   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:41:32 truenas k3s[29115]: E0119 22:41:32.803382   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:41:36 truenas k3s[29115]: time="2023-01-19T22:41:36+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:42 truenas k3s[29115]: time="2023-01-19T22:41:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:46 truenas k3s[29115]: I0119 22:41:46.803892   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:41:46 truenas k3s[29115]: E0119 22:41:46.804271   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:41:47 truenas k3s[29115]: time="2023-01-19T22:41:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:48 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:41:48.454+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:41:52 truenas k3s[29115]: time="2023-01-19T22:41:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:57 truenas k3s[29115]: time="2023-01-19T22:41:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:41:59 truenas k3s[29115]: I0119 22:41:59.803791   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:41:59 truenas k3s[29115]: E0119 22:41:59.804165   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:42:02 truenas k3s[29115]: time="2023-01-19T22:42:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:07 truenas k3s[29115]: time="2023-01-19T22:42:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:12 truenas k3s[29115]: time="2023-01-19T22:42:12+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:12 truenas k3s[29115]: I0119 22:42:12.803229   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:42:12 truenas k3s[29115]: E0119 22:42:12.803692   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:42:17 truenas k3s[29115]: time="2023-01-19T22:42:17+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:22 truenas k3s[29115]: time="2023-01-19T22:42:22+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:24 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:42:24.526+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:42:25 truenas k3s[29115]: I0119 22:42:25.804162   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:42:25 truenas k3s[29115]: E0119 22:42:25.804543   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:42:27 truenas k3s[29115]: time="2023-01-19T22:42:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:32 truenas k3s[29115]: time="2023-01-19T22:42:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:37 truenas k3s[29115]: time="2023-01-19T22:42:37+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:39 truenas k3s[29115]: I0119 22:42:39.803958   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:42:39 truenas k3s[29115]: E0119 22:42:39.804335   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:42:42 truenas k3s[29115]: time="2023-01-19T22:42:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:47 truenas k3s[29115]: time="2023-01-19T22:42:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:52 truenas k3s[29115]: time="2023-01-19T22:42:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:42:54 truenas k3s[29115]: I0119 22:42:54.803258   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:42:54 truenas k3s[29115]: E0119 22:42:54.803634   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:42:57 truenas k3s[29115]: time="2023-01-19T22:42:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:01 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:43:01.325+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:43:02 truenas k3s[29115]: time="2023-01-19T22:43:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:06 truenas k3s[29115]: I0119 22:43:06.803917   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:43:06 truenas k3s[29115]: E0119 22:43:06.804291   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:43:07 truenas k3s[29115]: time="2023-01-19T22:43:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:12 truenas k3s[29115]: time="2023-01-19T22:43:12+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:17 truenas k3s[29115]: time="2023-01-19T22:43:17+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:17 truenas k3s[29115]: I0119 22:43:17.803149   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:43:17 truenas k3s[29115]: E0119 22:43:17.803599   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:43:22 truenas k3s[29115]: time="2023-01-19T22:43:22+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:27 truenas k3s[29115]: time="2023-01-19T22:43:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:30 truenas k3s[29115]: I0119 22:43:30.803897   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:43:30 truenas k3s[29115]: E0119 22:43:30.804266   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:43:32 truenas k3s[29115]: time="2023-01-19T22:43:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:33 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:43:33.641+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:43:37 truenas k3s[29115]: time="2023-01-19T22:43:37+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:41 truenas k3s[29115]: I0119 22:43:41.803814   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:43:41 truenas k3s[29115]: E0119 22:43:41.804198   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:43:42 truenas k3s[29115]: time="2023-01-19T22:43:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:47 truenas k3s[29115]: time="2023-01-19T22:43:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:52 truenas k3s[29115]: time="2023-01-19T22:43:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:43:53 truenas k3s[29115]: I0119 22:43:53.803277   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:43:53 truenas k3s[29115]: E0119 22:43:53.803654   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:43:57 truenas k3s[29115]: time="2023-01-19T22:43:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:02 truenas k3s[29115]: time="2023-01-19T22:44:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:06 truenas k3s[29115]: I0119 22:44:06.803756   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:44:06 truenas k3s[29115]: E0119 22:44:06.804141   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:44:07 truenas k3s[29115]: time="2023-01-19T22:44:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:12 truenas k3s[29115]: time="2023-01-19T22:44:12+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:16 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:44:16.470+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:44:17 truenas k3s[29115]: time="2023-01-19T22:44:17+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:19 truenas k3s[29115]: I0119 22:44:19.803735   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:44:19 truenas k3s[29115]: E0119 22:44:19.804140   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:44:22 truenas k3s[29115]: time="2023-01-19T22:44:22+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:27 truenas k3s[29115]: time="2023-01-19T22:44:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:32 truenas k3s[29115]: time="2023-01-19T22:44:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:34 truenas k3s[29115]: I0119 22:44:34.802942   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:44:34 truenas k3s[29115]: E0119 22:44:34.803266   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:44:37 truenas k3s[29115]: time="2023-01-19T22:44:37+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:42 truenas k3s[29115]: time="2023-01-19T22:44:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:45 truenas k3s[29115]: I0119 22:44:45.803949   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:44:45 truenas k3s[29115]: E0119 22:44:45.804315   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:44:47 truenas k3s[29115]: time="2023-01-19T22:44:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:52 truenas k3s[29115]: time="2023-01-19T22:44:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:53 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:44:53.475+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:44:57 truenas k3s[29115]: time="2023-01-19T22:44:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:44:58 truenas k3s[29115]: I0119 22:44:58.803607   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:44:58 truenas k3s[29115]: E0119 22:44:58.803966   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:45:02 truenas k3s[29115]: time="2023-01-19T22:45:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:07 truenas k3s[29115]: time="2023-01-19T22:45:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:12 truenas k3s[29115]: time="2023-01-19T22:45:12+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:12 truenas k3s[29115]: I0119 22:45:12.803312   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:45:12 truenas k3s[29115]: E0119 22:45:12.803671   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:45:17 truenas k3s[29115]: time="2023-01-19T22:45:17+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:22 truenas k3s[29115]: time="2023-01-19T22:45:22+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:23 truenas k3s[29115]: I0119 22:45:23.803571   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:45:23 truenas k3s[29115]: E0119 22:45:23.803936   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:45:27 truenas k3s[29115]: time="2023-01-19T22:45:27+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:32 truenas k3s[29115]: time="2023-01-19T22:45:32+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:34 truenas k3s[29115]: I0119 22:45:34.803061   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:45:34 truenas k3s[29115]: E0119 22:45:34.803454   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:45:35 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:45:35.938+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:45:37 truenas k3s[29115]: time="2023-01-19T22:45:37+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:42 truenas k3s[29115]: time="2023-01-19T22:45:42+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:47 truenas k3s[29115]: time="2023-01-19T22:45:47+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:49 truenas k3s[29115]: I0119 22:45:49.803744   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:45:50 truenas k3s[29115]: E0119 22:45:50.207630   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:45:50 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:45:50 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:45:50 truenas k3s[29115]:  > containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:45:50 truenas k3s[29115]: E0119 22:45:50.207760   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:45:50 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:45:50 truenas k3s[29115]: E0119 22:45:50.207834   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:45:52 truenas k3s[29115]: time="2023-01-19T22:45:52+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:45:56 truenas k3s[29115]: I0119 22:45:56.717708   29115 scope.go:115] "RemoveContainer" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:45:57 truenas k3s[29115]: I0119 22:45:57.345947   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="c70b94015de757b765c55eb8b359cf66b4d25a796c3652985b9d2aa400c97ca0"
Jan 19 22:45:57 truenas k3s[29115]: I0119 22:45:57.346412   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:45:57 truenas k3s[29115]: E0119 22:45:57.346777   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:45:57 truenas k3s[29115]: time="2023-01-19T22:45:57+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:02 truenas k3s[29115]: time="2023-01-19T22:46:02+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:07 truenas k3s[29115]: time="2023-01-19T22:46:07+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:09 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:46:09.094+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:46:11 truenas k3s[29115]: I0119 22:46:11.804353   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:46:11 truenas k3s[29115]: E0119 22:46:11.805040   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:46:13 truenas k3s[29115]: time="2023-01-19T22:46:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:18 truenas k3s[29115]: time="2023-01-19T22:46:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:23 truenas k3s[29115]: time="2023-01-19T22:46:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:24 truenas k3s[29115]: I0119 22:46:24.803816   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:46:24 truenas k3s[29115]: E0119 22:46:24.804190   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:46:28 truenas k3s[29115]: time="2023-01-19T22:46:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:33 truenas k3s[29115]: time="2023-01-19T22:46:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:38 truenas k3s[29115]: time="2023-01-19T22:46:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:39 truenas k3s[29115]: I0119 22:46:39.803412   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:46:39 truenas k3s[29115]: E0119 22:46:39.803800   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:46:43 truenas k3s[29115]: time="2023-01-19T22:46:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:48 truenas k3s[29115]: time="2023-01-19T22:46:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:51 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:46:51.755+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:46:53 truenas k3s[29115]: time="2023-01-19T22:46:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:46:53 truenas k3s[29115]: I0119 22:46:53.804240   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:46:53 truenas k3s[29115]: E0119 22:46:53.804625   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:46:58 truenas k3s[29115]: time="2023-01-19T22:46:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:03 truenas k3s[29115]: time="2023-01-19T22:47:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:05 truenas k3s[29115]: I0119 22:47:05.803690   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:47:05 truenas k3s[29115]: E0119 22:47:05.804072   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:47:08 truenas k3s[29115]: time="2023-01-19T22:47:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:13 truenas k3s[29115]: time="2023-01-19T22:47:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:16 truenas k3s[29115]: I0119 22:47:16.803691   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:47:16 truenas k3s[29115]: E0119 22:47:16.804064   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:47:18 truenas k3s[29115]: time="2023-01-19T22:47:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:23 truenas k3s[29115]: time="2023-01-19T22:47:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:28 truenas k3s[29115]: time="2023-01-19T22:47:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:30 truenas k3s[29115]: I0119 22:47:30.803528   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:47:30 truenas k3s[29115]: E0119 22:47:30.804096   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:47:31 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:47:31.338+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:47:33 truenas k3s[29115]: time="2023-01-19T22:47:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:38 truenas k3s[29115]: time="2023-01-19T22:47:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:41 truenas k3s[29115]: I0119 22:47:41.804097   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:47:41 truenas k3s[29115]: E0119 22:47:41.804492   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:47:43 truenas k3s[29115]: time="2023-01-19T22:47:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:48 truenas k3s[29115]: time="2023-01-19T22:47:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:53 truenas k3s[29115]: time="2023-01-19T22:47:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:47:53 truenas k3s[29115]: I0119 22:47:53.803507   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:47:53 truenas k3s[29115]: E0119 22:47:53.803887   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:47:58 truenas k3s[29115]: time="2023-01-19T22:47:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:03 truenas k3s[29115]: time="2023-01-19T22:48:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:05 truenas k3s[29115]: I0119 22:48:05.803540   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:48:05 truenas k3s[29115]: E0119 22:48:05.803922   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:48:08 truenas k3s[29115]: time="2023-01-19T22:48:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:09 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:48:09.606+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:48:13 truenas k3s[29115]: time="2023-01-19T22:48:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:17 truenas k3s[29115]: I0119 22:48:17.803740   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:48:17 truenas k3s[29115]: E0119 22:48:17.804168   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:48:18 truenas k3s[29115]: time="2023-01-19T22:48:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:23 truenas k3s[29115]: time="2023-01-19T22:48:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:28 truenas k3s[29115]: time="2023-01-19T22:48:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:30 truenas k3s[29115]: I0119 22:48:30.803639   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:48:30 truenas k3s[29115]: E0119 22:48:30.804008   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:48:33 truenas k3s[29115]: time="2023-01-19T22:48:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:38 truenas k3s[29115]: time="2023-01-19T22:48:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:40 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:48:40.264+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:48:43 truenas k3s[29115]: time="2023-01-19T22:48:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:44 truenas k3s[29115]: I0119 22:48:44.803697   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:48:44 truenas k3s[29115]: E0119 22:48:44.804055   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:48:48 truenas k3s[29115]: time="2023-01-19T22:48:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:53 truenas k3s[29115]: time="2023-01-19T22:48:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:58 truenas k3s[29115]: time="2023-01-19T22:48:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:48:58 truenas k3s[29115]: I0119 22:48:58.803639   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:48:58 truenas k3s[29115]: E0119 22:48:58.804009   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:49:03 truenas k3s[29115]: time="2023-01-19T22:49:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:08 truenas k3s[29115]: time="2023-01-19T22:49:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:13 truenas k3s[29115]: time="2023-01-19T22:49:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:13 truenas k3s[29115]: I0119 22:49:13.803409   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:49:13 truenas k3s[29115]: E0119 22:49:13.803789   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:49:18 truenas k3s[29115]: time="2023-01-19T22:49:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:19 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:49:19.503+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:49:23 truenas k3s[29115]: time="2023-01-19T22:49:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:26 truenas k3s[29115]: I0119 22:49:26.803311   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:49:26 truenas k3s[29115]: E0119 22:49:26.803700   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:49:28 truenas k3s[29115]: time="2023-01-19T22:49:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:33 truenas k3s[29115]: time="2023-01-19T22:49:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:38 truenas k3s[29115]: time="2023-01-19T22:49:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:41 truenas k3s[29115]: I0119 22:49:41.804009   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:49:41 truenas k3s[29115]: E0119 22:49:41.804382   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:49:43 truenas k3s[29115]: time="2023-01-19T22:49:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:48 truenas k3s[29115]: time="2023-01-19T22:49:48+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:53 truenas k3s[29115]: I0119 22:49:53.803890   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:49:53 truenas k3s[29115]: E0119 22:49:53.804270   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:49:53 truenas k3s[29115]: time="2023-01-19T22:49:53+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:49:57 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:49:57.892+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:49:58 truenas k3s[29115]: time="2023-01-19T22:49:58+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:03 truenas k3s[29115]: time="2023-01-19T22:50:03+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:08 truenas k3s[29115]: I0119 22:50:08.802937   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:50:08 truenas k3s[29115]: E0119 22:50:08.803273   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:50:08 truenas k3s[29115]: time="2023-01-19T22:50:08+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:13 truenas k3s[29115]: time="2023-01-19T22:50:13+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:18 truenas k3s[29115]: time="2023-01-19T22:50:18+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:19 truenas k3s[29115]: I0119 22:50:19.803926   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:50:19 truenas k3s[29115]: E0119 22:50:19.804302   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:50:23 truenas k3s[29115]: time="2023-01-19T22:50:23+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:28 truenas k3s[29115]: time="2023-01-19T22:50:28+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:31 truenas k3s[29115]: I0119 22:50:31.803726   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:50:31 truenas k3s[29115]: E0119 22:50:31.804096   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:50:33 truenas k3s[29115]: time="2023-01-19T22:50:33+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:38 truenas k3s[29115]: time="2023-01-19T22:50:38+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:42 truenas k3s[29115]: I0119 22:50:42.803977   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:50:42 truenas k3s[29115]: E0119 22:50:42.804352   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:50:42 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:50:42.844+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:50:43 truenas k3s[29115]: time="2023-01-19T22:50:43+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:49 truenas k3s[29115]: time="2023-01-19T22:50:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:54 truenas k3s[29115]: time="2023-01-19T22:50:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:50:54 truenas k3s[29115]: I0119 22:50:54.803177   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:50:55 truenas k3s[29115]: E0119 22:50:55.162476   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:50:55 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:50:55 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:50:55 truenas k3s[29115]:  > containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:50:55 truenas k3s[29115]: E0119 22:50:55.162622   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:50:55 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:50:55 truenas k3s[29115]: E0119 22:50:55.162769   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:50:56 truenas k3s[29115]: I0119 22:50:56.887667   29115 scope.go:115] "RemoveContainer" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:50:57 truenas k3s[29115]: I0119 22:50:57.492458   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="019df4421ebda1c4f16203b0e6b431f1e16943fa5ec4d3d8345c4d221137cf8f"
Jan 19 22:50:57 truenas k3s[29115]: I0119 22:50:57.492792   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:50:57 truenas k3s[29115]: E0119 22:50:57.493091   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:50:59 truenas k3s[29115]: time="2023-01-19T22:50:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:04 truenas k3s[29115]: time="2023-01-19T22:51:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:09 truenas k3s[29115]: time="2023-01-19T22:51:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:12 truenas k3s[29115]: I0119 22:51:12.803646   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:51:12 truenas k3s[29115]: E0119 22:51:12.804022   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:51:14 truenas k3s[29115]: time="2023-01-19T22:51:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:19 truenas k3s[29115]: time="2023-01-19T22:51:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:24 truenas k3s[29115]: time="2023-01-19T22:51:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:24 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:51:24.620+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:51:24 truenas k3s[29115]: I0119 22:51:24.803385   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:51:24 truenas k3s[29115]: E0119 22:51:24.803965   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:51:29 truenas k3s[29115]: time="2023-01-19T22:51:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:34 truenas k3s[29115]: time="2023-01-19T22:51:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:38 truenas k3s[29115]: I0119 22:51:38.804142   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:51:38 truenas k3s[29115]: E0119 22:51:38.804538   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:51:39 truenas k3s[29115]: time="2023-01-19T22:51:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:44 truenas k3s[29115]: time="2023-01-19T22:51:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:49 truenas k3s[29115]: time="2023-01-19T22:51:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:51 truenas k3s[29115]: I0119 22:51:51.803687   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:51:51 truenas k3s[29115]: E0119 22:51:51.804261   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:51:54 truenas k3s[29115]: time="2023-01-19T22:51:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:51:59 truenas k3s[29115]: time="2023-01-19T22:51:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:04 truenas k3s[29115]: time="2023-01-19T22:52:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:06 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:52:06.442+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:52:06 truenas k3s[29115]: I0119 22:52:06.804057   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:52:06 truenas k3s[29115]: E0119 22:52:06.804574   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:52:09 truenas k3s[29115]: time="2023-01-19T22:52:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:14 truenas k3s[29115]: time="2023-01-19T22:52:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:19 truenas k3s[29115]: time="2023-01-19T22:52:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:20 truenas k3s[29115]: I0119 22:52:20.802980   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:52:20 truenas k3s[29115]: E0119 22:52:20.803362   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:52:24 truenas k3s[29115]: time="2023-01-19T22:52:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:29 truenas k3s[29115]: time="2023-01-19T22:52:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:32 truenas k3s[29115]: I0119 22:52:32.803050   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:52:32 truenas k3s[29115]: E0119 22:52:32.803431   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:52:34 truenas k3s[29115]: time="2023-01-19T22:52:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:39 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:52:39.114+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:52:39 truenas k3s[29115]: time="2023-01-19T22:52:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:44 truenas k3s[29115]: time="2023-01-19T22:52:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:44 truenas k3s[29115]: I0119 22:52:44.803040   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:52:44 truenas k3s[29115]: E0119 22:52:44.803445   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:52:49 truenas k3s[29115]: time="2023-01-19T22:52:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:54 truenas k3s[29115]: time="2023-01-19T22:52:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:59 truenas k3s[29115]: time="2023-01-19T22:52:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:52:59 truenas k3s[29115]: I0119 22:52:59.803140   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:52:59 truenas k3s[29115]: E0119 22:52:59.803504   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:53:04 truenas k3s[29115]: time="2023-01-19T22:53:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:09 truenas k3s[29115]: time="2023-01-19T22:53:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:12 truenas k3s[29115]: I0119 22:53:12.803500   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:53:12 truenas k3s[29115]: E0119 22:53:12.803874   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:53:14 truenas k3s[29115]: time="2023-01-19T22:53:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:14 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:53:14.769+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:53:19 truenas k3s[29115]: time="2023-01-19T22:53:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:24 truenas k3s[29115]: time="2023-01-19T22:53:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:27 truenas k3s[29115]: I0119 22:53:27.804020   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:53:27 truenas k3s[29115]: E0119 22:53:27.804425   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:53:29 truenas k3s[29115]: time="2023-01-19T22:53:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:34 truenas k3s[29115]: time="2023-01-19T22:53:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:39 truenas k3s[29115]: time="2023-01-19T22:53:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:42 truenas k3s[29115]: I0119 22:53:42.803456   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:53:42 truenas k3s[29115]: E0119 22:53:42.803897   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:53:44 truenas k3s[29115]: time="2023-01-19T22:53:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:49 truenas k3s[29115]: time="2023-01-19T22:53:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:51 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:53:51.881+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:53:54 truenas k3s[29115]: time="2023-01-19T22:53:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:53:54 truenas k3s[29115]: I0119 22:53:54.804162   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:53:54 truenas k3s[29115]: E0119 22:53:54.804542   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:53:59 truenas k3s[29115]: time="2023-01-19T22:53:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:04 truenas k3s[29115]: time="2023-01-19T22:54:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:08 truenas k3s[29115]: I0119 22:54:08.803387   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:54:08 truenas k3s[29115]: E0119 22:54:08.803760   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:54:09 truenas k3s[29115]: time="2023-01-19T22:54:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:14 truenas k3s[29115]: time="2023-01-19T22:54:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:19 truenas k3s[29115]: time="2023-01-19T22:54:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:22 truenas k3s[29115]: I0119 22:54:22.803977   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:54:22 truenas k3s[29115]: E0119 22:54:22.804358   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:54:24 truenas k3s[29115]: time="2023-01-19T22:54:24+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:29 truenas k3s[29115]: time="2023-01-19T22:54:29+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:34 truenas k3s[29115]: time="2023-01-19T22:54:34+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:35 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:54:35.522+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:54:37 truenas k3s[29115]: I0119 22:54:37.803921   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:54:37 truenas k3s[29115]: E0119 22:54:37.804359   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:54:39 truenas k3s[29115]: time="2023-01-19T22:54:39+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:44 truenas k3s[29115]: time="2023-01-19T22:54:44+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:49 truenas k3s[29115]: I0119 22:54:49.803473   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:54:49 truenas k3s[29115]: E0119 22:54:49.803858   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:54:49 truenas k3s[29115]: time="2023-01-19T22:54:49+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:54 truenas k3s[29115]: time="2023-01-19T22:54:54+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:54:59 truenas k3s[29115]: time="2023-01-19T22:54:59+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:04 truenas k3s[29115]: I0119 22:55:04.803783   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:55:04 truenas k3s[29115]: E0119 22:55:04.804153   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:55:04 truenas k3s[29115]: time="2023-01-19T22:55:04+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:09 truenas k3s[29115]: time="2023-01-19T22:55:09+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:14 truenas k3s[29115]: time="2023-01-19T22:55:14+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:18 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:55:18.403+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:55:19 truenas k3s[29115]: I0119 22:55:19.803892   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:55:19 truenas k3s[29115]: E0119 22:55:19.804258   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:55:19 truenas k3s[29115]: time="2023-01-19T22:55:19+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:25 truenas k3s[29115]: time="2023-01-19T22:55:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:30 truenas k3s[29115]: time="2023-01-19T22:55:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:30 truenas k3s[29115]: I0119 22:55:30.803547   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:55:30 truenas k3s[29115]: E0119 22:55:30.804071   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:55:35 truenas k3s[29115]: time="2023-01-19T22:55:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:40 truenas k3s[29115]: time="2023-01-19T22:55:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:43 truenas k3s[29115]: I0119 22:55:43.803036   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:55:43 truenas k3s[29115]: E0119 22:55:43.803447   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:55:45 truenas k3s[29115]: time="2023-01-19T22:55:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:50 truenas k3s[29115]: time="2023-01-19T22:55:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:54 truenas k3s[29115]: I0119 22:55:54.803997   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:55:54 truenas k3s[29115]: E0119 22:55:54.804385   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:55:55 truenas k3s[29115]: time="2023-01-19T22:55:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:55:58 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:55:58.243+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:56:00 truenas k3s[29115]: time="2023-01-19T22:56:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:05 truenas k3s[29115]: time="2023-01-19T22:56:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:07 truenas k3s[29115]: I0119 22:56:07.803351   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:56:08 truenas k3s[29115]: E0119 22:56:08.129382   29115 remote_runtime.go:474] "StartContainer from runtime service failed" err=<
Jan 19 22:56:08 truenas k3s[29115]:         rpc error: code = Unknown desc = failed to start container "2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:56:08 truenas k3s[29115]:         nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:56:08 truenas k3s[29115]:  > containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:56:08 truenas k3s[29115]: E0119 22:56:08.129498   29115 kuberuntime_manager.go:862] container &Container{Name:nvidia-device-plugin-ctr,Image:nvidia/k8s-device-plugin:v0.10.0,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:device-plugin,ReadOnly:false,MountPath:/var/lib/kubelet/device-plugins,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6l7hd,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,} start failed in pod nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8): RunContainerError: failed to start container "2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'
Jan 19 22:56:08 truenas k3s[29115]: nvidia-container-cli: initialization error: nvml error: driver not loaded: unknown
Jan 19 22:56:08 truenas k3s[29115]: E0119 22:56:08.129574   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with RunContainerError: \"failed to start container \\\"2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4\\\": Error response from daemon: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error running hook #0: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'legacy'\\nnvidia-container-cli: initialization error: nvml error: driver not loaded: unknown\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:56:10 truenas k3s[29115]: time="2023-01-19T22:56:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:15 truenas k3s[29115]: time="2023-01-19T22:56:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:18 truenas k3s[29115]: I0119 22:56:18.803016   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:56:18 truenas k3s[29115]: E0119 22:56:18.803398   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:56:20 truenas k3s[29115]: time="2023-01-19T22:56:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:25 truenas k3s[29115]: time="2023-01-19T22:56:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:29 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:56:29.241+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:56:30 truenas k3s[29115]: time="2023-01-19T22:56:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:32 truenas k3s[29115]: I0119 22:56:32.803726   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:56:32 truenas k3s[29115]: E0119 22:56:32.804122   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:56:35 truenas k3s[29115]: time="2023-01-19T22:56:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:40 truenas k3s[29115]: time="2023-01-19T22:56:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:45 truenas k3s[29115]: time="2023-01-19T22:56:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:46 truenas k3s[29115]: I0119 22:56:46.803638   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:56:46 truenas k3s[29115]: E0119 22:56:46.804009   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:56:50 truenas k3s[29115]: time="2023-01-19T22:56:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:55 truenas k3s[29115]: time="2023-01-19T22:56:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:56:57 truenas k3s[29115]: I0119 22:56:57.085304   29115 scope.go:115] "RemoveContainer" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:56:57 truenas k3s[29115]: I0119 22:56:57.587885   29115 pod_container_deletor.go:79] "Container not found in pod's containers" containerID="43cfd70145f2805042542fc5931384fd9405371644ff7ab45aae6b9dd70a2ab0"
Jan 19 22:56:57 truenas k3s[29115]: I0119 22:56:57.588324   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:56:57 truenas k3s[29115]: E0119 22:56:57.588695   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:57:00 truenas k3s[29115]: time="2023-01-19T22:57:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:05 truenas k3s[29115]: time="2023-01-19T22:57:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:10 truenas k3s[29115]: time="2023-01-19T22:57:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:11 truenas k3s[29115]: I0119 22:57:11.804230   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:57:11 truenas k3s[29115]: E0119 22:57:11.804656   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:57:12 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:57:12.262+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:57:15 truenas k3s[29115]: time="2023-01-19T22:57:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:20 truenas k3s[29115]: time="2023-01-19T22:57:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:25 truenas k3s[29115]: time="2023-01-19T22:57:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:26 truenas k3s[29115]: I0119 22:57:26.803782   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:57:26 truenas k3s[29115]: E0119 22:57:26.804218   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:57:30 truenas k3s[29115]: time="2023-01-19T22:57:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:35 truenas k3s[29115]: time="2023-01-19T22:57:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:37 truenas k3s[29115]: I0119 22:57:37.803711   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:57:37 truenas k3s[29115]: E0119 22:57:37.804101   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:57:40 truenas k3s[29115]: time="2023-01-19T22:57:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:45 truenas k3s[29115]: time="2023-01-19T22:57:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:50 truenas k3s[29115]: time="2023-01-19T22:57:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:57:50 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:57:50.741+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:57:52 truenas k3s[29115]: I0119 22:57:52.803454   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:57:52 truenas k3s[29115]: E0119 22:57:52.803818   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:57:55 truenas k3s[29115]: time="2023-01-19T22:57:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:00 truenas k3s[29115]: time="2023-01-19T22:58:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:05 truenas k3s[29115]: time="2023-01-19T22:58:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:06 truenas k3s[29115]: I0119 22:58:06.803765   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:58:06 truenas k3s[29115]: E0119 22:58:06.804138   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:58:10 truenas k3s[29115]: time="2023-01-19T22:58:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:15 truenas k3s[29115]: time="2023-01-19T22:58:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:17 truenas k3s[29115]: I0119 22:58:17.804142   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:58:17 truenas k3s[29115]: E0119 22:58:17.804620   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:58:20 truenas k3s[29115]: time="2023-01-19T22:58:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:24 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:58:24.913+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:58:25 truenas k3s[29115]: time="2023-01-19T22:58:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:30 truenas k3s[29115]: time="2023-01-19T22:58:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:31 truenas k3s[29115]: I0119 22:58:31.804298   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:58:31 truenas k3s[29115]: E0119 22:58:31.804693   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:58:35 truenas k3s[29115]: time="2023-01-19T22:58:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:40 truenas k3s[29115]: time="2023-01-19T22:58:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:44 truenas k3s[29115]: I0119 22:58:44.803216   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:58:44 truenas k3s[29115]: E0119 22:58:44.803638   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:58:45 truenas k3s[29115]: time="2023-01-19T22:58:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:50 truenas k3s[29115]: time="2023-01-19T22:58:50+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:55 truenas k3s[29115]: time="2023-01-19T22:58:55+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:58:57 truenas k3s[29115]: I0119 22:58:57.803966   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:58:57 truenas k3s[29115]: E0119 22:58:57.804344   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:59:00 truenas k3s[29115]: time="2023-01-19T22:59:00+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:05 truenas k3s[29115]: time="2023-01-19T22:59:05+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:06 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:59:06.501+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:59:10 truenas k3s[29115]: time="2023-01-19T22:59:10+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:11 truenas k3s[29115]: I0119 22:59:11.804665   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:59:11 truenas k3s[29115]: E0119 22:59:11.805255   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:59:15 truenas k3s[29115]: time="2023-01-19T22:59:15+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:20 truenas k3s[29115]: time="2023-01-19T22:59:20+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:23 truenas k3s[29115]: I0119 22:59:23.804083   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:59:23 truenas k3s[29115]: E0119 22:59:23.804488   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:59:25 truenas k3s[29115]: time="2023-01-19T22:59:25+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:30 truenas k3s[29115]: time="2023-01-19T22:59:30+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:34 truenas k3s[29115]: I0119 22:59:34.803750   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:59:34 truenas k3s[29115]: E0119 22:59:34.804117   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:59:35 truenas k3s[29115]: time="2023-01-19T22:59:35+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:40 truenas k3s[29115]: time="2023-01-19T22:59:40+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:45 truenas k3s[29115]: time="2023-01-19T22:59:45+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:46 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T22:59:46.062+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 22:59:46 truenas k3s[29115]: I0119 22:59:46.803699   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:59:46 truenas k3s[29115]: E0119 22:59:46.804079   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 22:59:51 truenas k3s[29115]: time="2023-01-19T22:59:51+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:56 truenas k3s[29115]: time="2023-01-19T22:59:56+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 22:59:58 truenas k3s[29115]: I0119 22:59:58.803764   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 22:59:58 truenas k3s[29115]: E0119 22:59:58.804133   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 23:00:01 truenas k3s[29115]: time="2023-01-19T23:00:01+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:06 truenas k3s[29115]: time="2023-01-19T23:00:06+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:11 truenas k3s[29115]: time="2023-01-19T23:00:11+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:13 truenas k3s[29115]: I0119 23:00:13.804048   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 23:00:13 truenas k3s[29115]: E0119 23:00:13.804444   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 23:00:16 truenas k3s[29115]: time="2023-01-19T23:00:16+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:21 truenas k3s[29115]: time="2023-01-19T23:00:21+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:24 truenas k3s[29115]: I0119 23:00:24.803610   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 23:00:24 truenas k3s[29115]: E0119 23:00:24.803983   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 23:00:26 truenas k3s[29115]: time="2023-01-19T23:00:26+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:26 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T23:00:26.521+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 23:00:31 truenas k3s[29115]: time="2023-01-19T23:00:31+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:36 truenas k3s[29115]: time="2023-01-19T23:00:36+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:37 truenas k3s[29115]: I0119 23:00:37.803699   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 23:00:37 truenas k3s[29115]: E0119 23:00:37.804080   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 23:00:41 truenas k3s[29115]: time="2023-01-19T23:00:41+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:46 truenas k3s[29115]: time="2023-01-19T23:00:46+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:51 truenas k3s[29115]: time="2023-01-19T23:00:51+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:00:51 truenas k3s[29115]: I0119 23:00:51.804638   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 23:00:51 truenas k3s[29115]: E0119 23:00:51.805015   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 23:00:56 truenas k3s[29115]: time="2023-01-19T23:00:56+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:01:00 truenas k3s[29115]: {"level":"warn","ts":"2023-01-19T23:01:00.340+0100","logger":"etcd-client","caller":"v3@v3.5.3-k3s1/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00057d180/kine.sock","attempt":0,"error":"rpc error: code = Unknown desc = no such table: dbstat"}
Jan 19 23:01:01 truenas k3s[29115]: time="2023-01-19T23:01:01+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:01:04 truenas k3s[29115]: I0119 23:01:04.803610   29115 scope.go:115] "RemoveContainer" containerID="2a54ee86c73968bb31c5b602dab7ce3dbac4a84f5d8b7086a45fd71736d773d4"
Jan 19 23:01:04 truenas k3s[29115]: E0119 23:01:04.803977   29115 pod_workers.go:965] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"nvidia-device-plugin-ctr\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=nvidia-device-plugin-ctr pod=nvidia-device-plugin-daemonset-f8pn2_kube-system(3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8)\"" pod="kube-system/nvidia-device-plugin-daemonset-f8pn2" podUID=3bcd9276-2a8f-4617-bfdd-7250b4cfe3d8
Jan 19 23:01:06 truenas k3s[29115]: time="2023-01-19T23:01:06+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:01:11 truenas k3s[29115]: time="2023-01-19T23:01:11+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
Jan 19 23:01:16 truenas k3s[29115]: time="2023-01-19T23:01:16+01:00" level=info msg="Using CNI configuration file /etc/cni/net.d/00-multus.conf"
debug finished in 1 seconds for K3s Logs (last 1000 lines)


