{"id":14470,"date":"2026-10-05T14:03:22","date_gmt":"2026-10-05T12:03:22","guid":{"rendered":"https:\/\/irg-net.com\/?p=14470"},"modified":"2026-10-06T11:18:34","modified_gmt":"2026-10-06T09:18:34","slug":"recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff","status":"publish","type":"post","link":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/","title":{"rendered":"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff"},"content":{"rendered":"<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img data-recalc-dims=\"1\" loading=\"lazy\" decoding=\"async\" width=\"314\" height=\"275\" src=\"https:\/\/i0.wp.com\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png?resize=314%2C275&#038;ssl=1\" alt=\"\" class=\"wp-image-14471\" style=\"width:225px;height:auto\" srcset=\"https:\/\/i0.wp.com\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png?w=314&amp;ssl=1 314w, https:\/\/i0.wp.com\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png?resize=300%2C263&amp;ssl=1 300w\" sizes=\"auto, (max-width: 314px) 100vw, 314px\" \/><\/figure>\n<\/div>\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The VxRail Manager VM had been rebooted and the VxRail pages in the vSphere Client no longer loaded. VxRail Manager runs its services as Kubernetes pods on the appliance itself, so the first look was at one of the pods that was obviously unhappy.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl describe pod do-vxrail-system\nName:             do-vxrail-system-xxxxxxxxxx-aaaaa\nNamespace:        helium\nStatus:           Running\nContainers:\n  do-vxrail-system:\n    Image:          do-main\/do-vxrail-system:2.14.26\n    Port:           5000\/TCP\n    State:          Waiting\n      Reason:       CrashLoopBackOff\n    Last State:     Terminated\n      Reason:       Error\n      Exit Code:    1\n      Started:      &lt;today> 08:37:26 +0000\n      Finished:     &lt;today> 08:37:26 +0000\n    Ready:          False\n    Restart Count:  12\n    Limits:\n      memory:  1024M\n    Liveness:  http-get http:\/\/:5000\/do-vxrail-system\/health-check delay=0s timeout=60s period=30s<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl describe pod do-vxrail-system<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Name:             do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">vxrail<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">system<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Namespace:        helium<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Status:           Running<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Containers:<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">vxrail<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">system:<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    Image:          do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">main<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">vxrail<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">system:<\/span><span style=\"color: #6CB6FF\">2.14<\/span><span style=\"color: #ADBAC7\">.26<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    Port:           <\/span><span style=\"color: #6CB6FF\">5000<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">TCP<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    State:          Waiting<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">      Reason:       CrashLoopBackOff<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    Last State:     Terminated<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">      Reason:       Error<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">      Exit Code:    <\/span><span style=\"color: #6CB6FF\">1<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">      Started:      <\/span><span style=\"color: #F47067\">&lt;<\/span><span style=\"color: #ADBAC7\">today<\/span><span style=\"color: #F47067\">&gt;<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">8<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">37<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">26<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">+<\/span><span style=\"color: #6CB6FF\">0000<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">      Finished:     <\/span><span style=\"color: #F47067\">&lt;<\/span><span style=\"color: #ADBAC7\">today<\/span><span style=\"color: #F47067\">&gt;<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">8<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">37<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">26<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">+<\/span><span style=\"color: #6CB6FF\">0000<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    Ready:          <\/span><span style=\"color: #6CB6FF\">False<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    Restart Count:  <\/span><span style=\"color: #6CB6FF\">12<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    Limits:<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">      memory:  <\/span><span style=\"color: #FF938A; font-style: italic\">1024M<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    Liveness:  http<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">get http:<\/span><span style=\"color: #F47067\">\/\/<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">5000<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">vxrail<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">system<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">health<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">check delay<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #FF938A; font-style: italic\">0s<\/span><span style=\"color: #ADBAC7\"> timeout<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #FF938A; font-style: italic\">60s<\/span><span style=\"color: #ADBAC7\"> period<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #FF938A; font-style: italic\">30s<\/span><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">Two details matter here. The container starts and finishes within the same second, and the exit code is 1. That rules out an out-of-memory kill (which would show OOMKilled) and a failing liveness probe (which needs minutes, not milliseconds). The application itself gives up immediately at startup, so the reason has to be in its log.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">More than 30 pods were crash-looping. The handful that were running had all restarted at the same moment, which is simply the reboot. When that many services fail together, the pod you started with is a casualty, not the cause. Deleting pods one by one at this point would have achieved nothing.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl get pods -n helium\nNAME                                 READY   STATUS             RESTARTS        AGE\napi-gateway-xxxxxxxxx-aaaaa          0\/1     Unknown            1               509d\nauth-service-xxxxxxxxxx-aaaaa        0\/1     CrashLoopBackOff   23 (45s ago)    509d\ncacheservice-xxxxxxxxxx-aaaaa        1\/1     Running            2 (66m ago)     509d\ncms-service-xxxxxxxxx-aaaaa          0\/1     CrashLoopBackOff   23 (36s ago)    509d\ncompute-service-xxxxxxxxxx-aaaaa     0\/1     CrashLoopBackOff   24 (42s ago)    509d\ndo-cluster-xxxxxxxxxx-aaaaa          0\/1     CrashLoopBackOff   22 (53s ago)    509d\ndo-host-xxxxxxxxxx-aaaaa             0\/1     CrashLoopBackOff   22 (38s ago)    509d\ndo-vxrail-system-xxxxxxxxxx-aaaaa    0\/1     CrashLoopBackOff   21 (63s ago)    509d\n...\nlockbox-xxxxxxxxx-aaaaa              0\/1     CrashLoopBackOff   19 (2m38s ago)  509d\nlogging-xxxxxxxxx-aaaaa              1\/1     Running            2 (47m ago)     509d\nproperty-collector-xxxxxxxxxx-aaaaa  0\/1     CrashLoopBackOff   18 (2m37s ago)  509d\nserviceregistry-xxxxxxxxx-aaaaa      1\/1     Running            2 (47m ago)     509d\nworkflow-engine-xxxxxxxxxx-aaaaa     0\/1     CrashLoopBackOff   17 (2m13s ago)  509d\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get pods -n helium<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">NAME<\/span><span style=\"color: #ADBAC7\">                                 <\/span><span style=\"color: #6CB6FF\">READY<\/span><span style=\"color: #ADBAC7\">   <\/span><span style=\"color: #6CB6FF\">STATUS<\/span><span style=\"color: #ADBAC7\">             <\/span><span style=\"color: #6CB6FF\">RESTARTS<\/span><span style=\"color: #ADBAC7\">        <\/span><span style=\"color: #6CB6FF\">AGE<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">api<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">gateway<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa          <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Unknown            <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">               <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">auth<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa        <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">23<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">45s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">cacheservice<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa        <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Running            <\/span><span style=\"color: #6CB6FF\">2<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">66m<\/span><span style=\"color: #ADBAC7\"> ago)     <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">cms<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa          <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">23<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">36s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa     <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">24<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">42s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">cluster<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa          <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">22<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">53s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">host<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa             <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">22<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">38s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">vxrail<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">system<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa    <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">21<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">63s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">...<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">lockbox<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa              <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">19<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">2m38s<\/span><span style=\"color: #ADBAC7\"> ago)  <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">logging<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa              <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Running            <\/span><span style=\"color: #6CB6FF\">2<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">47m<\/span><span style=\"color: #ADBAC7\"> ago)     <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">property<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">collector<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa  <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">18<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">2m37s<\/span><span style=\"color: #ADBAC7\"> ago)  <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">serviceregistry<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa      <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Running            <\/span><span style=\"color: #6CB6FF\">2<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">47m<\/span><span style=\"color: #ADBAC7\"> ago)     <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">workflow<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">engine<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa     <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">17<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">2m13s<\/span><span style=\"color: #ADBAC7\"> ago)  <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The &#8211;previous flag returns the log of the last terminated container instead of the one currently waiting. Addressing the deployment instead of the pod avoids typing the generated pod name.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl logs -n helium deploy\/do-vxrail-system --previous --tail=50\nREGISTERED:true\nRegistering do-vxrail-system.helium.svc.cluster.local.\nport:5000\ncurl: (7) Failed to connect to api-gateway port 8080 after 1 ms: Couldn't connect to server\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl logs -n helium deploy\/do-vxrail-system --previous --tail=50<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">REGISTERED<\/span><span style=\"color: #ADBAC7\">:true<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Registering do<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">vxrail<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">system.helium.svc.cluster.local.<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">port:<\/span><span style=\"color: #6CB6FF\">5000<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">curl: (<\/span><span style=\"color: #6CB6FF\">7<\/span><span style=\"color: #ADBAC7\">) Failed to connect to api<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">gateway port <\/span><span style=\"color: #6CB6FF\">8080<\/span><span style=\"color: #ADBAC7\"> after <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> ms: Couldn<\/span><span style=\"color: #96D0FF\">&#39;t connect to server<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The service tries to register itself at api-gateway on port 8080 and exits when that fails. The refusal after 1 ms is informative: the name resolves, but nothing is listening behind the service. That points away from DNS and toward a missing backend pod. The api-gateway pod is stuck in Unknown:<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl get pods -n helium | grep api-gateway\napi-gateway-xxxxxxxxx-aaaaa          0\/1     Unknown            1               509d\n \nvxm01:~ # kubectl get endpoints -n helium api-gateway\nNAME          ENDPOINTS   AGE\napi-gateway               509d\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get pods -n helium | grep api-gateway<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">api<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">gateway<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa          <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Unknown            <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">               <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get endpoints -n helium api-gateway<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">NAME<\/span><span style=\"color: #ADBAC7\">          <\/span><span style=\"color: #6CB6FF\">ENDPOINTS<\/span><span style=\"color: #ADBAC7\">   <\/span><span style=\"color: #6CB6FF\">AGE<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">api<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">gateway               <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The endpoints column is empty, which matches the refused connection. A describe of the pod showed all volumes and ConfigMaps in place and, at the bottom, Events: &lt;none&gt;. No image pull error, no failed mount. The kubelet was not even trying to start the pod any more; it had been left in Unknown across the reboot.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">With no concrete error to fix, the reasonable move is to delete the pod and let the ReplicaSet create a fresh one. We took a snapshot of the VxRail Manager VM first.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl delete pod -n helium api-gateway-xxxxxxxxx-aaaaa\npod \"api-gateway-xxxxxxxxx-aaaaa\" deleted\n \nvxm01:~ # kubectl get pods -n helium | grep api-gateway\napi-gateway-xxxxxxxxx-bbbbb          1\/1     Running            0               25s\n \nvxm01:~ # kubectl get endpoints -n helium api-gateway\nNAME          ENDPOINTS          AGE\napi-gateway   10.42.0.239:8080   509d\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl delete pod -n helium api-gateway-xxxxxxxxx-aaaaa<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">pod <\/span><span style=\"color: #96D0FF\">&quot;api-gateway-xxxxxxxxx-aaaaa&quot;<\/span><span style=\"color: #ADBAC7\"> deleted<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get pods -n helium | grep api-gateway<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">api<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">gateway<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">bbbbb          <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Running            <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #ADBAC7\">               <\/span><span style=\"color: #FF938A; font-style: italic\">25s<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get endpoints -n helium api-gateway<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">NAME<\/span><span style=\"color: #ADBAC7\">          <\/span><span style=\"color: #6CB6FF\">ENDPOINTS<\/span><span style=\"color: #ADBAC7\">          <\/span><span style=\"color: #6CB6FF\">AGE<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">api<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">gateway   <\/span><span style=\"color: #6CB6FF\">10.42<\/span><span style=\"color: #ADBAC7\">.0.239:<\/span><span style=\"color: #6CB6FF\">8080<\/span><span style=\"color: #ADBAC7\">   <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<p class=\"has-text-color has-link-color wp-elements-1 wp-block-paragraph\" style=\"color:#009e03\"><em><strong>If the delete hangs.<\/strong> A pod in Unknown can stay in Terminating. In that case kubectl delete pod &#8230; &#8211;force &#8211;grace-period=0 removes it.<\/em><\/p>\n<\/blockquote>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">Immediately afterwards, the list of failing pods looked exactly the same. That is expected. After more than 20 restarts Kubernetes waits five minutes between attempts, and the last attempts had all happened before the new gateway existed. Nothing needed to be done except wait.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl get pods -n helium | grep -v -E \"Running|Completed\"\nNAME                               READY   STATUS             RESTARTS        AGE\nauth-service-xxxxxxxxxx-aaaaa      0\/1     Error              27 (5m30s ago)  509d\ncompute-service-xxxxxxxxxx-aaaaa   0\/1     CrashLoopBackOff   28 (15s ago)    509d\ninfra-config-service-xxxxx-aaaaa   0\/1     CrashLoopBackOff   21 (4m56s ago)  509d\n \n(a few minutes later)\nNAME                               READY   STATUS             RESTARTS        AGE\ncompute-service-xxxxxxxxxx-aaaaa   0\/1     CrashLoopBackOff   29 (40s ago)    509d\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get pods -n helium | grep -v -E &quot;Running|Completed&quot;<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">NAME<\/span><span style=\"color: #ADBAC7\">                               <\/span><span style=\"color: #6CB6FF\">READY<\/span><span style=\"color: #ADBAC7\">   <\/span><span style=\"color: #6CB6FF\">STATUS<\/span><span style=\"color: #ADBAC7\">             <\/span><span style=\"color: #6CB6FF\">RESTARTS<\/span><span style=\"color: #ADBAC7\">        <\/span><span style=\"color: #6CB6FF\">AGE<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">auth<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa      <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Error              <\/span><span style=\"color: #6CB6FF\">27<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">5m30s<\/span><span style=\"color: #ADBAC7\"> ago)  <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa   <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">28<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">15s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">infra<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">config<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa   <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">21<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">4m56s<\/span><span style=\"color: #ADBAC7\"> ago)  <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">(a few minutes later)<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">NAME<\/span><span style=\"color: #ADBAC7\">                               <\/span><span style=\"color: #6CB6FF\">READY<\/span><span style=\"color: #ADBAC7\">   <\/span><span style=\"color: #6CB6FF\">STATUS<\/span><span style=\"color: #ADBAC7\">             <\/span><span style=\"color: #6CB6FF\">RESTARTS<\/span><span style=\"color: #ADBAC7\">        <\/span><span style=\"color: #6CB6FF\">AGE<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">aaaaa   <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     CrashLoopBackOff   <\/span><span style=\"color: #6CB6FF\">29<\/span><span style=\"color: #ADBAC7\"> (<\/span><span style=\"color: #FF938A; font-style: italic\">40s<\/span><span style=\"color: #ADBAC7\"> ago)    <\/span><span style=\"color: #FF938A; font-style: italic\">509d<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">From more than 30 down to one. It would have been easy to stop here, delete the last pod and call it done. But compute-service had already failed several times with the gateway up, so it had a reason of its own. The straggler points at another service:<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl logs -n helium deploy\/compute-service --previous --tail=50\n...\ncompute-service &#91;INFO&#93; db.go tryToOpenDB() (110): successfully opened DB\ncompute-service &#91;INFO&#93; db.go initDB() (64): Datasource connected: postgres:\/\/10.42.0.38:5432\/vxrail\ncompute-service &#91;INFO&#93; mq.go tryToConnectToMQ() (249): mq service connection success\ncompute-service &#91;INFO&#93; mq.go tryToWaitPropertyCollectorService() (182): http:\/\/property-collector:5000\/propertycollector\/health-check\ncompute-service &#91;WARNING&#93; mq.go tryToWaitPropertyCollectorService() (190): Get \"http:\/\/property-collector:5000\/propertycollector\/health-check\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\ncompute-service &#91;WARNING&#93; mq.go tryToWaitPropertyCollectorService() (191): property collector service seems not ready yet\ncompute-service &#91;INFO&#93; mq.go tryToWaitPropertyCollectorService() (192): sleep 10 seconds for 1 time\npanic: runtime error: invalid memory address or nil pointer dereference\n&#91;signal SIGSEGV: segmentation violation code=0x1 addr=0x40 pc=0x...&#93;\n \ngoroutine 1 &#91;running&#93;:\nvxrail\/compute-svc\/compute\/cmd\/app.(*App).tryToWaitPropertyCollectorService(...)\n        \/compute-svc\/compute\/cmd\/app\/mq.go:202 +0x26e\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl logs -n helium deploy\/compute-service --previous --tail=50<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">...<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service &#91;<\/span><span style=\"color: #6CB6FF\">INFO<\/span><span style=\"color: #ADBAC7\">&#93; db.go tryToOpenDB() (<\/span><span style=\"color: #6CB6FF\">110<\/span><span style=\"color: #ADBAC7\">): successfully opened <\/span><span style=\"color: #6CB6FF\">DB<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service &#91;<\/span><span style=\"color: #6CB6FF\">INFO<\/span><span style=\"color: #ADBAC7\">&#93; db.go initDB() (<\/span><span style=\"color: #6CB6FF\">64<\/span><span style=\"color: #ADBAC7\">): Datasource connected: postgres:<\/span><span style=\"color: #F47067\">\/\/<\/span><span style=\"color: #6CB6FF\">10.42<\/span><span style=\"color: #ADBAC7\">.0.38:<\/span><span style=\"color: #6CB6FF\">5432<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">vxrail<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service &#91;<\/span><span style=\"color: #6CB6FF\">INFO<\/span><span style=\"color: #ADBAC7\">&#93; mq.go tryToConnectToMQ() (<\/span><span style=\"color: #6CB6FF\">249<\/span><span style=\"color: #ADBAC7\">): mq service connection success<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service &#91;<\/span><span style=\"color: #6CB6FF\">INFO<\/span><span style=\"color: #ADBAC7\">&#93; mq.go tryToWaitPropertyCollectorService() (<\/span><span style=\"color: #6CB6FF\">182<\/span><span style=\"color: #ADBAC7\">): http:<\/span><span style=\"color: #F47067\">\/\/<\/span><span style=\"color: #6CB6FF\">property<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">collector:<\/span><span style=\"color: #6CB6FF\">5000<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">propertycollector<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">health<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">check<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service &#91;<\/span><span style=\"color: #6CB6FF\">WARNING<\/span><span style=\"color: #ADBAC7\">&#93; mq.go tryToWaitPropertyCollectorService() (<\/span><span style=\"color: #6CB6FF\">190<\/span><span style=\"color: #ADBAC7\">): Get <\/span><span style=\"color: #96D0FF\">&quot;http:\/\/property-collector:5000\/propertycollector\/health-check&quot;<\/span><span style=\"color: #ADBAC7\">: context deadline exceeded (Client.Timeout exceeded <\/span><span style=\"color: #F47067\">while<\/span><span style=\"color: #ADBAC7\"> awaiting headers)<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service &#91;<\/span><span style=\"color: #6CB6FF\">WARNING<\/span><span style=\"color: #ADBAC7\">&#93; mq.go tryToWaitPropertyCollectorService() (<\/span><span style=\"color: #6CB6FF\">191<\/span><span style=\"color: #ADBAC7\">): <\/span><span style=\"color: #6CB6FF\">property<\/span><span style=\"color: #ADBAC7\"> collector service seems <\/span><span style=\"color: #F47067\">not<\/span><span style=\"color: #ADBAC7\"> ready yet<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service &#91;<\/span><span style=\"color: #6CB6FF\">INFO<\/span><span style=\"color: #ADBAC7\">&#93; mq.go tryToWaitPropertyCollectorService() (<\/span><span style=\"color: #6CB6FF\">192<\/span><span style=\"color: #ADBAC7\">): sleep <\/span><span style=\"color: #6CB6FF\">10<\/span><span style=\"color: #ADBAC7\"> seconds <\/span><span style=\"color: #F47067\">for<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> time<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">panic: runtime error: invalid memory address <\/span><span style=\"color: #F47067\">or<\/span><span style=\"color: #ADBAC7\"> nil pointer dereference<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">&#91;signal <\/span><span style=\"color: #6CB6FF\">SIGSEGV<\/span><span style=\"color: #ADBAC7\">: segmentation violation code=<\/span><span style=\"color: #F47067\">0x<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> addr=<\/span><span style=\"color: #F47067\">0x<\/span><span style=\"color: #6CB6FF\">40<\/span><span style=\"color: #ADBAC7\"> pc=<\/span><span style=\"color: #FF938A; font-style: italic\">0x<\/span><span style=\"color: #6CB6FF\">...<\/span><span style=\"color: #ADBAC7\">&#93;<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">goroutine <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> &#91;running&#93;:<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxrail<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">svc<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">cmd<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">app.(<\/span><span style=\"color: #F47067\">*<\/span><span style=\"color: #ADBAC7\">App).tryToWaitPropertyCollectorService(<\/span><span style=\"color: #6CB6FF\">...<\/span><span style=\"color: #ADBAC7\">)<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">        <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">svc<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">cmd<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">app<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">mq.go:<\/span><span style=\"color: #6CB6FF\">202<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">+0x<\/span><span style=\"color: #6CB6FF\">26e<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">Database and message queue connect fine. The service then waits for the health check of property-collector, which does not answer within five seconds, and instead of retrying it dies with a nil pointer panic. The panic is the loud part of the log, but it is only a consequence. The real question is why property-collector does not answer, even though Kubernetes lists it as 1\/1 Running.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A direct call from another pod confirmed it. The command ran and returned nothing, because -s hides the timeout message:<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl exec -n helium deploy\/do-vxrail-system -- curl -s -m 15 http:\/\/property-collector:5000\/propertycollector\/health-check\nvxm01:~ #\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl exec -n helium deploy\/do-vxrail-system -- curl -s -m 15 http:\/\/property-collector:5000\/propertycollector\/health-check<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\">#<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The root cause in the property-collector log:<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl logs -n helium deploy\/property-collector --tail=50\nTraceback (most recent call last):\n  File \"...\/do_common\/connector\/vsphere\/soap_client.py\", line 111, in _update_shared_session\n    self._make_fresh_new_conn()\n  File \"...\/do_common\/connector\/vsphere\/do_smart_connect.py\", line 61, in DoSmartConnect\n    return SmartConnect(**kwargs)\n  File \"...\/pyVim\/connect.py\", line 985, in SmartConnect\n  ...\n  File \"\/usr\/lib64\/python3.11\/ssl.py\", line 1382, in do_handshake\n    self._sslobj.do_handshake()\nssl.SSLCertVerificationError: &#91;SSL: CERTIFICATE_VERIFY_FAILED&#93; certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)\nproperty-collector &#91;ERROR&#93; vc_credential.py _connect() (68): SSLCertVerificationError(1, '&#91;SSL: CERTIFICATE_VERIFY_FAILED&#93; certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)')\nproperty-collector &#91;WARNING&#93; utils.py call() (99): Retryed 1 times, raise Exception('Could not connect to host vcenter01.example.local:443 with username svc-vxrail@vsphere.local') error\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl logs -n helium deploy\/property-collector --tail=50<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Traceback (most recent call last):<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  File <\/span><span style=\"color: #96D0FF\">&quot;...\/do_common\/connector\/vsphere\/soap_client.py&quot;<\/span><span style=\"color: #ADBAC7\">, line <\/span><span style=\"color: #6CB6FF\">111<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #F47067\">in<\/span><span style=\"color: #ADBAC7\"> _update_shared_session<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    <\/span><span style=\"color: #6CB6FF\">self<\/span><span style=\"color: #ADBAC7\">._make_fresh_new_conn()<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  File <\/span><span style=\"color: #96D0FF\">&quot;...\/do_common\/connector\/vsphere\/do_smart_connect.py&quot;<\/span><span style=\"color: #ADBAC7\">, line <\/span><span style=\"color: #6CB6FF\">61<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #F47067\">in<\/span><span style=\"color: #ADBAC7\"> DoSmartConnect<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    <\/span><span style=\"color: #F47067\">return<\/span><span style=\"color: #ADBAC7\"> SmartConnect(<\/span><span style=\"color: #F47067\">**<\/span><span style=\"color: #ADBAC7\">kwargs)<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  File <\/span><span style=\"color: #96D0FF\">&quot;...\/pyVim\/connect.py&quot;<\/span><span style=\"color: #ADBAC7\">, line <\/span><span style=\"color: #6CB6FF\">985<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #F47067\">in<\/span><span style=\"color: #ADBAC7\"> SmartConnect<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  <\/span><span style=\"color: #6CB6FF\">...<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  File <\/span><span style=\"color: #96D0FF\">&quot;\/usr\/lib64\/python3.11\/ssl.py&quot;<\/span><span style=\"color: #ADBAC7\">, line <\/span><span style=\"color: #6CB6FF\">1382<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #F47067\">in<\/span><span style=\"color: #ADBAC7\"> do_handshake<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    <\/span><span style=\"color: #6CB6FF\">self<\/span><span style=\"color: #ADBAC7\">._sslobj.do_handshake()<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">ssl.SSLCertVerificationError: &#91;<\/span><span style=\"color: #6CB6FF\">SSL<\/span><span style=\"color: #ADBAC7\">: <\/span><span style=\"color: #6CB6FF\">CERTIFICATE_VERIFY_FAILED<\/span><span style=\"color: #ADBAC7\">&#93; certificate verify failed: unable to get local issuer certificate (_ssl.c:<\/span><span style=\"color: #6CB6FF\">1006<\/span><span style=\"color: #ADBAC7\">)<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">property<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">collector &#91;<\/span><span style=\"color: #6CB6FF\">ERROR<\/span><span style=\"color: #ADBAC7\">&#93; vc_credential.py _connect() (<\/span><span style=\"color: #6CB6FF\">68<\/span><span style=\"color: #ADBAC7\">): SSLCertVerificationError(<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #96D0FF\">&#39;&#91;SSL: CERTIFICATE_VERIFY_FAILED&#93; certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)&#39;<\/span><span style=\"color: #ADBAC7\">)<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">property<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">collector &#91;<\/span><span style=\"color: #6CB6FF\">WARNING<\/span><span style=\"color: #ADBAC7\">&#93; utils.py call() (<\/span><span style=\"color: #6CB6FF\">99<\/span><span style=\"color: #ADBAC7\">): Retryed <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> times, <\/span><span style=\"color: #F47067\">raise<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">Exception<\/span><span style=\"color: #ADBAC7\">(<\/span><span style=\"color: #96D0FF\">&#39;Could not connect to host vcenter01.example.local:443 with username svc-vxrail@vsphere.local&#39;<\/span><span style=\"color: #ADBAC7\">) error<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The last line reads like a credential problem, but the line above it says otherwise. unable to get local issuer certificate means the client does not have the CA that signed the server certificate. The VxRail Manager did not trust the vCenter certificate. The same check can be made with openssl against the trust store directory VxRail Manager uses. All of these commands are read-only.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # openssl s_client -connect vcenter01.example.local:443 -verify_return_error -brief \\\n    -CApath \/var\/lib\/vmware-marvin\/trust\/lin -verify_hostname vcenter01.example.local &lt;\/dev\/null\ndepth=0 C = XX, O = Example Corp, OU = IT, CN = vcenter01.example.local\nverify error:num=20:unable to get local issuer certificate\n...:error:0A000086:SSL routines:tls_post_process_server_certificate:certificate verify failed:...\n \nvxm01:~ # openssl s_client -connect vcenter01.example.local:443 &lt;\/dev\/null 2>\/dev\/null | openssl x509 -noout -issuer -dates\nissuer=C = XX, O = Example Corp, CN = Example Internal Issuing CA\nnotBefore=Jun  5 11:08:47 2025 GMT\nnotAfter=Jun  5 11:08:47 2027 GMT\n \nvxm01:~ # ls -l \/var\/lib\/vmware-marvin\/trust\/lin\/\n-rw-r--r-- 1 tcserver pivotal 1489 May 13  2025 aaaaaaaa.0\n-rw-r--r-- 1 tcserver pivotal  779 May 13  2025 aaaaaaaa.r0\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># openssl s_client -connect vcenter01.example.local:443 -verify_return_error -brief \\<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">CApath <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">var<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lib<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">vmware<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">marvin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">trust<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lin <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">verify_hostname vcenter01.example.local <\/span><span style=\"color: #F47067\">&lt;\/<\/span><span style=\"color: #ADBAC7\">dev<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">null<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">depth<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #ADBAC7\"> C <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">XX<\/span><span style=\"color: #ADBAC7\">, O <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> Example Corp, <\/span><span style=\"color: #6CB6FF\">OU<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">IT<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #6CB6FF\">CN<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> vcenter01.example.local<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">verify error:num<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #6CB6FF\">20<\/span><span style=\"color: #ADBAC7\">:unable to get local issuer certificate<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">...<\/span><span style=\"color: #ADBAC7\">:error:<\/span><span style=\"color: #FF938A; font-style: italic\">0A000086<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">SSL<\/span><span style=\"color: #ADBAC7\"> routines:tls_post_process_server_certificate:certificate verify failed:<\/span><span style=\"color: #6CB6FF\">...<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># openssl s_client -connect vcenter01.example.local:443 &lt;\/dev\/null 2&gt;\/dev\/null | openssl x509 -noout -issuer -dates<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">issuer<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\">C <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">XX<\/span><span style=\"color: #ADBAC7\">, O <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> Example Corp, <\/span><span style=\"color: #6CB6FF\">CN<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> Example Internal Issuing <\/span><span style=\"color: #6CB6FF\">CA<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">notBefore<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\">Jun  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">11<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">8<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">47<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">2025<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">GMT<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">notAfter<\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\">Jun  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">11<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">8<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">47<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">2027<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">GMT<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># ls -l \/var\/lib\/vmware-marvin\/trust\/lin\/<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> tcserver pivotal <\/span><span style=\"color: #6CB6FF\">1489<\/span><span style=\"color: #ADBAC7\"> May <\/span><span style=\"color: #6CB6FF\">13<\/span><span style=\"color: #ADBAC7\">  <\/span><span style=\"color: #6CB6FF\">2025<\/span><span style=\"color: #ADBAC7\"> aaaaaaaa.0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> tcserver pivotal  <\/span><span style=\"color: #6CB6FF\">779<\/span><span style=\"color: #ADBAC7\"> May <\/span><span style=\"color: #6CB6FF\">13<\/span><span style=\"color: #ADBAC7\">  <\/span><span style=\"color: #6CB6FF\">2025<\/span><span style=\"color: #ADBAC7\"> aaaaaaaa.r0<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">Three outputs, one story. The trust store holds a single CA from the day the cluster was deployed. About three weeks later the vCenter machine certificate was replaced with one issued by the internal company CA. Nobody imported that chain into the VxRail Manager, and nothing complained, because the running services kept their established sessions. Sixteen months later a reboot forced a new handshake, and the missing trust finally surfaced.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Dell documents the import in KB 000077894. From VxRail 7.0.480 on, the script is already on the appliance at \/mystic\/ssl\/cert_util.py and must be run as root.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # python3 \/mystic\/ssl\/cert_util.py --help\nRunning script with version 2024.07.11\nusage: cert_util.py &#91;-h&#93; &#91;-r&#93; &#91;-i&#93;\n \n  -r, --regencert  Regenerate VxRail Manager self signed cert.\n  -i, --vc_import  Import vCenter Root CA certificates to VxRail Manager.\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># python3 \/mystic\/ssl\/cert_util.py --help<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Running script <\/span><span style=\"color: #F47067\">with<\/span><span style=\"color: #ADBAC7\"> version <\/span><span style=\"color: #6CB6FF\">2024.07<\/span><span style=\"color: #ADBAC7\">.11<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">usage: cert_util.py &#91;<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">h&#93; &#91;<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r&#93; &#91;<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">i&#93;<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r, <\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">regencert  Regenerate VxRail Manager <\/span><span style=\"color: #6CB6FF\">self<\/span><span style=\"color: #ADBAC7\"> signed cert.<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">  <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">i, <\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">vc_import  Import vCenter Root <\/span><span style=\"color: #6CB6FF\">CA<\/span><span style=\"color: #ADBAC7\"> certificates to VxRail Manager.<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<p class=\"has-text-color has-link-color wp-elements-2 wp-block-paragraph\" style=\"color:#0ea300\"><em><strong>Mind the option.<\/strong> -i imports the vCenter CA certificates, which is what is needed here. -r regenerates the VxRail Manager&#8217;s own certificate, a different operation that was not the problem.<\/em><\/p>\n<\/blockquote>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The script cleans out the existing trust store entries before writing the new ones, so we copied the directory first.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # cp -a \/var\/lib\/vmware-marvin\/trust \/root\/trust-backup-$(date +%F)\n \nvxm01:~ # python3 \/mystic\/ssl\/cert_util.py -i\nRunning script with version 2024.07.11\nDownloaded root CA certificate zip from vcenter01.example.local\nVerify certificate against vCenter vcenter01.example.local\nFound certificates &#91;'certs\/lin\/aaaaaaaa.r0', 'certs\/lin\/bbbbbbbb.0', 'certs\/lin\/cccccccc.0', ...&#93; that can verify server certificate\nClean up existing certificates in \/var\/lib\/vmware-marvin\/trust\/\n - Removing \/var\/lib\/vmware-marvin\/trust\/lin\/aaaaaaaa.r0\n - Removing \/var\/lib\/vmware-marvin\/trust\/lin\/aaaaaaaa.0\nProcessing file &#91;2\/4&#93;: certs\/lin\/bbbbbbbb.0\nRoot CA certificate \/tmp\/certs\/lin is saved at \/var\/lib\/vmware-marvin\/trust\/.\n...\nDelete saved CRL info in cacheservice...\nRestarting vmware-marvin service...\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># cp -a \/var\/lib\/vmware-marvin\/trust \/root\/trust-backup-$(date +%F)<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># python3 \/mystic\/ssl\/cert_util.py -i<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Running script <\/span><span style=\"color: #F47067\">with<\/span><span style=\"color: #ADBAC7\"> version <\/span><span style=\"color: #6CB6FF\">2024.07<\/span><span style=\"color: #ADBAC7\">.11<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Downloaded root <\/span><span style=\"color: #6CB6FF\">CA<\/span><span style=\"color: #ADBAC7\"> certificate <\/span><span style=\"color: #6CB6FF\">zip<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">from<\/span><span style=\"color: #ADBAC7\"> vcenter01.example.local<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Verify certificate against vCenter vcenter01.example.local<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Found certificates &#91;<\/span><span style=\"color: #96D0FF\">&#39;certs\/lin\/aaaaaaaa.r0&#39;<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #96D0FF\">&#39;certs\/lin\/bbbbbbbb.0&#39;<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #96D0FF\">&#39;certs\/lin\/cccccccc.0&#39;<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #6CB6FF\">...<\/span><span style=\"color: #ADBAC7\">&#93; that can verify server certificate<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Clean up existing certificates <\/span><span style=\"color: #F47067\">in<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">var<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lib<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">vmware<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">marvin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">trust<\/span><span style=\"color: #F47067\">\/<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\"> Removing <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">var<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lib<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">vmware<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">marvin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">trust<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">aaaaaaaa.r0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\"> Removing <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">var<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lib<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">vmware<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">marvin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">trust<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">aaaaaaaa.0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Processing <\/span><span style=\"color: #F69D50\">file<\/span><span style=\"color: #ADBAC7\"> &#91;<\/span><span style=\"color: #6CB6FF\">2<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">4<\/span><span style=\"color: #ADBAC7\">&#93;: certs<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">bbbbbbbb.0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Root <\/span><span style=\"color: #6CB6FF\">CA<\/span><span style=\"color: #ADBAC7\"> certificate <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">tmp<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">certs<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lin <\/span><span style=\"color: #F47067\">is<\/span><span style=\"color: #ADBAC7\"> saved at <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">var<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lib<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">vmware<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">marvin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">trust<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">.<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">...<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Delete saved <\/span><span style=\"color: #6CB6FF\">CRL<\/span><span style=\"color: #ADBAC7\"> info <\/span><span style=\"color: #F47067\">in<\/span><span style=\"color: #ADBAC7\"> cacheservice<\/span><span style=\"color: #6CB6FF\">...<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Restarting vmware<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">marvin service<\/span><span style=\"color: #6CB6FF\">...<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">The line to look for is Found certificates [&#8230;] that can verify server certificate. If it is missing, vCenter itself does not publish the complete chain (for example a missing intermediate CA) and the CA certificates have to be imported by hand.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # ls -l \/var\/lib\/vmware-marvin\/trust\/lin\/\n-rw-r--r-- 1 tcserver pivotal 3073 Oct  5 10:07 bbbbbbbb.0\n-rw-r--r-- 1 tcserver pivotal 2130 Oct  5 10:07 dddddddd.0\n-rw-r--r-- 1 tcserver pivotal 1489 Oct  5 10:07 aaaaaaaa.0\n-rw-r--r-- 1 tcserver pivotal  779 Oct  5 10:07 aaaaaaaa.r0\n-rw-r--r-- 1 tcserver pivotal 2264 Oct  5 10:07 cccccccc.0\n-rw-r--r-- 1 root     root    1702 Oct  5 10:07 cccccccc.r0\n \nvxm01:~ # openssl s_client -connect vcenter01.example.local:443 -verify_return_error -brief \\\n    -CApath \/var\/lib\/vmware-marvin\/trust\/lin -verify_hostname vcenter01.example.local &lt;\/dev\/null\nCONNECTION ESTABLISHED\nProtocol version: TLSv1.2\nPeer certificate: C = XX, O = Example Corp, OU = IT, CN = vcenter01.example.local\nVerification: OK\nVerified peername: vcenter01.example.local\nDONE\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># ls -l \/var\/lib\/vmware-marvin\/trust\/lin\/<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> tcserver pivotal <\/span><span style=\"color: #6CB6FF\">3073<\/span><span style=\"color: #ADBAC7\"> Oct  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">10<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">7<\/span><span style=\"color: #ADBAC7\"> bbbbbbbb.0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> tcserver pivotal <\/span><span style=\"color: #6CB6FF\">2130<\/span><span style=\"color: #ADBAC7\"> Oct  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">10<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">7<\/span><span style=\"color: #ADBAC7\"> dddddddd.0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> tcserver pivotal <\/span><span style=\"color: #6CB6FF\">1489<\/span><span style=\"color: #ADBAC7\"> Oct  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">10<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">7<\/span><span style=\"color: #ADBAC7\"> aaaaaaaa.0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> tcserver pivotal  <\/span><span style=\"color: #6CB6FF\">779<\/span><span style=\"color: #ADBAC7\"> Oct  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">10<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">7<\/span><span style=\"color: #ADBAC7\"> aaaaaaaa.r0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> tcserver pivotal <\/span><span style=\"color: #6CB6FF\">2264<\/span><span style=\"color: #ADBAC7\"> Oct  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">10<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">7<\/span><span style=\"color: #ADBAC7\"> cccccccc.0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">rw<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\">r<\/span><span style=\"color: #FF938A; font-style: italic\">--<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\"> root     root    <\/span><span style=\"color: #6CB6FF\">1702<\/span><span style=\"color: #ADBAC7\"> Oct  <\/span><span style=\"color: #6CB6FF\">5<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">10<\/span><span style=\"color: #ADBAC7\">:<\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #FF938A; font-style: italic\">7<\/span><span style=\"color: #ADBAC7\"> cccccccc.r0<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># openssl s_client -connect vcenter01.example.local:443 -verify_return_error -brief \\<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">    <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">CApath <\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">var<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lib<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">vmware<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">marvin<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">trust<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">lin <\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">verify_hostname vcenter01.example.local <\/span><span style=\"color: #F47067\">&lt;\/<\/span><span style=\"color: #ADBAC7\">dev<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #ADBAC7\">null<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">CONNECTION<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">ESTABLISHED<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Protocol version: TLSv1.2<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Peer certificate: C <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">XX<\/span><span style=\"color: #ADBAC7\">, O <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> Example Corp, <\/span><span style=\"color: #6CB6FF\">OU<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #6CB6FF\">IT<\/span><span style=\"color: #ADBAC7\">, <\/span><span style=\"color: #6CB6FF\">CN<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #F47067\">=<\/span><span style=\"color: #ADBAC7\"> vcenter01.example.local<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Verification: <\/span><span style=\"color: #6CB6FF\">OK<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">Verified peername: vcenter01.example.local<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">DONE<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">Now Restart the two pods. compute-service waits for property-collector at startup, so the collector should be up and answering before the other one is restarted.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<div class=\"wp-block-kevinbatdorf-code-block-pro\" data-code-block-pro-font-family=\"Code-Pro-JetBrains-Mono\" style=\"font-size:14px;font-family:Code-Pro-JetBrains-Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,monospace;line-height:1.25rem;--cbp-tab-width:2;tab-size:var(--cbp-tab-width, 2)\"><span style=\"display:block;padding:16px 0 0 16px;margin-bottom:-1px;width:100%;text-align:left;background-color:#22272e\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"54\" height=\"14\" viewBox=\"0 0 54 14\"><g fill=\"none\" fill-rule=\"evenodd\" transform=\"translate(1 1)\"><circle cx=\"6\" cy=\"6\" r=\"6\" fill=\"#FF5F56\" stroke=\"#E0443E\" stroke-width=\".5\"><\/circle><circle cx=\"26\" cy=\"6\" r=\"6\" fill=\"#FFBD2E\" stroke=\"#DEA123\" stroke-width=\".5\"><\/circle><circle cx=\"46\" cy=\"6\" r=\"6\" fill=\"#27C93F\" stroke=\"#1AAB29\" stroke-width=\".5\"><\/circle><\/g><\/svg><\/span><span role=\"button\" tabindex=\"0\" style=\"color:#adbac7;display:none\" aria-label=\"Copy\" class=\"code-block-pro-copy-button\"><pre class=\"code-block-pro-copy-button-pre\" aria-hidden=\"true\"><textarea class=\"code-block-pro-copy-button-textarea\" tabindex=\"-1\" aria-hidden=\"true\" readonly>\nvxm01:~ # kubectl delete pod -n helium property-collector-xxxxxxxxxx-aaaaa\npod \"property-collector-xxxxxxxxxx-aaaaa\" deleted\nvxm01:~ # kubectl delete pod -n helium compute-service-xxxxxxxxxx-aaaaa\npod \"compute-service-xxxxxxxxxx-aaaaa\" deleted\n \nvxm01:~ # kubectl get pods -n helium | grep -E \"property-collector|compute-service\"\nproperty-collector-xxxxxxxxxx-bbbbb   1\/1     Running   0     85s\ncompute-service-xxxxxxxxxx-bbbbb      1\/1     Running   0     78s\n \nvxm01:~ # kubectl get pods -n helium | grep -v -E \"Running|Completed\"\nNAME                                  READY   STATUS    RESTARTS   AGE\n<\/textarea><\/pre><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"width:24px;height:24px\" fill=\"none\" viewBox=\"0 0 24 24\" stroke=\"currentColor\" stroke-width=\"2\"><path class=\"with-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M4.5 12.75l6 6 9-13.5\"><\/path><path class=\"without-check\" stroke-linecap=\"round\" stroke-linejoin=\"round\" d=\"M16.5 8.25V6a2.25 2.25 0 00-2.25-2.25H6A2.25 2.25 0 003.75 6v8.25A2.25 2.25 0 006 16.5h2.25m8.25-8.25H18a2.25 2.25 0 012.25 2.25V18A2.25 2.25 0 0118 20.25h-7.5A2.25 2.25 0 018.25 18v-1.5m8.25-8.25h-6a2.25 2.25 0 00-2.25 2.25v6\"><\/path><\/svg><\/span><pre class=\"shiki github-dark-dimmed\" style=\"background-color: #22272e\" tabindex=\"0\"><code><span class=\"line\"><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl delete pod -n helium property-collector-xxxxxxxxxx-aaaaa<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">pod <\/span><span style=\"color: #96D0FF\">&quot;property-collector-xxxxxxxxxx-aaaaa&quot;<\/span><span style=\"color: #ADBAC7\"> deleted<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl delete pod -n helium compute-service-xxxxxxxxxx-aaaaa<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">pod <\/span><span style=\"color: #96D0FF\">&quot;compute-service-xxxxxxxxxx-aaaaa&quot;<\/span><span style=\"color: #ADBAC7\"> deleted<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get pods -n helium | grep -E &quot;property-collector|compute-service&quot;<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">property<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">collector<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">bbbbb   <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Running   <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #ADBAC7\">     <\/span><span style=\"color: #FF938A; font-style: italic\">85s<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">compute<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">service<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">xxxxxxxxxx<\/span><span style=\"color: #F47067\">-<\/span><span style=\"color: #ADBAC7\">bbbbb      <\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #F47067\">\/<\/span><span style=\"color: #6CB6FF\">1<\/span><span style=\"color: #ADBAC7\">     Running   <\/span><span style=\"color: #6CB6FF\">0<\/span><span style=\"color: #ADBAC7\">     <\/span><span style=\"color: #FF938A; font-style: italic\">78s<\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\"> <\/span><\/span>\n<span class=\"line\"><span style=\"color: #ADBAC7\">vxm01:<\/span><span style=\"color: #F47067\">~<\/span><span style=\"color: #ADBAC7\"> <\/span><span style=\"color: #768390\"># kubectl get pods -n helium | grep -v -E &quot;Running|Completed&quot;<\/span><\/span>\n<span class=\"line\"><span style=\"color: #6CB6FF\">NAME<\/span><span style=\"color: #ADBAC7\">                                  <\/span><span style=\"color: #6CB6FF\">READY<\/span><span style=\"color: #ADBAC7\">   <\/span><span style=\"color: #6CB6FF\">STATUS<\/span><span style=\"color: #ADBAC7\">    <\/span><span style=\"color: #6CB6FF\">RESTARTS<\/span><span style=\"color: #ADBAC7\">   <\/span><span style=\"color: #6CB6FF\">AGE<\/span><\/span>\n<span class=\"line\"><\/span><\/code><\/pre><\/div>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p class=\"wp-block-paragraph\">An empty list. Every pod in the namespace was running again and the VxRail Page inside of vSphere was visible. That&#8217;s it from this Blog Post, if you have any questions use the comment section below or contact Dell Support.<\/p>\n\n\n\n<div style=\"height:34px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n","protected":false},"excerpt":{"rendered":"<p>The VxRail Manager VM had been rebooted and the VxRail pages in the vSphere Client no longer loaded. VxRail Manager runs its services as Kubernetes pods on the appliance itself, so the first look was at one of the pods that was obviously unhappy. Two details matter here. The container starts and finishes within the [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":true,"template":"","format":"standard","meta":{"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_feature_clip_id":0,"_jetpack_memberships_contains_paid_content":false,"footnotes":"","jetpack_post_was_ever_published":true},"categories":[1,18,48],"tags":[],"class_list":["post-14470","post","type-post","status-publish","format-standard","hentry","category-blog","category-dellemc","category-dell-vxrail"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.6 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff - irgNET<\/title>\n<meta name=\"description\" content=\"VxRail Manager pods stuck in CrashLoopBackOff after a reboot? A hung api-gateway pod and a missing vCenter CA in the trust store were the cause.\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff - irgNET\" \/>\n<meta property=\"og:description\" content=\"VxRail Manager pods stuck in CrashLoopBackOff after a reboot? A hung api-gateway pod and a missing vCenter CA in the trust store were the cause.\" \/>\n<meta property=\"og:url\" content=\"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/\" \/>\n<meta property=\"og:site_name\" content=\"irgNET\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-05T12:03:22+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2026-10-06T09:18:34+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png\" \/>\n<meta name=\"author\" content=\"H. Cemre G\u00fcnay\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:creator\" content=\"@irg_net\" \/>\n<meta name=\"twitter:site\" content=\"@irg_net\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"H. Cemre G\u00fcnay\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"4 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/\"},\"author\":{\"name\":\"H. Cemre G\u00fcnay\",\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#\\\/schema\\\/person\\\/71b46e79e8c12bdc8e610214a6621a66\"},\"headline\":\"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff\",\"datePublished\":\"2026-10-05T12:03:22+00:00\",\"dateModified\":\"2026-10-06T09:18:34+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/\"},\"wordCount\":854,\"commentCount\":0,\"publisher\":{\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#organization\"},\"image\":{\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/irg-net.com\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/image-27.png\",\"articleSection\":[\"Blog\",\"Dell\",\"Dell VxRail\"],\"inLanguage\":\"en\",\"potentialAction\":[{\"@type\":\"CommentAction\",\"name\":\"Comment\",\"target\":[\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#respond\"]}]},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/\",\"url\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/\",\"name\":\"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff - irgNET\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/irg-net.com\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/image-27.png\",\"datePublished\":\"2026-10-05T12:03:22+00:00\",\"dateModified\":\"2026-10-06T09:18:34+00:00\",\"description\":\"VxRail Manager pods stuck in CrashLoopBackOff after a reboot? A hung api-gateway pod and a missing vCenter CA in the trust store were the cause.\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#breadcrumb\"},\"inLanguage\":\"en\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en\",\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#primaryimage\",\"url\":\"https:\\\/\\\/i0.wp.com\\\/irg-net.com\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/image-27.png?fit=314%2C275&ssl=1\",\"contentUrl\":\"https:\\\/\\\/i0.wp.com\\\/irg-net.com\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/image-27.png?fit=314%2C275&ssl=1\",\"width\":314,\"height\":275},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/irg-net.com\\\/index.php\\\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#website\",\"url\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/\",\"name\":\"irgNET\",\"description\":\"a virtual Knowledge Base\",\"publisher\":{\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#organization\"},\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en\"},{\"@type\":\"Organization\",\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#organization\",\"name\":\"irg-NET\",\"url\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en\",\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#\\\/schema\\\/logo\\\/image\\\/\",\"url\":\"https:\\\/\\\/i2.wp.com\\\/irg-net.com\\\/wp-content\\\/uploads\\\/2021\\\/06\\\/cropped-FINAL-Removed-Alpha.png?fit=2048%2C2048&ssl=1\",\"contentUrl\":\"https:\\\/\\\/i2.wp.com\\\/irg-net.com\\\/wp-content\\\/uploads\\\/2021\\\/06\\\/cropped-FINAL-Removed-Alpha.png?fit=2048%2C2048&ssl=1\",\"width\":2048,\"height\":2048,\"caption\":\"irg-NET\"},\"image\":{\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#\\\/schema\\\/logo\\\/image\\\/\"},\"sameAs\":[\"https:\\\/\\\/x.com\\\/irg_net\",\"https:\\\/\\\/www.linkedin.com\\\/in\\\/h-cemre-g\u00fcnay-2b61601b4\\\/\",\"https:\\\/\\\/www.youtube.com\\\/channel\\\/UC8GknhO5YMAkAoVxppu6aaQ\"]},{\"@type\":\"Person\",\"@id\":\"https:\\\/\\\/irgnet-ga3w6nfbvw.live-website.com\\\/#\\\/schema\\\/person\\\/71b46e79e8c12bdc8e610214a6621a66\",\"name\":\"H. Cemre G\u00fcnay\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en\",\"@id\":\"https:\\\/\\\/secure.gravatar.com\\\/avatar\\\/15f5d90a0ec651cb5003605e56e8ba104f93bc11230562284db56c845df9d54d?s=96&r=g\",\"url\":\"https:\\\/\\\/secure.gravatar.com\\\/avatar\\\/15f5d90a0ec651cb5003605e56e8ba104f93bc11230562284db56c845df9d54d?s=96&r=g\",\"contentUrl\":\"https:\\\/\\\/secure.gravatar.com\\\/avatar\\\/15f5d90a0ec651cb5003605e56e8ba104f93bc11230562284db56c845df9d54d?s=96&r=g\",\"caption\":\"H. Cemre G\u00fcnay\"},\"sameAs\":[\"http:\\\/\\\/192.168.34.155\\\/wordpress\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff - irgNET","description":"VxRail Manager pods stuck in CrashLoopBackOff after a reboot? A hung api-gateway pod and a missing vCenter CA in the trust store were the cause.","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/","og_locale":"en_US","og_type":"article","og_title":"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff - irgNET","og_description":"VxRail Manager pods stuck in CrashLoopBackOff after a reboot? A hung api-gateway pod and a missing vCenter CA in the trust store were the cause.","og_url":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/","og_site_name":"irgNET","article_published_time":"2026-10-05T12:03:22+00:00","article_modified_time":"2026-10-06T09:18:34+00:00","og_image":[{"url":"https:\/\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png","type":"","width":"","height":""}],"author":"H. Cemre G\u00fcnay","twitter_card":"summary_large_image","twitter_creator":"@irg_net","twitter_site":"@irg_net","twitter_misc":{"Written by":"H. Cemre G\u00fcnay","Est. reading time":"4 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#article","isPartOf":{"@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/"},"author":{"name":"H. Cemre G\u00fcnay","@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#\/schema\/person\/71b46e79e8c12bdc8e610214a6621a66"},"headline":"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff","datePublished":"2026-10-05T12:03:22+00:00","dateModified":"2026-10-06T09:18:34+00:00","mainEntityOfPage":{"@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/"},"wordCount":854,"commentCount":0,"publisher":{"@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#organization"},"image":{"@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#primaryimage"},"thumbnailUrl":"https:\/\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png","articleSection":["Blog","Dell","Dell VxRail"],"inLanguage":"en","potentialAction":[{"@type":"CommentAction","name":"Comment","target":["https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#respond"]}]},{"@type":"WebPage","@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/","url":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/","name":"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff - irgNET","isPartOf":{"@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#primaryimage"},"image":{"@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#primaryimage"},"thumbnailUrl":"https:\/\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png","datePublished":"2026-10-05T12:03:22+00:00","dateModified":"2026-10-06T09:18:34+00:00","description":"VxRail Manager pods stuck in CrashLoopBackOff after a reboot? A hung api-gateway pod and a missing vCenter CA in the trust store were the cause.","breadcrumb":{"@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#breadcrumb"},"inLanguage":"en","potentialAction":[{"@type":"ReadAction","target":["https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/"]}]},{"@type":"ImageObject","inLanguage":"en","@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#primaryimage","url":"https:\/\/i0.wp.com\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png?fit=314%2C275&ssl=1","contentUrl":"https:\/\/i0.wp.com\/irg-net.com\/wp-content\/uploads\/2026\/10\/image-27.png?fit=314%2C275&ssl=1","width":314,"height":275},{"@type":"BreadcrumbList","@id":"https:\/\/irg-net.com\/index.php\/recovering-a-vxrail-manager-with-nearly-every-pod-in-crashloopbackoff\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/"},{"@type":"ListItem","position":2,"name":"Recovering a VxRail Manager with nearly every pod in CrashLoopBackOff"}]},{"@type":"WebSite","@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#website","url":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/","name":"irgNET","description":"a virtual Knowledge Base","publisher":{"@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en"},{"@type":"Organization","@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#organization","name":"irg-NET","url":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/","logo":{"@type":"ImageObject","inLanguage":"en","@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#\/schema\/logo\/image\/","url":"https:\/\/i2.wp.com\/irg-net.com\/wp-content\/uploads\/2021\/06\/cropped-FINAL-Removed-Alpha.png?fit=2048%2C2048&ssl=1","contentUrl":"https:\/\/i2.wp.com\/irg-net.com\/wp-content\/uploads\/2021\/06\/cropped-FINAL-Removed-Alpha.png?fit=2048%2C2048&ssl=1","width":2048,"height":2048,"caption":"irg-NET"},"image":{"@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/x.com\/irg_net","https:\/\/www.linkedin.com\/in\/h-cemre-g\u00fcnay-2b61601b4\/","https:\/\/www.youtube.com\/channel\/UC8GknhO5YMAkAoVxppu6aaQ"]},{"@type":"Person","@id":"https:\/\/irgnet-ga3w6nfbvw.live-website.com\/#\/schema\/person\/71b46e79e8c12bdc8e610214a6621a66","name":"H. Cemre G\u00fcnay","image":{"@type":"ImageObject","inLanguage":"en","@id":"https:\/\/secure.gravatar.com\/avatar\/15f5d90a0ec651cb5003605e56e8ba104f93bc11230562284db56c845df9d54d?s=96&r=g","url":"https:\/\/secure.gravatar.com\/avatar\/15f5d90a0ec651cb5003605e56e8ba104f93bc11230562284db56c845df9d54d?s=96&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/15f5d90a0ec651cb5003605e56e8ba104f93bc11230562284db56c845df9d54d?s=96&r=g","caption":"H. Cemre G\u00fcnay"},"sameAs":["http:\/\/192.168.34.155\/wordpress"]}]}},"jetpack_sharing_enabled":true,"jetpack-related-posts":[],"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/posts\/14470","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/comments?post=14470"}],"version-history":[{"count":2,"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/posts\/14470\/revisions"}],"predecessor-version":[{"id":14473,"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/posts\/14470\/revisions\/14473"}],"wp:attachment":[{"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/media?parent=14470"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/categories?post=14470"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/irg-net.com\/index.php\/wp-json\/wp\/v2\/tags?post=14470"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}