<html xmlns="http://www.w3.org/TR/REC-html40" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:w="urn:schemas-microsoft-com:office:word">
<head>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
<meta content="Microsoft Word 15 (filtered medium)" name="Generator"/>
<!--[if !mso]><style>v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style><![endif]--><style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
p.xmsonormal, li.xmsonormal, div.xmsonormal
{mso-style-name:x_msonormal;
margin:0in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.xmsolistparagraph, li.xmsolistparagraph, div.xmsolistparagraph
{mso-style-name:x_msolistparagraph;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.xxmsonormal, li.xxmsonormal, div.xxmsonormal
{mso-style-name:x_xmsonormal;
margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
p.xxxmsonormal, li.xxxmsonormal, div.xxxmsonormal
{mso-style-name:x_xxmsonormal;
margin:0in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
span.EmailStyle27
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:22900815;
mso-list-type:hybrid;
mso-list-template-ids:-454004700 849382296 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l0:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;
mso-fareast-font-family:Calibri;
mso-bidi-font-family:"Times New Roman";}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l1
{mso-list-id:847250463;
mso-list-template-ids:2032854298;}
@list l1:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2
{mso-list-id:1297376193;
mso-list-template-ids:1751163650;}
@list l2:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level2
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level3
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level5
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level6
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level8
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level9
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3
{mso-list-id:1407608752;
mso-list-template-ids:1269586964;}
@list l3:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level2
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level3
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level5
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level6
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level8
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level9
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4
{mso-list-id:1587499284;
mso-list-template-ids:1757948636;}
@list l4:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level2
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level3
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level5
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level6
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level8
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4:level9
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-US" link="#0563C1" vlink="purple">
<div class="WordSection1">
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>Hi Eric,<o:p></o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'><o:p> </o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>I was able to figure out the issue. Thanks for the pointers.<o:p></o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'><o:p> </o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>While bringing up data network, I had used the below command for the both the workers as I wanted sriov interfaces.<o:p></o:p></span></p>
<ul style="margin-top:0in" type="disc">
<li class="MsoListParagraph" style="margin-left:0in;mso-list:l0 level1 lfo5"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>system host-if-modify $NODE -n sriov0 -c
<b>pci-sriov</b> -N 8 --vf-driver=netdevice $DATA0IFUUID<o:p></o:p></span></li></ul>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>I m not sure if this is the reason the why worker nodes were in disabled and offline state.<o:p></o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'><o:p> </o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>When I changed the class from
<b>pci-sriov</b> to “<b>data</b>”, worker nodes are enabled and operational status is available.<o:p></o:p></span></p>
<ul style="margin-top:0in" type="disc">
<li class="MsoListParagraph" style="margin-left:0in;mso-list:l0 level1 lfo5"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>system host-if-modify -m 1500 -n data0 -c
<b>data</b> worker-0 2ded1706-1cd3-43d3-80d2-c7e0ce2b79c<o:p></o:p></span></li></ul>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'><o:p> </o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>Do you see any problem in first command.<o:p></o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'><o:p> </o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>Regards,<o:p></o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>Sriram<o:p></o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'><o:p> </o:p></span></p>
<p class="MsoNormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>From:</span></b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'> MacDonald, Eric <Eric.MacDonald@windriver.com>
<br/>
<b>Sent:</b> Tuesday, October 6, 2020 12:36 AM<br/>
<b>To:</b> Dharwadkar, Sriram <Sriram.Dharwadkar@commscope.com><br/>
<b>Subject:</b> Re: [Starlingx-discuss] [Distributed StarlingX-4.0] - Controller-0 in edge cloud shows availability as degraded<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<p class="MsoNormal" style="mso-line-height-alt:.75pt;background:white"><!--removed inky banner--></p></div><p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal" style="background:white"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif;color:black;background:white'>Sriram,</span><span style='font-family:"Calibri",sans-serif;color:black'><o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'><o:p> </o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'>For the worker node configuration failures, look or Error Warn logs in /var/log/puppet/* logs.<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'><o:p> </o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'>Those logs should help you understand what config failed.<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'><o:p> </o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'>Eric.<o:p></o:p></span></p>
</div>
<div align="center" class="MsoNormal" style="text-align:center">
<hr align="center" size="2" width="98%"/>
</div>
<div id="divRplyFwdMsg">
<p class="MsoNormal"><b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif;color:black'>From:</span></b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif;color:black'> Dharwadkar, Sriram <<a href="mailto:Sriram.Dharwadkar@commscope.com">Sriram.Dharwadkar@commscope.com</a>><br/>
<b>Sent:</b> Monday, October 5, 2020 2:57 PM<br/>
<b>To:</b> MacDonald, Eric <<a href="mailto:Eric.MacDonald@windriver.com">Eric.MacDonald@windriver.com</a>><br/>
<b>Subject:</b> RE: [Starlingx-discuss] [Distributed StarlingX-4.0] - Controller-0 in edge cloud shows availability as degraded</span>
<o:p></o:p></p>
<div>
<p class="MsoNormal"> <o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="xmsonormal">Hi Eric,<o:p></o:p></p>
<p class="xmsonormal"> <o:p></o:p></p>
<p class="xmsonormal">Thanks for the explanation. <o:p></o:p></p>
<p class="xmsonormal"> <o:p></o:p></p>
<p class="xmsonormal">Initially I was planning to bring up 2 controller (All in one stand alone) + 1 worker node. For this kind of configuration, we would need high capacity h/w for all the nodes which may not be available. That’s the reason I deployed 1 standard
controller -with storage + 2 worker nodes configuration. <o:p></o:p></p>
<p class="xmsonormal">Controller node came up w/o any issues, worker nodes also came up after system host-unlock worker-0 and worker-1. But I could see these alarms<o:p></o:p></p>
<p class="xmsonormal"> <o:p></o:p></p>
<ul style="margin-top:0in" type="disc">
<li class="xmsolistparagraph" style="margin-left:0in;mso-list:l1 level1 lfo1">worker-1 experienced a service-affecting failure. Auto-recovery | host=worker-1 | critical | 2020-10-05T |<o:p></o:p></li></ul>
<p class="xmsonormal" style="text-indent:.5in">in progress. Manual Lock and Unlock may be required if auto-recovery is unsuccessful<o:p></o:p></p>
<ul style="margin-top:0in" type="disc">
<li class="xmsolistparagraph" style="margin-left:0in;mso-list:l3 level1 lfo2">worker-1 experienced a configuration failure.<o:p></o:p></li></ul>
<p class="xmsonormal"> <o:p></o:p></p>
<ul style="margin-top:0in" type="disc">
<li class="xmsolistparagraph" style="margin-left:0in;mso-list:l4 level1 lfo3">worker-0 experienced a service-affecting failure. Auto-recovery | host=worker-0 | critical | 2020-10-05T |<o:p></o:p></li></ul>
<p class="xmsonormal"> in progress. Manual Lock and Unlock may be required if auto-recovery is unsuccessful<o:p></o:p></p>
<ul style="margin-top:0in" type="disc">
<li class="xmsolistparagraph" style="margin-left:0in;mso-list:l2 level1 lfo4">worker-0 experienced a configuration failure.<o:p></o:p></li></ul>
<p class="xmsonormal"> <o:p></o:p></p>
<p class="xmsonormal">I did lock and unlock of nodes. After nodes came up, I still see the same issue.
<o:p></o:p></p>
<p class="xmsonormal">How to check for configuration failures?<o:p></o:p></p>
<p class="xmsonormal">Also there is taint on both the worker nodes “Taints: services=disabled:NoExecute”, which I could see using kubectl describe node worker-0 and kubectl describe node worker-1.
<o:p></o:p></p>
<p class="xmsonormal">So, I don’t see Sriov CNI and Sriov DP being deployed in worker nodes.<o:p></o:p></p>
<p class="xmsonormal">Regards,<o:p></o:p></p>
<p class="xmsonormal">Sriram <o:p></o:p></p>
<p class="xmsonormal"> <o:p></o:p></p>
<p class="xmsonormal"> <o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="xmsonormal"><b>From:</b> MacDonald, Eric <<a href="mailto:Eric.MacDonald@windriver.com">Eric.MacDonald@windriver.com</a>>
<br/>
<b>Sent:</b> Monday, October 5, 2020 9:14 PM<br/>
<b>To:</b> Dharwadkar, Sriram <<a href="mailto:Sriram.Dharwadkar@commscope.com">Sriram.Dharwadkar@commscope.com</a>><br/>
<b>Subject:</b> Re: [Starlingx-discuss] [Distributed StarlingX-4.0] - Controller-0 in edge cloud shows availability as degraded<o:p></o:p></p>
</div>
</div>
<p class="xmsonormal"> <o:p></o:p></p>
<div>
<p class="xmsonormal"> <o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="color:black;background:white">Sriram,</span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black"> </span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black">The AIO controller has a lot more work to do during startup as it contains both control and compute functions.</span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black">As a result, you may temporarily see CPU resource utilization logs during the latter stages of the provisioning.</span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black"> </span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black">Now that you went to standard config, you might see another degrade for a short period of time following the unlock of the second controller during initial filesystem
sync.</span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black"> </span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black">Again, degrade is general, but if you see a degrade there should be an alarm that represents the reason for the degrade.</span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black"> </span><o:p></o:p></p>
</div>
<div>
<p class="xmsonormal" style="background:white"><span style="font-size:12.0pt;color:black">Eric.</span><o:p></o:p></p>
</div>
<div align="center" class="MsoNormal" style="text-align:center"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>
<hr align="center" size="1" width="98%"/>
</span></div>
<div id="x_divRplyFwdMsg">
<p class="xmsonormal"><b><span style="color:black">From:</span></b><span style="color:black"> Dharwadkar, Sriram <<a href="mailto:Sriram.Dharwadkar@commscope.com">Sriram.Dharwadkar@commscope.com</a>><br/>
<b>Sent:</b> Monday, October 5, 2020 10:44 AM<br/>
<b>To:</b> MacDonald, Eric <<a href="mailto:Eric.MacDonald@windriver.com">Eric.MacDonald@windriver.com</a>><br/>
<b>Subject:</b> RE: [Starlingx-discuss] [Distributed StarlingX-4.0] - Controller-0 in edge cloud shows availability as degraded</span>
<o:p></o:p></p>
<div>
<p class="xmsonormal"> <o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="xxmsonormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>Thanks Eric.
</span><o:p></o:p></p>
<p class="xxmsonormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>I reinstalled the system with iso again. I didn’t see this error again with different configuration (I selected standard controller, instead of all in one duplex).</span><o:p></o:p></p>
<p class="xxmsonormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>I will check for alarms in case of any issues further.</span><o:p></o:p></p>
<p class="xxmsonormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'> </span><o:p></o:p></p>
<p class="xxmsonormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>Regards,</span><o:p></o:p></p>
<p class="xxmsonormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>Sriram</span><o:p></o:p></p>
<p class="xxmsonormal"><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'> </span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="xxmsonormal"><b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'>From:</span></b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif'> MacDonald, Eric <<a href="mailto:Eric.MacDonald@windriver.com">Eric.MacDonald@windriver.com</a>>
<br/>
<b>Sent:</b> Monday, October 5, 2020 5:55 PM<br/>
<b>To:</b> Dharwadkar, Sriram <<a href="mailto:Sriram.Dharwadkar@commscope.com">Sriram.Dharwadkar@commscope.com</a>><br/>
<b>Subject:</b> Re: [Starlingx-discuss] [Distributed StarlingX-4.0] - Controller-0 in edge cloud shows availability as degraded</span><o:p></o:p></p>
</div>
</div>
<p class="xxmsonormal"> <o:p></o:p></p>
<div>
<p class="xxmsonormal"> <o:p></o:p></p>
</div>
<div>
<p class="xxmsonormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'>What does 'fm alarm-list' show ?</span><o:p></o:p></p>
</div>
<div>
<p class="xxmsonormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'> </span><o:p></o:p></p>
</div>
<div>
<p class="xxmsonormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'>As a general rule ; if a host is degraded there should be an alarm raised for that degraded condition.</span><o:p></o:p></p>
</div>
<div>
<p class="xxmsonormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'> </span><o:p></o:p></p>
</div>
<div>
<p class="xxmsonormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'>Eric MacDonald</span><o:p></o:p></p>
</div>
<div>
<p class="xxmsonormal" style="background:white"><span style='font-family:"Calibri",sans-serif;color:black'>StarlingX Maintenance</span><o:p></o:p></p>
</div>
<div align="center" class="MsoNormal" style="text-align:center">
<hr align="center" size="1" width="98%"/>
</div>
<div id="x_x_divRplyFwdMsg">
<p class="xxmsonormal"><b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif;color:black'>From:</span></b><span style='font-size:11.0pt;font-family:"Calibri",sans-serif;color:black'> Dharwadkar, Sriram <<a href="mailto:Sriram.Dharwadkar@commscope.com">Sriram.Dharwadkar@commscope.com</a>><br/>
<b>Sent:</b> Monday, October 5, 2020 4:00 AM<br/>
<b>To:</b> <a href="mailto:starlingx-discuss@lists.starlingx.io">starlingx-discuss@lists.starlingx.io</a> <<a href="mailto:starlingx-discuss@lists.starlingx.io">starlingx-discuss@lists.starlingx.io</a>><br/>
<b>Subject:</b> [Starlingx-discuss] [Distributed StarlingX-4.0] - Controller-0 in edge cloud shows availability as degraded</span>
<o:p></o:p></p>
<div>
<p class="xxmsonormal"> <o:p></o:p></p>
</div>
</div>
<div>
<div>
<p class="xxxmsonormal">Hi,<o:p></o:p></p>
<p class="xxxmsonormal"> <o:p></o:p></p>
<p class="xxxmsonormal"><span style="font-size:12.0pt;color:#222222;background:white">I have installed distributed starlingx 4.0 in "All in one Duplex" mode. There are two nodes in the central cloud and two in the edge cloud. Central cloud is up and running.
</span><o:p></o:p></p>
<p class="xxxmsonormal"><span style="font-size:12.0pt;color:#222222;background:white">For the edge cloud configuration, in the bootstrap override file, I have configured the private registry. From the central cloud, I was able to add the edge cloud. Images
required for starlingX installation are downloaded from private registry and installation goes through w/o any issues.
</span><o:p></o:p></p>
<p class="xxxmsonormal"><span style="font-size:12.0pt;color:#222222;background:white"> </span><o:p></o:p></p>
<p class="xxxmsonormal">[sysadmin@controller-0 ~(keystone_admin)]$ dcmanager subcloud list<o:p></o:p></p>
<p class="xxxmsonormal">+----+------+------------+--------------+---------------+---------+<o:p></o:p></p>
<p class="xxxmsonormal">| id | name | management | availability | deploy status | sync |<o:p></o:p></p>
<p class="xxxmsonormal">+----+------+------------+--------------+---------------+---------+<o:p></o:p></p>
<p class="xxxmsonormal">| 47 | edge | unmanaged | offline | complete | unknown |<o:p></o:p></p>
<p class="xxxmsonormal">+----+------+------------+--------------+---------------+---------+<o:p></o:p></p>
<p class="xxxmsonormal">[sysadmin@controller-0 ~(keystone_admin)]$<o:p></o:p></p>
<p class="xxxmsonormal"> <o:p></o:p></p>
<p class="xxxmsonormal">Then I followed the steps mentioned in the document <a href="https://secure-web.cisco.com/18J4qBN9kVYaO0J69T77cfP7fEjqwtAkIFtWZbtdqn4HlYyYbtMkCwP5oK0Yk_eMzqR98tqYvWy0Qaac8jSP7Q0UM7AN8WsL5vG0Yg-0K24wPsPpY8dxyPIqnVcTbc6aZKklkk6wfv5MMxoElG_wXSHKluGnA2vOSOf5oH2BWCnIJk1v90YZi52kR91w7N3UJ3L_NCSG2yDQyb8wCmq0CnSeLYqW0BdekK4u2vjekFUXhQ5ER0E8j9aBWWCoBpoX2CMA88i24PIuojQyTKJUXAx9A5DXyIuy2XV8NXplGFHnA4uKPH3KpupU7Porkz4TU/https%3A%2F%2Fdocs.starlingx.io%2Fdeploy_install_guides%2Fr4_release%2Fbare_metal%2Faio_duplex_install_kubernetes.html%23configure-controller-0">
https://docs.starlingx.io/deploy_install_guides/r4_release/bare_metal/aio_duplex_install_kubernetes.html#configure-controller-0</a><o:p></o:p></p>
<p class="xxxmsonormal">And finally did unlock of controller-0. System went for reboot and it came up successfully. After I see availability as “degraded”<o:p></o:p></p>
<p class="xxxmsonormal"> <o:p></o:p></p>
<p class="xxxmsonormal">[sysadmin@controller-0 log(keystone_admin)]$ system host-list<o:p></o:p></p>
<p class="xxxmsonormal">+----+--------------+-------------+----------------+-------------+--------------+<o:p></o:p></p>
<p class="xxxmsonormal">| id | hostname | personality | administrative | operational | availability |<o:p></o:p></p>
<p class="xxxmsonormal">+----+--------------+-------------+----------------+-------------+--------------+<o:p></o:p></p>
<p class="xxxmsonormal">| 1 | controller-0 | controller | unlocked | enabled |
<b>degraded</b> |<o:p></o:p></p>
<p class="xxxmsonormal">+----+--------------+-------------+----------------+-------------+--------------+<o:p></o:p></p>
<p class="xxxmsonormal"> <o:p></o:p></p>
<p class="xxxmsonormal">Tail -f /var/log/sysinv.log shows – prerequisites not met..<o:p></o:p></p>
<p class="xxxmsonormal">ysinv 2020-10-05 07:57:12.271 96246 INFO ceph_client [-] Result: {u'waiting': [], u'has_failed': False, u'state': u'success', u'is_waiting': False, u'running': [], u'failed': [], u'finished': [{u'outb': u'{"fsid":"50634828-68b2-43c4-aaa0-ebf53f6e675a","health":{"checks":{},"status":"HEALTH_OK","overall_status":"HEALTH_WARN"},"election_epoch":7,"quorum":[0],"quorum_names":["controller"],"monmap":{"epoch":1,"fsid":"50634828-68b2-43c4-aaa0-ebf53f6e675a","modified":"2020-10-05
06:53:11.461060","created":"2020-10-05 06:53:11.461060","features":{"persistent":["kraken","luminous","mimic","osdmap-prune"],"optional":[]},"mons":[{"rank":0,"name":"controller","addr":"192.168.22.101:6789/0","public_addr":"192.168.22.101:6789/0"}]},"osdmap":{"osdmap":{"epoch":10,"num_osds":1,"num_up_osds":1,"num_in_osds":1,"full":false,"nearfull":false,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[],"num_pgs":0,"num_pools":0,"num_objects":0,"data_bytes":0,"bytes_used":112181248,"bytes_avail":1197865828352,"bytes_total":1197978009600},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":48,"active_gid":24132,"active_name":"controller-0","active_addr":"192.168.22.102:6804/93283","available":true,"standbys":[],"modules":["restful"],"available_modules":[{"name":"balancer","can_run":true,"error_string":""},{"name":"dashboard","can_run":false,"error_string":"Frontend
assets not found: incomplete build?"},{"name":"hello","can_run":true,"error_string":""},{"name":"iostat","can_run":true,"error_string":""},{"name":"localpool","can_run":true,"error_string":""},{"name":"prometheus","can_run":true,"error_string":""},{"name":"restful","can_run":true,"error_string":""},{"name":"selftest","can_run":true,"error_string":""},{"name":"smart","can_run":true,"error_string":""},{"name":"status","can_run":true,"error_string":""},{"name":"telegraf","can_run":true,"error_string":""},{"name":"telemetry","can_run":true,"error_string":""},{"name":"zabbix","can_run":true,"error_string":""}],"services":{"restful":"<a href="https://secure-web.cisco.com/1-3m0_FGRL2G021BGsNLb_f3jpLYppw2gODW7dMb6VvFuoueSl4fyedH82VkkCK6Ae8SzWg-igdi1OyjS0YrO9j82utfiYEYgoTxjfP2KErzt6lrGH6znNUjgTdaKCfg1xklskKvk_LLCOtuDSJVAzI0H6EfvbG3XoTgDTSWPud-Fa5zMZzldvYre7baJ4onf2Koj2ofEbG_zPpw0bIiNIRuDOXKMDhdio-xRpBEsX7dyIdnPW6Bf_Ee7W79GKKa57hG5rJuP5LwPHwUkDDXIM2v5MYDtblOLhDSQawNs83BnecirGDByae6H02rDhviH/https%3A%2F%2Fcontroller-0%3A7999%2F">https://controller-0:7999/</a>"}},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}\n',
u'outs': u'', u'command': u'status format=json'}], u'is_finished': True, u'id': u'140404196232080'}<o:p></o:p></p>
<p class="xxxmsonormal"><b>sysinv 2020-10-05 07:57:12.284 96246 INFO sysinv.conductor.manager [-] Platform managed application platform-integ-apps: Prerequisites not met.</b><o:p></o:p></p>
<p class="xxxmsonormal">sysinv 2020-10-05 07:57:12.286 96246 INFO sysinv.conductor.manager [-] Platform managed application oidc-auth-apps: Prerequisites not met.<o:p></o:p></p>
<p class="xxxmsonormal">sysinv 2020-10-05 07:57:12.291 96246 INFO sysinv.api.controllers.v1.rest_api [-] GET cmd:http://localhost:30001/nfvi-plugins/v1/sw-update hdr:{'Content-type': 'application/json', 'User-Agent': 'sysinv/1.0'} payload:None<o:p></o:p></p>
<p class="xxxmsonormal">sysinv 2020-10-05 07:57:12.293 96246 INFO sysinv.ap<o:p></o:p></p>
<p class="xxxmsonormal"> <o:p></o:p></p>
<p class="xxxmsonormal">I’m not sure why availability is shown as degraded. Any help would be appreciated. Let me know if any logs are required.
<o:p></o:p></p>
<p class="xxxmsonormal"> <o:p></o:p></p>
<p class="xxxmsonormal">Regards,<o:p></o:p></p>
<p class="xxxmsonormal">Sriram<o:p></o:p></p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</body>
</html>